diff options
149 files changed, 5820 insertions, 4958 deletions
diff --git a/.luacheckrc b/.luacheckrc index c242bddd8..353bee41d 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -80,3 +80,4 @@ files['/**/test/functional/'].ignore = {'631'} max_string_line_length = 500 max_comment_line_length = 500 +max_line_length = 140 diff --git a/lualib/lua_auth_results.lua b/lualib/lua_auth_results.lua index d10e1081c..8c907d9b7 100644 --- a/lualib/lua_auth_results.lua +++ b/lualib/lua_auth_results.lua @@ -93,7 +93,7 @@ local function gen_auth_results(task, settings) else common.symbols[sym] = s if not auth_results[auth_type] then - auth_results[auth_type] = {key} + auth_results[auth_type] = { key } else table.insert(auth_results[auth_type], key) end @@ -111,7 +111,7 @@ local function gen_auth_results(task, settings) -- dkim=neutral (body hash did not verify) header.d=example.com header.s=sel header.b=fA8VVvJ8; -- dkim=neutral (body hash did not verify) header.d=example.com header.s=sel header.b=f8pM8o90; - for _,dres in ipairs(dkim_results) do + for _, dres in ipairs(dkim_results) do local ar_string = 'none' if dres.result == 'reject' then @@ -168,7 +168,7 @@ local function gen_auth_results(task, settings) hdr = hdr .. ' (policy=' .. lua_util.maybe_smtp_quote_value(opts[2]) .. ')' hdr = hdr .. ' header.from=' .. lua_util.maybe_smtp_quote_value(opts[1]) elseif key ~= 'none' then - local t = {opts[1]:match('^([^%s]+) : (.*)$')} + local t = { opts[1]:match('^([^%s]+) : (.*)$') } if #t > 0 then local dom = t[1] local rsn = t[2] @@ -197,7 +197,7 @@ local function gen_auth_results(task, settings) -- Main type local sender local sender_type - local smtp_from = task:get_from({'smtp','orig'}) + local smtp_from = task:get_from({ 'smtp', 'orig' }) if smtp_from and smtp_from[1] and @@ -241,26 +241,25 @@ local function gen_auth_results(task, settings) hdr = string.format('%s=%s', auth_type, key) end - table.insert(hdr_parts, hdr) end end end local u = task:get_user() - local smtp_from = task:get_from({'smtp','orig'}) + local smtp_from = task:get_from({ 'smtp', 'orig' }) if u and smtp_from then - local hdr = {[1] = 'auth=pass'} + local hdr = { [1] = 'auth=pass' } if settings['add_smtp_user'] then - table.insert(hdr,'smtp.auth=' .. lua_util.maybe_smtp_quote_value(u)) + table.insert(hdr, 'smtp.auth=' .. lua_util.maybe_smtp_quote_value(u)) end if smtp_from[1]['addr'] then - table.insert(hdr,'smtp.mailfrom=' .. lua_util.maybe_smtp_quote_value(smtp_from[1]['addr'])) + table.insert(hdr, 'smtp.mailfrom=' .. lua_util.maybe_smtp_quote_value(smtp_from[1]['addr'])) end - table.insert(hdr_parts, table.concat(hdr,' ')) + table.insert(hdr_parts, table.concat(hdr, ' ')) end if #hdr_parts > 0 then @@ -287,12 +286,12 @@ local function parse_ar_element(elt) local S = lpeg.S local V = lpeg.V local C = lpeg.C - local space = S(" ")^0 - local doublequoted = space * P'"' * ((1 - S'"\r\n\f\\') + (P'\\' * 1))^0 * '"' * space - local comment = space * P{ "(" * ((1 - S"()") + V(1))^0 * ")" } * space - local name = C((1 - S('=(" '))^1) * space + local space = S(" ") ^ 0 + local doublequoted = space * P '"' * ((1 - S '"\r\n\f\\') + (P '\\' * 1)) ^ 0 * '"' * space + local comment = space * P { "(" * ((1 - S "()") + V(1)) ^ 0 * ")" } * space + local name = C((1 - S('=(" ')) ^ 1) * space local pair = lpeg.Cg(name * "=" * space * name) * space - aar_elt_grammar = lpeg.Cf(lpeg.Ct("") * (pair + comment + doublequoted)^1, rawset) + aar_elt_grammar = lpeg.Cf(lpeg.Ct("") * (pair + comment + doublequoted) ^ 1, rawset) end return aar_elt_grammar:match(elt) diff --git a/lualib/lua_aws.lua b/lualib/lua_aws.lua index 5de61eb0c..e6c4b29a2 100644 --- a/lualib/lua_aws.lua +++ b/lualib/lua_aws.lua @@ -68,13 +68,12 @@ end local function save_cached_key(date_str, secret_key, region, service, req_type, key) local numdate = tonumber(date_str) -- expire old buckets - for k,_ in pairs(cached_keys) do + for k, _ in pairs(cached_keys) do if k < numdate then cached_keys[k] = nil end end - local bucket = cached_keys[tonumber(date_str)] local idx = string.format('%s.%s.%s.%s', secret_key, region, service, req_type) @@ -113,7 +112,7 @@ local function aws_signing_key(date_str, secret_key, region, service, req_type) end local hmac1 = rspamd_crypto_hash.create_specific_keyed("AWS4" .. secret_key, "sha256", date_str):bin() - local hmac2 = rspamd_crypto_hash.create_specific_keyed(hmac1, "sha256",region):bin() + local hmac2 = rspamd_crypto_hash.create_specific_keyed(hmac1, "sha256", region):bin() local hmac3 = rspamd_crypto_hash.create_specific_keyed(hmac2, "sha256", service):bin() local final_key = rspamd_crypto_hash.create_specific_keyed(hmac3, "sha256", req_type):bin() @@ -155,7 +154,7 @@ local function aws_canon_request_hash(method, uri, headers_to_sign, hex_hash) end, headers_to_sign)) local header_names = lua_util.keys(hdr_canon) table.sort(header_names) - for _,hn in ipairs(header_names) do + for _, hn in ipairs(header_names) do local v = hdr_canon[hn] lua_util.debugm(N, 'update signature with the header %s, %s', hn, v) @@ -165,19 +164,25 @@ local function aws_canon_request_hash(method, uri, headers_to_sign, hex_hash) lua_util.debugm(N, 'headers list to sign: %s', hdrs_list) sha_ctx:update(string.format('\n%s\n%s', hdrs_list, hex_hash)) - return sha_ctx:hex(),hdrs_list + return sha_ctx:hex(), hdrs_list end exports.aws_canon_request_hash = aws_canon_request_hash -local aws_authorization_hdr_args_schema = ts.shape{ +local aws_authorization_hdr_args_schema = ts.shape { date = ts.string + ts['nil'] / today_canonical, secret_key = ts.string, - method = ts.string + ts['nil'] / function() return 'GET' end, + method = ts.string + ts['nil'] / function() + return 'GET' + end, uri = ts.string, region = ts.string, - service = ts.string + ts['nil'] / function() return 's3' end, - req_type = ts.string + ts['nil'] / function() return 'aws4_request' end, + service = ts.string + ts['nil'] / function() + return 's3' + end, + req_type = ts.string + ts['nil'] / function() + return 'aws4_request' + end, headers = ts.map_of(ts.string, ts.string), key_id = ts.string, } @@ -199,9 +204,9 @@ ts.shape{ -- --]] local function aws_authorization_hdr(tbl, transformed) - local res,err + local res, err if not transformed then - res,err = aws_authorization_hdr_args_schema:transform(tbl) + res, err = aws_authorization_hdr_args_schema:transform(tbl) assert(res, err) else res = tbl @@ -210,7 +215,7 @@ local function aws_authorization_hdr(tbl, transformed) local signing_key = aws_signing_key(res.date, res.secret_key, res.region, res.service, res.req_type) assert(signing_key ~= nil) - local signed_sha,signed_hdrs = aws_canon_request_hash(res.method, res.uri, + local signed_sha, signed_hdrs = aws_canon_request_hash(res.method, res.uri, res.headers) if not signed_sha then @@ -224,7 +229,7 @@ local function aws_authorization_hdr(tbl, transformed) lua_util.debugm(N, "string to sign: %s", string_to_sign) local hmac = rspamd_crypto_hash.create_specific_keyed(signing_key, 'sha256', string_to_sign):hex() lua_util.debugm(N, "hmac: %s", hmac) - local auth_hdr = string.format('AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/%s,'.. + local auth_hdr = string.format('AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/%s,' .. 'SignedHeaders=%s,Signature=%s', res.key_id, res.date, res.region, res.service, res.req_type, signed_hdrs, hmac) @@ -255,7 +260,7 @@ This method returns new/modified in place table of the headers -- --]] local function aws_request_enrich(tbl, content) - local res,err = aws_authorization_hdr_args_schema:transform(tbl) + local res, err = aws_authorization_hdr_args_schema:transform(tbl) assert(res, err) local content_sha256 = rspamd_crypto_hash.create_specific('sha256', content):hex() local hdrs = res.headers @@ -281,7 +286,7 @@ local test_request_hdrs = { assert(aws_canon_request_hash('GET', '/test.txt', test_request_hdrs) == '7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972') -assert(aws_authorization_hdr{ +assert(aws_authorization_hdr { date = '20130524', region = 'us-east-1', headers = test_request_hdrs, diff --git a/lualib/lua_bayes_learn.lua b/lualib/lua_bayes_learn.lua index 7782d8196..ea97db6f8 100644 --- a/lualib/lua_bayes_learn.lua +++ b/lualib/lua_bayes_learn.lua @@ -40,7 +40,7 @@ exports.can_learn = function(task, is_spam, is_unlearn) end if in_class then - return false,string.format( + return false, string.format( 'already in class %s; probability %.2f%%', cl, math.abs((prob - 0.5) * 200.0)) end @@ -56,7 +56,7 @@ exports.autolearn = function(task, conf) local mime_rcpts = 'undef' local mr = task:get_recipients('mime') if mr then - for _,r in ipairs(mr) do + for _, r in ipairs(mr) do if mime_rcpts == 'undef' then mime_rcpts = r.addr else @@ -76,8 +76,8 @@ exports.autolearn = function(task, conf) end -- We have autolearn config so let's figure out what is requested - local verdict,score = lua_verdict.get_specific_verdict("bayes", task) - local learn_spam,learn_ham = false, false + local verdict, score = lua_verdict.get_specific_verdict("bayes", task) + local learn_spam, learn_ham = false, false if verdict == 'passthrough' then -- No need to autolearn @@ -117,12 +117,14 @@ exports.autolearn = function(task, conf) local ham_learns = task:get_mempool():get_variable('ham_learns', 'int64') or 0 local min_balance = 0.9 - if conf.min_balance then min_balance = conf.min_balance end + if conf.min_balance then + min_balance = conf.min_balance + end if spam_learns > 0 or ham_learns > 0 then local max_ratio = 1.0 / min_balance local spam_learns_ratio = spam_learns / (ham_learns + 1) - if spam_learns_ratio > max_ratio and learn_spam then + if spam_learns_ratio > max_ratio and learn_spam then lua_util.debugm(N, task, 'skip learning spam, balance is not satisfied: %s < %s; %s spam learns; %s ham learns', spam_learns_ratio, min_balance, spam_learns, ham_learns) @@ -130,7 +132,7 @@ exports.autolearn = function(task, conf) end local ham_learns_ratio = ham_learns / (spam_learns + 1) - if ham_learns_ratio > max_ratio and learn_ham then + if ham_learns_ratio > max_ratio and learn_ham then lua_util.debugm(N, task, 'skip learning ham, balance is not satisfied: %s < %s; %s spam learns; %s ham learns', ham_learns_ratio, min_balance, spam_learns, ham_learns) diff --git a/lualib/lua_cfg_transform.lua b/lualib/lua_cfg_transform.lua index 419ebf0f6..d6243ade1 100644 --- a/lualib/lua_cfg_transform.lua +++ b/lualib/lua_cfg_transform.lua @@ -32,24 +32,24 @@ local function metric_pairs(t) local function gen_keys(tbl) if implicit_array then - for _,v in ipairs(tbl) do + for _, v in ipairs(tbl) do if v.name then - table.insert(keys, {v.name, v}) + table.insert(keys, { v.name, v }) v.name = nil else -- Very tricky to distinguish: -- group {name = "foo" ... } + group "blah" { ... } - for gr_name,gr in pairs(v) do + for gr_name, gr in pairs(v) do if type(gr_name) ~= 'number' then -- We can also have implicit arrays here local gr_implicit = is_implicit(gr) if gr_implicit then - for _,gr_elt in ipairs(gr) do - table.insert(keys, {gr_name, gr_elt}) + for _, gr_elt in ipairs(gr) do + table.insert(keys, { gr_name, gr_elt }) end else - table.insert(keys, {gr_name, gr}) + table.insert(keys, { gr_name, gr }) end end end @@ -57,20 +57,20 @@ local function metric_pairs(t) end else if tbl.name then - table.insert(keys, {tbl.name, tbl}) + table.insert(keys, { tbl.name, tbl }) tbl.name = nil else - for k,v in pairs(tbl) do + for k, v in pairs(tbl) do if type(k) ~= 'number' then -- We can also have implicit arrays here local sym_implicit = is_implicit(v) if sym_implicit then - for _,elt in ipairs(v) do - table.insert(keys, {k, elt}) + for _, elt in ipairs(v) do + table.insert(keys, { k, elt }) end else - table.insert(keys, {k, v}) + table.insert(keys, { k, v }) end end end @@ -91,18 +91,26 @@ local function metric_pairs(t) end local function group_transform(cfg, k, v) - if v.name then k = v.name end + if v.name then + k = v.name + end local new_group = { symbols = {} } - if v.enabled then new_group.enabled = v.enabled end - if v.disabled then new_group.disabled = v.disabled end - if v.max_score then new_group.max_score = v.max_score end + if v.enabled then + new_group.enabled = v.enabled + end + if v.disabled then + new_group.disabled = v.disabled + end + if v.max_score then + new_group.max_score = v.max_score + end if v.symbol then - for sk,sv in metric_pairs(v.symbol) do + for sk, sv in metric_pairs(v.symbol) do if sv.name then sk = sv.name sv.name = nil -- Remove field @@ -112,7 +120,9 @@ local function group_transform(cfg, k, v) end end - if not cfg.group then cfg.group = {} end + if not cfg.group then + cfg.group = {} + end if cfg.group[k] then cfg.group[k] = lua_util.override_defaults(cfg.group[k], new_group) @@ -153,7 +163,9 @@ local function test_groups(groups) for gr_name, gr in pairs(groups) do if not gr.symbols then local cnt = 0 - for _,_ in pairs(gr) do cnt = cnt + 1 end + for _, _ in pairs(gr) do + cnt = cnt + 1 + end if cnt == 0 then logger.debugx('group %s is empty', gr_name) @@ -205,9 +217,9 @@ end -- merged group definition local function merge_groups(groups) local ret = {} - for k,gr in pairs(groups) do + for k, gr in pairs(groups) do if type(k) == 'number' then - for key,sec in pairs(gr) do + for key, sec in pairs(gr) do ret[key] = sec end else @@ -228,7 +240,7 @@ local function check_statistics_sanity() if rspamd_util.file_exists(local_stat) and rspamd_util.file_exists(local_bayes) then - logger.warnx(rspamd_config, 'conflicting files %s and %s are found: '.. + logger.warnx(rspamd_config, 'conflicting files %s and %s are found: ' .. 'Rspamd classifier configuration might be broken!', local_stat, local_bayes) end end @@ -237,7 +249,7 @@ end local function surbl_section_convert(cfg, section) local rbl_section = cfg.rbl.rbls local wl = section.whitelist - for name,value in pairs(section.rules or {}) do + for name, value in pairs(section.rules or {}) do if rbl_section[name] then logger.warnx(rspamd_config, 'conflicting names in surbl and rbl rules: %s, prefer surbl rule!', name) @@ -251,13 +263,21 @@ local function surbl_section_convert(cfg, section) converted.whitelist = wl end - for k,v in pairs(value) do + for k, v in pairs(value) do local skip = false -- Rename - if k == 'suffix' then k = 'rbl' end - if k == 'ips' then k = 'returncodes' end - if k == 'bits' then k = 'returnbits' end - if k == 'noip' then k = 'no_ip' end + if k == 'suffix' then + k = 'rbl' + end + if k == 'ips' then + k = 'returncodes' + end + if k == 'bits' then + k = 'returnbits' + end + if k == 'noip' then + k = 'no_ip' + end -- Crappy legacy if k == 'options' then if v == 'noip' or v == 'no_ip' then @@ -292,7 +312,7 @@ end local function emails_section_convert(cfg, section) local rbl_section = cfg.rbl.rbls local wl = section.whitelist - for name,value in pairs(section.rules or {}) do + for name, value in pairs(section.rules or {}) do if rbl_section[name] then logger.warnx(rspamd_config, 'conflicting names in emails and rbl rules: %s, prefer emails rule!', name) @@ -306,15 +326,27 @@ local function emails_section_convert(cfg, section) converted.whitelist = wl end - for k,v in pairs(value) do + for k, v in pairs(value) do local skip = false -- Rename - if k == 'dnsbl' then k = 'rbl' end - if k == 'check_replyto' then k = 'replyto' end - if k == 'hashlen' then k = 'hash_len' end - if k == 'encoding' then k = 'hash_format' end - if k == 'domain_only' then k = 'emails_domainonly' end - if k == 'delimiter' then k = 'emails_delimiter' end + if k == 'dnsbl' then + k = 'rbl' + end + if k == 'check_replyto' then + k = 'replyto' + end + if k == 'hashlen' then + k = 'hash_len' + end + if k == 'encoding' then + k = 'hash_format' + end + if k == 'domain_only' then + k = 'emails_domainonly' + end + if k == 'delimiter' then + k = 'emails_delimiter' + end if k == 'skip_body' then skip = true if v then @@ -365,14 +397,14 @@ return function(cfg) logger.errx('no actions defined') else -- Perform sanity check for actions - local actions_defs = {'no action', 'no_action', -- In case if that's added - 'greylist', 'add header', 'add_header', - 'rewrite subject', 'rewrite_subject', 'quarantine', - 'reject', 'discard'} + local actions_defs = { 'no action', 'no_action', -- In case if that's added + 'greylist', 'add header', 'add_header', + 'rewrite subject', 'rewrite_subject', 'quarantine', + 'reject', 'discard' } if not cfg.actions['no action'] and not cfg.actions['no_action'] and - not cfg.actions['accept'] then - for _,d in ipairs(actions_defs) do + not cfg.actions['accept'] then + for _, d in ipairs(actions_defs) do if cfg.actions[d] then local action_score = nil @@ -387,7 +419,7 @@ return function(cfg) elseif type(action_score) == 'number' and action_score < 0 then cfg.actions['no_action'] = cfg.actions[d] - 0.001 logger.infox(rspamd_config, 'set no_action score to: %s, as action %s has negative score', - cfg.actions['no_action'], d) + cfg.actions['no_action'], d) break end end @@ -401,7 +433,7 @@ return function(cfg) actions_set['grow_factor'] = true actions_set['subject'] = true - for k,_ in pairs(cfg.actions) do + for k, _ in pairs(cfg.actions) do if not actions_set[k] then logger.warnx(rspamd_config, 'unknown element in actions section: %s', k) end @@ -417,18 +449,18 @@ return function(cfg) 'reject', 'discard' } - for i=1,(#actions_order - 1) do + for i = 1, (#actions_order - 1) do local act = actions_order[i] if cfg.actions[act] and type(cfg.actions[act]) == 'number' then local score = cfg.actions[act] - for j=i+1,#actions_order do + for j = i + 1, #actions_order do local next_act = actions_order[j] if cfg.actions[next_act] and type(cfg.actions[next_act]) == 'number' then local next_score = cfg.actions[next_act] if next_score <= score then - logger.errx(rspamd_config, 'invalid actions thresholds order: action %s (%s) must have lower '.. + logger.errx(rspamd_config, 'invalid actions thresholds order: action %s (%s) must have lower ' .. 'score than action %s (%s)', act, score, next_act, next_score) ret = false end @@ -475,7 +507,9 @@ return function(cfg) -- Again: legacy stuff :( if not cfg.dkim.sign_headers then local sec = cfg.dkim_signing - if sec and sec[1] then sec = cfg.dkim_signing[1] end + if sec and sec[1] then + sec = cfg.dkim_signing[1] + end if sec and sec.sign_headers then cfg.dkim.sign_headers = sec.sign_headers @@ -483,7 +517,7 @@ return function(cfg) end -- DKIM signing/ARC legacy - for _, mod in ipairs({'dkim_signing', 'arc'}) do + for _, mod in ipairs({ 'dkim_signing', 'arc' }) do if cfg[mod] then if cfg[mod].auth_only ~= nil then if cfg[mod].sign_authenticated ~= nil then @@ -502,10 +536,10 @@ return function(cfg) end -- Try to find some obvious issues with configuration - for k,v in pairs(cfg) do - if type(v) == 'table' and v[k] and type (v[k]) == 'table' then + for k, v in pairs(cfg) do + if type(v) == 'table' and v[k] and type(v[k]) == 'table' then logger.errx('nested section: %s { %s { ... } }, it is likely a configuration error', - k, k) + k, k) end end @@ -531,9 +565,13 @@ return function(cfg) } end - if not cfg.reputation.rules then cfg.reputation.rules = {} end + if not cfg.reputation.rules then + cfg.reputation.rules = {} + end - if not fun.any(function(_, v) return v.selector and v.selector.ip end, + if not fun.any(function(_, v) + return v.selector and v.selector.ip + end, cfg.reputation.rules) then logger.infox(rspamd_config, 'attach ip reputation element to use it') diff --git a/lualib/lua_clickhouse.lua b/lualib/lua_clickhouse.lua index 7317cc392..28366d28a 100644 --- a/lualib/lua_clickhouse.lua +++ b/lualib/lua_clickhouse.lua @@ -35,7 +35,7 @@ local function escape_spaces(query) end local function ch_number(a) - if (a+2^52)-2^52 == a then + if (a + 2 ^ 52) - 2 ^ 52 == a then -- Integer return tostring(math.floor(a)) end @@ -59,7 +59,7 @@ end -- Converts an array to a string suitable for clickhouse local function array_to_string(ar) - for i,elt in ipairs(ar) do + for i, elt in ipairs(ar) do local t = type(elt) if t == 'string' then ar[i] = string.format('\'%s\'', clickhouse_quote(elt)) @@ -76,7 +76,7 @@ end -- Converts a row into TSV, taking extra care about arrays local function row_to_tsv(row) - for i,elt in ipairs(row) do + for i, elt in ipairs(row) do local t = type(elt) if t == 'table' then row[i] = '[' .. array_to_string(elt) .. ']' @@ -107,9 +107,9 @@ local function parse_clickhouse_response_json_eachrow(params, data, row_cb) local parser = ucl.parser() local res, err if type(s) == 'string' then - res,err = parser:parse_string(s) + res, err = parser:parse_string(s) else - res,err = parser:parse_text(s) + res, err = parser:parse_text(s) end if not res then @@ -151,9 +151,9 @@ local function parse_clickhouse_response_json(params, data) local res, err if type(s) == 'string' then - res,err = parser:parse_string(s) + res, err = parser:parse_string(s) else - res,err = parser:parse_text(s) + res, err = parser:parse_text(s) end if not res then @@ -169,14 +169,16 @@ local function parse_clickhouse_response_json(params, data) return 'bad json', {} end - return nil,json + return nil, json end -- Helper to generate HTTP closure local function mk_http_select_cb(upstream, params, ok_cb, fail_cb, row_cb) local function http_cb(err_message, code, data, _) if code ~= 200 or err_message then - if not err_message then err_message = data end + if not err_message then + err_message = data + end local ip_addr = upstream:get_addr():to_string(true) if fail_cb then @@ -205,8 +207,8 @@ local function mk_http_select_cb(upstream, params, ok_cb, fail_cb, row_cb) else local ip_addr = upstream:get_addr():to_string(true) rspamd_logger.errx(params.log_obj, - "request failed on clickhouse server %s: %s", - ip_addr, 'failed to parse reply') + "request failed on clickhouse server %s: %s", + ip_addr, 'failed to parse reply') end end end @@ -219,7 +221,9 @@ end local function mk_http_insert_cb(upstream, params, ok_cb, fail_cb) local function http_cb(err_message, code, data, _) if code ~= 200 or err_message then - if not err_message then err_message = data end + if not err_message then + err_message = data + end local ip_addr = upstream:get_addr():to_string(true) if fail_cb then @@ -234,7 +238,7 @@ local function mk_http_insert_cb(upstream, params, ok_cb, fail_cb) upstream:ok() if ok_cb then - local err,parsed = parse_clickhouse_response_json(data) + local err, parsed = parse_clickhouse_response_json(data) if err then fail_cb(params, err, data) @@ -273,10 +277,12 @@ end -- @example -- --]] -exports.select = function (upstream, settings, params, query, ok_cb, fail_cb, row_cb) +exports.select = function(upstream, settings, params, query, ok_cb, fail_cb, row_cb) local http_params = {} - for k,v in pairs(params) do http_params[k] = v end + for k, v in pairs(params) do + http_params[k] = v + end http_params.callback = mk_http_select_cb(upstream, http_params, ok_cb, fail_cb, row_cb) http_params.gzip = settings.use_gzip @@ -327,10 +333,12 @@ end -- @example -- --]] -exports.select_sync = function (upstream, settings, params, query, row_cb) +exports.select_sync = function(upstream, settings, params, query, row_cb) local http_params = {} - for k,v in pairs(params) do http_params[k] = v end + for k, v in pairs(params) do + http_params[k] = v + end http_params.gzip = settings.use_gzip http_params.mime_type = 'text/plain' @@ -388,11 +396,13 @@ end -- @example -- --]] -exports.insert = function (upstream, settings, params, query, rows, - ok_cb, fail_cb) +exports.insert = function(upstream, settings, params, query, rows, + ok_cb, fail_cb) local http_params = {} - for k,v in pairs(params) do http_params[k] = v end + for k, v in pairs(params) do + http_params[k] = v + end http_params.callback = mk_http_insert_cb(upstream, http_params, ok_cb, fail_cb) http_params.gzip = settings.use_gzip @@ -402,7 +412,7 @@ exports.insert = function (upstream, settings, params, query, rows, http_params.user = settings.user http_params.password = settings.password http_params.method = 'POST' - http_params.body = {rspamd_text.fromtable(rows, '\n'), '\n'} + http_params.body = { rspamd_text.fromtable(rows, '\n'), '\n' } http_params.log_obj = params.task or params.config if not http_params.url then @@ -441,11 +451,13 @@ end -- @example -- --]] -exports.generic = function (upstream, settings, params, query, +exports.generic = function(upstream, settings, params, query, ok_cb, fail_cb) local http_params = {} - for k,v in pairs(params) do http_params[k] = v end + for k, v in pairs(params) do + http_params[k] = v + end http_params.callback = mk_http_insert_cb(upstream, http_params, ok_cb, fail_cb) http_params.gzip = settings.use_gzip @@ -488,10 +500,12 @@ end -- @example -- --]] -exports.generic_sync = function (upstream, settings, params, query) +exports.generic_sync = function(upstream, settings, params, query) local http_params = {} - for k,v in pairs(params) do http_params[k] = v end + for k, v in pairs(params) do + http_params[k] = v + end http_params.gzip = settings.use_gzip http_params.mime_type = 'text/plain' @@ -521,10 +535,10 @@ exports.generic_sync = function (upstream, settings, params, query) return response.content, response else lua_util.debugm(N, http_params.log_obj, "clickhouse generic response: %1", response) - local e,obj = parse_clickhouse_response_json(params, response.content) + local e, obj = parse_clickhouse_response_json(params, response.content) if e then - return e,nil + return e, nil end return nil, obj end diff --git a/lualib/lua_content/ical.lua b/lualib/lua_content/ical.lua index 72db634ee..d018a8596 100644 --- a/lualib/lua_content/ical.lua +++ b/lualib/lua_content/ical.lua @@ -23,11 +23,15 @@ local ical_grammar local function gen_grammar() if not ical_grammar then local wsp = l.S(" \t\v\f") - local crlf = (l.P"\r"^-1 * l.P"\n") + l.P"\r" - local eol = (crlf * #crlf) + (crlf - (crlf^-1 * wsp)) - local name = l.C((l.P(1) - (l.P":"))^1) / function(v) return (v:gsub("[\n\r]+%s","")) end - local value = l.C((l.P(1) - eol)^0) / function(v) return (v:gsub("[\n\r]+%s","")) end - ical_grammar = name * ":" * wsp^0 * value * eol^-1 + local crlf = (l.P "\r" ^ -1 * l.P "\n") + l.P "\r" + local eol = (crlf * #crlf) + (crlf - (crlf ^ -1 * wsp)) + local name = l.C((l.P(1) - (l.P ":")) ^ 1) / function(v) + return (v:gsub("[\n\r]+%s", "")) + end + local value = l.C((l.P(1) - eol) ^ 0) / function(v) + return (v:gsub("[\n\r]+%s", "")) + end + ical_grammar = name * ":" * wsp ^ 0 * value * eol ^ -1 end return ical_grammar @@ -38,13 +42,15 @@ local exports = {} local function extract_text_data(specific) local fun = require "fun" - local tbl = fun.totable(fun.map(function(e) return e[2]:lower() end, specific.elts)) + local tbl = fun.totable(fun.map(function(e) + return e[2]:lower() + end, specific.elts)) return table.concat(tbl, '\n') end -- Keys that can have visible urls -local url_keys = lua_util.list_to_hash{ +local url_keys = lua_util.list_to_hash { 'description', 'location', 'summary', @@ -55,7 +61,7 @@ local url_keys = lua_util.list_to_hash{ } local function process_ical(input, mpart, task) - local control={n='\n', r=''} + local control = { n = '\n', r = '' } local rspamd_url = require "rspamd_url" local escaper = l.Ct((gen_grammar() / function(key, value) value = value:gsub("\\(.)", control) @@ -65,17 +71,17 @@ local function process_ical(input, mpart, task) local local_urls = rspamd_url.all(task:get_mempool(), value) if local_urls and #local_urls > 0 then - for _,u in ipairs(local_urls) do + for _, u in ipairs(local_urls) do lua_util.debugm(N, task, 'ical: found URL in ical key "%s": %s', - key, tostring(u)) + key, tostring(u)) task:inject_url(u, mpart) end end end lua_util.debugm(N, task, 'ical: ical key %s = "%s"', key, value) - return {key, value} - end)^1) + return { key, value } + end) ^ 1) local elts = escaper:match(input) diff --git a/lualib/lua_content/init.lua b/lualib/lua_content/init.lua index 1a13311b1..701d223a5 100644 --- a/lualib/lua_content/init.lua +++ b/lualib/lua_content/init.lua @@ -26,21 +26,21 @@ local lua_util = require "lua_util" local content_modules = { ical = { - mime_type = {"text/calendar", "application/calendar"}, + mime_type = { "text/calendar", "application/calendar" }, module = require "lua_content/ical", - extensions = {'ics'}, + extensions = { 'ics' }, output = "text" }, vcf = { - mime_type = {"text/vcard", "application/vcard"}, + mime_type = { "text/vcard", "application/vcard" }, module = require "lua_content/vcard", - extensions = {'vcf'}, + extensions = { 'vcf' }, output = "text" }, pdf = { mime_type = "application/pdf", module = require "lua_content/pdf", - extensions = {'pdf'}, + extensions = { 'pdf' }, output = "table" }, } @@ -51,20 +51,20 @@ local modules_by_extension local function init() modules_by_mime_type = {} modules_by_extension = {} - for k,v in pairs(content_modules) do + for k, v in pairs(content_modules) do if v.mime_type then if type(v.mime_type) == 'table' then - for _,mt in ipairs(v.mime_type) do - modules_by_mime_type[mt] = {k, v} + for _, mt in ipairs(v.mime_type) do + modules_by_mime_type[mt] = { k, v } end else - modules_by_mime_type[v.mime_type] = {k, v} + modules_by_mime_type[v.mime_type] = { k, v } end end if v.extensions then - for _,ext in ipairs(v.extensions) do - modules_by_extension[ext] = {k, v} + for _, ext in ipairs(v.extensions) do + modules_by_extension[ext] = { k, v } end end end @@ -106,5 +106,4 @@ exports.maybe_process_mime_part = function(part, task) end - return exports
\ No newline at end of file diff --git a/lualib/lua_content/pdf.lua b/lualib/lua_content/pdf.lua index 0677fab06..a8b6e984a 100644 --- a/lualib/lua_content/pdf.lua +++ b/lualib/lua_content/pdf.lua @@ -147,10 +147,10 @@ local function compile_tries() rspamd_trie.flags.no_start) local function compile_pats(patterns, indexes, compile_flags) local strs = {} - for what,data in pairs(patterns) do - for i,pat in ipairs(data.patterns) do + for what, data in pairs(patterns) do + for i, pat in ipairs(data.patterns) do strs[#strs + 1] = pat - indexes[#indexes + 1] = {what, data, pat, i} + indexes[#indexes + 1] = { what, data, pat, i } end end @@ -175,7 +175,7 @@ local function generic_grammar_elts() local S = lpeg.S local V = lpeg.V local C = lpeg.C - local D = R'09' -- Digits + local D = R '09' -- Digits local grammar_elts = {} @@ -214,37 +214,37 @@ local function generic_grammar_elts() end local function pdf_id_unescape(s) - return (s:gsub('#%d%d', function (cc) + return (s:gsub('#%d%d', function(cc) return string.char(tonumber(cc:sub(2), 16)) end)) end - local delim = S'()<>[]{}/%' - grammar_elts.ws = S'\0 \r\n\t\f' - local hex = R'af' + R'AF' + D + local delim = S '()<>[]{}/%' + grammar_elts.ws = S '\0 \r\n\t\f' + local hex = R 'af' + R 'AF' + D -- Comments. - local eol = P'\r\n' + '\n' - local line = (1 - S'\r\n\f')^0 * eol^-1 - grammar_elts.comment = P'%' * line + local eol = P '\r\n' + '\n' + local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1 + grammar_elts.comment = P '%' * line -- Numbers. - local sign = S'+-'^-1 - local decimal = D^1 - local float = D^1 * P'.' * D^0 + P'.' * D^1 + local sign = S '+-' ^ -1 + local decimal = D ^ 1 + local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1 grammar_elts.number = C(sign * (float + decimal)) / tonumber -- String - grammar_elts.str = P{ "(" * C(((1 - S"()\\") + (P '\\' * 1) + V(1))^0) / pdf_string_unescape * ")" } - grammar_elts.hexstr = P{"<" * C(hex^0) / pdf_hexstring_unescape * ">"} + grammar_elts.str = P { "(" * C(((1 - S "()\\") + (P '\\' * 1) + V(1)) ^ 0) / pdf_string_unescape * ")" } + grammar_elts.hexstr = P { "<" * C(hex ^ 0) / pdf_hexstring_unescape * ">" } -- Identifier - grammar_elts.id = P{'/' * C((1-(delim + grammar_elts.ws))^1) / pdf_id_unescape} + grammar_elts.id = P { '/' * C((1 - (delim + grammar_elts.ws)) ^ 1) / pdf_id_unescape } -- Booleans (who care about them?) grammar_elts.boolean = C(P("true") + P("false")) -- Stupid references - grammar_elts.ref = lpeg.Ct{lpeg.Cc("%REF%") * C(D^1) * " " * C(D^1) * " " * "R"} + grammar_elts.ref = lpeg.Ct { lpeg.Cc("%REF%") * C(D ^ 1) * " " * C(D ^ 1) * " " * "R" } return grammar_elts end @@ -255,16 +255,16 @@ local function gen_outer_grammar() local V = lpeg.V local gen = generic_grammar_elts() - return lpeg.P{ + return lpeg.P { "EXPR"; - EXPR = gen.ws^0 * V("ELT")^0 * gen.ws^0, + EXPR = gen.ws ^ 0 * V("ELT") ^ 0 * gen.ws ^ 0, ELT = V("ARRAY") + V("DICT") + V("ATOM"), - ATOM = gen.ws^0 * (gen.comment + gen.boolean + gen.ref + - gen.number + V("STRING") + gen.id) * gen.ws^0, - DICT = "<<" * gen.ws^0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR")^0, rawset) * gen.ws^0 * ">>", - KV_PAIR = lpeg.Cg(gen.id * gen.ws^0 * V("ELT") * gen.ws^0), - ARRAY = "[" * gen.ws^0 * lpeg.Ct(V("ELT")^0) * gen.ws^0 * "]", - STRING = lpeg.P{gen.str + gen.hexstr}, + ATOM = gen.ws ^ 0 * (gen.comment + gen.boolean + gen.ref + + gen.number + V("STRING") + gen.id) * gen.ws ^ 0, + DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>", + KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ELT") * gen.ws ^ 0), + ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ELT") ^ 0) * gen.ws ^ 0 * "]", + STRING = lpeg.P { gen.str + gen.hexstr }, } end @@ -274,7 +274,7 @@ local function gen_graphics_unary() local S = lpeg.S return P("q") + P("Q") + P("h") - + S("WSsFfBb") * P("*")^0 + P("n") + + S("WSsFfBb") * P("*") ^ 0 + P("n") end local function gen_graphics_binary() @@ -317,29 +317,29 @@ local function gen_text_grammar() local text_quote_op = P('"') local font_op = P("Tf") - return lpeg.P{ + return lpeg.P { "EXPR"; - EXPR = gen.ws^0 * lpeg.Ct(V("COMMAND")^0), + EXPR = gen.ws ^ 0 * lpeg.Ct(V("COMMAND") ^ 0), COMMAND = (V("UNARY") + V("BINARY") + V("TERNARY") + V("NARY") + V("TEXT") + - V("FONT") + gen.comment) * gen.ws^0, + V("FONT") + gen.comment) * gen.ws ^ 0, UNARY = unary_ops, - BINARY = V("ARG") / empty * gen.ws^1 * binary_ops, - TERNARY = V("ARG") / empty * gen.ws^1 * V("ARG") / empty * gen.ws^1 * ternary_ops, - NARY = (gen.number / 0 * gen.ws^1)^1 * (gen.id / empty * gen.ws^0)^-1 * nary_op, + BINARY = V("ARG") / empty * gen.ws ^ 1 * binary_ops, + TERNARY = V("ARG") / empty * gen.ws ^ 1 * V("ARG") / empty * gen.ws ^ 1 * ternary_ops, + NARY = (gen.number / 0 * gen.ws ^ 1) ^ 1 * (gen.id / empty * gen.ws ^ 0) ^ -1 * nary_op, ARG = V("ARRAY") + V("DICT") + V("ATOM"), ATOM = (gen.comment + gen.boolean + gen.ref + gen.number + V("STRING") + gen.id), - DICT = "<<" * gen.ws^0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR")^0, rawset) * gen.ws^0 * ">>", - KV_PAIR = lpeg.Cg(gen.id * gen.ws^0 * V("ARG") * gen.ws^0), - ARRAY = "[" * gen.ws^0 * lpeg.Ct(V("ARG")^0) * gen.ws^0 * "]", - STRING = lpeg.P{gen.str + gen.hexstr}, - TEXT = (V("TEXT_ARG") * gen.ws^1 * text_binary_op) + - (V("ARG") / 0 * gen.ws^1 * V("ARG") / 0 * gen.ws^1 * V("TEXT_ARG") * gen.ws^1 * text_quote_op), - FONT = (V("FONT_ARG") * gen.ws^1 * (gen.number / 0) * gen.ws^1 * font_op), + DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>", + KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ARG") * gen.ws ^ 0), + ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ARG") ^ 0) * gen.ws ^ 0 * "]", + STRING = lpeg.P { gen.str + gen.hexstr }, + TEXT = (V("TEXT_ARG") * gen.ws ^ 1 * text_binary_op) + + (V("ARG") / 0 * gen.ws ^ 1 * V("ARG") / 0 * gen.ws ^ 1 * V("TEXT_ARG") * gen.ws ^ 1 * text_quote_op), + FONT = (V("FONT_ARG") * gen.ws ^ 1 * (gen.number / 0) * gen.ws ^ 1 * font_op), FONT_ARG = lpeg.Ct(lpeg.Cc("%font%") * gen.id), TEXT_ARG = lpeg.Ct(V("STRING")) + V("TEXT_ARRAY"), TEXT_ARRAY = "[" * - lpeg.Ct(((gen.ws^0 * (gen.ws^0 * (gen.number / 0)^0 * gen.ws^0 * (gen.str + gen.hexstr)))^1)) * gen.ws^0 * "]", + lpeg.Ct(((gen.ws ^ 0 * (gen.ws ^ 0 * (gen.number / 0) ^ 0 * gen.ws ^ 0 * (gen.str + gen.hexstr))) ^ 1)) * gen.ws ^ 0 * "]", } end @@ -393,7 +393,7 @@ local function maybe_apply_filter(dict, data, pdf, task) if dict.Filter then local filt = dict.Filter if type(filt) == 'string' then - filt = {filt} + filt = { filt } end if dict.DecodeParms then @@ -401,19 +401,21 @@ local function maybe_apply_filter(dict, data, pdf, task) if type(decode_params) == 'table' then if decode_params.Predictor then - return nil,'predictor exists' + return nil, 'predictor exists' end end end - for _,f in ipairs(filt) do + for _, f in ipairs(filt) do uncompressed = apply_pdf_filter(uncompressed, f) - if not uncompressed then break end + if not uncompressed then + break + end end end - return uncompressed,nil + return uncompressed, nil end -- Conditionally extract stream data from object and attach it as obj.uncompressed @@ -428,7 +430,7 @@ local function maybe_extract_object_stream(obj, pdf, task) tonumber(maybe_dereference_object(dict.Length, pdf, task)) or 0) local real_stream = obj.stream.data:span(1, len) - local uncompressed,filter_err = maybe_apply_filter(dict, real_stream, pdf, task) + local uncompressed, filter_err = maybe_apply_filter(dict, real_stream, pdf, task) if uncompressed then obj.uncompressed = uncompressed @@ -442,7 +444,6 @@ local function maybe_extract_object_stream(obj, pdf, task) end end - local function parse_object_grammar(obj, task, pdf) -- Parse grammar local obj_dict_span @@ -453,7 +454,7 @@ local function parse_object_grammar(obj, task, pdf) end if obj_dict_span:len() < config.max_processing_size then - local ret,obj_or_err = pcall(pdf_outer_grammar.match, pdf_outer_grammar, obj_dict_span) + local ret, obj_or_err = pcall(pdf_outer_grammar.match, pdf_outer_grammar, obj_dict_span) if ret then if obj.stream then @@ -669,11 +670,11 @@ process_dict = function(task, pdf, obj, dict) if contents and type(contents) == 'table' then if contents[1] == '%REF%' then -- Single reference - contents = {contents} + contents = { contents } end obj.contents = {} - for _,c in ipairs(contents) do + for _, c in ipairs(contents) do local cobj = maybe_dereference_object(c, pdf, task) if cobj and type(cobj) == 'table' then obj.contents[#obj.contents + 1] = cobj @@ -719,25 +720,25 @@ process_dict = function(task, pdf, obj, dict) ---[[Disabled fonts extraction - local fonts = obj.resources.Font - if fonts and type(fonts) == 'table' then - obj.fonts = {} - for k,v in pairs(fonts) do - obj.fonts[k] = maybe_dereference_object(v, pdf, task) + --[[Disabled fonts extraction + local fonts = obj.resources.Font + if fonts and type(fonts) == 'table' then + obj.fonts = {} + for k,v in pairs(fonts) do + obj.fonts[k] = maybe_dereference_object(v, pdf, task) - if obj.fonts[k] then - local font = obj.fonts[k] + if obj.fonts[k] then + local font = obj.fonts[k] - if config.text_extraction then - process_font(task, pdf, font, k) - lua_util.debugm(N, task, 'found font "%s" for object %s:%s -> %s', - k, obj.major, obj.minor, font) + if config.text_extraction then + process_font(task, pdf, font, k) + lua_util.debugm(N, task, 'found font "%s" for object %s:%s -> %s', + k, obj.major, obj.minor, font) + end + end end end - end - end -]] + ]] lua_util.debugm(N, task, 'found resources for object %s:%s (%s): %s', obj.major, obj.minor, obj.type, obj.resources) @@ -783,8 +784,8 @@ local compound_obj_grammar local function compound_obj_grammar_gen() if not compound_obj_grammar then local gen = generic_grammar_elts() - compound_obj_grammar = gen.ws^0 * (gen.comment * gen.ws^1)^0 * - lpeg.Ct(lpeg.Ct(gen.number * gen.ws^1 * gen.number * gen.ws^0)^1) + compound_obj_grammar = gen.ws ^ 0 * (gen.comment * gen.ws ^ 1) ^ 0 * + lpeg.Ct(lpeg.Ct(gen.number * gen.ws ^ 1 * gen.number * gen.ws ^ 0) ^ 1) end return compound_obj_grammar @@ -798,8 +799,8 @@ local function pdf_compound_object_unpack(_, uncompressed, pdf, task, first) lua_util.debugm(N, task, 'compound elts (chunk length %s): %s', #uncompressed, elts) - for i,pair in ipairs(elts) do - local obj_number,offset = pair[1], pair[2] + for i, pair in ipairs(elts) do + local obj_number, offset = pair[1], pair[2] offset = offset + first if offset < #uncompressed then @@ -833,7 +834,7 @@ end -- PDF 1.5 ObjStmt local function extract_pdf_compound_objects(task, pdf) - for i,obj in ipairs(pdf.objects or {}) do + for i, obj in ipairs(pdf.objects or {}) do if i > 0 and i % 100 == 0 then local now = rspamd_util.get_ticks() @@ -894,7 +895,9 @@ local function extract_outer_objects(task, input, pdf) -- Also get the starting span and try to match it versus obj re to get numbers local obj_line_potential = first - 32 - if obj_line_potential < 1 then obj_line_potential = 1 end + if obj_line_potential < 1 then + obj_line_potential = 1 + end local prev_obj_end = pdf.end_objects[end_pos - 1] if end_pos > 1 and prev_obj_end >= obj_line_potential and prev_obj_end < first then obj_line_potential = prev_obj_end + 1 @@ -941,7 +944,7 @@ local function attach_pdf_streams(task, input, pdf) max_start_pos = math.min(config.max_pdf_objects, #pdf.start_streams) max_end_pos = math.min(config.max_pdf_objects, #pdf.end_streams) - for _,obj in ipairs(pdf.objects) do + for _, obj in ipairs(pdf.objects) do while start_pos <= max_start_pos and end_pos <= max_end_pos do local first = pdf.start_streams[start_pos] local last = pdf.end_streams[end_pos] @@ -957,7 +960,9 @@ local function attach_pdf_streams(task, input, pdf) -- Strip the first \n while first < last do local chr = input:byte(first) - if chr ~= 13 and chr ~= 10 then break end + if chr ~= 13 and chr ~= 10 then + break + end first = first + 1 end local len = last - first @@ -1000,7 +1005,7 @@ local function postprocess_pdf_objects(task, input, pdf) -- Now we have objects and we need to attach streams that are in bounds attach_pdf_streams(task, input, pdf) -- Parse grammar for outer objects - for i,obj in ipairs(pdf.objects) do + for i, obj in ipairs(pdf.objects) do if i > 0 and i % 100 == 0 then local now = rspamd_util.get_ticks() @@ -1031,7 +1036,7 @@ local function postprocess_pdf_objects(task, input, pdf) end -- Now we might probably have all objects being processed - for i,obj in ipairs(pdf.objects) do + for i, obj in ipairs(pdf.objects) do if obj.dict then -- Types processing if i > 0 and i % 100 == 0 then @@ -1076,10 +1081,10 @@ local function offsets_to_blocks(starts, ends, out) end local function search_text(task, pdf) - for _,obj in ipairs(pdf.objects) do + for _, obj in ipairs(pdf.objects) do if obj.type == 'Page' and obj.contents then local text = {} - for _,tobj in ipairs(obj.contents) do + for _, tobj in ipairs(obj.contents) do maybe_extract_object_stream(tobj, pdf, task) local matches = pdf_text_trie:match(tobj.uncompressed or '') if matches then @@ -1087,20 +1092,20 @@ local function search_text(task, pdf) local starts = {} local ends = {} - for npat,matched_positions in pairs(matches) do + for npat, matched_positions in pairs(matches) do if npat == 1 then - for _,pos in ipairs(matched_positions) do + for _, pos in ipairs(matched_positions) do starts[#starts + 1] = pos end else - for _,pos in ipairs(matched_positions) do + for _, pos in ipairs(matched_positions) do ends[#ends + 1] = pos end end end offsets_to_blocks(starts, ends, text_blocks) - for _,bl in ipairs(text_blocks) do + for _, bl in ipairs(text_blocks) do if bl.len > 2 then -- To remove \s+ET\b pattern (it can leave trailing space or not but it doesn't matter) bl.len = bl.len - 2 @@ -1111,7 +1116,7 @@ local function search_text(task, pdf) -- tobj.major, tobj.minor, bl.data) if bl.len < config.max_processing_size then - local ret,obj_or_err = pcall(pdf_text_grammar.match, pdf_text_grammar, + local ret, obj_or_err = pcall(pdf_text_grammar.match, pdf_text_grammar, bl.data) if ret then @@ -1147,13 +1152,13 @@ local function search_urls(task, pdf, mpart) return end - for k,v in pairs(dict) do + for k, v in pairs(dict) do if type(v) == 'table' then recursive_object_traverse(obj, v, rec + 1) elseif k == 'URI' then v = maybe_dereference_object(v, pdf, task) if type(v) == 'string' then - local url = rspamd_url.create(task:get_mempool(), v, {'content'}) + local url = rspamd_url.create(task:get_mempool(), v, { 'content' }) if url then lua_util.debugm(N, task, 'found url %s in object %s:%s', @@ -1165,7 +1170,7 @@ local function search_urls(task, pdf, mpart) end end - for _,obj in ipairs(pdf.objects) do + for _, obj in ipairs(pdf.objects) do if obj.dict and type(obj.dict) == 'table' then recursive_object_traverse(obj, obj.dict, 0) end @@ -1193,10 +1198,10 @@ local function process_pdf(input, mpart, task) -- Output object that excludes all internal stuff local pdf_output = lua_util.shallowcopy(pdf_object) local grouped_processors = {} - for npat,matched_positions in pairs(matches) do + for npat, matched_positions in pairs(matches) do local index = pdf_indexes[npat] - local proc_key,loc_npat = index[1], index[4] + local proc_key, loc_npat = index[1], index[4] if not grouped_processors[proc_key] then grouped_processors[proc_key] = { @@ -1206,16 +1211,18 @@ local function process_pdf(input, mpart, task) end local proc = grouped_processors[proc_key] -- Fill offsets - for _,pos in ipairs(matched_positions) do - proc.offsets[#proc.offsets + 1] = {pos, loc_npat} + for _, pos in ipairs(matched_positions) do + proc.offsets[#proc.offsets + 1] = { pos, loc_npat } end end - for name,processor in pairs(grouped_processors) do + for name, processor in pairs(grouped_processors) do -- Sort by offset lua_util.debugm(N, task, "pdf: process group %s with %s matches", name, #processor.offsets) - table.sort(processor.offsets, function(e1, e2) return e1[1] < e2[1] end) + table.sort(processor.offsets, function(e1, e2) + return e1[1] < e2[1] + end) processor.processor_func(input, task, processor.offsets, pdf_object, pdf_output) end @@ -1254,7 +1261,7 @@ local function process_pdf(input, mpart, task) end else -- All hashes - for h,sc in pairs(pdf_object.scripts) do + for h, sc in pairs(pdf_object.scripts) do if config.min_js_fuzzy and #sc.data >= config.min_js_fuzzy then lua_util.debugm(N, task, "pdf: add fuzzy hash from JavaScript: %s; size = %s; object: %s:%s", sc.hash, @@ -1323,7 +1330,7 @@ processors.suspicious = function(input, task, positions, pdf_object, pdf_output) local nencoded = 0 local close_encoded = 0 local last_encoded - for _,match in ipairs(positions) do + for _, match in ipairs(positions) do if match[2] == 1 then -- netsh suspicious_factor = suspicious_factor + 0.5 @@ -1386,7 +1393,7 @@ local function generic_table_inserter(positions, pdf_object, output_key) pdf_object[output_key] = {} end local shift = #pdf_object[output_key] - for i,pos in ipairs(positions) do + for i, pos in ipairs(positions) do pdf_object[output_key][i + shift] = pos[1] end end diff --git a/lualib/lua_content/vcard.lua b/lualib/lua_content/vcard.lua index 993a0828f..ed14412ea 100644 --- a/lualib/lua_content/vcard.lua +++ b/lualib/lua_content/vcard.lua @@ -24,11 +24,15 @@ local vcard_grammar local function gen_grammar() if not vcard_grammar then local wsp = l.S(" \t\v\f") - local crlf = (l.P"\r"^-1 * l.P"\n") + l.P"\r" - local eol = (crlf * #crlf) + (crlf - (crlf^-1 * wsp)) - local name = l.C((l.P(1) - (l.P":"))^1) / function(v) return (v:gsub("[\n\r]+%s","")) end - local value = l.C((l.P(1) - eol)^0) / function(v) return (v:gsub("[\n\r]+%s","")) end - vcard_grammar = name * ":" * wsp^0 * value * eol^-1 + local crlf = (l.P "\r" ^ -1 * l.P "\n") + l.P "\r" + local eol = (crlf * #crlf) + (crlf - (crlf ^ -1 * wsp)) + local name = l.C((l.P(1) - (l.P ":")) ^ 1) / function(v) + return (v:gsub("[\n\r]+%s", "")) + end + local value = l.C((l.P(1) - eol) ^ 0) / function(v) + return (v:gsub("[\n\r]+%s", "")) + end + vcard_grammar = name * ":" * wsp ^ 0 * value * eol ^ -1 end return vcard_grammar @@ -37,7 +41,7 @@ end local exports = {} local function process_vcard(input, mpart, task) - local control={n='\n', r=''} + local control = { n = '\n', r = '' } local rspamd_url = require "rspamd_url" local escaper = l.Ct((gen_grammar() / function(key, value) value = value:gsub("\\(.)", control) @@ -45,7 +49,7 @@ local function process_vcard(input, mpart, task) local local_urls = rspamd_url.all(task:get_mempool(), value) if local_urls and #local_urls > 0 then - for _,u in ipairs(local_urls) do + for _, u in ipairs(local_urls) do lua_util.debugm(N, task, 'vcard: found URL in vcard %s', tostring(u)) task:inject_url(u, mpart) @@ -53,8 +57,8 @@ local function process_vcard(input, mpart, task) end lua_util.debugm(N, task, 'vcard: vcard key %s = "%s"', key, value) - return {key, value} - end)^1) + return { key, value } + end) ^ 1) local elts = escaper:match(input) @@ -64,7 +68,9 @@ local function process_vcard(input, mpart, task) return { tag = 'vcard', - extract_text = function() return nil end, -- NYI + extract_text = function() + return nil + end, -- NYI elts = elts } end diff --git a/lualib/lua_dkim_tools.lua b/lualib/lua_dkim_tools.lua index 719b52067..1f396fbc4 100644 --- a/lualib/lua_dkim_tools.lua +++ b/lualib/lua_dkim_tools.lua @@ -27,7 +27,9 @@ local function check_violation(N, task, domain) -- Check for DKIM_REJECT local sym_check = 'R_DKIM_REJECT' - if N == 'arc' then sym_check = 'ARC_REJECT' end + if N == 'arc' then + sym_check = 'ARC_REJECT' + end if task:has_symbol(sym_check) then local sym = task:get_symbol(sym_check)[1] logger.infox(task, 'skip signing for %s: violation %s found: %s', @@ -92,7 +94,7 @@ local function parse_dkim_http_headers(N, task, settings) if not (domain and selector and key) then logger.errx(task, 'missing required headers to sign email') - return false,{} + return false, {} end -- Now check if we need to check the existing auth @@ -114,23 +116,23 @@ local function parse_dkim_http_headers(N, task, settings) end lua_util.debugm(N, task, 'no sign header %s', headers.sign_header) - return false,{} + return false, {} end local function prepare_dkim_signing(N, task, settings) local is_local, is_sign_networks, is_authed if settings.use_http_headers then - local res,tbl = parse_dkim_http_headers(N, task, settings) + local res, tbl = parse_dkim_http_headers(N, task, settings) if not res then if not settings.allow_headers_fallback then - return res,{} + return res, {} else lua_util.debugm(N, task, 'failed to read http headers, fallback to normal schema') end else - return res,tbl + return res, tbl end end @@ -139,13 +141,13 @@ local function prepare_dkim_signing(N, task, settings) local ret = settings.sign_condition(task) if not ret then - return false,{} + return false, {} end if ret[1] then - return true,ret + return true, ret else - return true,{ret} + return true, { ret } end end @@ -163,19 +165,19 @@ local function prepare_dkim_signing(N, task, settings) if metric_action == 'reject' or metric_action == 'drop' then -- No need to sign what we are already rejecting/dropping lua_util.debugm(N, task, 'task result is already %s, no need to sign', metric_action) - return false,{} + return false, {} end if metric_action == 'soft reject' then -- Same here, we are going to delay an email, signing is just a waste of time lua_util.debugm(N, task, 'task result is %s, skip signing', metric_action) - return false,{} + return false, {} end -- For spam actions, there is no clear distinction if metric_action ~= 'no action' and type(settings.skip_spam_sign) == 'boolean' and settings.skip_spam_sign then lua_util.debugm(N, task, 'task result is %s, no need to sign', metric_action) - return false,{} + return false, {} end end @@ -191,7 +193,7 @@ local function prepare_dkim_signing(N, task, settings) lua_util.debugm(N, task, 'mail was sent to us') else lua_util.debugm(N, task, 'mail is ineligible for signing') - return false,{} + return false, {} end local efrom = task:get_from('smtp') @@ -199,7 +201,7 @@ local function prepare_dkim_signing(N, task, settings) if #(((efrom or E)[1] or E).addr or '') == 0 then if not settings.allow_envfrom_empty then lua_util.debugm(N, task, 'empty envelope from not allowed') - return false,{} + return false, {} else empty_envelope = true end @@ -208,7 +210,7 @@ local function prepare_dkim_signing(N, task, settings) local hfrom = task:get_from('mime') if not settings.allow_hdrfrom_multiple and (hfrom or E)[2] then lua_util.debugm(N, task, 'multiple header from not allowed') - return false,{} + return false, {} end local eto = task:get_recipients(0) @@ -258,13 +260,13 @@ local function prepare_dkim_signing(N, task, settings) lua_util.debugm(N, task, 'skip signing: is_sign_network: %s, is_authed: %s, is_local: %s', is_sign_networks, is_authed, is_local) - return false,{} + return false, {} end if not hfrom or not hfrom[1] or not hfrom[1].addr then lua_util.debugm(N, task, 'signing_table: cannot get data when no header from is presented') - return false,{} + return false, {} end local sign_entry = settings.signing_table:get_key(hfrom[1].addr) @@ -277,8 +279,8 @@ local function prepare_dkim_signing(N, task, settings) end if settings.key_table then - -- Now search in key table - local key_entry = settings.key_table:get_key(sign_entry) + -- Now search in key table + local key_entry = settings.key_table:get_key(sign_entry) if key_entry then local parts = lua_util.str_split(key_entry, ':') @@ -290,7 +292,7 @@ local function prepare_dkim_signing(N, task, settings) if not selector then logger.errx(task, 'no selector defined for sign_entry %s, key_entry %s', sign_entry, key_entry) - return false,{} + return false, {} end local res = { @@ -310,7 +312,7 @@ local function prepare_dkim_signing(N, task, settings) hdom, selector, res.domain) end - return true,{res} + return true, { res } elseif #parts == 3 then -- domain, selector, key local selector = parts[2] @@ -332,11 +334,11 @@ local function prepare_dkim_signing(N, task, settings) hdom, selector, res.domain) end - return true,{res} + return true, { res } else logger.errx(task, 'invalid key entry for sign entry %s: %s; when signing %s domain', sign_entry, key_entry, hdom) - return false,{} + return false, {} end elseif settings.use_vault then -- Sign table is presented, the rest is covered by vault @@ -349,17 +351,17 @@ local function prepare_dkim_signing(N, task, settings) else logger.errx(task, 'missing key entry for sign entry %s; when signing %s domain', sign_entry, hdom) - return false,{} + return false, {} end else logger.errx(task, 'cannot get key entry for signing entry %s, when signing %s domain', sign_entry, hdom) - return false,{} + return false, {} end else lua_util.debugm(N, task, 'signing_table: no entry for %s', hfrom[1].addr) - return false,{} + return false, {} end else if settings.use_domain_sign_networks and is_sign_networks then @@ -409,7 +411,7 @@ local function prepare_dkim_signing(N, task, settings) if not dkim_domain then lua_util.debugm(N, task, 'could not extract dkim domain') - return false,{} + return false, {} end if settings.use_esld then @@ -435,7 +437,7 @@ local function prepare_dkim_signing(N, task, settings) lua_util.debugm(N, task, 'domain mismatch allowed for empty envelope: %1 != %2', hdom, edom) else lua_util.debugm(N, task, 'domain mismatch not allowed: %1 != %2', hdom, edom) - return false,{} + return false, {} end end end @@ -443,14 +445,14 @@ local function prepare_dkim_signing(N, task, settings) if auser and not settings.allow_username_mismatch then if not udom then lua_util.debugm(N, task, 'couldnt find domain in username') - return false,{} + return false, {} end if settings.use_esld then udom = rspamd_util.get_tld(udom) end if udom ~= dkim_domain then lua_util.debugm(N, task, 'user domain mismatch') - return false,{} + return false, {} end end @@ -465,8 +467,8 @@ local function prepare_dkim_signing(N, task, settings) } else lua_util.debugm(N, task, 'domain %s is not designated for vault', - dkim_domain) - return false,{} + dkim_domain) + return false, {} end else -- TODO: try every domain in the vault @@ -480,7 +482,7 @@ local function prepare_dkim_signing(N, task, settings) if settings.domain[dkim_domain] then -- support old style selector/paths if settings.domain[dkim_domain].selector or - settings.domain[dkim_domain].path then + settings.domain[dkim_domain].path then local k = {} k.selector = settings.domain[dkim_domain].selector k.key = settings.domain[dkim_domain].path @@ -500,7 +502,7 @@ local function prepare_dkim_signing(N, task, settings) if ret then table.insert(p, k) lua_util.debugm(N, task, 'using mempool selector %s with key %s', - k.selector, k.key) + k.selector, k.key) end end @@ -524,7 +526,7 @@ local function prepare_dkim_signing(N, task, settings) if #p == 0 and not settings.try_fallback then lua_util.debugm(N, task, 'dkim unconfigured and fallback disabled') - return false,{} + return false, {} end if not settings.use_redis then @@ -533,18 +535,18 @@ local function prepare_dkim_signing(N, task, settings) end insert_or_update_prop(N, task, p, 'selector', - 'default selector', settings.selector) + 'default selector', settings.selector) if settings.check_violation then if not check_violation(N, task, p.domain) then - return false,{} + return false, {} end end insert_or_update_prop(N, task, p, 'domain', 'dkim_domain', - dkim_domain) + dkim_domain) - return true,p + return true, p end exports.prepare_dkim_signing = prepare_dkim_signing @@ -575,11 +577,11 @@ exports.sign_using_redis = function(N, task, settings, selectors, sign_func, err false, -- is write redis_key_cb, --callback 'HGET', -- command - {settings.key_prefix, rk} -- arguments + { settings.key_prefix, rk } -- arguments ) if not rret then err_func(task, - string.format( "cannot make request to load DKIM key for %s", rk)) + string.format("cannot make request to load DKIM key for %s", rk)) end end @@ -601,7 +603,7 @@ exports.sign_using_redis = function(N, task, settings, selectors, sign_func, err false, -- is write redis_selector_cb, --callback 'HGET', -- command - {settings.selector_prefix, p.domain} -- arguments + { settings.selector_prefix, p.domain } -- arguments ) if not rret then err_func(task, string.format("cannot make Redis request to load DKIM selector for domain %s", @@ -627,7 +629,7 @@ exports.sign_using_vault = function(N, task, settings, selectors, sign_func, err full_url, err, body)) else local parser = ucl.parser() - local res,parser_err = parser:parse_string(body) + local res, parser_err = parser:parse_string(body) if not res then err_func(task, string.format('vault reply for %s (data=%s) cannot be parsed: %s', full_url, body, parser_err)) @@ -677,7 +679,7 @@ exports.sign_using_vault = function(N, task, settings, selectors, sign_func, err end end - local ret = http.request{ + local ret = http.request { task = task, url = full_url, callback = vault_callback, @@ -712,20 +714,20 @@ exports.process_signing_settings = function(N, settings, opts) local lua_maps = require "lua_maps" -- Used to convert plain options to the maps local maps_opts = { - sign_networks = {'radix', 'DKIM signing networks'}, - path_map = {'map', 'Paths to DKIM signing keys'}, - selector_map = {'map', 'DKIM selectors'}, - signing_table = {'glob', 'DKIM signing table'}, - key_table = {'glob', 'DKIM keys table'}, - vault_domains = {'glob', 'DKIM signing domains in vault'}, - whitelisted_signers_map = {'set', 'ARC trusted signers domains'} + sign_networks = { 'radix', 'DKIM signing networks' }, + path_map = { 'map', 'Paths to DKIM signing keys' }, + selector_map = { 'map', 'DKIM selectors' }, + signing_table = { 'glob', 'DKIM signing table' }, + key_table = { 'glob', 'DKIM keys table' }, + vault_domains = { 'glob', 'DKIM signing domains in vault' }, + whitelisted_signers_map = { 'set', 'ARC trusted signers domains' } } - for k,v in pairs(opts) do + for k, v in pairs(opts) do local maybe_map = maps_opts[k] if maybe_map then settings[k] = lua_maps.map_add_from_ucl(v, maybe_map[1], maybe_map[2]) elseif k == 'sign_condition' then - local ret,f = lua_util.callback_from_string(v) + local ret, f = lua_util.callback_from_string(v) if ret then settings[k] = f else diff --git a/lualib/lua_ffi/common.lua b/lualib/lua_ffi/common.lua index 4455c580e..4076cfad4 100644 --- a/lualib/lua_ffi/common.lua +++ b/lualib/lua_ffi/common.lua @@ -21,7 +21,7 @@ limitations under the License. local ffi = require 'ffi' -ffi.cdef[[ +ffi.cdef [[ struct GString { char *str; size_t len; diff --git a/lualib/lua_ffi/dkim.lua b/lualib/lua_ffi/dkim.lua index 16d7c11a1..e4592c2d2 100644 --- a/lualib/lua_ffi/dkim.lua +++ b/lualib/lua_ffi/dkim.lua @@ -21,7 +21,7 @@ limitations under the License. local ffi = require 'ffi' -ffi.cdef[[ +ffi.cdef [[ struct rspamd_dkim_sign_context_s; struct rspamd_dkim_key_s; struct rspamd_task; @@ -69,23 +69,22 @@ local function load_sign_key(what, format) elseif format == 'raw' then format = ffi.C.RSPAMD_DKIM_KEY_RAW else - return nil,'unknown key format' + return nil, 'unknown key format' end end return ffi.C.rspamd_dkim_sign_key_load(what, #what, format, nil) end -local default_dkim_headers = -"(o)from:(o)sender:(o)reply-to:(o)subject:(o)date:(o)message-id:" .. -"(o)to:(o)cc:(o)mime-version:(o)content-type:(o)content-transfer-encoding:" .. -"resent-to:resent-cc:resent-from:resent-sender:resent-message-id:" .. -"(o)in-reply-to:(o)references:list-id:list-owner:list-unsubscribe:" .. -"list-subscribe:list-post:(o)openpgp:(o)autocrypt" +local default_dkim_headers = "(o)from:(o)sender:(o)reply-to:(o)subject:(o)date:(o)message-id:" .. + "(o)to:(o)cc:(o)mime-version:(o)content-type:(o)content-transfer-encoding:" .. + "resent-to:resent-cc:resent-from:resent-sender:resent-message-id:" .. + "(o)in-reply-to:(o)references:list-id:list-owner:list-unsubscribe:" .. + "list-subscribe:list-post:(o)openpgp:(o)autocrypt" local function create_sign_context(task, privkey, dkim_headers, sign_type) if not task or not privkey then - return nil,'invalid arguments' + return nil, 'invalid arguments' end if not dkim_headers then @@ -103,10 +102,9 @@ local function create_sign_context(task, privkey, dkim_headers, sign_type) elseif sign_type == 'arc-seal' then sign_type = ffi.C.RSPAMD_DKIM_ARC_SEAL else - return nil,'invalid sign type' + return nil, 'invalid sign type' end - return ffi.C.rspamd_create_dkim_sign_context(task:topointer(), privkey, 1, 1, dkim_headers, sign_type, nil) end @@ -114,17 +112,23 @@ end local function do_sign(task, sign_context, selector, domain, expire, len, arc_idx) if not task or not sign_context or not selector or not domain then - return nil,'invalid arguments' + return nil, 'invalid arguments' end - if not expire then expire = 0 end - if not len then len = 0 end - if not arc_idx then arc_idx = 0 end + if not expire then + expire = 0 + end + if not len then + len = 0 + end + if not arc_idx then + arc_idx = 0 + end local gstring = ffi.C.rspamd_dkim_sign(task:topointer(), selector, domain, expire, len, arc_idx, nil, sign_context) if not gstring then - return nil,'cannot sign' + return nil, 'cannot sign' end local ret = ffi.string(gstring.str, gstring.len) diff --git a/lualib/lua_ffi/init.lua b/lualib/lua_ffi/init.lua index 6e61462f9..efbbc7ab6 100644 --- a/lualib/lua_ffi/init.lua +++ b/lualib/lua_ffi/init.lua @@ -31,7 +31,7 @@ if type(jit) == 'table' then return o ~= NULL end else - local ret,result_or_err = pcall(require, 'ffi') + local ret, result_or_err = pcall(require, 'ffi') if not ret then return {} @@ -51,7 +51,7 @@ exports.dkim = require "lua_ffi/dkim" exports.spf = require "lua_ffi/spf" exports.linalg = require "lua_ffi/linalg" -for k,v in pairs(ffi) do +for k, v in pairs(ffi) do -- Preserve all stuff to use lua_ffi as ffi itself exports[k] = v end diff --git a/lualib/lua_ffi/linalg.lua b/lualib/lua_ffi/linalg.lua index 6a5c53df0..2df488afe 100644 --- a/lualib/lua_ffi/linalg.lua +++ b/lualib/lua_ffi/linalg.lua @@ -23,15 +23,15 @@ local ffi = require 'ffi' local exports = {} -ffi.cdef[[ +ffi.cdef [[ void kad_sgemm_simple(int trans_A, int trans_B, int M, int N, int K, const float *A, const float *B, float *C); bool kad_ssyev_simple (int N, float *A, float *output); ]] local function table_to_ffi(a, m, n) local a_conv = ffi.new("float[?]", m * n) - for i=1,m or #a do - for j=1,n or #a[1] do + for i = 1, m or #a do + for j = 1, n or #a[1] do a_conv[(i - 1) * n + (j - 1)] = a[i][j] end end @@ -41,9 +41,9 @@ end local function ffi_to_table(a, m, n) local res = {} - for i=0,m-1 do + for i = 0, m - 1 do res[i + 1] = {} - for j=0,n-1 do + for j = 0, n - 1 do res[i + 1][j + 1] = a[i * n + j] end end @@ -75,7 +75,7 @@ exports.eigen = function(a, n) local res = ffi.new("float[?]", n) if ffi.C.kad_ssyev_simple(n, ffi.cast('float*', a), res) then - return res,a + return res, a end return nil diff --git a/lualib/lua_ffi/spf.lua b/lualib/lua_ffi/spf.lua index 783870eac..0f982f2f9 100644 --- a/lualib/lua_ffi/spf.lua +++ b/lualib/lua_ffi/spf.lua @@ -21,7 +21,7 @@ limitations under the License. local ffi = require 'ffi' -ffi.cdef[[ +ffi.cdef [[ enum spf_mech_e { SPF_FAIL, SPF_SOFT_FAIL, @@ -112,7 +112,7 @@ local function spf_resolve(task, cb) local digstr = ffi.new("char[64]") ffi.C.rspamd_snprintf(digstr, 64, "0x%xuL", rec.digest) res.digest = ffi.string(digstr) - for i = 1,nelts do + for i = 1, nelts do res.addrs[i] = spf_addr_tolua(elts[i - 1]) end diff --git a/lualib/lua_fuzzy.lua b/lualib/lua_fuzzy.lua index 332e828ac..f941436eb 100644 --- a/lualib/lua_fuzzy.lua +++ b/lualib/lua_fuzzy.lua @@ -38,7 +38,7 @@ local policies = { min_width = 500, min_length = 64, text_multiplier = 4.0, -- divide min_bytes by 4 for texts - mime_types = {"application/*"}, + mime_types = { "application/*" }, scan_archives = true, short_text_direct_hash = true, text_shingles = true, @@ -48,7 +48,7 @@ local policies = { local default_policy = policies.recommended -local policy_schema = ts.shape{ +local policy_schema = ts.shape { min_bytes = ts.number + ts.string / tonumber, min_height = ts.number + ts.string / tonumber, min_width = ts.number + ts.string / tonumber, @@ -61,7 +61,6 @@ local policy_schema = ts.shape{ skip_images = ts.boolean, } - local exports = {} @@ -74,7 +73,7 @@ exports.register_policy = function(name, policy) rspamd_logger.warnx(rspamd_config, "overriding policy %s", name) end - local parsed_policy,err = policy_schema:transform(policy) + local parsed_policy, err = policy_schema:transform(policy) if not parsed_policy then rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule policy %s: %s', @@ -160,7 +159,7 @@ local function check_length(task, part, rule) end local function check_text_part(task, part, rule, text) - local allow_direct,allow_shingles = false,false + local allow_direct, allow_shingles = false, false local id = part:get_id() lua_util.debugm(N, task, 'check text part %s', id) @@ -200,7 +199,7 @@ local function check_text_part(task, part, rule, text) allow_direct = check_length(task, part, rule) end - return allow_direct,allow_shingles + return allow_direct, allow_shingles end --local function has_sane_text_parts(task) @@ -211,7 +210,7 @@ end local function check_image_part(task, part, rule, image) if rule.skip_images then lua_util.debugm(N, task, 'skip image part as images are disabled') - return false,false + return false, false end local id = part:get_id() @@ -227,7 +226,7 @@ local function check_image_part(task, part, rule, image) if height and width then if height < min_height or width < min_width then lua_util.debugm(N, task, 'skip image part %s as it does not meet minimum sizes: %sx%s < %sx%s', - id, width, height, min_width, min_height) + id, width, height, min_width, min_height) return false, false else lua_util.debugm(N, task, 'allow image part %s: %sx%s', @@ -236,18 +235,20 @@ local function check_image_part(task, part, rule, image) end end - return check_length(task, part, rule),false + return check_length(task, part, rule), false end local function mime_types_check(task, part, rule) - local t,st = part:get_type() + local t, st = part:get_type() - if not t then return false, false end + if not t then + return false, false + end local ct = string.format('%s/%s', t, st) local detected_ct - t,st = part:get_detected_type() + t, st = part:get_detected_type() if t then detected_ct = string.format('%s/%s', t, st) else @@ -266,12 +267,12 @@ local function mime_types_check(task, part, rule) end opts = fun.tomap(fun.map(function(opt) local elts = lua_util.str_split(opt, ':') - return elts[1],elts[2] + return elts[1], elts[2] end, opts)) if opts[id] and opts[id] == '-' then lua_util.debugm(N, task, 'explicitly check binary part %s: bad mime type %s', id, ct) - return check_length(task, part, rule),false + return check_length(task, part, rule), false end if rule.mime_types then @@ -285,13 +286,13 @@ local function mime_types_check(task, part, rule) end, rule.mime_types) then lua_util.debugm(N, task, 'found mime type match for part %s: %s (%s detected)', id, ct, detected_ct) - return check_length(task, part, rule),false + return check_length(task, part, rule), false end return false, false end - return false,false + return false, false end exports.check_mime_part = function(task, part, rule_id) @@ -300,7 +301,7 @@ exports.check_mime_part = function(task, part, rule_id) if not rule then rspamd_logger.errx(task, 'cannot find rule with id %s', rule_id) - return false,false + return false, false end if part:is_text() then @@ -315,7 +316,7 @@ exports.check_mime_part = function(task, part, rule_id) -- Always send archives lua_util.debugm(N, task, 'check archive part %s', part:get_id()) - return true,false + return true, false end if part:is_specific() then @@ -323,7 +324,7 @@ exports.check_mime_part = function(task, part, rule_id) if type(sp) == 'table' and sp.fuzzy_hashes then lua_util.debugm(N, task, 'check specific part %s', part:get_id()) - return true,false + return true, false end end @@ -331,7 +332,7 @@ exports.check_mime_part = function(task, part, rule_id) return mime_types_check(task, part, rule) end - return false,false + return false, false end exports.cleanup_rules = function() diff --git a/lualib/lua_lexer.lua b/lualib/lua_lexer.lua index 324641013..54bbd7c04 100644 --- a/lualib/lua_lexer.lua +++ b/lualib/lua_lexer.lua @@ -22,36 +22,38 @@ local lpeg = require "lpeg" local P = lpeg.P local R = lpeg.R local S = lpeg.S -local D = R'09' -- Digits +local D = R '09' -- Digits local I = R('AZ', 'az', '\127\255') + '_' -- Identifiers local B = -(I + D) -- Word boundary local EOS = -lpeg.P(1) -- end of string -- Pattern for long strings and long comments. -local longstring = #(P'[[' + (P'[' * P'='^0 * '[')) * P(function(input, index) +local longstring = #(P '[[' + (P '[' * P '=' ^ 0 * '[')) * P(function(input, index) local level = input:match('^%[(=*)%[', index) if level then local _, last = input:find(']' .. level .. ']', index, true) - if last then return last + 1 end + if last then + return last + 1 + end end end) -- String literals. -local singlequoted = P"'" * ((1 - S"'\r\n\f\\") + (P'\\' * 1))^0 * "'" -local doublequoted = P'"' * ((1 - S'"\r\n\f\\') + (P'\\' * 1))^0 * '"' +local singlequoted = P "'" * ((1 - S "'\r\n\f\\") + (P '\\' * 1)) ^ 0 * "'" +local doublequoted = P '"' * ((1 - S '"\r\n\f\\') + (P '\\' * 1)) ^ 0 * '"' -- Comments. -local eol = P'\r\n' + '\n' -local line = (1 - S'\r\n\f')^0 * eol^-1 -local singleline = P'--' * line -local multiline = P'--' * longstring +local eol = P '\r\n' + '\n' +local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1 +local singleline = P '--' * line +local multiline = P '--' * longstring -- Numbers. -local sign = S'+-'^-1 -local decimal = D^1 -local hexadecimal = P'0' * S'xX' * R('09', 'AF', 'af') ^ 1 -local float = D^1 * P'.' * D^0 + P'.' * D^1 -local maybeexp = (float + decimal) * (S'eE' * sign * D^1)^-1 +local sign = S '+-' ^ -1 +local decimal = D ^ 1 +local hexadecimal = P '0' * S 'xX' * R('09', 'AF', 'af') ^ 1 +local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1 +local maybeexp = (float + decimal) * (S 'eE' * sign * D ^ 1) ^ -1 local function compile_keywords(keywords) local list = {} @@ -74,26 +76,26 @@ local function compile_keywords(keywords) end -- Identifiers -local ident = I * (I + D)^0 -local expr = ('.' * ident)^0 +local ident = I * (I + D) ^ 0 +local expr = ('.' * ident) ^ 0 local patterns = { - {'whitespace', S'\r\n\f\t\v '^1}, - {'constant', (P'true' + 'false' + 'nil') * B}, - {'string', singlequoted + doublequoted + longstring}, - {'comment', multiline + singleline}, - {'number', hexadecimal + maybeexp}, - {'operator', P'not' + '...' + 'and' + '..' + '~=' + '==' + '>=' + '<=' - + 'or' + S']{=>^[<;)*(%}+-:,/.#'}, - {'keyword', compile_keywords([[ + { 'whitespace', S '\r\n\f\t\v ' ^ 1 }, + { 'constant', (P 'true' + 'false' + 'nil') * B }, + { 'string', singlequoted + doublequoted + longstring }, + { 'comment', multiline + singleline }, + { 'number', hexadecimal + maybeexp }, + { 'operator', P 'not' + '...' + 'and' + '..' + '~=' + '==' + '>=' + '<=' + + 'or' + S ']{=>^[<;)*(%}+-:,/.#' }, + { 'keyword', compile_keywords([[ break do else elseif end for function if in local repeat return then until while - ]])}, - {'identifier', lpeg.Cmt(ident, + ]]) }, + { 'identifier', lpeg.Cmt(ident, function(input, index) return expr:match(input, index) end) }, - {'error', 1}, + { 'error', 1 }, } local compiled @@ -101,7 +103,7 @@ local compiled local function compile_patterns() if not compiled then local function process(elt) - local n,grammar = elt[1],elt[2] + local n, grammar = elt[1], elt[2] return lpeg.Cc(n) * lpeg.P(grammar) * lpeg.Cp() end local any = process(patterns[1]) @@ -151,7 +153,7 @@ exports.lex_to_table = function(input) local out = {} for kind, text, lnum, cnum in exports.gmatch(input) do - out[#out + 1] = {kind, text, lnum, cnum} + out[#out + 1] = { kind, text, lnum, cnum } end return out diff --git a/lualib/lua_magic/heuristics.lua b/lualib/lua_magic/heuristics.lua index 98cfb0eee..b8a1b4188 100644 --- a/lualib/lua_magic/heuristics.lua +++ b/lualib/lua_magic/heuristics.lua @@ -28,18 +28,18 @@ local fun = require "fun" local N = "lua_magic" local msoffice_trie local msoffice_patterns = { - doc = {[[WordDocument]]}, - xls = {[[Workbook]], [[Book]]}, - ppt = {[[PowerPoint Document]], [[Current User]]}, - vsd = {[[VisioDocument]]}, + doc = { [[WordDocument]] }, + xls = { [[Workbook]], [[Book]] }, + ppt = { [[PowerPoint Document]], [[Current User]] }, + vsd = { [[VisioDocument]] }, } local msoffice_trie_clsid local msoffice_clsids = { - doc = {[[0609020000000000c000000000000046]]}, - xls = {[[1008020000000000c000000000000046]], [[2008020000000000c000000000000046]]}, - ppt = {[[108d81649b4fcf1186ea00aa00b929e8]]}, - msg = {[[46f0060000000000c000000000000046]], [[0b0d020000000000c000000000000046]]}, - msi = {[[84100c0000000000c000000000000046]]}, + doc = { [[0609020000000000c000000000000046]] }, + xls = { [[1008020000000000c000000000000046]], [[2008020000000000c000000000000046]] }, + ppt = { [[108d81649b4fcf1186ea00aa00b929e8]] }, + msg = { [[46f0060000000000c000000000000046]], [[0b0d020000000000c000000000000046]] }, + msi = { [[84100c0000000000c000000000000046]] }, } local zip_trie local zip_patterns = { @@ -54,37 +54,37 @@ local zip_patterns = { [[mimetypeapplication/vnd\.oasis\.opendocument\.formula]], [[mimetypeapplication/vnd\.oasis\.opendocument\.chart]] }, - odp = {[[mimetypeapplication/vnd\.oasis\.opendocument\.presentation]]}, - epub = {[[epub\+zip]]}, - asice = {[[mimetypeapplication/vnd\.etsi\.asic-e\+zipPK]]}, - asics = {[[mimetypeapplication/vnd\.etsi\.asic-s\+zipPK]]}, + odp = { [[mimetypeapplication/vnd\.oasis\.opendocument\.presentation]] }, + epub = { [[epub\+zip]] }, + asice = { [[mimetypeapplication/vnd\.etsi\.asic-e\+zipPK]] }, + asics = { [[mimetypeapplication/vnd\.etsi\.asic-s\+zipPK]] }, } local txt_trie local txt_patterns = { html = { - {[=[(?i)<html[\s>]]=], 32}, - {[[(?i)<script\b]], 20}, -- Commonly used by spammers - {[[<script\s+type="text\/javascript">]], 31}, -- Another spammy pattern - {[[(?i)<\!DOCTYPE HTML\b]], 33}, - {[[(?i)<body\b]], 20}, - {[[(?i)<table\b]], 20}, - {[[(?i)<a\s]], 10}, - {[[(?i)<p\b]], 10}, - {[[(?i)<div\b]], 10}, - {[[(?i)<span\b]], 10}, + { [=[(?i)<html[\s>]]=], 32 }, + { [[(?i)<script\b]], 20 }, -- Commonly used by spammers + { [[<script\s+type="text\/javascript">]], 31 }, -- Another spammy pattern + { [[(?i)<\!DOCTYPE HTML\b]], 33 }, + { [[(?i)<body\b]], 20 }, + { [[(?i)<table\b]], 20 }, + { [[(?i)<a\s]], 10 }, + { [[(?i)<p\b]], 10 }, + { [[(?i)<div\b]], 10 }, + { [[(?i)<span\b]], 10 }, }, csv = { - {[[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]], 20} + { [[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]], 20 } }, ics = { - {[[^BEGIN:VCALENDAR\r?\n]], 40}, + { [[^BEGIN:VCALENDAR\r?\n]], 40 }, }, vcf = { - {[[^BEGIN:VCARD\r?\n]], 40}, + { [[^BEGIN:VCARD\r?\n]], 40 }, }, xml = { - {[[<\?xml\b.+\?>]], 31}, + { [[<\?xml\b.+\?>]], 31 }, } } @@ -103,11 +103,11 @@ local function compile_tries() rspamd_trie.flags.no_start) local function compile_pats(patterns, indexes, transform_func, compile_flags) local strs = {} - for ext,pats in pairs(patterns) do - for _,pat in ipairs(pats) do + for ext, pats in pairs(patterns) do + for _, pat in ipairs(pats) do -- These are utf16 strings in fact... strs[#strs + 1] = transform_func(pat) - indexes[#indexes + 1] = {ext, pat} + indexes[#indexes + 1] = { ext, pat } end end @@ -120,12 +120,14 @@ local function compile_tries() return '^' .. table.concat( fun.totable( - fun.map(function(c) return c .. [[\x{00}]] end, + fun.map(function(c) + return c .. [[\x{00}]] + end, fun.iter(pat)))) end local function msoffice_clsid_transform(pat) local hex_table = {} - for i=1,#pat,2 do + for i = 1, #pat, 2 do local subc = pat:sub(i, i + 1) hex_table[#hex_table + 1] = string.format('\\x{%s}', subc) end @@ -140,10 +142,14 @@ local function compile_tries() msoffice_clsid_transform) -- Misc zip patterns at the initial fragment zip_trie = compile_pats(zip_patterns, zip_patterns_indexes, - function(pat) return pat end) + function(pat) + return pat + end) -- Text patterns at the initial fragment txt_trie = compile_pats(txt_patterns, txt_patterns_indexes, - function(pat_tbl) return pat_tbl[1] end, + function(pat_tbl) + return pat_tbl[1] + end, bit.bor(rspamd_trie.flags.re, rspamd_trie.flags.dot_all, rspamd_trie.flags.no_start)) @@ -160,12 +166,13 @@ local function detect_ole_format(input, log_obj, _, part) return nil end - local bom,sec_size = rspamd_util.unpack('<I2<I2', input:span(29, 4)) + local bom, sec_size = rspamd_util.unpack('<I2<I2', input:span(29, 4)) if bom == 0xFFFE then bom = '<' else lua_util.debugm(N, log_obj, "bom file!: %s", bom) - bom = '>'; sec_size = bit.bswap(sec_size) + bom = '>'; + sec_size = bit.bswap(sec_size) end if sec_size < 7 or sec_size > 31 then @@ -194,39 +201,39 @@ local function detect_ole_format(input, log_obj, _, part) -- Extract clsid local matches = msoffice_trie_clsid:match(input:span(offset + 80, 16)) if matches then - for n,_ in pairs(matches) do + for n, _ in pairs(matches) do if msoffice_clsid_indexes[n] then lua_util.debugm(N, log_obj, "found valid clsid for %s", msoffice_clsid_indexes[n][1]) - return true,msoffice_clsid_indexes[n][1] + return true, msoffice_clsid_indexes[n][1] end end end - return true,nil + return true, nil elseif dtype == 2 then local matches = msoffice_trie:match(input:span(offset, 64)) if matches then - for n,_ in pairs(matches) do + for n, _ in pairs(matches) do if msoffice_patterns_indexes[n] then - return true,msoffice_patterns_indexes[n][1] + return true, msoffice_patterns_indexes[n][1] end end end - return true,nil + return true, nil elseif dtype >= 0 and dtype < 5 then -- Bad type - return true,nil + return true, nil end end - return false,nil + return false, nil end repeat - local res,ext = process_dir_entry(directory_offset) + local res, ext = process_dir_entry(directory_offset) if res and ext then - return ext,60 + return ext, 60 end if not res then @@ -247,7 +254,7 @@ local function process_top_detected(res) return res[ex1] > res[ex2] end) - return extensions[1],res[extensions[1]] + return extensions[1], res[extensions[1]] end return nil @@ -276,7 +283,7 @@ local function detect_archive_flaw(part, arch, log_obj, _) if arch_type == 'zip' then -- Find specific files/folders in zip file local files = arch:get_files(100) or {} - for _,file in ipairs(files) do + for _, file in ipairs(files) do if file == '[Content_Types].xml' then add_msoffice_confidence(10) elseif file:sub(1, 3) == 'xl/' then @@ -292,10 +299,10 @@ local function detect_archive_flaw(part, arch, log_obj, _) end end - local ext,weight = process_top_detected(res) + local ext, weight = process_top_detected(res) if weight >= 40 then - return ext,weight + return ext, weight end -- Apply misc Zip detection logic @@ -306,32 +313,34 @@ local function detect_archive_flaw(part, arch, log_obj, _) local matches = zip_trie:match(start_span) if matches then - for n,_ in pairs(matches) do + for n, _ in pairs(matches) do if zip_patterns_indexes[n] then lua_util.debugm(N, log_obj, "found zip pattern for %s", zip_patterns_indexes[n][1]) - return zip_patterns_indexes[n][1],40 + return zip_patterns_indexes[n][1], 40 end end end end end - return arch_type:lower(),40 + return arch_type:lower(), 40 end local csv_grammar -- Returns a grammar that will count commas local function get_csv_grammar() if not csv_grammar then - local lpeg = require'lpeg' + local lpeg = require 'lpeg' - local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P'""' / '"')^0) * '"' + - lpeg.C((1 - lpeg.S',\n"')^0) + local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P '""' / '"') ^ 0) * '"' + + lpeg.C((1 - lpeg.S ',\n"') ^ 0) - csv_grammar = lpeg.Cf(lpeg.Cc(0) * field * lpeg.P( (lpeg.P(',') + - lpeg.P('\t')) * field)^1 * (lpeg.S'\r\n' + -1), - function(acc) return acc + 1 end) + csv_grammar = lpeg.Cf(lpeg.Cc(0) * field * lpeg.P((lpeg.P(',') + + lpeg.P('\t')) * field) ^ 1 * (lpeg.S '\r\n' + -1), + function(acc) + return acc + 1 + end) end return csv_grammar @@ -402,17 +411,17 @@ exports.text_part_heuristic = function(part, log_obj, _) while b >= 127 and idx < len do -- utf8 part if bit.band(b, 0xe0) == 0xc0 and remain > 1 and - bit.band(bytes[idx + 1], 0xc0) == 0x80 then - return true,1 + bit.band(bytes[idx + 1], 0xc0) == 0x80 then + return true, 1 elseif bit.band(b, 0xf0) == 0xe0 and remain > 2 and - bit.band(bytes[idx + 1], 0xc0) == 0x80 and - bit.band(bytes[idx + 2], 0xc0) == 0x80 then - return true,2 + bit.band(bytes[idx + 1], 0xc0) == 0x80 and + bit.band(bytes[idx + 2], 0xc0) == 0x80 then + return true, 2 elseif bit.band(b, 0xf8) == 0xf0 and remain > 3 and - bit.band(bytes[idx + 1], 0xc0) == 0x80 and - bit.band(bytes[idx + 2], 0xc0) == 0x80 and - bit.band(bytes[idx + 3], 0xc0) == 0x80 then - return true,3 + bit.band(bytes[idx + 1], 0xc0) == 0x80 and + bit.band(bytes[idx + 2], 0xc0) == 0x80 and + bit.band(bytes[idx + 3], 0xc0) == 0x80 then + return true, 3 end n8bit = n8bit + 1 @@ -422,10 +431,10 @@ exports.text_part_heuristic = function(part, log_obj, _) end if n8bit >= 3 then - return true,n8bit + return true, n8bit end - return false,0 + return false, 0 end -- Convert to string as LuaJIT can optimise string.sub (and fun.iter) but not C calls @@ -439,7 +448,7 @@ exports.text_part_heuristic = function(part, log_obj, _) if (b < 0x20) and not (b == 0x0d or b == 0x0a or b == 0x09) then non_printable = non_printable + 1 elseif b >= 127 then - local c,nskip = rough_8bit_check(bytes, i, tlen - i, tlen) + local c, nskip = rough_8bit_check(bytes, i, tlen - i, tlen) if not c then non_printable = non_printable + 1 @@ -462,7 +471,7 @@ exports.text_part_heuristic = function(part, log_obj, _) local parent = part:get_parent() if parent then - local parent_type,parent_subtype = parent:get_type() + local parent_type, parent_subtype = parent:get_type() if parent_type == 'multipart' and parent_subtype == 'encrypted' then -- Skip text heuristics for encrypted parts @@ -473,7 +482,7 @@ exports.text_part_heuristic = function(part, log_obj, _) end local content = part:get_content() - local mtype,msubtype = part:get_type() + local mtype, msubtype = part:get_type() local clen = #content local is_text @@ -495,8 +504,8 @@ exports.text_part_heuristic = function(part, log_obj, _) if matches then -- Require at least 2 occurrences of those patterns - for n,positions in pairs(matches) do - local ext,weight = txt_patterns_indexes[n][1], txt_patterns_indexes[n][2][2] + for n, positions in pairs(matches) do + local ext, weight = txt_patterns_indexes[n][1], txt_patterns_indexes[n][2][2] if ext then res[ext] = (res[ext] or 0) + weight * #positions lua_util.debugm(N, log_obj, "found txt pattern for %s: %s, total: %s; %s/%s announced", @@ -504,7 +513,7 @@ exports.text_part_heuristic = function(part, log_obj, _) end end - if res.html and res.html >= 40 then + if res.html and res.html >= 40 then -- HTML has priority over something like js... return 'html', res.html end @@ -525,7 +534,7 @@ exports.text_part_heuristic = function(part, log_obj, _) -- Content type stuff if (mtype == 'text' or mtype == 'application') and - (msubtype == 'html' or msubtype == 'xhtml+xml') then + (msubtype == 'html' or msubtype == 'xhtml+xml') then return 'html', 21 end @@ -539,12 +548,12 @@ exports.text_part_heuristic = function(part, log_obj, _) local function has_extension(file, ext) local ext_len = ext:len() return file:len() > ext_len + 1 - and file:sub(-ext_len):lower() == ext - and file:sub(-ext_len - 1, -ext_len - 1) == '.' + and file:sub(-ext_len):lower() == ext + and file:sub(-ext_len - 1, -ext_len - 1) == '.' end if fname and (has_extension(fname, 'htm') or has_extension(fname, 'html')) then - return 'html',21 + return 'html', 21 end if mtype ~= 'text' then @@ -552,7 +561,7 @@ exports.text_part_heuristic = function(part, log_obj, _) return nil end - return 'txt',40 + return 'txt', 40 end end end @@ -569,7 +578,7 @@ exports.pdf_format_heuristic = function(input, log_obj, pos, part) weight = weight + 30 end - return 'pdf',weight + return 'pdf', weight end exports.pe_part_heuristic = function(input, log_obj, pos, part) @@ -590,7 +599,7 @@ exports.pe_part_heuristic = function(input, log_obj, pos, part) return end - return 'exe',30 + return 'exe', 30 end return exports diff --git a/lualib/lua_magic/init.lua b/lualib/lua_magic/init.lua index 4ad027168..38bfddbf2 100644 --- a/lualib/lua_magic/init.lua +++ b/lualib/lua_magic/init.lua @@ -70,7 +70,6 @@ local function process_patterns(log_obj) str, pattern.ext) end - if max_short_offset < match.position then max_short_offset = match.position end @@ -93,10 +92,10 @@ local function process_patterns(log_obj) end if not compiled_patterns then - for ext,pattern in pairs(patterns) do + for ext, pattern in pairs(patterns) do assert(types[ext], 'not found type: ' .. ext) pattern.ext = ext - for _,match in ipairs(pattern.matches) do + for _, match in ipairs(pattern.matches) do if match.string then if match.relative_position and not match.position then match.position = match.relative_position + #match.string @@ -111,7 +110,7 @@ local function process_patterns(log_obj) elseif match.hex then local hex_table = {} - for i=1,#match.hex,2 do + for i = 1, #match.hex, 2 do local subc = match.hex:sub(i, i + 1) hex_table[#hex_table + 1] = string.format('\\x{%s}', subc) end @@ -131,15 +130,21 @@ local function process_patterns(log_obj) compile_flags = bit.bor(compile_flags, rspamd_trie.flags.single_match) compile_flags = bit.bor(compile_flags, rspamd_trie.flags.no_start) compiled_patterns = rspamd_trie.create(fun.totable( - fun.map(function(t) return t[1] end, processed_patterns)), + fun.map(function(t) + return t[1] + end, processed_patterns)), compile_flags ) compiled_short_patterns = rspamd_trie.create(fun.totable( - fun.map(function(t) return t[1] end, short_patterns)), + fun.map(function(t) + return t[1] + end, short_patterns)), compile_flags ) compiled_tail_patterns = rspamd_trie.create(fun.totable( - fun.map(function(t) return t[1] end, tail_patterns)), + fun.map(function(t) + return t[1] + end, tail_patterns)), compile_flags ) @@ -167,24 +172,36 @@ local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_ res[ext] = res[ext] + 1 end - lua_util.debugm(N, log_obj,'add pattern for %s, weight %s, total weight %s', + lua_util.debugm(N, log_obj, 'add pattern for %s, weight %s, total weight %s', ext, weight, res[ext]) end local function match_position(pos, expected) - local cmp = function(a, b) return a == b end + local cmp = function(a, b) + return a == b + end if type(expected) == 'table' then -- Something like {'>', 0} if expected[1] == '>' then - cmp = function(a, b) return a > b end + cmp = function(a, b) + return a > b + end elseif expected[1] == '>=' then - cmp = function(a, b) return a >= b end + cmp = function(a, b) + return a >= b + end elseif expected[1] == '<' then - cmp = function(a, b) return a < b end + cmp = function(a, b) + return a < b + end elseif expected[1] == '<=' then - cmp = function(a, b) return a <= b end + cmp = function(a, b) + return a <= b + end elseif expected[1] == '!=' then - cmp = function(a, b) return a ~= b end + cmp = function(a, b) + return a ~= b + end end expected = expected[2] end @@ -196,7 +213,7 @@ local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_ return cmp(pos, expected) end - for npat,matched_positions in pairs(matches) do + for npat, matched_positions in pairs(matches) do local pat_data = processed_tbl[npat] local pattern = pat_data[3] local match = pat_data[2] @@ -205,12 +222,12 @@ local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_ if match.position then local position = match.position - for _,pos in ipairs(matched_positions) do + for _, pos in ipairs(matched_positions) do lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)', pattern.ext, pos, offset) if match_position(pos + offset, position) then if match.heuristic then - local ext,weight = match.heuristic(input, log_obj, pos + offset, part) + local ext, weight = match.heuristic(input, log_obj, pos + offset, part) if ext then add_result(weight, ext) @@ -226,9 +243,9 @@ local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_ -- Match all positions local all_right = true local matched_pos = 0 - for _,position in ipairs(match.positions) do + for _, position in ipairs(match.positions) do local matched = false - for _,pos in ipairs(matched_positions) do + for _, pos in ipairs(matched_positions) do lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)', pattern.ext, pos, offset) if not match_position(pos + offset, position) then @@ -245,7 +262,7 @@ local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_ if all_right then if match.heuristic then - local ext,weight = match.heuristic(input, log_obj, matched_pos + offset, part) + local ext, weight = match.heuristic(input, log_obj, matched_pos + offset, part) if ext then add_result(weight, ext) @@ -269,14 +286,16 @@ local function process_detected(res) return res[ex1] > res[ex2] end) - return extensions,res[extensions[1]] + return extensions, res[extensions[1]] end return nil end exports.detect = function(part, log_obj) - if not log_obj then log_obj = rspamd_config end + if not log_obj then + log_obj = rspamd_config + end local input = part:get_content() local res = {} @@ -286,7 +305,6 @@ exports.detect = function(part, log_obj) input = rspamd_text.fromstring(input) end - if type(input) == 'userdata' then local inplen = #input @@ -303,18 +321,17 @@ exports.detect = function(part, log_obj) compiled_short_patterns, short_patterns, log_obj, res, part) -- Check if we have enough data or go to long patterns - local extensions,confidence = process_detected(res) + local extensions, confidence = process_detected(res) if extensions and #extensions > 0 and confidence > 30 then -- We are done on short patterns - return extensions[1],types[extensions[1]] + return extensions[1], types[extensions[1]] end -- No way, let's check data in chunks or just the whole input if it is small enough if #input > exports.chunk_size * 3 then -- Chunked version as input is too long - local chunk1, chunk2 = - input:span(1, exports.chunk_size * 2), + local chunk1, chunk2 = input:span(1, exports.chunk_size * 2), input:span(inplen - exports.chunk_size, exports.chunk_size) local offset1, offset2 = 0, inplen - exports.chunk_size @@ -335,7 +352,7 @@ exports.detect = function(part, log_obj) local extensions = process_detected(res) if extensions and #extensions > 0 then - return extensions[1],types[extensions[1]] + return extensions[1], types[extensions[1]] end -- Nothing found @@ -343,22 +360,22 @@ exports.detect = function(part, log_obj) end exports.detect_mime_part = function(part, log_obj) - local ext,weight = heuristics.mime_part_heuristic(part, log_obj) + local ext, weight = heuristics.mime_part_heuristic(part, log_obj) if ext and weight and weight > 20 then - return ext,types[ext] + return ext, types[ext] end ext = exports.detect(part, log_obj) if ext then - return ext,types[ext] + return ext, types[ext] end -- Text/html and other parts - ext,weight = heuristics.text_part_heuristic(part, log_obj) + ext, weight = heuristics.text_part_heuristic(part, log_obj) if ext and weight and weight > 20 then - return ext,types[ext] + return ext, types[ext] end end diff --git a/lualib/lua_magic/patterns.lua b/lualib/lua_magic/patterns.lua index b1eb5ad4d..971ddd95f 100644 --- a/lualib/lua_magic/patterns.lua +++ b/lualib/lua_magic/patterns.lua @@ -27,13 +27,13 @@ local patterns = { matches = { { string = [[%PDF-[12]\.\d]], - position = {'<=', 1024}, + position = { '<=', 1024 }, weight = 60, heuristic = heuristics.pdf_format_heuristic }, { string = [[%FDF-[12]\.\d]], - position = {'<=', 1024}, + position = { '<=', 1024 }, weight = 60, heuristic = heuristics.pdf_format_heuristic }, @@ -103,7 +103,7 @@ local patterns = { -- PE part { string = [[PE\x{00}\x{00}]], - position = {'>=', 0x3c + 4}, + position = { '>=', 0x3c + 4 }, weight = 15, heuristic = heuristics.pe_part_heuristic, } @@ -131,7 +131,7 @@ local patterns = { matches = { { string = [[(?i)@\s*ECHO\s+OFF]], - position = {'>=', 0}, + position = { '>=', 0 }, weight = 60, }, } @@ -189,7 +189,7 @@ local patterns = { matches = { { hex = [[4d53434600000000]], -- Can be anywhere for SFX :( - position = {'>=', 8}, + position = { '>=', 8 }, weight = 60, }, } @@ -268,7 +268,7 @@ local patterns = { matches = { { string = [[\x{01}CD001\x{01}]], - position = {'>=', 0x8000 + 7}, -- first 32k is unused + position = { '>=', 0x8000 + 7 }, -- first 32k is unused weight = 60, }, } diff --git a/lualib/lua_maps.lua b/lualib/lua_maps.lua index d65481177..baaed5b50 100644 --- a/lualib/lua_maps.lua +++ b/lualib/lua_maps.lua @@ -36,15 +36,15 @@ local function map_hash_key(data, mtype) return st:hex() end -local function starts(where,st) - return string.sub(where,1,string.len(st))==st +local function starts(where, st) + return string.sub(where, 1, string.len(st)) == st end -local function cut_prefix(where,st) - return string.sub(where,#st + 1) +local function cut_prefix(where, st) + return string.sub(where, #st + 1) end -local function maybe_adjust_type(data,mtype) +local function maybe_adjust_type(data, mtype) local function check_prefix(prefix, t) if starts(data, prefix) then data = cut_prefix(data, prefix) @@ -57,41 +57,40 @@ local function maybe_adjust_type(data,mtype) end local known_types = { - {'regexp;', 'regexp'}, - {'re;', 'regexp'}, - {'regexp_multi;', 'regexp_multi'}, - {'re_multi;', 'regexp_multi'}, - {'glob;', 'glob'}, - {'glob_multi;', 'glob_multi'}, - {'radix;', 'radix'}, - {'ipnet;', 'radix'}, - {'set;', 'set'}, - {'hash;', 'hash'}, - {'plain;', 'hash'}, - {'cdb;', 'cdb'}, - {'cdb:/', 'cdb'}, + { 'regexp;', 'regexp' }, + { 're;', 'regexp' }, + { 'regexp_multi;', 'regexp_multi' }, + { 're_multi;', 'regexp_multi' }, + { 'glob;', 'glob' }, + { 'glob_multi;', 'glob_multi' }, + { 'radix;', 'radix' }, + { 'ipnet;', 'radix' }, + { 'set;', 'set' }, + { 'hash;', 'hash' }, + { 'plain;', 'hash' }, + { 'cdb;', 'cdb' }, + { 'cdb:/', 'cdb' }, } if mtype == 'callback' then return mtype end - for _,t in ipairs(known_types) do + for _, t in ipairs(known_types) do if check_prefix(t[1], t[2]) then - return data,mtype + return data, mtype end end -- No change - return data,mtype + return data, mtype end - -local external_map_schema = ts.shape{ +local external_map_schema = ts.shape { external = ts.equivalent(true), -- must be true backend = ts.string, -- where to get data, required - method = ts.one_of{"body", "header", "query"}, -- how to pass input - encode = ts.one_of{"json", "messagepack"}:is_optional(), -- how to encode input (if relevant) + method = ts.one_of { "body", "header", "query" }, -- how to pass input + encode = ts.one_of { "json", "messagepack" }:is_optional(), -- how to encode input (if relevant) timeout = (ts.number + ts.string / lua_util.parse_time_interval):is_optional(), } @@ -100,7 +99,9 @@ local ucl = require "ucl" local function url_encode_string(str) str = string.gsub(str, "([^%w _%%%-%.~])", - function(c) return string.format("%%%02X", string.byte(c)) end) + function(c) + return string.format("%%%02X", string.byte(c)) + end) str = string.gsub(str, " ", "+") return str end @@ -147,7 +148,7 @@ local function query_external_map(map_config, upstreams, key, callback, task) -- query/header and no encode if map_config.method == 'query' then local params_table = {} - for k,v in pairs(key) do + for k, v in pairs(key) do if type(v) == 'string' then table.insert(params_table, string.format('%s=%s', url_encode_string(k), url_encode_string(v))) end @@ -176,7 +177,7 @@ local function query_external_map(map_config, upstreams, key, callback, task) end end - local ret = rspamd_http.request{ + local ret = rspamd_http.request { task = task, url = url, callback = map_callback, @@ -261,7 +262,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) end if type(opt) == 'string' then - opt,mtype = maybe_adjust_type(opt, mtype) + opt, mtype = maybe_adjust_type(opt, mtype) local cache_key = map_hash_key(opt, mtype) if not callback and maps_cache[cache_key] then rspamd_logger.infox(rspamd_config, 'reuse url for %s(%s)', @@ -270,7 +271,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) return maps_cache[cache_key] end -- We have a single string, so we treat it as a map - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = opt, @@ -287,7 +288,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) local cache_key = lua_util.table_digest(opt) if not callback and maps_cache[cache_key] then rspamd_logger.infox(rspamd_config, 'reuse url for complex map definition %s: %s', - cache_key:sub(1,8), description) + cache_key:sub(1, 8), description) return maps_cache[cache_key] end @@ -295,8 +296,8 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) if opt[1] then -- Adjust each element if needed local adjusted - for i,source in ipairs(opt) do - local nsrc,ntype = maybe_adjust_type(source, mtype) + for i, source in ipairs(opt) do + local nsrc, ntype = maybe_adjust_type(source, mtype) if mtype ~= ntype then if not adjusted then @@ -322,7 +323,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) end else -- Plain table - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = opt, @@ -339,7 +340,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) elseif mtype == 'regexp' or mtype == 'glob' then if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then -- Plain table - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = opt, @@ -353,7 +354,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) return ret end else - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = { @@ -373,7 +374,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) else if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then -- Plain table - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = opt, @@ -390,7 +391,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) local data = {} local nelts = 0 -- Plain array of keys, count merely numeric elts - for _,elt in ipairs(opt) do + for _, elt in ipairs(opt) do if type(elt) == 'string' then -- Numeric table if mtype == 'hash' then @@ -421,7 +422,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) return nil end ret.foreach = function(_, func) - for k,v in pairs(ret.__data) do + for k, v in pairs(ret.__data) do if not func(k, v) then return false end @@ -449,7 +450,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) else if opt.external then -- External map definition, missing fields are handled by schema - local parse_res,parse_err = external_map_schema(opt) + local parse_res, parse_err = external_map_schema(opt) if parse_res then ret.__upstreams = lua_util.http_upstreams_by_url(rspamd_config:get_mempool(), opt.backend) @@ -471,14 +472,14 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) else -- Adjust lua specific augmentations in a trivial case if type(opt.url) == 'string' then - local nsrc,ntype = maybe_adjust_type(opt.url, mtype) + local nsrc, ntype = maybe_adjust_type(opt.url, mtype) if nsrc and ntype then opt.url = nsrc mtype = ntype end end -- We have some non-trivial object so let C code to deal with it somehow... - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = opt, @@ -526,7 +527,9 @@ local function rspamd_maybe_check_map(key, what) local fun = require "fun" if type(what) == "table" then - return fun.any(function(elt) return rspamd_maybe_check_map(key, elt) end, what) + return fun.any(function(elt) + return rspamd_maybe_check_map(key, elt) + end, what) end if type(rspamd_maps) == "table" then local mn @@ -572,7 +575,7 @@ exports.fill_config_maps = function(mname, opts, map_defs) rspamd_logger.errx(rspamd_config, 'map add error %s for module %s', k, mname) return false end - opts[k..'_orig'] = opts[k] + opts[k .. '_orig'] = opts[k] opts[k] = map elseif not v.optional then rspamd_logger.errx(rspamd_config, 'cannot find non optional map %s for module %s', k, mname) @@ -583,27 +586,27 @@ exports.fill_config_maps = function(mname, opts, map_defs) return true end -local direct_map_schema = ts.shape{ -- complex object +local direct_map_schema = ts.shape { -- complex object name = ts.string:is_optional(), description = ts.string:is_optional(), selector_alias = ts.string:is_optional(), -- an optional alias for the selectos framework timeout = ts.number, data = ts.array_of(ts.string):is_optional(), -- Tableshape has no options support for something like key1 or key2? - upstreams = ts.one_of{ + upstreams = ts.one_of { ts.string, ts.array_of(ts.string), - }:is_optional(), - url = ts.one_of{ + } :is_optional(), + url = ts.one_of { ts.string, ts.array_of(ts.string), - }:is_optional(), + } :is_optional(), } -exports.map_schema = ts.one_of{ +exports.map_schema = ts.one_of { ts.string, -- 'http://some_map' ts.array_of(ts.string), -- ['foo', 'bar'] - ts.one_of{direct_map_schema, external_map_schema} + ts.one_of { direct_map_schema, external_map_schema } } return exports diff --git a/lualib/lua_maps_expressions.lua b/lualib/lua_maps_expressions.lua index 0d030b4c4..996de99c0 100644 --- a/lualib/lua_maps_expressions.lua +++ b/lualib/lua_maps_expressions.lua @@ -64,7 +64,7 @@ local function process_func(elt, task) if values then if type(values) == 'table' then - for _,val in ipairs(values) do + for _, val in ipairs(values) do if res == 0 then match_rule(val) end @@ -80,17 +80,16 @@ local function process_func(elt, task) local res = elt.expr:process(process_atom) if res > 0 then - return res,matched + return res, matched end return nil end - -exports.schema = ts.shape{ +exports.schema = ts.shape { expression = ts.string, rules = ts.array_of( - ts.shape{ + ts.shape { selector = ts.string, map = lua_maps.map_schema, } @@ -119,7 +118,9 @@ exports.schema = ts.shape{ -- --]] local function create(cfg, obj, module_name) - if not module_name then module_name = 'lua_maps_expressions' end + if not module_name then + module_name = 'lua_maps_expressions' + end if not obj or not obj.rules or not obj.expression then rspamd_logger.errx(cfg, 'cannot add maps combination for module %s: required elements are missing', @@ -133,7 +134,7 @@ local function create(cfg, obj, module_name) module_name = module_name } - for name,rule in pairs(obj.rules) do + for name, rule in pairs(obj.rules) do local sel = lua_selectors.create_selector_closure(cfg, rule.selector) if not sel then @@ -201,7 +202,7 @@ local function create(cfg, obj, module_name) ret.expr = expr if obj.symbol then - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual,ghost', name = obj.symbol, score = 0.0, diff --git a/lualib/lua_meta.lua b/lualib/lua_meta.lua index 2de26cf44..340d89ee8 100644 --- a/lualib/lua_meta.lua +++ b/lualib/lua_meta.lua @@ -42,13 +42,13 @@ local function meta_size_function(task) } local size = task:get_size() - for i = 1,#sizes do + for i = 1, #sizes do if sizes[i] >= size then - return {(1.0 * i) / #sizes} + return { (1.0 * i) / #sizes } end end - return {0} + return { 0 } end local function meta_images_function(task) @@ -60,7 +60,7 @@ local function meta_images_function(task) local nsmall = 0 if images then - for _,img in ipairs(images) do + for _, img in ipairs(images) do if img:get_type() == 'png' then npng = npng + 1 elseif img:get_type() == 'jpeg' then @@ -87,7 +87,7 @@ local function meta_images_function(task) nlarge = 1.0 * nlarge / ntotal nsmall = 1.0 * nsmall / ntotal end - return {ntotal,njpg,npng,nlarge,nsmall} + return { ntotal, njpg, npng, nlarge, nsmall } end local function meta_nparts_function(task) @@ -103,7 +103,7 @@ local function meta_nparts_function(task) local parts = task:get_parts() if parts then - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do if p:is_attachment() then nattachments = nattachments + 1 end @@ -111,7 +111,7 @@ local function meta_nparts_function(task) end end - return {(1.0 * ntextparts)/totalparts, (1.0 * nattachments)/totalparts} + return { (1.0 * ntextparts) / totalparts, (1.0 * nattachments) / totalparts } end local function meta_encoding_function(task) @@ -120,7 +120,7 @@ local function meta_encoding_function(task) local tp = task:get_text_parts() if tp and #tp > 0 then - for _,p in ipairs(tp) do + for _, p in ipairs(tp) do if p:is_utf() then nutf = nutf + 1 else @@ -128,10 +128,10 @@ local function meta_encoding_function(task) end end - return {nutf / #tp, nother / #tp} + return { nutf / #tp, nother / #tp } end - return {0, 0} + return { 0, 0 } end local function meta_recipients_function(task) @@ -145,10 +145,14 @@ local function meta_recipients_function(task) nsmtp = #(task:get_recipients('smtp')) end - if nmime > 0 then nmime = 1.0 / nmime end - if nsmtp > 0 then nsmtp = 1.0 / nsmtp end + if nmime > 0 then + nmime = 1.0 / nmime + end + if nsmtp > 0 then + nsmtp = 1.0 / nsmtp + end - return {nmime,nsmtp} + return { nmime, nsmtp } end local function meta_received_function(task) @@ -180,7 +184,9 @@ local function meta_received_function(task) secure_factor = secure_factor + 1.0 end end, - fun.filter(function(rc) return not rc.flags or not rc.flags['artificial'] end, rh)) + fun.filter(function(rc) + return not rc.flags or not rc.flags['artificial'] + end, rh)) if ntotal > 0 then invalid_factor = invalid_factor / ntotal @@ -193,16 +199,16 @@ local function meta_received_function(task) end end - return {count_factor, invalid_factor, time_factor, secure_factor} + return { count_factor, invalid_factor, time_factor, secure_factor } end local function meta_urls_function(task) - local has_urls,nurls = task:has_urls() + local has_urls, nurls = task:has_urls() if has_urls and nurls > 0 then - return {1.0 / nurls} + return { 1.0 / nurls } end - return {0} + return { 0 } end local function meta_words_function(task) @@ -224,7 +230,7 @@ local function meta_words_function(task) 20, } - for i = 1,#lens do + for i = 1, #lens do if lens[i] >= avg_len then ret_len = (1.0 * i) / #lens break @@ -241,7 +247,7 @@ local function meta_words_function(task) 0, -- capital characters rate 0, -- numeric characters } - for _,p in ipairs(tp) do + for _, p in ipairs(tp) do local stats = p:get_stats() local len = p:get_length() @@ -266,7 +272,7 @@ local function meta_words_function(task) divisor = #tp end - for _,wr in ipairs(wres) do + for _, wr in ipairs(wres) do table.insert(ret, wr / divisor) end @@ -401,7 +407,7 @@ local metafunctions = { }, } -local meta_schema = ts.shape{ +local meta_schema = ts.shape { cb = ts.func, ninputs = ts.number, names = ts.array_of(ts.string), @@ -413,8 +419,8 @@ local metatokens_by_name = {} local function fill_metatokens_by_name() metatokens_by_name = {} - for _,mt in ipairs(metafunctions) do - for i=1,mt.ninputs do + for _, mt in ipairs(metafunctions) do + for i = 1, mt.ninputs do local name = mt.names[i] metatokens_by_name[name] = function(task) @@ -429,8 +435,8 @@ local function calculate_digest() local cr = require "rspamd_cryptobox_hash" local h = cr.create() - for _,mt in ipairs(metafunctions) do - for i=1,mt.ninputs do + for _, mt in ipairs(metafunctions) do + for i = 1, mt.ninputs do local name = mt.names[i] h:update(name) end @@ -450,9 +456,9 @@ local function rspamd_gen_metatokens(task, names) if cached then return cached else - for _,mt in ipairs(metafunctions) do + for _, mt in ipairs(metafunctions) do local ct = mt.cb(task) - for i,tok in ipairs(ct) do + for i, tok in ipairs(ct) do lua_util.debugm(N, task, "metatoken: %s = %s", mt.names[i], tok) if tok ~= tok or tok == math.huge then @@ -468,7 +474,7 @@ local function rspamd_gen_metatokens(task, names) end else - for _,n in ipairs(names) do + for _, n in ipairs(names) do if metatokens_by_name[n] then local tok = metatokens_by_name[n](task) if tok ~= tok or tok == math.huge then @@ -484,7 +490,7 @@ local function rspamd_gen_metatokens(task, names) end return metatokens - end +end exports.rspamd_gen_metatokens = rspamd_gen_metatokens exports.gen_metatokens = rspamd_gen_metatokens @@ -492,9 +498,9 @@ exports.gen_metatokens = rspamd_gen_metatokens local function rspamd_gen_metatokens_table(task) local metatokens = {} - for _,mt in ipairs(metafunctions) do + for _, mt in ipairs(metafunctions) do local ct = mt.cb(task) - for i,tok in ipairs(ct) do + for i, tok in ipairs(ct) do if tok ~= tok or tok == math.huge then logger.errx(task, 'metatoken %s returned %s; replace it with 0 for sanity', mt.names[i], tok) @@ -514,7 +520,7 @@ exports.gen_metatokens_table = rspamd_gen_metatokens_table local function rspamd_count_metatokens() local ipairs = ipairs local total = 0 - for _,mt in ipairs(metafunctions) do + for _, mt in ipairs(metafunctions) do total = total + mt.ninputs end diff --git a/lualib/lua_mime.lua b/lualib/lua_mime.lua index 038dccceb..0f5aa75c0 100644 --- a/lualib/lua_mime.lua +++ b/lualib/lua_mime.lua @@ -64,7 +64,7 @@ local function do_append_footer(task, part, footer, is_multipart, out, state) end if is_multipart then - out[#out + 1] = string.format('Content-Type: %s; charset=utf-8%s'.. + out[#out + 1] = string.format('Content-Type: %s; charset=utf-8%s' .. 'Content-Transfer-Encoding: %s', ct, newline_s, cte) out[#out + 1] = '' @@ -79,11 +79,11 @@ local function do_append_footer(task, part, footer, is_multipart, out, state) if content:sub(-(nlen), nlen + 1) == double_nline then -- content without last newline content = content:sub(-(#newline_s), #newline_s + 1) .. footer - out[#out + 1] = {encode_func(content), true} + out[#out + 1] = { encode_func(content), true } out[#out + 1] = '' else content = content .. footer - out[#out + 1] = {encode_func(content), true} + out[#out + 1] = { encode_func(content), true } out[#out + 1] = '' end @@ -156,7 +156,7 @@ exports.add_text_footer = function(task, html_footer, text_footer) local boundaries = {} local cur_boundary - for _,part in ipairs(task:get_parts()) do + for _, part in ipairs(task:get_parts()) do local boundary = part:get_boundary() if part:is_multipart() then if cur_boundary then @@ -169,7 +169,7 @@ exports.add_text_footer = function(task, html_footer, text_footer) local rh = part:get_raw_headers() if #rh > 0 then - out[#out + 1] = {rh, true} + out[#out + 1] = { rh, true } end elseif part:is_message() then if boundary then @@ -184,14 +184,14 @@ exports.add_text_footer = function(task, html_footer, text_footer) boundary) end - out[#out + 1] = {part:get_raw_headers(), true} + out[#out + 1] = { part:get_raw_headers(), true } else local append_footer = false local skip_footer = part:is_attachment() local parent = part:get_parent() if parent then - local t,st = parent:get_type() + local t, st = parent:get_type() if t == 'multipart' and st == 'signed' then -- Do not modify signed parts @@ -230,8 +230,8 @@ exports.add_text_footer = function(task, html_footer, text_footer) do_append_footer(task, part, append_footer, parent and parent:is_multipart(), out, state) else - out[#out + 1] = {part:get_raw_headers(), true} - out[#out + 1] = {part:get_raw_content(), false} + out[#out + 1] = { part:get_raw_headers(), true } + out[#out + 1] = { part:get_raw_content(), false } end end end @@ -285,38 +285,40 @@ local function do_replacement (task, part, mp, replacements, -- sort matches and form the table: -- start .. end for inclusion position local matches_flattened = {} - for npat,matches in pairs(match_pos) do - for _,m in ipairs(matches) do - table.insert(matches_flattened, {m, npat}) + for npat, matches in pairs(match_pos) do + for _, m in ipairs(matches) do + table.insert(matches_flattened, { m, npat }) end end -- Handle the case of empty match if #matches_flattened == 0 then - out[#out + 1] = {part:get_raw_headers(), true} - out[#out + 1] = {part:get_raw_content(), false} + out[#out + 1] = { part:get_raw_headers(), true } + out[#out + 1] = { part:get_raw_content(), false } return end if is_multipart then - out[#out + 1] = {string.format('Content-Type: %s; charset="utf-8"%s'.. + out[#out + 1] = { string.format('Content-Type: %s; charset="utf-8"%s' .. 'Content-Transfer-Encoding: %s', - ct, newline_s, cte), true} - out[#out + 1] = {'', true} + ct, newline_s, cte), true } + out[#out + 1] = { '', true } else state.new_cte = cte end state.has_matches = true -- now sort flattened by start of match and eliminate all overlaps - table.sort(matches_flattened, function(m1, m2) return m1[1][1] < m2[1][1] end) + table.sort(matches_flattened, function(m1, m2) + return m1[1][1] < m2[1][1] + end) - for i=1,#matches_flattened - 1 do + for i = 1, #matches_flattened - 1 do local st = matches_flattened[i][1][1] -- current start of match local e = matches_flattened[i][1][2] -- current end of match local max_npat = matches_flattened[i][2] - for j=i+1,#matches_flattened do + for j = i + 1, #matches_flattened do if matches_flattened[j][1][1] == st then -- overlap if matches_flattened[j][1][2] > e then @@ -329,7 +331,7 @@ local function do_replacement (task, part, mp, replacements, end end -- Maximum overlap for all matches - for j=i,#matches_flattened do + for j = i, #matches_flattened do if matches_flattened[j][1][1] == st then if e > matches_flattened[j][1][2] then matches_flattened[j][1][2] = e @@ -341,7 +343,7 @@ local function do_replacement (task, part, mp, replacements, end end -- Off-by one: match returns 0 based positions while we use 1 based in Lua - for _,m in ipairs(matches_flattened) do + for _, m in ipairs(matches_flattened) do m[1][1] = m[1][1] + 1 m[1][2] = m[1][2] + 1 end @@ -354,7 +356,7 @@ local function do_replacement (task, part, mp, replacements, local cur_start = 1 local fragments = {} - for _,m in ipairs(matches_flattened) do + for _, m in ipairs(matches_flattened) do if m[1][1] >= cur_start then fragments[#fragments + 1] = content:sub(cur_start, m[1][1] - 1) fragments[#fragments + 1] = replacements[m[2]] @@ -368,11 +370,11 @@ local function do_replacement (task, part, mp, replacements, end -- Final stuff - out[#out + 1] = {encode_func(rspamd_text.fromtable(fragments)), false} + out[#out + 1] = { encode_func(rspamd_text.fromtable(fragments)), false } else -- No matches - out[#out + 1] = {part:get_raw_headers(), true} - out[#out + 1] = {part:get_raw_content(), false} + out[#out + 1] = { part:get_raw_headers(), true } + out[#out + 1] = { part:get_raw_content(), false } end end @@ -429,12 +431,12 @@ exports.multipattern_text_replace = function(task, mp, replacements) local boundaries = {} local cur_boundary - for _,part in ipairs(task:get_parts()) do + for _, part in ipairs(task:get_parts()) do local boundary = part:get_boundary() if part:is_multipart() then if cur_boundary then - out[#out + 1] = {string.format('--%s', - boundaries[#boundaries]), true} + out[#out + 1] = { string.format('--%s', + boundaries[#boundaries]), true } end boundaries[#boundaries + 1] = boundary or '--XXX' @@ -442,28 +444,28 @@ exports.multipattern_text_replace = function(task, mp, replacements) local rh = part:get_raw_headers() if #rh > 0 then - out[#out + 1] = {rh, true} + out[#out + 1] = { rh, true } end elseif part:is_message() then if boundary then if cur_boundary and boundary ~= cur_boundary then -- Need to close boundary - out[#out + 1] = {string.format('--%s--', - boundaries[#boundaries]), true} + out[#out + 1] = { string.format('--%s--', + boundaries[#boundaries]), true } table.remove(boundaries) cur_boundary = nil end - out[#out + 1] = {string.format('--%s', - boundary), true} + out[#out + 1] = { string.format('--%s', + boundary), true } end - out[#out + 1] = {part:get_raw_headers(), true} + out[#out + 1] = { part:get_raw_headers(), true } else local skip_replacement = part:is_attachment() local parent = part:get_parent() if parent then - local t,st = parent:get_type() + local t, st = parent:get_type() if t == 'multipart' and st == 'signed' then -- Do not modify signed parts @@ -477,13 +479,13 @@ exports.multipattern_text_replace = function(task, mp, replacements) if boundary then if cur_boundary and boundary ~= cur_boundary then -- Need to close boundary - out[#out + 1] = {string.format('--%s--', - boundaries[#boundaries]), true} + out[#out + 1] = { string.format('--%s--', + boundaries[#boundaries]), true } table.remove(boundaries) cur_boundary = boundary end - out[#out + 1] = {string.format('--%s', - boundary), true} + out[#out + 1] = { string.format('--%s', + boundary), true } end if not skip_replacement then @@ -491,8 +493,8 @@ exports.multipattern_text_replace = function(task, mp, replacements) parent and parent:is_multipart(), out, state) else -- Append as is - out[#out + 1] = {part:get_raw_headers(), true} - out[#out + 1] = {part:get_raw_content(), false} + out[#out + 1] = { part:get_raw_headers(), true } + out[#out + 1] = { part:get_raw_content(), false } end end end @@ -500,9 +502,9 @@ exports.multipattern_text_replace = function(task, mp, replacements) -- Close remaining local b = table.remove(boundaries) while b do - out[#out + 1] = {string.format('--%s--', b), true} + out[#out + 1] = { string.format('--%s--', b), true } if #boundaries > 0 then - out[#out + 1] = {'', true} + out[#out + 1] = { '', true } end b = table.remove(boundaries) end @@ -519,7 +521,9 @@ end --]] exports.modify_headers = function(task, hdr_alterations, mode) -- Assume default mode compatibility - if not mode then mode = 'compat' end + if not mode then + mode = 'compat' + end local add = hdr_alterations.add or {} local remove = hdr_alterations.remove or {} @@ -531,7 +535,7 @@ exports.modify_headers = function(task, hdr_alterations, mode) add_headers[hname] = {} end if not hdr_flattened[hname] then - hdr_flattened[hname] = {add = {}} + hdr_flattened[hname] = { add = {} } end local add_tbl = hdr_flattened[hname].add if hdr.value then @@ -539,9 +543,9 @@ exports.modify_headers = function(task, hdr_alterations, mode) order = (tonumber(hdr.order) or -1), value = hdr.value, }) - table.insert(add_tbl, {tonumber(hdr.order) or -1, hdr.value}) + table.insert(add_tbl, { tonumber(hdr.order) or -1, hdr.value }) elseif type(hdr) == 'table' then - for _,v in ipairs(hdr) do + for _, v in ipairs(hdr) do flatten_add_header(hname, v) end elseif type(hdr) == 'string' then @@ -549,7 +553,7 @@ exports.modify_headers = function(task, hdr_alterations, mode) order = -1, value = hdr, }) - table.insert(add_tbl, {-1, hdr}) + table.insert(add_tbl, { -1, hdr }) else logger.errx(task, 'invalid modification of header: %s', hdr) end @@ -561,19 +565,18 @@ exports.modify_headers = function(task, hdr_alterations, mode) end if hdr_alterations.order then -- Get headers alterations ordered - for _,hname in ipairs(hdr_alterations.order) do + for _, hname in ipairs(hdr_alterations.order) do flatten_add_header(hname, add[hname]) end else - for hname,hdr in pairs(add) do + for hname, hdr in pairs(add) do flatten_add_header(hname, hdr) end end - - for hname,hdr in pairs(remove) do + for hname, hdr in pairs(remove) do if not hdr_flattened[hname] then - hdr_flattened[hname] = {remove = {}} + hdr_flattened[hname] = { remove = {} } end if not hdr_flattened[hname].remove then hdr_flattened[hname].remove = {} @@ -582,7 +585,7 @@ exports.modify_headers = function(task, hdr_alterations, mode) if type(hdr) == 'number' then table.insert(remove_tbl, hdr) else - for _,num in ipairs(hdr) do + for _, num in ipairs(hdr) do table.insert(remove_tbl, num) end end @@ -590,15 +593,19 @@ exports.modify_headers = function(task, hdr_alterations, mode) if mode == 'compat' then -- Clear empty alterations in the compat mode - if add_headers and not next(add_headers) then add_headers = nil end - if hdr_alterations.remove and not next(hdr_alterations.remove) then hdr_alterations.remove = nil end + if add_headers and not next(add_headers) then + add_headers = nil + end + if hdr_alterations.remove and not next(hdr_alterations.remove) then + hdr_alterations.remove = nil + end end task:set_milter_reply({ add_headers = add_headers, remove_headers = hdr_alterations.remove }) - for hname,flat_rules in pairs(hdr_flattened) do + for hname, flat_rules in pairs(hdr_flattened) do task:modify_header(hname, flat_rules) end end @@ -611,7 +618,9 @@ exports.message_to_ucl = function(task, stringify_content) local E = {} local maybe_stringify_f = stringify_content and - tostring or function(t) return t end + tostring or function(t) + return t + end local result = { size = task:get_size(), digest = task:get_digest(), @@ -643,7 +652,7 @@ exports.message_to_ucl = function(task, stringify_content) local parts = task:get_parts() or E result.parts = {} - for _,part in ipairs(parts) do + for _, part in ipairs(parts) do if not part:is_multipart() and not part:is_message() then local p = { size = part:get_length(), @@ -651,7 +660,7 @@ exports.message_to_ucl = function(task, stringify_content) detected_type = string.format('%s/%s', part:get_detected_type()), filename = part:get_filename(), content = maybe_stringify_f(part:get_content()), - headers = part:get_headers(true) or E, + headers = part:get_headers(true) or E, boundary = part:get_enclosing_boundary(), } table.insert(result.parts, p) @@ -659,7 +668,7 @@ exports.message_to_ucl = function(task, stringify_content) -- Service part: multipart container or message/rfc822 local p = { type = string.format('%s/%s', part:get_type()), - headers = part:get_headers(true) or E, + headers = part:get_headers(true) or E, boundary = part:get_enclosing_boundary(), size = 0, } @@ -683,8 +692,8 @@ exports.message_to_ucl_schema = function() local ts = require("tableshape").types local function headers_schema() - return ts.shape{ - order = ts.integer:describe('Header order in a message'), + return ts.shape { + order = ts.integer:describe('Header order in a message'), raw = ts.string:describe('Raw header value'):is_optional(), empty_separator = ts.boolean:describe('Whether header has an empty separator'), separator = ts.string:describe('Separator between a header and a value'), @@ -696,8 +705,8 @@ exports.message_to_ucl_schema = function() end local function part_schema() - return ts.shape{ - content = ts.string:describe('Decoded content'):is_optional(), + return ts.shape { + content = ts.string:describe('Decoded content'):is_optional(), multipart_boundary = ts.string:describe('Multipart service boundary'):is_optional(), size = ts.integer:describe('Size of the part'), type = ts.string:describe('Announced type'):is_optional(), @@ -709,10 +718,10 @@ exports.message_to_ucl_schema = function() end local function email_addr_schema() - return ts.shape{ - addr = ts.string:describe('Parsed address'):is_optional(), + return ts.shape { + addr = ts.string:describe('Parsed address'):is_optional(), raw = ts.string:describe('Raw address'), - flags = ts.shape{ + flags = ts.shape { valid = ts.boolean:describe('Valid address'):is_optional(), ip = ts.boolean:describe('IP like address'):is_optional(), braced = ts.boolean:describe('Have braces around address'):is_optional(), @@ -721,13 +730,13 @@ exports.message_to_ucl_schema = function() backslash = ts.boolean:describe('Backslash in address'):is_optional(), ['8bit'] = ts.boolean:describe('8 bit characters in address'):is_optional(), }, - user = ts.string:describe('Parsed user part'):is_optional(), - name = ts.string:describe('Displayed name'):is_optional(), - domain = ts.string:describe('Parsed domain part'):is_optional(), + user = ts.string:describe('Parsed user part'):is_optional(), + name = ts.string:describe('Displayed name'):is_optional(), + domain = ts.string:describe('Parsed domain part'):is_optional(), } end local function envelope_schema() - return ts.shape{ + return ts.shape { from_smtp = email_addr_schema():describe('SMTP from'):is_optional(), recipients_smtp = ts.array_of(email_addr_schema()):describe('SMTP recipients'):is_optional(), helo = ts.string:describe('SMTP Helo'):is_optional(), @@ -737,12 +746,12 @@ exports.message_to_ucl_schema = function() } end - return ts.shape{ + return ts.shape { headers = ts.array_of(headers_schema()), parts = ts.array_of(part_schema()), digest = ts.pattern(string.format('^%s$', string.rep('%x', 32))) - :describe('Message digest'), - newlines = ts.one_of({"cr", "lf", "crlf"}):describe('Newlines type'), + :describe('Message digest'), + newlines = ts.one_of({ "cr", "lf", "crlf" }):describe('Newlines type'), size = ts.integer:describe('Size of the message in bytes'), envelope = envelope_schema() } diff --git a/lualib/lua_mime_types.lua b/lualib/lua_mime_types.lua index 6bb20f1d9..ba55f9740 100644 --- a/lualib/lua_mime_types.lua +++ b/lualib/lua_mime_types.lua @@ -23,723 +23,723 @@ local exports = {} -- All mime extensions with corresponding content types exports.full_extensions_map = { - {"323", "text/h323"}, - {"3g2", "video/3gpp2"}, - {"3gp", "video/3gpp"}, - {"3gp2", "video/3gpp2"}, - {"3gpp", "video/3gpp"}, - {"7z", {"application/x-7z-compressed", "application/7z"}}, - {"aa", "audio/audible"}, - {"AAC", "audio/aac"}, - {"aaf", "application/octet-stream"}, - {"aax", "audio/vnd.audible.aax"}, - {"ac3", "audio/ac3"}, - {"aca", "application/octet-stream"}, - {"accda", "application/msaccess.addin"}, - {"accdb", "application/msaccess"}, - {"accdc", "application/msaccess.cab"}, - {"accde", "application/msaccess"}, - {"accdr", "application/msaccess.runtime"}, - {"accdt", "application/msaccess"}, - {"accdw", "application/msaccess.webapplication"}, - {"accft", "application/msaccess.ftemplate"}, - {"acx", "application/internet-property-stream"}, - {"AddIn", "text/xml"}, - {"ade", "application/msaccess"}, - {"adobebridge", "application/x-bridge-url"}, - {"adp", "application/msaccess"}, - {"ADT", "audio/vnd.dlna.adts"}, - {"ADTS", "audio/aac"}, - {"afm", "application/octet-stream"}, - {"ai", "application/postscript"}, - {"aif", "audio/aiff"}, - {"aifc", "audio/aiff"}, - {"aiff", "audio/aiff"}, - {"air", "application/vnd.adobe.air-application-installer-package+zip"}, - {"amc", "application/mpeg"}, - {"anx", "application/annodex"}, - {"apk", "application/vnd.android.package-archive" }, - {"application", "application/x-ms-application"}, - {"art", "image/x-jg"}, - {"asa", "application/xml"}, - {"asax", "application/xml"}, - {"ascx", "application/xml"}, - {"asd", "application/octet-stream"}, - {"asf", "video/x-ms-asf"}, - {"ashx", "application/xml"}, - {"asi", "application/octet-stream"}, - {"asm", "text/plain"}, - {"asmx", "application/xml"}, - {"aspx", "application/xml"}, - {"asr", "video/x-ms-asf"}, - {"asx", "video/x-ms-asf"}, - {"atom", "application/atom+xml"}, - {"au", "audio/basic"}, - {"avi", "video/x-msvideo"}, - {"axa", "audio/annodex"}, - {"axs", "application/olescript"}, - {"axv", "video/annodex"}, - {"bas", "text/plain"}, - {"bcpio", "application/x-bcpio"}, - {"bin", "application/octet-stream"}, - {"bmp", {"image/bmp", "image/x-ms-bmp"}}, - {"c", "text/plain"}, - {"cab", "application/octet-stream"}, - {"caf", "audio/x-caf"}, - {"calx", "application/vnd.ms-office.calx"}, - {"cat", "application/vnd.ms-pki.seccat"}, - {"cc", "text/plain"}, - {"cd", "text/plain"}, - {"cdda", "audio/aiff"}, - {"cdf", "application/x-cdf"}, - {"cer", "application/x-x509-ca-cert"}, - {"cfg", "text/plain"}, - {"chm", "application/octet-stream"}, - {"class", "application/x-java-applet"}, - {"clp", "application/x-msclip"}, - {"cmd", "text/plain"}, - {"cmx", "image/x-cmx"}, - {"cnf", "text/plain"}, - {"cod", "image/cis-cod"}, - {"config", "application/xml"}, - {"contact", "text/x-ms-contact"}, - {"coverage", "application/xml"}, - {"cpio", "application/x-cpio"}, - {"cpp", "text/plain"}, - {"crd", "application/x-mscardfile"}, - {"crl", "application/pkix-crl"}, - {"crt", "application/x-x509-ca-cert"}, - {"cs", "text/plain"}, - {"csdproj", "text/plain"}, - {"csh", "application/x-csh"}, - {"csproj", "text/plain"}, - {"css", "text/css"}, - {"csv", {"application/vnd.ms-excel", "text/csv", "text/plain"}}, - {"cur", "application/octet-stream"}, - {"cxx", "text/plain"}, - {"dat", {"application/octet-stream", "application/ms-tnef"}}, - {"datasource", "application/xml"}, - {"dbproj", "text/plain"}, - {"dcr", "application/x-director"}, - {"def", "text/plain"}, - {"deploy", "application/octet-stream"}, - {"der", "application/x-x509-ca-cert"}, - {"dgml", "application/xml"}, - {"dib", "image/bmp"}, - {"dif", "video/x-dv"}, - {"dir", "application/x-director"}, - {"disco", "text/xml"}, - {"divx", "video/divx"}, - {"dll", "application/x-msdownload"}, - {"dll.config", "text/xml"}, - {"dlm", "text/dlm"}, - {"doc", "application/msword"}, - {"docm", "application/vnd.ms-word.document.macroEnabled.12"}, - {"docx", { - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/msword", - "application/vnd.ms-word.document.12", - "application/octet-stream", - }}, - {"dot", "application/msword"}, - {"dotm", "application/vnd.ms-word.template.macroEnabled.12"}, - {"dotx", "application/vnd.openxmlformats-officedocument.wordprocessingml.template"}, - {"dsp", "application/octet-stream"}, - {"dsw", "text/plain"}, - {"dtd", "text/xml"}, - {"dtsConfig", "text/xml"}, - {"dv", "video/x-dv"}, - {"dvi", "application/x-dvi"}, - {"dwf", "drawing/x-dwf"}, - {"dwg", {"application/acad", "image/vnd.dwg"}}, - {"dwp", "application/octet-stream"}, - {"dxf", "application/x-dxf" }, - {"dxr", "application/x-director"}, - {"eml", "message/rfc822"}, - {"emz", "application/octet-stream"}, - {"eot", "application/vnd.ms-fontobject"}, - {"eps", "application/postscript"}, - {"etl", "application/etl"}, - {"etx", "text/x-setext"}, - {"evy", "application/envoy"}, - {"exe", { - "application/x-dosexec", - "application/x-msdownload", - "application/x-executable", - }}, - {"exe.config", "text/xml"}, - {"fdf", "application/vnd.fdf"}, - {"fif", "application/fractals"}, - {"filters", "application/xml"}, - {"fla", "application/octet-stream"}, - {"flac", "audio/flac"}, - {"flr", "x-world/x-vrml"}, - {"flv", "video/x-flv"}, - {"fsscript", "application/fsharp-script"}, - {"fsx", "application/fsharp-script"}, - {"generictest", "application/xml"}, - {"gif", "image/gif"}, - {"gpx", "application/gpx+xml"}, - {"group", "text/x-ms-group"}, - {"gsm", "audio/x-gsm"}, - {"gtar", "application/x-gtar"}, - {"gz", {"application/gzip", "application/x-gzip", "application/tlsrpt+gzip"}}, - {"h", "text/plain"}, - {"hdf", "application/x-hdf"}, - {"hdml", "text/x-hdml"}, - {"hhc", "application/x-oleobject"}, - {"hhk", "application/octet-stream"}, - {"hhp", "application/octet-stream"}, - {"hlp", "application/winhlp"}, - {"hpp", "text/plain"}, - {"hqx", "application/mac-binhex40"}, - {"hta", "application/hta"}, - {"htc", "text/x-component"}, - {"htm", "text/html"}, - {"html", "text/html"}, - {"htt", "text/webviewhtml"}, - {"hxa", "application/xml"}, - {"hxc", "application/xml"}, - {"hxd", "application/octet-stream"}, - {"hxe", "application/xml"}, - {"hxf", "application/xml"}, - {"hxh", "application/octet-stream"}, - {"hxi", "application/octet-stream"}, - {"hxk", "application/xml"}, - {"hxq", "application/octet-stream"}, - {"hxr", "application/octet-stream"}, - {"hxs", "application/octet-stream"}, - {"hxt", "text/html"}, - {"hxv", "application/xml"}, - {"hxw", "application/octet-stream"}, - {"hxx", "text/plain"}, - {"i", "text/plain"}, - {"ico", "image/x-icon"}, - {"ics", {"text/calendar", "application/ics", "application/octet-stream"}}, - {"idl", "text/plain"}, - {"ief", "image/ief"}, - {"iii", "application/x-iphone"}, - {"inc", "text/plain"}, - {"inf", "application/octet-stream"}, - {"ini", "text/plain"}, - {"inl", "text/plain"}, - {"ins", "application/x-internet-signup"}, - {"ipa", "application/x-itunes-ipa"}, - {"ipg", "application/x-itunes-ipg"}, - {"ipproj", "text/plain"}, - {"ipsw", "application/x-itunes-ipsw"}, - {"iqy", "text/x-ms-iqy"}, - {"isp", "application/x-internet-signup"}, - {"ite", "application/x-itunes-ite"}, - {"itlp", "application/x-itunes-itlp"}, - {"itms", "application/x-itunes-itms"}, - {"itpc", "application/x-itunes-itpc"}, - {"IVF", "video/x-ivf"}, - {"jar", "application/java-archive"}, - {"java", "application/octet-stream"}, - {"jck", "application/liquidmotion"}, - {"jcz", "application/liquidmotion"}, - {"jfif", {"image/jpeg", "image/pjpeg"}}, - {"jnlp", "application/x-java-jnlp-file"}, - {"jpb", "application/octet-stream"}, - {"jpe", {"image/jpeg", "image/pjpeg"}}, - {"jpeg", {"image/jpeg", "image/pjpeg"}}, - {"jpg", {"image/jpeg", "image/pjpeg"}}, - {"js", "application/javascript"}, - {"json", "application/json"}, - {"jsx", "text/jscript"}, - {"jsxbin", "text/plain"}, - {"latex", "application/x-latex"}, - {"library-ms", "application/windows-library+xml"}, - {"lit", "application/x-ms-reader"}, - {"loadtest", "application/xml"}, - {"lpk", "application/octet-stream"}, - {"lsf", "video/x-la-asf"}, - {"lst", "text/plain"}, - {"lsx", "video/x-la-asf"}, - {"lzh", "application/octet-stream"}, - {"m13", "application/x-msmediaview"}, - {"m14", "application/x-msmediaview"}, - {"m1v", "video/mpeg"}, - {"m2t", "video/vnd.dlna.mpeg-tts"}, - {"m2ts", "video/vnd.dlna.mpeg-tts"}, - {"m2v", "video/mpeg"}, - {"m3u", "audio/x-mpegurl"}, - {"m3u8", "audio/x-mpegurl"}, - {"m4a", {"audio/m4a", "audio/x-m4a"}}, - {"m4b", "audio/m4b"}, - {"m4p", "audio/m4p"}, - {"m4r", "audio/x-m4r"}, - {"m4v", "video/x-m4v"}, - {"mac", "image/x-macpaint"}, - {"mak", "text/plain"}, - {"man", "application/x-troff-man"}, - {"manifest", "application/x-ms-manifest"}, - {"map", "text/plain"}, - {"master", "application/xml"}, - {"mbox", "application/mbox"}, - {"mda", "application/msaccess"}, - {"mdb", "application/x-msaccess"}, - {"mde", "application/msaccess"}, - {"mdp", "application/octet-stream"}, - {"me", "application/x-troff-me"}, - {"mfp", "application/x-shockwave-flash"}, - {"mht", "message/rfc822"}, - {"mhtml", "message/rfc822"}, - {"mid", "audio/mid"}, - {"midi", "audio/mid"}, - {"mix", "application/octet-stream"}, - {"mk", "text/plain"}, - {"mmf", "application/x-smaf"}, - {"mno", "text/xml"}, - {"mny", "application/x-msmoney"}, - {"mod", "video/mpeg"}, - {"mov", "video/quicktime"}, - {"movie", "video/x-sgi-movie"}, - {"mp2", "video/mpeg"}, - {"mp2v", "video/mpeg"}, - {"mp3", {"audio/mpeg", "audio/mpeg3", "audio/mp3", "audio/x-mpeg-3"}}, - {"mp4", "video/mp4"}, - {"mp4v", "video/mp4"}, - {"mpa", "video/mpeg"}, - {"mpe", "video/mpeg"}, - {"mpeg", "video/mpeg"}, - {"mpf", "application/vnd.ms-mediapackage"}, - {"mpg", "video/mpeg"}, - {"mpp", "application/vnd.ms-project"}, - {"mpv2", "video/mpeg"}, - {"mqv", "video/quicktime"}, - {"ms", "application/x-troff-ms"}, - {"msg", "application/vnd.ms-outlook"}, - {"msi", {"application/x-msi", "application/octet-stream"}}, - {"mso", "application/octet-stream"}, - {"mts", "video/vnd.dlna.mpeg-tts"}, - {"mtx", "application/xml"}, - {"mvb", "application/x-msmediaview"}, - {"mvc", "application/x-miva-compiled"}, - {"mxp", "application/x-mmxp"}, - {"nc", "application/x-netcdf"}, - {"nsc", "video/x-ms-asf"}, - {"nws", "message/rfc822"}, - {"ocx", "application/octet-stream"}, - {"oda", "application/oda"}, - {"odb", "application/vnd.oasis.opendocument.database"}, - {"odc", "application/vnd.oasis.opendocument.chart"}, - {"odf", "application/vnd.oasis.opendocument.formula"}, - {"odg", "application/vnd.oasis.opendocument.graphics"}, - {"odh", "text/plain"}, - {"odi", "application/vnd.oasis.opendocument.image"}, - {"odl", "text/plain"}, - {"odm", "application/vnd.oasis.opendocument.text-master"}, - {"odp", "application/vnd.oasis.opendocument.presentation"}, - {"ods", "application/vnd.oasis.opendocument.spreadsheet"}, - {"odt", "application/vnd.oasis.opendocument.text"}, - {"oga", "audio/ogg"}, - {"ogg", "audio/ogg"}, - {"ogv", "video/ogg"}, - {"ogx", "application/ogg"}, - {"one", "application/onenote"}, - {"onea", "application/onenote"}, - {"onepkg", "application/onenote"}, - {"onetmp", "application/onenote"}, - {"onetoc", "application/onenote"}, - {"onetoc2", "application/onenote"}, - {"opus", "audio/ogg"}, - {"orderedtest", "application/xml"}, - {"osdx", "application/opensearchdescription+xml"}, - {"otf", "application/font-sfnt"}, - {"otg", "application/vnd.oasis.opendocument.graphics-template"}, - {"oth", "application/vnd.oasis.opendocument.text-web"}, - {"otp", "application/vnd.oasis.opendocument.presentation-template"}, - {"ots", "application/vnd.oasis.opendocument.spreadsheet-template"}, - {"ott", "application/vnd.oasis.opendocument.text-template"}, - {"oxt", "application/vnd.openofficeorg.extension"}, - {"p10", "application/pkcs10"}, - {"p12", "application/x-pkcs12"}, - {"p7b", "application/x-pkcs7-certificates"}, - {"p7c", "application/pkcs7-mime"}, - {"p7m", "application/pkcs7-mime", "application/x-pkcs7-mime"}, - {"p7r", "application/x-pkcs7-certreqresp"}, - {"p7s", {"application/pkcs7-signature", "application/x-pkcs7-signature", "text/plain"}}, - {"pbm", "image/x-portable-bitmap"}, - {"pcast", "application/x-podcast"}, - {"pct", "image/pict"}, - {"pcx", "application/octet-stream"}, - {"pcz", "application/octet-stream"}, - {"pdf", "application/pdf"}, - {"pfb", "application/octet-stream"}, - {"pfm", "application/octet-stream"}, - {"pfx", "application/x-pkcs12"}, - {"pgm", "image/x-portable-graymap"}, - {"pic", "image/pict"}, - {"pict", "image/pict"}, - {"pkgdef", "text/plain"}, - {"pkgundef", "text/plain"}, - {"pko", "application/vnd.ms-pki.pko"}, - {"pls", "audio/scpls"}, - {"pma", "application/x-perfmon"}, - {"pmc", "application/x-perfmon"}, - {"pml", "application/x-perfmon"}, - {"pmr", "application/x-perfmon"}, - {"pmw", "application/x-perfmon"}, - {"png", "image/png"}, - {"pnm", "image/x-portable-anymap"}, - {"pnt", "image/x-macpaint"}, - {"pntg", "image/x-macpaint"}, - {"pnz", "image/png"}, - {"pot", "application/vnd.ms-powerpoint"}, - {"potm", "application/vnd.ms-powerpoint.template.macroEnabled.12"}, - {"potx", "application/vnd.openxmlformats-officedocument.presentationml.template"}, - {"ppa", "application/vnd.ms-powerpoint"}, - {"ppam", "application/vnd.ms-powerpoint.addin.macroEnabled.12"}, - {"ppm", "image/x-portable-pixmap"}, - {"pps", "application/vnd.ms-powerpoint"}, - {"ppsm", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12"}, - {"ppsx", "application/vnd.openxmlformats-officedocument.presentationml.slideshow"}, - {"ppt", "application/vnd.ms-powerpoint"}, - {"pptm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12"}, - {"pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, - {"prf", "application/pics-rules"}, - {"prm", "application/octet-stream"}, - {"prx", "application/octet-stream"}, - {"ps", "application/postscript"}, - {"psc1", "application/PowerShell"}, - {"psd", "application/octet-stream"}, - {"psess", "application/xml"}, - {"psm", "application/octet-stream"}, - {"psp", "application/octet-stream"}, - {"pst", "application/vnd.ms-outlook"}, - {"pub", "application/x-mspublisher"}, - {"pwz", "application/vnd.ms-powerpoint"}, - {"qht", "text/x-html-insertion"}, - {"qhtm", "text/x-html-insertion"}, - {"qt", "video/quicktime"}, - {"qti", "image/x-quicktime"}, - {"qtif", "image/x-quicktime"}, - {"qtl", "application/x-quicktimeplayer"}, - {"qxd", "application/octet-stream"}, - {"ra", "audio/x-pn-realaudio"}, - {"ram", "audio/x-pn-realaudio"}, - {"rar", {"application/x-rar-compressed", "application/x-rar", "application/rar", "application/octet-stream"}}, - {"ras", "image/x-cmu-raster"}, - {"rat", "application/rat-file"}, - {"rc", "text/plain"}, - {"rc2", "text/plain"}, - {"rct", "text/plain"}, - {"rdlc", "application/xml"}, - {"reg", "text/plain"}, - {"resx", "application/xml"}, - {"rf", "image/vnd.rn-realflash"}, - {"rgb", "image/x-rgb"}, - {"rgs", "text/plain"}, - {"rm", "application/vnd.rn-realmedia"}, - {"rmi", "audio/mid"}, - {"rmp", "application/vnd.rn-rn_music_package"}, - {"roff", "application/x-troff"}, - {"rpm", "audio/x-pn-realaudio-plugin"}, - {"rqy", "text/x-ms-rqy"}, - {"rtf", {"application/rtf","application/msword", "text/richtext", "text/rtf"}}, - {"rtx", "text/richtext"}, - {"rvt", "application/octet-stream" }, - {"ruleset", "application/xml"}, - {"s", "text/plain"}, - {"safariextz", "application/x-safari-safariextz"}, - {"scd", "application/x-msschedule"}, - {"scr", "text/plain"}, - {"sct", "text/scriptlet"}, - {"sd2", "audio/x-sd2"}, - {"sdp", "application/sdp"}, - {"sea", "application/octet-stream"}, - {"searchConnector-ms", "application/windows-search-connector+xml"}, - {"setpay", "application/set-payment-initiation"}, - {"setreg", "application/set-registration-initiation"}, - {"settings", "application/xml"}, - {"sgimb", "application/x-sgimb"}, - {"sgml", "text/sgml"}, - {"sh", "application/x-sh"}, - {"shar", "application/x-shar"}, - {"shtml", "text/html"}, - {"sit", "application/x-stuffit"}, - {"sitemap", "application/xml"}, - {"skin", "application/xml"}, - {"skp", "application/x-koan" }, - {"sldm", "application/vnd.ms-powerpoint.slide.macroEnabled.12"}, - {"sldx", "application/vnd.openxmlformats-officedocument.presentationml.slide"}, - {"slk", "application/vnd.ms-excel"}, - {"sln", "text/plain"}, - {"slupkg-ms", "application/x-ms-license"}, - {"smd", "audio/x-smd"}, - {"smi", "application/octet-stream"}, - {"smx", "audio/x-smd"}, - {"smz", "audio/x-smd"}, - {"snd", "audio/basic"}, - {"snippet", "application/xml"}, - {"snp", "application/octet-stream"}, - {"sol", "text/plain"}, - {"sor", "text/plain"}, - {"spc", "application/x-pkcs7-certificates"}, - {"spl", "application/futuresplash"}, - {"spx", "audio/ogg"}, - {"src", "application/x-wais-source"}, - {"srf", "text/plain"}, - {"SSISDeploymentManifest", "text/xml"}, - {"ssm", "application/streamingmedia"}, - {"sst", "application/vnd.ms-pki.certstore"}, - {"stl", "application/vnd.ms-pki.stl"}, - {"sv4cpio", "application/x-sv4cpio"}, - {"sv4crc", "application/x-sv4crc"}, - {"svc", "application/xml"}, - {"svg", "image/svg+xml"}, - {"swf", "application/x-shockwave-flash"}, - {"step", "application/step"}, - {"stp", "application/step"}, - {"t", "application/x-troff"}, - {"tar", "application/x-tar"}, - {"tcl", "application/x-tcl"}, - {"testrunconfig", "application/xml"}, - {"testsettings", "application/xml"}, - {"tex", "application/x-tex"}, - {"texi", "application/x-texinfo"}, - {"texinfo", "application/x-texinfo"}, - {"tgz", "application/x-compressed"}, - {"thmx", "application/vnd.ms-officetheme"}, - {"thn", "application/octet-stream"}, - {"tif", {"image/tiff", "application/octet-stream"}}, - {"tiff", "image/tiff"}, - {"tlh", "text/plain"}, - {"tli", "text/plain"}, - {"toc", "application/octet-stream"}, - {"tr", "application/x-troff"}, - {"trm", "application/x-msterminal"}, - {"trx", "application/xml"}, - {"ts", "video/vnd.dlna.mpeg-tts"}, - {"tsv", "text/tab-separated-values"}, - {"ttf", "application/font-sfnt"}, - {"tts", "video/vnd.dlna.mpeg-tts"}, - {"txt", "text/plain"}, - {"u32", "application/octet-stream"}, - {"uls", "text/iuls"}, - {"user", "text/plain"}, - {"ustar", "application/x-ustar"}, - {"vb", "text/plain"}, - {"vbdproj", "text/plain"}, - {"vbk", "video/mpeg"}, - {"vbproj", "text/plain"}, - {"vbs", "text/vbscript"}, - {"vcf", {"text/x-vcard", "text/vcard"}}, - {"vcproj", "application/xml"}, - {"vcs", "text/plain"}, - {"vcxproj", "application/xml"}, - {"vddproj", "text/plain"}, - {"vdp", "text/plain"}, - {"vdproj", "text/plain"}, - {"vdx", "application/vnd.ms-visio.viewer"}, - {"vml", "text/xml"}, - {"vscontent", "application/xml"}, - {"vsct", "text/xml"}, - {"vsd", "application/vnd.visio"}, - {"vsi", "application/ms-vsi"}, - {"vsix", "application/vsix"}, - {"vsixlangpack", "text/xml"}, - {"vsixmanifest", "text/xml"}, - {"vsmdi", "application/xml"}, - {"vspscc", "text/plain"}, - {"vss", "application/vnd.visio"}, - {"vsscc", "text/plain"}, - {"vssettings", "text/xml"}, - {"vssscc", "text/plain"}, - {"vst", "application/vnd.visio"}, - {"vstemplate", "text/xml"}, - {"vsto", "application/x-ms-vsto"}, - {"vsw", "application/vnd.visio"}, - {"vsx", "application/vnd.visio"}, - {"vtx", "application/vnd.visio"}, - {"wav", {"audio/wav", "audio/vnd.wave", "audio/x-wav"}}, - {"wave", "audio/wav"}, - {"wax", "audio/x-ms-wax"}, - {"wbk", "application/msword"}, - {"wbmp", "image/vnd.wap.wbmp"}, - {"wcm", "application/vnd.ms-works"}, - {"wdb", "application/vnd.ms-works"}, - {"wdp", "image/vnd.ms-photo"}, - {"webarchive", "application/x-safari-webarchive"}, - {"webm", "video/webm"}, - {"webp", "image/webp"}, - {"webtest", "application/xml"}, - {"wiq", "application/xml"}, - {"wiz", "application/msword"}, - {"wks", "application/vnd.ms-works"}, - {"WLMP", "application/wlmoviemaker"}, - {"wlpginstall", "application/x-wlpg-detect"}, - {"wlpginstall3", "application/x-wlpg3-detect"}, - {"wm", "video/x-ms-wm"}, - {"wma", "audio/x-ms-wma"}, - {"wmd", "application/x-ms-wmd"}, - {"wmf", {"application/x-msmetafile", "image/wmf", "image/x-wmf"}}, - {"wml", "text/vnd.wap.wml"}, - {"wmlc", "application/vnd.wap.wmlc"}, - {"wmls", "text/vnd.wap.wmlscript"}, - {"wmlsc", "application/vnd.wap.wmlscriptc"}, - {"wmp", "video/x-ms-wmp"}, - {"wmv", "video/x-ms-wmv"}, - {"wmx", "video/x-ms-wmx"}, - {"wmz", "application/x-ms-wmz"}, - {"woff", "application/font-woff"}, - {"wpl", "application/vnd.ms-wpl"}, - {"wps", "application/vnd.ms-works"}, - {"wri", "application/x-mswrite"}, - {"wrl", "x-world/x-vrml"}, - {"wrz", "x-world/x-vrml"}, - {"wsc", "text/scriptlet"}, - {"wsdl", "text/xml"}, - {"wvx", "video/x-ms-wvx"}, - {"x", "application/directx"}, - {"xaf", "x-world/x-vrml"}, - {"xaml", "application/xaml+xml"}, - {"xap", "application/x-silverlight-app"}, - {"xbap", "application/x-ms-xbap"}, - {"xbm", "image/x-xbitmap"}, - {"xdr", "text/plain"}, - {"xht", "application/xhtml+xml"}, - {"xhtml", "application/xhtml+xml"}, - {"xla", "application/vnd.ms-excel"}, - {"xlam", "application/vnd.ms-excel.addin.macroEnabled.12"}, - {"xlc", "application/vnd.ms-excel"}, - {"xld", "application/vnd.ms-excel"}, - {"xlk", "application/vnd.ms-excel"}, - {"xll", "application/vnd.ms-excel"}, - {"xlm", "application/vnd.ms-excel"}, - {"xls", { - "application/excel", - "application/vnd.ms-excel", - "application/vnd.ms-office", - "application/x-excel", - "application/octet-stream" - }}, - {"xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12"}, - {"xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12"}, - {"xlsx", { - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.ms-excel.12", - "application/octet-stream" - }}, - {"xlt", "application/vnd.ms-excel"}, - {"xltm", "application/vnd.ms-excel.template.macroEnabled.12"}, - {"xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template"}, - {"xlw", "application/vnd.ms-excel"}, - {"xml", {"application/xml", "text/xml", "application/octet-stream"}}, - {"xmp", "application/octet-stream" }, - {"xmta", "application/xml"}, - {"xof", "x-world/x-vrml"}, - {"XOML", "text/plain"}, - {"xpm", "image/x-xpixmap"}, - {"xps", "application/vnd.ms-xpsdocument"}, - {"xrm-ms", "text/xml"}, - {"xsc", "application/xml"}, - {"xsd", "text/xml"}, - {"xsf", "text/xml"}, - {"xsl", "text/xml"}, - {"xslt", "text/xml"}, - {"xsn", "application/octet-stream"}, - {"xss", "application/xml"}, - {"xspf", "application/xspf+xml"}, - {"xtp", "application/octet-stream"}, - {"xwd", "image/x-xwindowdump"}, - {"z", "application/x-compress"}, - {"zip", { - "application/zip", - "application/x-zip-compressed", - "application/octet-stream" - }}, - {"zlib", "application/zlib"}, + { "323", "text/h323" }, + { "3g2", "video/3gpp2" }, + { "3gp", "video/3gpp" }, + { "3gp2", "video/3gpp2" }, + { "3gpp", "video/3gpp" }, + { "7z", { "application/x-7z-compressed", "application/7z" } }, + { "aa", "audio/audible" }, + { "AAC", "audio/aac" }, + { "aaf", "application/octet-stream" }, + { "aax", "audio/vnd.audible.aax" }, + { "ac3", "audio/ac3" }, + { "aca", "application/octet-stream" }, + { "accda", "application/msaccess.addin" }, + { "accdb", "application/msaccess" }, + { "accdc", "application/msaccess.cab" }, + { "accde", "application/msaccess" }, + { "accdr", "application/msaccess.runtime" }, + { "accdt", "application/msaccess" }, + { "accdw", "application/msaccess.webapplication" }, + { "accft", "application/msaccess.ftemplate" }, + { "acx", "application/internet-property-stream" }, + { "AddIn", "text/xml" }, + { "ade", "application/msaccess" }, + { "adobebridge", "application/x-bridge-url" }, + { "adp", "application/msaccess" }, + { "ADT", "audio/vnd.dlna.adts" }, + { "ADTS", "audio/aac" }, + { "afm", "application/octet-stream" }, + { "ai", "application/postscript" }, + { "aif", "audio/aiff" }, + { "aifc", "audio/aiff" }, + { "aiff", "audio/aiff" }, + { "air", "application/vnd.adobe.air-application-installer-package+zip" }, + { "amc", "application/mpeg" }, + { "anx", "application/annodex" }, + { "apk", "application/vnd.android.package-archive" }, + { "application", "application/x-ms-application" }, + { "art", "image/x-jg" }, + { "asa", "application/xml" }, + { "asax", "application/xml" }, + { "ascx", "application/xml" }, + { "asd", "application/octet-stream" }, + { "asf", "video/x-ms-asf" }, + { "ashx", "application/xml" }, + { "asi", "application/octet-stream" }, + { "asm", "text/plain" }, + { "asmx", "application/xml" }, + { "aspx", "application/xml" }, + { "asr", "video/x-ms-asf" }, + { "asx", "video/x-ms-asf" }, + { "atom", "application/atom+xml" }, + { "au", "audio/basic" }, + { "avi", "video/x-msvideo" }, + { "axa", "audio/annodex" }, + { "axs", "application/olescript" }, + { "axv", "video/annodex" }, + { "bas", "text/plain" }, + { "bcpio", "application/x-bcpio" }, + { "bin", "application/octet-stream" }, + { "bmp", { "image/bmp", "image/x-ms-bmp" } }, + { "c", "text/plain" }, + { "cab", "application/octet-stream" }, + { "caf", "audio/x-caf" }, + { "calx", "application/vnd.ms-office.calx" }, + { "cat", "application/vnd.ms-pki.seccat" }, + { "cc", "text/plain" }, + { "cd", "text/plain" }, + { "cdda", "audio/aiff" }, + { "cdf", "application/x-cdf" }, + { "cer", "application/x-x509-ca-cert" }, + { "cfg", "text/plain" }, + { "chm", "application/octet-stream" }, + { "class", "application/x-java-applet" }, + { "clp", "application/x-msclip" }, + { "cmd", "text/plain" }, + { "cmx", "image/x-cmx" }, + { "cnf", "text/plain" }, + { "cod", "image/cis-cod" }, + { "config", "application/xml" }, + { "contact", "text/x-ms-contact" }, + { "coverage", "application/xml" }, + { "cpio", "application/x-cpio" }, + { "cpp", "text/plain" }, + { "crd", "application/x-mscardfile" }, + { "crl", "application/pkix-crl" }, + { "crt", "application/x-x509-ca-cert" }, + { "cs", "text/plain" }, + { "csdproj", "text/plain" }, + { "csh", "application/x-csh" }, + { "csproj", "text/plain" }, + { "css", "text/css" }, + { "csv", { "application/vnd.ms-excel", "text/csv", "text/plain" } }, + { "cur", "application/octet-stream" }, + { "cxx", "text/plain" }, + { "dat", { "application/octet-stream", "application/ms-tnef" } }, + { "datasource", "application/xml" }, + { "dbproj", "text/plain" }, + { "dcr", "application/x-director" }, + { "def", "text/plain" }, + { "deploy", "application/octet-stream" }, + { "der", "application/x-x509-ca-cert" }, + { "dgml", "application/xml" }, + { "dib", "image/bmp" }, + { "dif", "video/x-dv" }, + { "dir", "application/x-director" }, + { "disco", "text/xml" }, + { "divx", "video/divx" }, + { "dll", "application/x-msdownload" }, + { "dll.config", "text/xml" }, + { "dlm", "text/dlm" }, + { "doc", "application/msword" }, + { "docm", "application/vnd.ms-word.document.macroEnabled.12" }, + { "docx", { + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/msword", + "application/vnd.ms-word.document.12", + "application/octet-stream", + } }, + { "dot", "application/msword" }, + { "dotm", "application/vnd.ms-word.template.macroEnabled.12" }, + { "dotx", "application/vnd.openxmlformats-officedocument.wordprocessingml.template" }, + { "dsp", "application/octet-stream" }, + { "dsw", "text/plain" }, + { "dtd", "text/xml" }, + { "dtsConfig", "text/xml" }, + { "dv", "video/x-dv" }, + { "dvi", "application/x-dvi" }, + { "dwf", "drawing/x-dwf" }, + { "dwg", { "application/acad", "image/vnd.dwg" } }, + { "dwp", "application/octet-stream" }, + { "dxf", "application/x-dxf" }, + { "dxr", "application/x-director" }, + { "eml", "message/rfc822" }, + { "emz", "application/octet-stream" }, + { "eot", "application/vnd.ms-fontobject" }, + { "eps", "application/postscript" }, + { "etl", "application/etl" }, + { "etx", "text/x-setext" }, + { "evy", "application/envoy" }, + { "exe", { + "application/x-dosexec", + "application/x-msdownload", + "application/x-executable", + } }, + { "exe.config", "text/xml" }, + { "fdf", "application/vnd.fdf" }, + { "fif", "application/fractals" }, + { "filters", "application/xml" }, + { "fla", "application/octet-stream" }, + { "flac", "audio/flac" }, + { "flr", "x-world/x-vrml" }, + { "flv", "video/x-flv" }, + { "fsscript", "application/fsharp-script" }, + { "fsx", "application/fsharp-script" }, + { "generictest", "application/xml" }, + { "gif", "image/gif" }, + { "gpx", "application/gpx+xml" }, + { "group", "text/x-ms-group" }, + { "gsm", "audio/x-gsm" }, + { "gtar", "application/x-gtar" }, + { "gz", { "application/gzip", "application/x-gzip", "application/tlsrpt+gzip" } }, + { "h", "text/plain" }, + { "hdf", "application/x-hdf" }, + { "hdml", "text/x-hdml" }, + { "hhc", "application/x-oleobject" }, + { "hhk", "application/octet-stream" }, + { "hhp", "application/octet-stream" }, + { "hlp", "application/winhlp" }, + { "hpp", "text/plain" }, + { "hqx", "application/mac-binhex40" }, + { "hta", "application/hta" }, + { "htc", "text/x-component" }, + { "htm", "text/html" }, + { "html", "text/html" }, + { "htt", "text/webviewhtml" }, + { "hxa", "application/xml" }, + { "hxc", "application/xml" }, + { "hxd", "application/octet-stream" }, + { "hxe", "application/xml" }, + { "hxf", "application/xml" }, + { "hxh", "application/octet-stream" }, + { "hxi", "application/octet-stream" }, + { "hxk", "application/xml" }, + { "hxq", "application/octet-stream" }, + { "hxr", "application/octet-stream" }, + { "hxs", "application/octet-stream" }, + { "hxt", "text/html" }, + { "hxv", "application/xml" }, + { "hxw", "application/octet-stream" }, + { "hxx", "text/plain" }, + { "i", "text/plain" }, + { "ico", "image/x-icon" }, + { "ics", { "text/calendar", "application/ics", "application/octet-stream" } }, + { "idl", "text/plain" }, + { "ief", "image/ief" }, + { "iii", "application/x-iphone" }, + { "inc", "text/plain" }, + { "inf", "application/octet-stream" }, + { "ini", "text/plain" }, + { "inl", "text/plain" }, + { "ins", "application/x-internet-signup" }, + { "ipa", "application/x-itunes-ipa" }, + { "ipg", "application/x-itunes-ipg" }, + { "ipproj", "text/plain" }, + { "ipsw", "application/x-itunes-ipsw" }, + { "iqy", "text/x-ms-iqy" }, + { "isp", "application/x-internet-signup" }, + { "ite", "application/x-itunes-ite" }, + { "itlp", "application/x-itunes-itlp" }, + { "itms", "application/x-itunes-itms" }, + { "itpc", "application/x-itunes-itpc" }, + { "IVF", "video/x-ivf" }, + { "jar", "application/java-archive" }, + { "java", "application/octet-stream" }, + { "jck", "application/liquidmotion" }, + { "jcz", "application/liquidmotion" }, + { "jfif", { "image/jpeg", "image/pjpeg" } }, + { "jnlp", "application/x-java-jnlp-file" }, + { "jpb", "application/octet-stream" }, + { "jpe", { "image/jpeg", "image/pjpeg" } }, + { "jpeg", { "image/jpeg", "image/pjpeg" } }, + { "jpg", { "image/jpeg", "image/pjpeg" } }, + { "js", "application/javascript" }, + { "json", "application/json" }, + { "jsx", "text/jscript" }, + { "jsxbin", "text/plain" }, + { "latex", "application/x-latex" }, + { "library-ms", "application/windows-library+xml" }, + { "lit", "application/x-ms-reader" }, + { "loadtest", "application/xml" }, + { "lpk", "application/octet-stream" }, + { "lsf", "video/x-la-asf" }, + { "lst", "text/plain" }, + { "lsx", "video/x-la-asf" }, + { "lzh", "application/octet-stream" }, + { "m13", "application/x-msmediaview" }, + { "m14", "application/x-msmediaview" }, + { "m1v", "video/mpeg" }, + { "m2t", "video/vnd.dlna.mpeg-tts" }, + { "m2ts", "video/vnd.dlna.mpeg-tts" }, + { "m2v", "video/mpeg" }, + { "m3u", "audio/x-mpegurl" }, + { "m3u8", "audio/x-mpegurl" }, + { "m4a", { "audio/m4a", "audio/x-m4a" } }, + { "m4b", "audio/m4b" }, + { "m4p", "audio/m4p" }, + { "m4r", "audio/x-m4r" }, + { "m4v", "video/x-m4v" }, + { "mac", "image/x-macpaint" }, + { "mak", "text/plain" }, + { "man", "application/x-troff-man" }, + { "manifest", "application/x-ms-manifest" }, + { "map", "text/plain" }, + { "master", "application/xml" }, + { "mbox", "application/mbox" }, + { "mda", "application/msaccess" }, + { "mdb", "application/x-msaccess" }, + { "mde", "application/msaccess" }, + { "mdp", "application/octet-stream" }, + { "me", "application/x-troff-me" }, + { "mfp", "application/x-shockwave-flash" }, + { "mht", "message/rfc822" }, + { "mhtml", "message/rfc822" }, + { "mid", "audio/mid" }, + { "midi", "audio/mid" }, + { "mix", "application/octet-stream" }, + { "mk", "text/plain" }, + { "mmf", "application/x-smaf" }, + { "mno", "text/xml" }, + { "mny", "application/x-msmoney" }, + { "mod", "video/mpeg" }, + { "mov", "video/quicktime" }, + { "movie", "video/x-sgi-movie" }, + { "mp2", "video/mpeg" }, + { "mp2v", "video/mpeg" }, + { "mp3", { "audio/mpeg", "audio/mpeg3", "audio/mp3", "audio/x-mpeg-3" } }, + { "mp4", "video/mp4" }, + { "mp4v", "video/mp4" }, + { "mpa", "video/mpeg" }, + { "mpe", "video/mpeg" }, + { "mpeg", "video/mpeg" }, + { "mpf", "application/vnd.ms-mediapackage" }, + { "mpg", "video/mpeg" }, + { "mpp", "application/vnd.ms-project" }, + { "mpv2", "video/mpeg" }, + { "mqv", "video/quicktime" }, + { "ms", "application/x-troff-ms" }, + { "msg", "application/vnd.ms-outlook" }, + { "msi", { "application/x-msi", "application/octet-stream" } }, + { "mso", "application/octet-stream" }, + { "mts", "video/vnd.dlna.mpeg-tts" }, + { "mtx", "application/xml" }, + { "mvb", "application/x-msmediaview" }, + { "mvc", "application/x-miva-compiled" }, + { "mxp", "application/x-mmxp" }, + { "nc", "application/x-netcdf" }, + { "nsc", "video/x-ms-asf" }, + { "nws", "message/rfc822" }, + { "ocx", "application/octet-stream" }, + { "oda", "application/oda" }, + { "odb", "application/vnd.oasis.opendocument.database" }, + { "odc", "application/vnd.oasis.opendocument.chart" }, + { "odf", "application/vnd.oasis.opendocument.formula" }, + { "odg", "application/vnd.oasis.opendocument.graphics" }, + { "odh", "text/plain" }, + { "odi", "application/vnd.oasis.opendocument.image" }, + { "odl", "text/plain" }, + { "odm", "application/vnd.oasis.opendocument.text-master" }, + { "odp", "application/vnd.oasis.opendocument.presentation" }, + { "ods", "application/vnd.oasis.opendocument.spreadsheet" }, + { "odt", "application/vnd.oasis.opendocument.text" }, + { "oga", "audio/ogg" }, + { "ogg", "audio/ogg" }, + { "ogv", "video/ogg" }, + { "ogx", "application/ogg" }, + { "one", "application/onenote" }, + { "onea", "application/onenote" }, + { "onepkg", "application/onenote" }, + { "onetmp", "application/onenote" }, + { "onetoc", "application/onenote" }, + { "onetoc2", "application/onenote" }, + { "opus", "audio/ogg" }, + { "orderedtest", "application/xml" }, + { "osdx", "application/opensearchdescription+xml" }, + { "otf", "application/font-sfnt" }, + { "otg", "application/vnd.oasis.opendocument.graphics-template" }, + { "oth", "application/vnd.oasis.opendocument.text-web" }, + { "otp", "application/vnd.oasis.opendocument.presentation-template" }, + { "ots", "application/vnd.oasis.opendocument.spreadsheet-template" }, + { "ott", "application/vnd.oasis.opendocument.text-template" }, + { "oxt", "application/vnd.openofficeorg.extension" }, + { "p10", "application/pkcs10" }, + { "p12", "application/x-pkcs12" }, + { "p7b", "application/x-pkcs7-certificates" }, + { "p7c", "application/pkcs7-mime" }, + { "p7m", "application/pkcs7-mime", "application/x-pkcs7-mime" }, + { "p7r", "application/x-pkcs7-certreqresp" }, + { "p7s", { "application/pkcs7-signature", "application/x-pkcs7-signature", "text/plain" } }, + { "pbm", "image/x-portable-bitmap" }, + { "pcast", "application/x-podcast" }, + { "pct", "image/pict" }, + { "pcx", "application/octet-stream" }, + { "pcz", "application/octet-stream" }, + { "pdf", "application/pdf" }, + { "pfb", "application/octet-stream" }, + { "pfm", "application/octet-stream" }, + { "pfx", "application/x-pkcs12" }, + { "pgm", "image/x-portable-graymap" }, + { "pic", "image/pict" }, + { "pict", "image/pict" }, + { "pkgdef", "text/plain" }, + { "pkgundef", "text/plain" }, + { "pko", "application/vnd.ms-pki.pko" }, + { "pls", "audio/scpls" }, + { "pma", "application/x-perfmon" }, + { "pmc", "application/x-perfmon" }, + { "pml", "application/x-perfmon" }, + { "pmr", "application/x-perfmon" }, + { "pmw", "application/x-perfmon" }, + { "png", "image/png" }, + { "pnm", "image/x-portable-anymap" }, + { "pnt", "image/x-macpaint" }, + { "pntg", "image/x-macpaint" }, + { "pnz", "image/png" }, + { "pot", "application/vnd.ms-powerpoint" }, + { "potm", "application/vnd.ms-powerpoint.template.macroEnabled.12" }, + { "potx", "application/vnd.openxmlformats-officedocument.presentationml.template" }, + { "ppa", "application/vnd.ms-powerpoint" }, + { "ppam", "application/vnd.ms-powerpoint.addin.macroEnabled.12" }, + { "ppm", "image/x-portable-pixmap" }, + { "pps", "application/vnd.ms-powerpoint" }, + { "ppsm", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12" }, + { "ppsx", "application/vnd.openxmlformats-officedocument.presentationml.slideshow" }, + { "ppt", "application/vnd.ms-powerpoint" }, + { "pptm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12" }, + { "pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation" }, + { "prf", "application/pics-rules" }, + { "prm", "application/octet-stream" }, + { "prx", "application/octet-stream" }, + { "ps", "application/postscript" }, + { "psc1", "application/PowerShell" }, + { "psd", "application/octet-stream" }, + { "psess", "application/xml" }, + { "psm", "application/octet-stream" }, + { "psp", "application/octet-stream" }, + { "pst", "application/vnd.ms-outlook" }, + { "pub", "application/x-mspublisher" }, + { "pwz", "application/vnd.ms-powerpoint" }, + { "qht", "text/x-html-insertion" }, + { "qhtm", "text/x-html-insertion" }, + { "qt", "video/quicktime" }, + { "qti", "image/x-quicktime" }, + { "qtif", "image/x-quicktime" }, + { "qtl", "application/x-quicktimeplayer" }, + { "qxd", "application/octet-stream" }, + { "ra", "audio/x-pn-realaudio" }, + { "ram", "audio/x-pn-realaudio" }, + { "rar", { "application/x-rar-compressed", "application/x-rar", "application/rar", "application/octet-stream" } }, + { "ras", "image/x-cmu-raster" }, + { "rat", "application/rat-file" }, + { "rc", "text/plain" }, + { "rc2", "text/plain" }, + { "rct", "text/plain" }, + { "rdlc", "application/xml" }, + { "reg", "text/plain" }, + { "resx", "application/xml" }, + { "rf", "image/vnd.rn-realflash" }, + { "rgb", "image/x-rgb" }, + { "rgs", "text/plain" }, + { "rm", "application/vnd.rn-realmedia" }, + { "rmi", "audio/mid" }, + { "rmp", "application/vnd.rn-rn_music_package" }, + { "roff", "application/x-troff" }, + { "rpm", "audio/x-pn-realaudio-plugin" }, + { "rqy", "text/x-ms-rqy" }, + { "rtf", { "application/rtf", "application/msword", "text/richtext", "text/rtf" } }, + { "rtx", "text/richtext" }, + { "rvt", "application/octet-stream" }, + { "ruleset", "application/xml" }, + { "s", "text/plain" }, + { "safariextz", "application/x-safari-safariextz" }, + { "scd", "application/x-msschedule" }, + { "scr", "text/plain" }, + { "sct", "text/scriptlet" }, + { "sd2", "audio/x-sd2" }, + { "sdp", "application/sdp" }, + { "sea", "application/octet-stream" }, + { "searchConnector-ms", "application/windows-search-connector+xml" }, + { "setpay", "application/set-payment-initiation" }, + { "setreg", "application/set-registration-initiation" }, + { "settings", "application/xml" }, + { "sgimb", "application/x-sgimb" }, + { "sgml", "text/sgml" }, + { "sh", "application/x-sh" }, + { "shar", "application/x-shar" }, + { "shtml", "text/html" }, + { "sit", "application/x-stuffit" }, + { "sitemap", "application/xml" }, + { "skin", "application/xml" }, + { "skp", "application/x-koan" }, + { "sldm", "application/vnd.ms-powerpoint.slide.macroEnabled.12" }, + { "sldx", "application/vnd.openxmlformats-officedocument.presentationml.slide" }, + { "slk", "application/vnd.ms-excel" }, + { "sln", "text/plain" }, + { "slupkg-ms", "application/x-ms-license" }, + { "smd", "audio/x-smd" }, + { "smi", "application/octet-stream" }, + { "smx", "audio/x-smd" }, + { "smz", "audio/x-smd" }, + { "snd", "audio/basic" }, + { "snippet", "application/xml" }, + { "snp", "application/octet-stream" }, + { "sol", "text/plain" }, + { "sor", "text/plain" }, + { "spc", "application/x-pkcs7-certificates" }, + { "spl", "application/futuresplash" }, + { "spx", "audio/ogg" }, + { "src", "application/x-wais-source" }, + { "srf", "text/plain" }, + { "SSISDeploymentManifest", "text/xml" }, + { "ssm", "application/streamingmedia" }, + { "sst", "application/vnd.ms-pki.certstore" }, + { "stl", "application/vnd.ms-pki.stl" }, + { "sv4cpio", "application/x-sv4cpio" }, + { "sv4crc", "application/x-sv4crc" }, + { "svc", "application/xml" }, + { "svg", "image/svg+xml" }, + { "swf", "application/x-shockwave-flash" }, + { "step", "application/step" }, + { "stp", "application/step" }, + { "t", "application/x-troff" }, + { "tar", "application/x-tar" }, + { "tcl", "application/x-tcl" }, + { "testrunconfig", "application/xml" }, + { "testsettings", "application/xml" }, + { "tex", "application/x-tex" }, + { "texi", "application/x-texinfo" }, + { "texinfo", "application/x-texinfo" }, + { "tgz", "application/x-compressed" }, + { "thmx", "application/vnd.ms-officetheme" }, + { "thn", "application/octet-stream" }, + { "tif", { "image/tiff", "application/octet-stream" } }, + { "tiff", "image/tiff" }, + { "tlh", "text/plain" }, + { "tli", "text/plain" }, + { "toc", "application/octet-stream" }, + { "tr", "application/x-troff" }, + { "trm", "application/x-msterminal" }, + { "trx", "application/xml" }, + { "ts", "video/vnd.dlna.mpeg-tts" }, + { "tsv", "text/tab-separated-values" }, + { "ttf", "application/font-sfnt" }, + { "tts", "video/vnd.dlna.mpeg-tts" }, + { "txt", "text/plain" }, + { "u32", "application/octet-stream" }, + { "uls", "text/iuls" }, + { "user", "text/plain" }, + { "ustar", "application/x-ustar" }, + { "vb", "text/plain" }, + { "vbdproj", "text/plain" }, + { "vbk", "video/mpeg" }, + { "vbproj", "text/plain" }, + { "vbs", "text/vbscript" }, + { "vcf", { "text/x-vcard", "text/vcard" } }, + { "vcproj", "application/xml" }, + { "vcs", "text/plain" }, + { "vcxproj", "application/xml" }, + { "vddproj", "text/plain" }, + { "vdp", "text/plain" }, + { "vdproj", "text/plain" }, + { "vdx", "application/vnd.ms-visio.viewer" }, + { "vml", "text/xml" }, + { "vscontent", "application/xml" }, + { "vsct", "text/xml" }, + { "vsd", "application/vnd.visio" }, + { "vsi", "application/ms-vsi" }, + { "vsix", "application/vsix" }, + { "vsixlangpack", "text/xml" }, + { "vsixmanifest", "text/xml" }, + { "vsmdi", "application/xml" }, + { "vspscc", "text/plain" }, + { "vss", "application/vnd.visio" }, + { "vsscc", "text/plain" }, + { "vssettings", "text/xml" }, + { "vssscc", "text/plain" }, + { "vst", "application/vnd.visio" }, + { "vstemplate", "text/xml" }, + { "vsto", "application/x-ms-vsto" }, + { "vsw", "application/vnd.visio" }, + { "vsx", "application/vnd.visio" }, + { "vtx", "application/vnd.visio" }, + { "wav", { "audio/wav", "audio/vnd.wave", "audio/x-wav" } }, + { "wave", "audio/wav" }, + { "wax", "audio/x-ms-wax" }, + { "wbk", "application/msword" }, + { "wbmp", "image/vnd.wap.wbmp" }, + { "wcm", "application/vnd.ms-works" }, + { "wdb", "application/vnd.ms-works" }, + { "wdp", "image/vnd.ms-photo" }, + { "webarchive", "application/x-safari-webarchive" }, + { "webm", "video/webm" }, + { "webp", "image/webp" }, + { "webtest", "application/xml" }, + { "wiq", "application/xml" }, + { "wiz", "application/msword" }, + { "wks", "application/vnd.ms-works" }, + { "WLMP", "application/wlmoviemaker" }, + { "wlpginstall", "application/x-wlpg-detect" }, + { "wlpginstall3", "application/x-wlpg3-detect" }, + { "wm", "video/x-ms-wm" }, + { "wma", "audio/x-ms-wma" }, + { "wmd", "application/x-ms-wmd" }, + { "wmf", { "application/x-msmetafile", "image/wmf", "image/x-wmf" } }, + { "wml", "text/vnd.wap.wml" }, + { "wmlc", "application/vnd.wap.wmlc" }, + { "wmls", "text/vnd.wap.wmlscript" }, + { "wmlsc", "application/vnd.wap.wmlscriptc" }, + { "wmp", "video/x-ms-wmp" }, + { "wmv", "video/x-ms-wmv" }, + { "wmx", "video/x-ms-wmx" }, + { "wmz", "application/x-ms-wmz" }, + { "woff", "application/font-woff" }, + { "wpl", "application/vnd.ms-wpl" }, + { "wps", "application/vnd.ms-works" }, + { "wri", "application/x-mswrite" }, + { "wrl", "x-world/x-vrml" }, + { "wrz", "x-world/x-vrml" }, + { "wsc", "text/scriptlet" }, + { "wsdl", "text/xml" }, + { "wvx", "video/x-ms-wvx" }, + { "x", "application/directx" }, + { "xaf", "x-world/x-vrml" }, + { "xaml", "application/xaml+xml" }, + { "xap", "application/x-silverlight-app" }, + { "xbap", "application/x-ms-xbap" }, + { "xbm", "image/x-xbitmap" }, + { "xdr", "text/plain" }, + { "xht", "application/xhtml+xml" }, + { "xhtml", "application/xhtml+xml" }, + { "xla", "application/vnd.ms-excel" }, + { "xlam", "application/vnd.ms-excel.addin.macroEnabled.12" }, + { "xlc", "application/vnd.ms-excel" }, + { "xld", "application/vnd.ms-excel" }, + { "xlk", "application/vnd.ms-excel" }, + { "xll", "application/vnd.ms-excel" }, + { "xlm", "application/vnd.ms-excel" }, + { "xls", { + "application/excel", + "application/vnd.ms-excel", + "application/vnd.ms-office", + "application/x-excel", + "application/octet-stream" + } }, + { "xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12" }, + { "xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12" }, + { "xlsx", { + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel.12", + "application/octet-stream" + } }, + { "xlt", "application/vnd.ms-excel" }, + { "xltm", "application/vnd.ms-excel.template.macroEnabled.12" }, + { "xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template" }, + { "xlw", "application/vnd.ms-excel" }, + { "xml", { "application/xml", "text/xml", "application/octet-stream" } }, + { "xmp", "application/octet-stream" }, + { "xmta", "application/xml" }, + { "xof", "x-world/x-vrml" }, + { "XOML", "text/plain" }, + { "xpm", "image/x-xpixmap" }, + { "xps", "application/vnd.ms-xpsdocument" }, + { "xrm-ms", "text/xml" }, + { "xsc", "application/xml" }, + { "xsd", "text/xml" }, + { "xsf", "text/xml" }, + { "xsl", "text/xml" }, + { "xslt", "text/xml" }, + { "xsn", "application/octet-stream" }, + { "xss", "application/xml" }, + { "xspf", "application/xspf+xml" }, + { "xtp", "application/octet-stream" }, + { "xwd", "image/x-xwindowdump" }, + { "z", "application/x-compress" }, + { "zip", { + "application/zip", + "application/x-zip-compressed", + "application/octet-stream" + } }, + { "zlib", "application/zlib" }, } -- Used to match extension by content type exports.reversed_extensions_map = { - ["text/html"] = "html", - ["text/css"] = "css", - ["text/xml"] = "xml", - ["image/gif"] = "gif", - ["image/jpeg"] = "jpeg", - ["application/javascript"] = "js", - ["application/atom+xml"] = "atom", - ["application/rss+xml"] = "rss", - ["application/csv"] = "csv", - ["text/mathml"] = "mml", - ["text/plain"] = "txt", - ["text/vnd.sun.j2me.app-descriptor"] = "jad", - ["text/vnd.wap.wml"] = "wml", - ["text/x-component"] = "htc", - ["image/png"] = "png", - ["image/svg+xml"] = "svg", - ["image/tiff"] = "tiff", - ["image/vnd.wap.wbmp"] = "wbmp", - ["image/webp"] = "webp", - ["image/x-icon"] = "ico", - ["image/x-jng"] = "jng", - ["image/x-ms-bmp"] = "bmp", - ["font/woff"] = "woff", - ["font/woff2"] = "woff2", - ["application/java-archive"] = "jar", - ["application/json"] = "json", - ["application/mac-binhex40"] = "hqx", - ["application/msword"] = "doc", - ["application/pdf"] = "pdf", - ["application/postscript"] = "ps", - ["application/rtf"] = "rtf", - ["application/vnd.apple.mpegurl"] = "m3u8", - ["application/vnd.google-earth.kml+xml"] = "kml", - ["application/vnd.google-earth.kmz"] = "kmz", - ["application/vnd.ms-excel"] = "xls", - ["application/vnd.ms-fontobject"] = "eot", - ["application/vnd.ms-powerpoint"] = "ppt", - ["application/vnd.oasis.opendocument.graphics"] = "odg", - ["application/vnd.oasis.opendocument.presentation"] = "odp", - ["application/vnd.oasis.opendocument.spreadsheet"] = "ods", - ["application/vnd.oasis.opendocument.text"] = "odt", - ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "pptx", - ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "xlsx", - ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "docx", - ["application/x-7z-compressed"] = "7z", - ["application/x-cocoa"] = "cco", - ["application/x-java-archive-diff"] = "jardiff", - ["application/x-java-jnlp-file"] = "jnlp", - ["application/x-makeself"] = "run", - ["application/x-perl"] = "pl", - ["application/x-pilot"] = "pdb", - ["application/x-rar-compressed"] = "rar", - ["application/x-redhat-package-manager"] = "rpm", - ["application/x-sea"] = "sea", - ["application/x-shockwave-flash"] = "swf", - ["application/x-stuffit"] = "sit", - ["application/x-tcl"] = "tcl", - ["application/x-x509-ca-cert"] = "crt", - ["application/x-xpinstall"] = "xpi", - ["application/xhtml+xml"] = "xhtml", - ["application/xspf+xml"] = "xspf", - ["application/zip"] = "zip", - ["application/x-dosexec"] = "exe", - ["application/x-msdownload"] = "exe", - ["application/x-executable"] = "exe", - ["text/x-msdos-batch"] = "bat", + ["text/html"] = "html", + ["text/css"] = "css", + ["text/xml"] = "xml", + ["image/gif"] = "gif", + ["image/jpeg"] = "jpeg", + ["application/javascript"] = "js", + ["application/atom+xml"] = "atom", + ["application/rss+xml"] = "rss", + ["application/csv"] = "csv", + ["text/mathml"] = "mml", + ["text/plain"] = "txt", + ["text/vnd.sun.j2me.app-descriptor"] = "jad", + ["text/vnd.wap.wml"] = "wml", + ["text/x-component"] = "htc", + ["image/png"] = "png", + ["image/svg+xml"] = "svg", + ["image/tiff"] = "tiff", + ["image/vnd.wap.wbmp"] = "wbmp", + ["image/webp"] = "webp", + ["image/x-icon"] = "ico", + ["image/x-jng"] = "jng", + ["image/x-ms-bmp"] = "bmp", + ["font/woff"] = "woff", + ["font/woff2"] = "woff2", + ["application/java-archive"] = "jar", + ["application/json"] = "json", + ["application/mac-binhex40"] = "hqx", + ["application/msword"] = "doc", + ["application/pdf"] = "pdf", + ["application/postscript"] = "ps", + ["application/rtf"] = "rtf", + ["application/vnd.apple.mpegurl"] = "m3u8", + ["application/vnd.google-earth.kml+xml"] = "kml", + ["application/vnd.google-earth.kmz"] = "kmz", + ["application/vnd.ms-excel"] = "xls", + ["application/vnd.ms-fontobject"] = "eot", + ["application/vnd.ms-powerpoint"] = "ppt", + ["application/vnd.oasis.opendocument.graphics"] = "odg", + ["application/vnd.oasis.opendocument.presentation"] = "odp", + ["application/vnd.oasis.opendocument.spreadsheet"] = "ods", + ["application/vnd.oasis.opendocument.text"] = "odt", + ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "pptx", + ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "xlsx", + ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "docx", + ["application/x-7z-compressed"] = "7z", + ["application/x-cocoa"] = "cco", + ["application/x-java-archive-diff"] = "jardiff", + ["application/x-java-jnlp-file"] = "jnlp", + ["application/x-makeself"] = "run", + ["application/x-perl"] = "pl", + ["application/x-pilot"] = "pdb", + ["application/x-rar-compressed"] = "rar", + ["application/x-redhat-package-manager"] = "rpm", + ["application/x-sea"] = "sea", + ["application/x-shockwave-flash"] = "swf", + ["application/x-stuffit"] = "sit", + ["application/x-tcl"] = "tcl", + ["application/x-x509-ca-cert"] = "crt", + ["application/x-xpinstall"] = "xpi", + ["application/xhtml+xml"] = "xhtml", + ["application/xspf+xml"] = "xspf", + ["application/zip"] = "zip", + ["application/x-dosexec"] = "exe", + ["application/x-msdownload"] = "exe", + ["application/x-executable"] = "exe", + ["text/x-msdos-batch"] = "bat", - ["audio/midi"] = "mid", - ["audio/mpeg"] = "mp3", - ["audio/ogg"] = "ogg", - ["audio/x-m4a"] = "m4a", - ["audio/x-realaudio"] = "ra", - ["video/3gpp"] = "3gpp", - ["video/mp2t"] = "ts", - ["video/mp4"] = "mp4", - ["video/mpeg"] = "mpeg", - ["video/quicktime"] = "mov", - ["video/webm"] = "webm", - ["video/x-flv"] = "flv", - ["video/x-m4v"] = "m4v", - ["video/x-mng"] = "mng", - ["video/x-ms-asf"] = "asx", - ["video/x-ms-wmv"] = "wmv", - ["video/x-msvideo"] = "avi", + ["audio/midi"] = "mid", + ["audio/mpeg"] = "mp3", + ["audio/ogg"] = "ogg", + ["audio/x-m4a"] = "m4a", + ["audio/x-realaudio"] = "ra", + ["video/3gpp"] = "3gpp", + ["video/mp2t"] = "ts", + ["video/mp4"] = "mp4", + ["video/mpeg"] = "mpeg", + ["video/quicktime"] = "mov", + ["video/webm"] = "webm", + ["video/x-flv"] = "flv", + ["video/x-m4v"] = "m4v", + ["video/x-mng"] = "mng", + ["video/x-ms-asf"] = "asx", + ["video/x-ms-wmv"] = "wmv", + ["video/x-msvideo"] = "avi", } return exports diff --git a/lualib/lua_scanners/avast.lua b/lualib/lua_scanners/avast.lua index 5c54f8c40..7e778979a 100644 --- a/lualib/lua_scanners/avast.lua +++ b/lualib/lua_scanners/avast.lua @@ -189,7 +189,7 @@ local function avast_check(task, content, digest, rule, maybe_part) -- Define callbacks - avast_helo_cb = function (merr, mdata, conn) + avast_helo_cb = function(merr, mdata, conn) -- Called when we have established a connection but not read anything tcp_conn = conn @@ -204,7 +204,6 @@ local function avast_check(task, content, digest, rule, maybe_part) end end - avast_scan_cb = function(merr) -- Called when we have send request to avast and are waiting for reply if no_connection_error(merr) then diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua index f984864e7..fc99ab0b9 100644 --- a/lualib/lua_scanners/clamav.lua +++ b/lualib/lua_scanners/clamav.lua @@ -137,7 +137,7 @@ local function clamav_check(task, content, digest, rule, maybe_part) local vname = string.match(data, 'stream: (.+) FOUND') if string.find(vname, '^Heuristics%.Encrypted') then rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix) - common.yield_result(task, rule, 'File is encrypted: '.. vname, + common.yield_result(task, rule, 'File is encrypted: ' .. vname, 0.0, 'encrypted', maybe_part) cached = 'ENCRYPTED' elseif string.find(vname, '^Heuristics%.OLE2%.ContainsMacros') then @@ -146,7 +146,7 @@ local function clamav_check(task, content, digest, rule, maybe_part) cached = 'MACRO' elseif string.find(vname, '^Heuristics%.Limits%.Exceeded') then rspamd_logger.errx(task, '%s: ClamAV Limits Exceeded', rule.log_prefix) - common.yield_result(task, rule, 'Limits Exceeded: '.. vname, 0.0, + common.yield_result(task, rule, 'Limits Exceeded: ' .. vname, 0.0, 'fail', maybe_part) elseif vname then common.yield_result(task, rule, vname, 1.0, nil, maybe_part) diff --git a/lualib/lua_scanners/cloudmark.lua b/lualib/lua_scanners/cloudmark.lua index 5993c4fe2..b07f238ea 100644 --- a/lualib/lua_scanners/cloudmark.lua +++ b/lualib/lua_scanners/cloudmark.lua @@ -84,7 +84,7 @@ local function cloudmark_preload(rule, cfg, ev_base, _) http.request({ ev_base = ev_base, config = cfg, - url = cloudmark_url(rule, addr, '/score/v2/max-message-size'), + url = cloudmark_url(rule, addr, '/score/v2/max-message-size'), callback = max_message_size_cb, }) end @@ -142,8 +142,8 @@ local function cloudmark_config(opts) if cloudmark_conf.upstreams then - cloudmark_conf.symbols = {{ symbol = cloudmark_conf.symbol_spam, score = 5.0 }} - cloudmark_conf.preloads = {cloudmark_preload} + cloudmark_conf.symbols = { { symbol = cloudmark_conf.symbol_spam, score = 5.0 } } + cloudmark_conf.preloads = { cloudmark_preload } lua_util.add_debug_alias('external_services', cloudmark_conf.name) return cloudmark_conf end @@ -162,7 +162,7 @@ local function table_to_multipart_body(tbl, boundary) local seen_data = false local out = {} - for k,v in pairs(tbl) do + for k, v in pairs(tbl) do if v.data then seen_data = true table.insert(out, string.format('--%s\r\n', boundary)) @@ -229,7 +229,7 @@ local function parse_cloudmark_reply(task, rule, body) if rule.add_headers and type(obj.appendHeaders) == 'table' then local headers_add = fun.tomap(fun.map(function(h) - return h.headerField,{ + return h.headerField, { order = 1, value = h.body } end, obj.appendHeaders)) @@ -275,7 +275,9 @@ local function cloudmark_check(task, content, digest, rule, maybe_part) local rcpt_to = task:get_recipients('smtp') if rcpt_to then request['rcptTo'] = { - data = table.concat(fun.totable(fun.map(function(r) return r.addr end, rcpt_to)), ',') + data = table.concat(fun.totable(fun.map(function(r) + return r.addr + end, rcpt_to)), ',') } end @@ -325,9 +327,9 @@ local function cloudmark_check(task, content, digest, rule, maybe_part) http.request(request_data) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. 'exceed', rule.log_prefix) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '.. + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' .. 'retransmits exceed') upstream:fail() end @@ -337,7 +339,9 @@ local function cloudmark_check(task, content, digest, rule, maybe_part) cloudmark_requery() else -- Parse the response - if upstream then upstream:ok() end + if upstream then + upstream:ok() + end if code ~= 200 then rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) @@ -360,7 +364,7 @@ local function cloudmark_check(task, content, digest, rule, maybe_part) end return { - type = {'cloudmark', 'scanner'}, + type = { 'cloudmark', 'scanner' }, description = 'Cloudmark cartridge interface', configure = cloudmark_config, check = cloudmark_check, diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua index c2e314d39..11f5e1f87 100644 --- a/lualib/lua_scanners/common.lua +++ b/lualib/lua_scanners/common.lua @@ -42,7 +42,9 @@ local function log_clean(task, rule, msg) end local function match_patterns(default_sym, found, patterns, dyn_weight) - if type(patterns) ~= 'table' then return default_sym, dyn_weight end + if type(patterns) ~= 'table' then + return default_sym, dyn_weight + end if not patterns[1] then for sym, pat in pairs(patterns) do if pat:match(found) then @@ -71,7 +73,7 @@ local function yield_result(task, rule, vname, dyn_weight, is_fail, maybe_part) local flags if type(vname) == 'string' then - threat_table = {vname} + threat_table = { vname } elseif type(vname) == 'table' then threat_table = vname end @@ -82,7 +84,9 @@ local function yield_result(task, rule, vname, dyn_weight, is_fail, maybe_part) patterns = rule.patterns symbol = rule.symbol threat_info = rule.detection_category .. 'found' - if not dyn_weight then dyn_weight = 1.0 end + if not dyn_weight then + dyn_weight = 1.0 + end elseif is_fail == 'fail' then patterns = rule.patterns_fail symbol = rule.symbol_fail @@ -100,7 +104,6 @@ local function yield_result(task, rule, vname, dyn_weight, is_fail, maybe_part) dyn_weight = 1.0 end - for _, tm in ipairs(threat_table) do local symname, symscore = match_patterns(symbol, tm, patterns, dyn_weight) if rule.whitelist and rule.whitelist:get_key(tm) then @@ -136,7 +139,9 @@ end local function message_not_too_large(task, content, rule) local max_size = tonumber(rule.max_size) - if not max_size then return true end + if not max_size then + return true + end if #content > max_size then rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)", rule.log_prefix, #content, max_size) @@ -147,7 +152,9 @@ end local function message_not_too_small(task, content, rule) local min_size = tonumber(rule.min_size) - if not min_size then return true end + if not min_size then + return true + end if #content < min_size then rspamd_logger.infox(task, "skip %s check as it is too small: %s (%s is allowed)", rule.log_prefix, #content, min_size) @@ -171,7 +178,7 @@ local function message_min_words(task, rule) if not text_part_above_limit then rspamd_logger.infox(task, '%s: #words in all text parts is below text_part_min_words limit: %s', - rule.log_prefix, rule.text_part_min_words) + rule.log_prefix, rule.text_part_min_words) end return text_part_above_limit @@ -192,7 +199,7 @@ local function dynamic_scan(task, rule) -- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, "result is already reject") -- return false --elseif metric_result[1] > metric_result[2]*2 then - if metric_result[1] > metric_result[2]*2 then + if metric_result[1] > metric_result[2] * 2 then rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'score > 2 * reject_level: ' .. metric_result[1]) return false elseif has_pre_result and metric_action == 'reject' then @@ -236,7 +243,7 @@ local function need_check(task, content, rule, digest, fn, maybe_part) else lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s', - rule.log_prefix, key, threat_string[1]) + rule.log_prefix, key, threat_string[1]) end uncached = false else @@ -251,10 +258,10 @@ local function need_check(task, content, rule, digest, fn, maybe_part) local f_dynamic_scan = dynamic_scan(task, rule) if uncached and - f_message_not_too_large and - f_message_not_too_small and - f_message_min_words and - f_dynamic_scan then + f_message_not_too_large and + f_message_not_too_small and + f_message_min_words and + f_dynamic_scan then fn() @@ -272,7 +279,7 @@ local function need_check(task, content, rule, digest, fn, maybe_part) false, -- is write redis_av_cb, --callback 'GET', -- command - {key} -- arguments) + { key } -- arguments) ) then return true end @@ -284,7 +291,9 @@ end local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part) local key = digest - if not dyn_weight then dyn_weight = 1.0 end + if not dyn_weight then + dyn_weight = 1.0 + end local function redis_set_cb(err) -- Do nothing @@ -293,7 +302,7 @@ local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part) rule.detection_category, to_save, key, err) else lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s - ttl %s', - rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire) + rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire) end end @@ -301,7 +310,7 @@ local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part) to_save = table.concat(to_save, '\v') end - local value_tbl = {to_save, dyn_weight} + local value_tbl = { to_save, dyn_weight } if maybe_part and rule.show_attachments and maybe_part:get_filename() then local fname = maybe_part:get_filename() table.insert(value_tbl, fname) @@ -381,24 +390,24 @@ local function gen_extension(fname) local ext = {} for n = 1, 2 do - ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil + ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil end - return ext[1],ext[2],filename_parts + return ext[1], ext[2], filename_parts end local function check_parts_match(task, rule) local filter_func = function(p) - local mtype,msubtype = p:get_type() + local mtype, msubtype = p:get_type() local detected_ext = p:get_detected_ext() local fname = p:get_filename() local ext, ext2 if rule.scan_all_mime_parts == false then - -- check file extension and filename regex matching + -- check file extension and filename regex matching --lua_util.debugm(rule.name, task, '%s: filename: |%s|%s|', rule.log_prefix, fname) if fname ~= nil then - ext,ext2 = gen_extension(fname) + ext, ext2 = gen_extension(fname) --lua_util.debugm(rule.name, task, '%s: extension, fname: |%s|%s|%s|', rule.log_prefix, ext, ext2, fname) if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext') or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then @@ -433,8 +442,8 @@ local function check_parts_match(task, rule) if p:is_archive() then local arch = p:get_archive() local filelist = arch:get_files_full(1000) - for _,f in ipairs(filelist) do - ext,ext2 = gen_extension(f.name) + for _, f in ipairs(filelist) do + ext, ext2 = gen_extension(f.name) if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext') or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then lua_util.debugm(rule.name, task, '%s: extension matched in archive: |%s|%s|', rule.log_prefix, ext, ext2) @@ -488,7 +497,7 @@ local function check_metric_results(task, rule) if rule.symbol_type == 'postfilter' and metric_action == 'reject' then return true, 'result is already reject' - elseif metric_result[1] > metric_result[2]*2 then + elseif metric_result[1] > metric_result[2] * 2 then return true, 'score > 2 * reject_level: ' .. metric_result[1] elseif has_pre_result and metric_action == 'reject' then return true, 'pre_result reject is set' diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua index 5cf0ef4a6..8d5e9e10a 100644 --- a/lualib/lua_scanners/dcc.lua +++ b/lualib/lua_scanners/dcc.lua @@ -90,7 +90,7 @@ local function dcc_check(task, content, digest, rule) local upstream = rule.upstreams:get_upstream_round_robin() local addr = upstream:get_addr() local retransmits = rule.retransmits - local client = rule.client + local client = rule.client local client_ip = task:get_from_ip() if client_ip and client_ip:is_valid() then @@ -116,7 +116,8 @@ local function dcc_check(task, content, digest, rule) local rcpts = task:get_recipients(); if rcpts then local dcc_recipients = table.concat(fun.totable(fun.map(function(rcpt) - return rcpt['addr'] end, + return rcpt['addr'] + end, rcpts)), '\n') if dcc_recipients then envrcpt = dcc_recipients @@ -164,8 +165,8 @@ local function dcc_check(task, content, digest, rule) fuz2_max = 999999, }) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. - 'exceed', rule.log_prefix) + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed', rule.log_prefix) common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail') end end @@ -176,14 +177,14 @@ local function dcc_check(task, content, digest, rule) else -- Parse the response - local _,_,result,disposition,header = tostring(data):find("(.-)\n(.-)\n(.-)$") + local _, _, result, disposition, header = tostring(data):find("(.-)\n(.-)\n(.-)$") lua_util.debugm(rule.name, task, 'DCC result=%1 disposition=%2 header="%3"', result, disposition, header) if header then -- Unfold header header = header:gsub('\r?\n%s*', ' ') - local _,_,info = header:find("; (.-)$") + local _, _, info = header:find("; (.-)$") if (result == 'R') then -- Reject common.yield_result(task, rule, info, rule.default_score) @@ -194,63 +195,65 @@ local function dcc_check(task, content, digest, rule) dcc_requery() elseif result == 'A' then - local opts = {} - local score = 0.0 - info = info:lower() - local rep = info:match('rep=([^=%s]+)') + local opts = {} + local score = 0.0 + info = info:lower() + local rep = info:match('rep=([^=%s]+)') - -- Adjust reputation if available - if rep then rep = tonumber(rep) end - if not rep then - rep = 1.0 + -- Adjust reputation if available + if rep then + rep = tonumber(rep) + end + if not rep then + rep = 1.0 + end + + local function check_threshold(what, num, lim) + local rnum + if num == 'many' then + rnum = lim + else + rnum = tonumber(num) end - local function check_threshold(what, num, lim) - local rnum - if num == 'many' then - rnum = lim - else - rnum = tonumber(num) - end - - if rnum and rnum >= lim then - opts[#opts + 1] = string.format('%s=%s', what, num) - score = score + rep / 3.0 - end + if rnum and rnum >= lim then + opts[#opts + 1] = string.format('%s=%s', what, num) + score = score + rep / 3.0 end + end - info = info:lower() - local body = info:match('body=([^=%s]+)') + info = info:lower() + local body = info:match('body=([^=%s]+)') - if body then - check_threshold('body', body, rule.body_max) - end + if body then + check_threshold('body', body, rule.body_max) + end - local fuz1 = info:match('fuz1=([^=%s]+)') + local fuz1 = info:match('fuz1=([^=%s]+)') - if fuz1 then - check_threshold('fuz1', fuz1, rule.fuz1_max) - end + if fuz1 then + check_threshold('fuz1', fuz1, rule.fuz1_max) + end - local fuz2 = info:match('fuz2=([^=%s]+)') + local fuz2 = info:match('fuz2=([^=%s]+)') - if fuz2 then - check_threshold('fuz2', fuz2, rule.fuz2_max) - end + if fuz2 then + check_threshold('fuz2', fuz2, rule.fuz2_max) + end - if #opts > 0 and score > 0 then - task:insert_result(rule.symbol_bulk, - score, - opts) - common.save_cache(task, digest, rule, opts, score) + if #opts > 0 and score > 0 then + task:insert_result(rule.symbol_bulk, + score, + opts) + common.save_cache(task, digest, rule, opts, score) + else + common.save_cache(task, digest, rule, 'OK') + if rule.log_clean then + rspamd_logger.infox(task, '%s: clean, returned result A - info: %s', + rule.log_prefix, info) else - common.save_cache(task, digest, rule, 'OK') - if rule.log_clean then - rspamd_logger.infox(task, '%s: clean, returned result A - info: %s', - rule.log_prefix, info) - else - lua_util.debugm(rule.name, task, '%s: returned result A - info: %s', - rule.log_prefix, info) + lua_util.debugm(rule.name, task, '%s: returned result A - info: %s', + rule.log_prefix, info) end end elseif result == 'G' then @@ -302,7 +305,7 @@ local function dcc_check(task, content, digest, rule) end return { - type = {'dcc','bulk', 'hash', 'scanner'}, + type = { 'dcc', 'bulk', 'hash', 'scanner' }, description = 'dcc bulk scanner', configure = dcc_config, check = dcc_check, diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua index d2153f7ac..5a469c327 100644 --- a/lualib/lua_scanners/fprot.lua +++ b/lualib/lua_scanners/fprot.lua @@ -84,7 +84,9 @@ local function fprot_check(task, content, digest, rule, maybe_part) local addr = upstream:get_addr() local retransmits = rule.retransmits local scan_id = task:get_queue_id() - if not scan_id then scan_id = task:get_uid() end + if not scan_id then + scan_id = task:get_uid() + end local header = string.format('SCAN STREAM %s SIZE %d\n', scan_id, #content) local footer = '\n' diff --git a/lualib/lua_scanners/icap.lua b/lualib/lua_scanners/icap.lua index ba571b13d..16459b06e 100644 --- a/lualib/lua_scanners/icap.lua +++ b/lualib/lua_scanners/icap.lua @@ -91,7 +91,6 @@ local common = require "lua_scanners/common" local rspamd_util = require "rspamd_util" local rspamd_version = rspamd_version - local N = 'icap' local function icap_config(opts) @@ -151,8 +150,8 @@ local function icap_config(opts) end icap_conf.upstreams = upstream_list.create(rspamd_config, - icap_conf.servers, - icap_conf.default_port) + icap_conf.servers, + icap_conf.default_port) if icap_conf.upstreams then lua_util.add_debug_alias('external_services', icap_conf.name) @@ -160,7 +159,7 @@ local function icap_config(opts) end rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', - icap_conf.servers) + icap_conf.servers) return nil end @@ -180,7 +179,7 @@ local function icap_check(task, content, digest, rule, maybe_part) rspamd_version('main'), rspamd_version('id'), rspamd_util.get_hostname(), - string.sub(task:get_uid(), 1,6)) + string.sub(task:get_uid(), 1, 6)) end -- Build the icap queries @@ -196,9 +195,9 @@ local function icap_check(task, content, digest, rule, maybe_part) end local respond_headers = { - -- Add main RESPMOD header before any other - string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme), - string.format('Host: %s\r\n', addr:to_string()), + -- Add main RESPMOD header before any other + string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme), + string.format('Host: %s\r\n', addr:to_string()), } local size = tonumber(#content) @@ -221,7 +220,7 @@ local function icap_check(task, content, digest, rule, maybe_part) addr = upstream:get_addr() lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', - rule.log_prefix, addr, addr:get_port()) + rule.log_prefix, addr, addr:get_port()) tcp_options.host = addr:to_string() tcp_options.port = addr:get_port() @@ -232,8 +231,8 @@ local function icap_check(task, content, digest, rule, maybe_part) tcp.request(tcp_options) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. - 'exceed - error: %s', rule.log_prefix, err_m or '') + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed - error: %s', rule.log_prefix, err_m or '') common.yield_result(task, rule, string.format('failed - error: %s', err_m), 0.0, 'fail', maybe_part) end @@ -309,9 +308,13 @@ local function icap_check(task, content, digest, rule, maybe_part) end table.insert(respond_headers, '\r\n') - for _,h in ipairs(resp_req_headers) do table.insert(respond_headers, h) end + for _, h in ipairs(resp_req_headers) do + table.insert(respond_headers, h) + end table.insert(respond_headers, '\r\n') - for _,h in ipairs(resp_http_headers) do table.insert(respond_headers, h) end + for _, h in ipairs(resp_http_headers) do + table.insert(respond_headers, h) + end table.insert(respond_headers, '\r\n') table.insert(respond_headers, chunked_size .. '\r\n') table.insert(respond_headers, content) @@ -333,7 +336,7 @@ local function icap_check(task, content, digest, rule, maybe_part) elseif string.find(s, '^HTTP') then icap_headers['http'] = tostring(s) elseif string.find(s, '[%a%d-+]-:') then - local _,_,key,value = tostring(s):find("([%a%d-+]-):%s?(.+)") + local _, _, key, value = tostring(s):find("([%a%d-+]-):%s?(.+)") if key ~= nil then icap_headers[key:lower()] = tostring(value) end @@ -349,7 +352,7 @@ local function icap_check(task, content, digest, rule, maybe_part) if maybe_split and string.find(icap_threat, ',') then local threats = lua_util.str_split(string.gsub(icap_threat, "%s", ""), ',') or {} - for _,v in ipairs(threats) do + for _, v in ipairs(threats) do table.insert(threat_table, v) end else @@ -421,8 +424,7 @@ local function icap_check(task, content, digest, rule, maybe_part) -- Generic ICAP Headers if headers['x-infection-found'] then - local _,_,icap_type,_,icap_threat = - headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$") + local _, _, icap_type, _, icap_threat = headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$") -- Type=2 is typical for scan error returns if icap_type and icap_type == '2' then @@ -435,8 +437,8 @@ local function icap_check(task, content, digest, rule, maybe_part) lua_util.debugm(rule.name, task, '%s: icap X-Infection-Found: %s', rule.log_prefix, icap_threat) threat_table_add(icap_threat, false) - -- stupid workaround for unuseable x-infection-found header - -- but also x-virus-name set (McAfee Web Gateway 9) + -- stupid workaround for unuseable x-infection-found header + -- but also x-virus-name set (McAfee Web Gateway 9) elseif not icap_threat and headers['x-virus-name'] then threat_table_add(headers['x-virus-name'], true) else @@ -450,7 +452,7 @@ local function icap_check(task, content, digest, rule, maybe_part) lua_util.debugm(rule.name, task, '%s: icap X-Virus-ID: %s', rule.log_prefix, headers['x-virus-id']) threat_table_add(headers['x-virus-id'], true) - -- FSecure X-Headers + -- FSecure X-Headers elseif headers['x-fsecure-scan-result'] and headers['x-fsecure-scan-result'] ~= "clean" then local infected_filename = "" @@ -468,23 +470,23 @@ local function icap_check(task, content, digest, rule, maybe_part) rule.log_prefix, infection_name, infected_filename) threat_table_add(infection_name, true) - -- McAfee Web Gateway manual extra headers + -- McAfee Web Gateway manual extra headers elseif headers['x-mwg-block-reason'] and headers['x-mwg-block-reason'] ~= "" then threat_table_add(headers['x-mwg-block-reason'], false) - -- Sophos SAVDI special http headers + -- Sophos SAVDI special http headers elseif headers['x-blocked'] and headers['x-blocked'] ~= "" then threat_table_add(headers['x-blocked'], false) elseif headers['x-block-reason'] and headers['x-block-reason'] ~= "" then threat_table_add(headers['x-block-reason'], false) - -- last try HTTP [4]xx return + -- last try HTTP [4]xx return elseif headers.http and string.find(headers.http, '^HTTP%/[12]%.. [4]%d%d') then threat_table_add( - string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) + string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) elseif rule.use_http_3xx_as_threat and headers.http and string.find(headers.http, '^HTTP%/[12]%.. [3]%d%d') - then + then threat_table_add( - string.format("pseudo-virus (redirect): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) + string.format("pseudo-virus (redirect): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) end if #threat_table > 0 then @@ -515,7 +517,7 @@ local function icap_check(task, content, digest, rule, maybe_part) end else rspamd_logger.errx(task, '%s: unhandled response |%s|', - rule.log_prefix, string.gsub(result, "\r\n", ", ")) + rule.log_prefix, string.gsub(result, "\r\n", ", ")) common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_http_headers.icap), 0.0, 'fail', maybe_part) end @@ -536,13 +538,13 @@ local function icap_check(task, content, digest, rule, maybe_part) -- Threat found - close connection connection:close() elseif not icap_header_result - and rule.use_http_result_header - and icap_headers.encapsulated - and not string.find(icap_headers.encapsulated, 'null%-body=0') - then + and rule.use_http_result_header + and icap_headers.encapsulated + and not string.find(icap_headers.encapsulated, 'null%-body=0') + then -- Try to read encapsulated HTTP Headers lua_util.debugm(rule.name, task, '%s: no ICAP virus header found - try HTTP headers', - rule.log_prefix) + rule.log_prefix) connection:add_read(icap_r_respond_http_cb, '\r\n\r\n') else connection:close() @@ -569,7 +571,7 @@ local function icap_check(task, content, digest, rule, maybe_part) return false else rspamd_logger.errx(task, '%s: unhandled response |%s|', - rule.log_prefix, string.gsub(result, "\r\n", ", ")) + rule.log_prefix, string.gsub(result, "\r\n", ", ")) common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_headers.icap), 0.0, 'fail', maybe_part) end @@ -600,7 +602,9 @@ local function icap_check(task, content, digest, rule, maybe_part) if rule.x_client_header then local client = task:get_from_ip() - if client then add_respond_header('X-Client-IP', client:to_string()) end + if client then + add_respond_header('X-Client-IP', client:to_string()) + end end -- F-Secure extra headers @@ -608,19 +612,23 @@ local function icap_check(task, content, digest, rule, maybe_part) if rule.x_rcpt_header then local rcpt_to = task:get_principal_recipient() - if rcpt_to then add_respond_header('X-Rcpt-To', rcpt_to) end + if rcpt_to then + add_respond_header('X-Rcpt-To', rcpt_to) + end end if rule.x_from_header then local mail_from = task:get_principal_recipient() - if mail_from and mail_from[1] then add_respond_header('X-Rcpt-To', mail_from[1].addr) end + if mail_from and mail_from[1] then + add_respond_header('X-Rcpt-To', mail_from[1].addr) + end end end if icap_headers.connection and icap_headers.connection:lower() == 'close' then lua_util.debugm(rule.name, task, '%s: OPTIONS request Connection: %s - using new connection', - rule.log_prefix, icap_headers.connection) + rule.log_prefix, icap_headers.connection) connection:close() tcp_options.callback = icap_w_respond_cb tcp_options.data = get_respond_query() @@ -631,13 +639,13 @@ local function icap_check(task, content, digest, rule, maybe_part) else rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s', - rule.log_prefix, icap_headers['methods']) + rule.log_prefix, icap_headers['methods']) common.yield_result(task, rule, 'NO RESPMOD', 0.0, 'fail', maybe_part) end else rspamd_logger.errx(task, '%s: OPTIONS query failed: %s', - rule.log_prefix, icap_headers.icap or "-") + rule.log_prefix, icap_headers.icap or "-") common.yield_result(task, rule, 'OPTIONS query failed', 0.0, 'fail', maybe_part) end @@ -682,7 +690,7 @@ local function icap_check(task, content, digest, rule, maybe_part) end return { - type = {N, 'virus', 'virus', 'scanner'}, + type = { N, 'virus', 'virus', 'scanner' }, description = 'generic icap antivirus', configure = icap_config, check = icap_check, diff --git a/lualib/lua_scanners/init.lua b/lualib/lua_scanners/init.lua index 20bea7a47..e47cebeec 100644 --- a/lualib/lua_scanners/init.lua +++ b/lualib/lua_scanners/init.lua @@ -25,7 +25,7 @@ local exports = { } local function require_scanner(name) - local sc = require ("lua_scanners/" .. name) + local sc = require("lua_scanners/" .. name) exports[sc.name or name] = sc end @@ -65,7 +65,9 @@ exports.filter = function(t) return fun.tomap(fun.filter(function(_, elt) return type(elt) == 'table' and elt.type and ( (type(elt.type) == 'string' and elt.type == t) or - (type(elt.type) == 'table' and fun.any(function(tt) return tt == t end, elt.type)) + (type(elt.type) == 'table' and fun.any(function(tt) + return tt == t + end, elt.type)) ) end, exports)) end diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua index 69aaad7e5..d52cef041 100644 --- a/lualib/lua_scanners/kaspersky_av.lua +++ b/lualib/lua_scanners/kaspersky_av.lua @@ -107,7 +107,6 @@ local function kaspersky_check(task, content, digest, rule, maybe_part) rspamd_util.close_file(message_fd) end) - local function kaspersky_callback(err, data) if err then diff --git a/lualib/lua_scanners/kaspersky_se.lua b/lualib/lua_scanners/kaspersky_se.lua index b0031ad69..5e0f2eaad 100644 --- a/lualib/lua_scanners/kaspersky_se.lua +++ b/lualib/lua_scanners/kaspersky_se.lua @@ -125,7 +125,7 @@ local function kaspersky_se_check(task, content, digest, rule, maybe_part) local req_body if rule.use_files then - local fname = string.format('%s/%s.tmp', + local fname = string.format('%s/%s.tmp', rule.tmpdir, rspamd_util.random_hex(32)) local message_fd = rspamd_util.create_file(fname) @@ -187,9 +187,9 @@ local function kaspersky_se_check(task, content, digest, rule, maybe_part) http.request(request_data) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. 'exceed', rule.log_prefix) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '.. + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' .. 'retransmits exceed') end end @@ -198,7 +198,9 @@ local function kaspersky_se_check(task, content, digest, rule, maybe_part) requery() else -- Parse the response - if upstream then upstream:ok() end + if upstream then + upstream:ok() + end if code ~= 200 then rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) @@ -242,7 +244,7 @@ local function kaspersky_se_check(task, content, digest, rule, maybe_part) if why == 'PASSWORD PROTECTED' then rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix) - common.yield_result(task, rule, 'File is encrypted: '.. why, + common.yield_result(task, rule, 'File is encrypted: ' .. why, 0.0, 'encrypted', maybe_part) cached = 'ENCRYPTED' else diff --git a/lualib/lua_scanners/oletools.lua b/lualib/lua_scanners/oletools.lua index 8513699a0..378e09467 100644 --- a/lualib/lua_scanners/oletools.lua +++ b/lualib/lua_scanners/oletools.lua @@ -122,7 +122,7 @@ local function oletools_check(task, content, digest, rule, maybe_part) callback = oletools_callback, }) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. 'exceed - err: %s', rule.log_prefix, error) common.yield_result(task, rule, 'failed to scan, maximum retransmits exceed - err: ' .. error, @@ -137,9 +137,9 @@ local function oletools_check(task, content, digest, rule, maybe_part) else json_response = json_response .. tostring(data) - if not string.find(json_response, '\t\n\n\t') and #data == 8192 then + if not string.find(json_response, '\t\n\n\t') and #data == 8192 then lua_util.debugm(rule.name, task, '%s: no stop word: add_read - #json: %s / current packet: %s', - rule.log_prefix, #json_response, #data) + rule.log_prefix, #json_response, #data) conn:add_read(oletools_callback) else @@ -243,7 +243,7 @@ local function oletools_check(task, content, digest, rule, maybe_part) rule.log_prefix, v.message) --common.yield_result(task, rule, 'failed - err: ' .. v.error, 0.0, 'fail') - elseif type(v.analysis) == 'table' and type(v.macros) == 'table' then + elseif type(v.analysis) == 'table' and type(v.macros) == 'table' then -- analysis + macro found - evaluate response if type(v.analysis) == 'table' and #v.analysis == 0 and #v.macros == 0 then @@ -259,20 +259,20 @@ local function oletools_check(task, content, digest, rule, maybe_part) lua_util.debugm(rule.name, task, '%s: type: %s', rule.log_prefix, result[2]['type']) - for _,m in ipairs(v.macros) do - lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, '.. + for _, m in ipairs(v.macros) do + lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, ' .. 'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename) end - for _,a in ipairs(v.analysis) do - lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, '.. + for _, a in ipairs(v.analysis) do + lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, ' .. 'description: %s', rule.log_prefix, a.type, a.keyword, a.description) if a.type == 'AutoExec' then analysis_cat_table.autoexec = 'A' table.insert(analysis_keyword_table, a.keyword) elseif a.type == 'Suspicious' then if rule.extended == true or - (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings') + (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings') then analysis_cat_table.suspicious = 'S' table.insert(analysis_keyword_table, a.keyword) @@ -312,7 +312,7 @@ local function oletools_check(task, content, digest, rule, maybe_part) local analysis_cat_table_keys_sorted = lua_util.keys(analysis_cat_table) table.sort(analysis_cat_table_keys_sorted) - for _,v in ipairs(analysis_cat_table_keys_sorted) do + for _, v in ipairs(analysis_cat_table_keys_sorted) do table.insert(analysis_cat_table_values_sorted, analysis_cat_table[v]) end @@ -361,7 +361,7 @@ local function oletools_check(task, content, digest, rule, maybe_part) end return { - type = {N, 'attachment scanner', 'hash', 'scanner'}, + type = { N, 'attachment scanner', 'hash', 'scanner' }, description = 'oletools office macro scanner', configure = oletools_config, check = oletools_check, diff --git a/lualib/lua_scanners/p0f.lua b/lualib/lua_scanners/p0f.lua index c5ccea1fe..7785f83ce 100644 --- a/lualib/lua_scanners/p0f.lua +++ b/lualib/lua_scanners/p0f.lua @@ -30,8 +30,8 @@ local common = require "lua_scanners/common" -- SEE: https://github.com/p0f/p0f/blob/v3.06b/docs/README#L317 local S = { BAD_QUERY = 0x0, - OK = 0x10, - NO_MATCH = 0x20 + OK = 0x10, + NO_MATCH = 0x20 } local N = 'p0f' @@ -49,7 +49,7 @@ local function p0f_check(task, ip, rule) end local function trim(...) - local vars = {...} + local vars = { ... } for k, v in ipairs(vars) do -- skip numbers, trim only strings @@ -73,22 +73,22 @@ local function p0f_check(task, ip, rule) -- API response must be 232 bytes long if #data ~= 232 then rspamd_logger.errx(task, 'malformed response from p0f on %s, %s bytes', - rule.socket, #data) + rule.socket, #data) common.yield_result(task, rule, 'Malformed Response: ' .. rule.socket, - 0.0, 'fail') + 0.0, 'fail') return end local _, status, _, _, _, uptime_min, _, _, _, distance, _, _, os_name, - os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack( + os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack( 'I4I4I4I4I4I4I4I4I4hbbc32c32c32c32c32c32', data)) if status ~= S.OK then if status == S.BAD_QUERY then rspamd_logger.errx(task, 'malformed p0f query on %s', rule.socket) common.yield_result(task, rule, 'Malformed Query: ' .. rule.socket, - 0.0, 'fail') + 0.0, 'fail') end return @@ -97,19 +97,19 @@ local function p0f_check(task, ip, rule) local os_string = #os_name == 0 and 'unknown' or os_name .. ' ' .. os_flavor task:get_mempool():set_variable('os_fingerprint', os_string, link_type, - uptime_min, distance) + uptime_min, distance) if link_type and #link_type > 0 then common.yield_result(task, rule, { os_string, 'link=' .. link_type, - 'distance=' .. distance}, + 'distance=' .. distance }, 0.0) else common.yield_result(task, rule, { os_string, 'link=unknown', - 'distance=' .. distance}, + 'distance=' .. distance }, 0.0) end @@ -153,7 +153,7 @@ local function p0f_check(task, ip, rule) end local query = rspamd_util.pack('I4 I1 c16', 0x50304601, - ip:get_version(), ip2bin(ip)) + ip:get_version(), ip2bin(ip)) tcp.request({ host = rule.socket, @@ -176,12 +176,12 @@ local function p0f_check(task, ip, rule) if rule.redis_params then local key = rule.prefix .. ip:to_string() ret = lua_redis.redis_make_request(task, - rule.redis_params, - key, - false, - redis_get_cb, - 'GET', - { key } + rule.redis_params, + key, + false, + redis_get_cb, + 'GET', + { key } ) end @@ -219,7 +219,7 @@ local function p0f_config(opts) end return { - type = {N, 'fingerprint', 'scanner'}, + type = { N, 'fingerprint', 'scanner' }, description = 'passive OS fingerprinter', configure = p0f_config, check = p0f_check, diff --git a/lualib/lua_scanners/pyzor.lua b/lualib/lua_scanners/pyzor.lua index 78250a3ad..75c1b4a15 100644 --- a/lualib/lua_scanners/pyzor.lua +++ b/lualib/lua_scanners/pyzor.lua @@ -28,7 +28,7 @@ local rspamd_logger = require "rspamd_logger" local common = require "lua_scanners/common" local N = 'pyzor' -local categories = {'pyzor','bulk', 'hash', 'scanner'} +local categories = { 'pyzor', 'bulk', 'hash', 'scanner' } local function pyzor_config(opts) @@ -174,10 +174,10 @@ local function pyzor_check(task, content, digest, rule) end end - if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then - rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix) - return - end + if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then + rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix) + return + end tcp.request({ task = task, @@ -197,7 +197,6 @@ local function pyzor_check(task, content, digest, rule) end end - return { type = categories, description = 'pyzor bulk scanner', diff --git a/lualib/lua_scanners/razor.lua b/lualib/lua_scanners/razor.lua index 7de4c84eb..fcc0a8e3a 100644 --- a/lualib/lua_scanners/razor.lua +++ b/lualib/lua_scanners/razor.lua @@ -80,7 +80,6 @@ local function razor_config(opts) return nil end - local function razor_check(task, content, digest, rule) local function razor_check_uncached () local upstream = rule.upstreams:get_upstream_round_robin() @@ -96,14 +95,14 @@ local function razor_check(task, content, digest, rule) retransmits = retransmits - 1 lua_util.debugm(rule.name, task, '%s: Request Error: %s - retries left: %s', - rule.log_prefix, err, retransmits) + rule.log_prefix, err, retransmits) -- Select a different upstream! upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', - rule.log_prefix, addr, addr:get_port()) + rule.log_prefix, addr, addr:get_port()) tcp.request({ task = task, @@ -116,8 +115,8 @@ local function razor_check(task, content, digest, rule) callback = razor_callback, }) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. - 'exceed', rule.log_prefix) + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed', rule.log_prefix) common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail') end end @@ -148,7 +147,7 @@ local function razor_check(task, content, digest, rule) end common.save_cache(task, digest, rule, 'OK', rule.default_score) else - rspamd_logger.errx(task,"%s - unknown response from razorfy: %s", addr:to_string(), threat_string) + rspamd_logger.errx(task, "%s - unknown response from razorfy: %s", addr:to_string(), threat_string) end end @@ -174,7 +173,7 @@ local function razor_check(task, content, digest, rule) end return { - type = {'razor','spam', 'hash', 'scanner'}, + type = { 'razor', 'spam', 'hash', 'scanner' }, description = 'razor bulk scanner', configure = razor_config, check = razor_check, diff --git a/lualib/lua_scanners/savapi.lua b/lualib/lua_scanners/savapi.lua index 430009df3..08f7b6682 100644 --- a/lualib/lua_scanners/savapi.lua +++ b/lualib/lua_scanners/savapi.lua @@ -116,13 +116,13 @@ local function savapi_check(task, content, digest, rule) local function savapi_fin_cb(err, conn) local vnames_reordered = {} -- Swap table - for virus,_ in pairs(vnames) do + for virus, _ in pairs(vnames) do table.insert(vnames_reordered, virus) end lua_util.debugm(rule.name, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered) if #vnames_reordered > 0 then local vname = {} - for _,virus in ipairs(vnames_reordered) do + for _, virus in ipairs(vnames_reordered) do table.insert(vname, virus) end @@ -181,8 +181,8 @@ local function savapi_check(task, content, digest, rule) if string.find(result, '100 PRODUCT') then lua_util.debugm(rule.name, task, "%s: scanning file: %s", rule['type'], fname) - conn:add_write(savapi_scan1_cb, {string.format('SCAN %s\n', - fname)}) + conn:add_write(savapi_scan1_cb, { string.format('SCAN %s\n', + fname) }) else rspamd_logger.errx(task, '%s: invalid product id %s', rule['type'], rule['product_id']) @@ -217,7 +217,7 @@ local function savapi_check(task, content, digest, rule) upstream = upstream, timeout = rule['timeout'], callback = savapi_callback_init, - stop_pattern = {'\n'}, + stop_pattern = { '\n' }, }) else rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) @@ -228,7 +228,7 @@ local function savapi_check(task, content, digest, rule) -- 100 SAVAPI:4.0 greeting if string.find(result, '100') then - conn:add_write(savapi_greet1_cb, {string.format('SET PRODUCT %s\n', rule['product_id'])}) + conn:add_write(savapi_greet1_cb, { string.format('SET PRODUCT %s\n', rule['product_id']) }) end end end @@ -240,7 +240,7 @@ local function savapi_check(task, content, digest, rule) upstream = upstream, timeout = rule['timeout'], callback = savapi_callback_init, - stop_pattern = {'\n'}, + stop_pattern = { '\n' }, }) end diff --git a/lualib/lua_scanners/spamassassin.lua b/lualib/lua_scanners/spamassassin.lua index c4fd0d2e5..f425924d5 100644 --- a/lualib/lua_scanners/spamassassin.lua +++ b/lualib/lua_scanners/spamassassin.lua @@ -71,8 +71,8 @@ local function spamassassin_config(opts) end spamassassin_conf.upstreams = upstream_list.create(rspamd_config, - spamassassin_conf.servers, - spamassassin_conf.default_port) + spamassassin_conf.servers, + spamassassin_conf.default_port) if spamassassin_conf.upstreams then lua_util.add_debug_alias('external_services', spamassassin_conf.N) @@ -80,7 +80,7 @@ local function spamassassin_config(opts) end rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', - spamassassin_conf.servers) + spamassassin_conf.servers) return nil end @@ -95,7 +95,7 @@ local function spamassassin_check(task, content, digest, rule) local request_data = { "HEADERS SPAMC/1.5\r\n", "User: root\r\n", - "Content-length: ".. #content .. "\r\n", + "Content-length: " .. #content .. "\r\n", "\r\n", content, } @@ -110,14 +110,14 @@ local function spamassassin_check(task, content, digest, rule) retransmits = retransmits - 1 lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s', - rule.log_prefix, error, retransmits) + rule.log_prefix, error, retransmits) -- Select a different upstream! upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s', - rule.log_prefix, addr, addr:get_port()) + rule.log_prefix, addr, addr:get_port()) tcp.request({ task = task, @@ -129,8 +129,8 @@ local function spamassassin_check(task, content, digest, rule) callback = spamassassin_callback, }) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. - 'exceed - err: %s', rule.log_prefix, error) + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed - err: %s', rule.log_prefix, error) common.yield_result(task, rule, 'failed to scan and retransmits exceed: ' .. error, 0.0, 'fail') end end @@ -164,7 +164,7 @@ local function spamassassin_check(task, content, digest, rule) end lua_util.debugm(rule.N, task, '%s: spam_score: %s, symbols: %s, int spam_score: |%s|, type spam_score: |%s|', - rule.log_prefix, spam_score, symbols, tonumber(spam_score), type(spam_score)) + rule.log_prefix, spam_score, symbols, tonumber(spam_score), type(spam_score)) if tonumber(spam_score) > 0 and #symbols > 0 and symbols ~= "none" then @@ -205,7 +205,7 @@ local function spamassassin_check(task, content, digest, rule) end return { - type = {N,'spam', 'scanner'}, + type = { N, 'spam', 'scanner' }, description = 'spamassassin spam scanner', configure = spamassassin_config, check = spamassassin_check, diff --git a/lualib/lua_scanners/vadesecure.lua b/lualib/lua_scanners/vadesecure.lua index b912eb46f..826573a89 100644 --- a/lualib/lua_scanners/vadesecure.lua +++ b/lualib/lua_scanners/vadesecure.lua @@ -85,17 +85,17 @@ local function vade_config(opts) score = 8.0, description = 'VadeSecure decided message to be phishing' }, - commercial = { + commercial = { symbol = 'VADE_COMMERCIAL', score = 0.0, description = 'VadeSecure decided message to be commercial message' }, - community = { + community = { symbol = 'VADE_COMMUNITY', score = 0.0, description = 'VadeSecure decided message to be community message' }, - transactional = { + transactional = { symbol = 'VADE_TRANSACTIONAL', score = 0.0, description = 'VadeSecure decided message to be transactional message' @@ -250,9 +250,9 @@ local function vade_check(task, content, digest, rule, maybe_part) http.request(request_data) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. 'exceed', rule.log_prefix) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '.. + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' .. 'retransmits exceed') end end @@ -261,7 +261,9 @@ local function vade_check(task, content, digest, rule, maybe_part) vade_requery() else -- Parse the response - if upstream then upstream:ok() end + if upstream then + upstream:ok() + end if code ~= 200 then rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) @@ -341,7 +343,7 @@ local function vade_check(task, content, digest, rule, maybe_part) end return { - type = {'vadesecure', 'scanner'}, + type = { 'vadesecure', 'scanner' }, description = 'VadeSecure Filterd interface', configure = vade_config, check = vade_check, diff --git a/lualib/lua_scanners/virustotal.lua b/lualib/lua_scanners/virustotal.lua index e53795b84..d937c4128 100644 --- a/lualib/lua_scanners/virustotal.lua +++ b/lualib/lua_scanners/virustotal.lua @@ -123,7 +123,7 @@ local function virustotal_check(task, content, digest, rule, maybe_part) else local ucl = require "ucl" local parser = ucl.parser() - local res,json_err = parser:parse_string(body) + local res, json_err = parser:parse_string(body) lua_util.debugm(rule.name, task, '%s: got reply data: "%s"', rule.log_prefix, body) diff --git a/lualib/lua_selectors/common.lua b/lualib/lua_selectors/common.lua index c2fe9b58f..7b2372da8 100644 --- a/lualib/lua_selectors/common.lua +++ b/lualib/lua_selectors/common.lua @@ -21,8 +21,8 @@ local cr_hash = require 'rspamd_cryptobox_hash' local blake2b_key = cr_hash.create_specific('blake2'):update('rspamd'):bin() local function digest_schema() - return {ts.one_of{'hex', 'base32', 'bleach32', 'rbase32', 'base64'}:is_optional(), - ts.one_of{'blake2', 'sha256', 'sha1', 'sha512', 'md5'}:is_optional()} + return { ts.one_of { 'hex', 'base32', 'bleach32', 'rbase32', 'base64' }:is_optional(), + ts.one_of { 'blake2', 'sha256', 'sha1', 'sha512', 'md5' }:is_optional() } end exports.digest_schema = digest_schema @@ -66,7 +66,6 @@ local function create_digest(data, args) return encode_digest(h, args) end - local function get_cached_or_raw_digest(task, idx, mime_part, args) if #args == 0 then -- Optimise as we already have this hash in the API diff --git a/lualib/lua_selectors/extractors.lua b/lualib/lua_selectors/extractors.lua index 484adec56..81dfa9d4d 100644 --- a/lualib/lua_selectors/extractors.lua +++ b/lualib/lua_selectors/extractors.lua @@ -30,7 +30,9 @@ local function gen_exclude_flags_filter(exclude_flags) return function(u) local got_flags = u:get_flags() for _, flag in ipairs(exclude_flags) do - if got_flags[flag] then return false end + if got_flags[flag] then + return false + end end return true end @@ -44,11 +46,11 @@ local extractors = { return args[1], 'string' end - return '','string' + return '', 'string' end, ['description'] = [[Return value from function's argument or an empty string, For example, `id('Something')` returns a string 'Something']], - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, -- Similar but for making lists ['list'] = { @@ -57,7 +59,7 @@ For example, `id('Something')` returns a string 'Something']], return fun.map(tostring, args), 'string_list' end - return {},'string_list' + return {}, 'string_list' end, ['description'] = [[Return a list from function's arguments or an empty list, For example, `list('foo', 'bar')` returns a list {'foo', 'bar'}]], @@ -66,7 +68,9 @@ For example, `list('foo', 'bar')` returns a list {'foo', 'bar'}]], ['ip'] = { ['get_value'] = function(task) local ip = task:get_ip() - if ip and ip:is_valid() then return ip,'userdata' end + if ip and ip:is_valid() then + return ip, 'userdata' + end return nil end, ['description'] = [[Get source IP address]], @@ -81,7 +85,7 @@ For example, `list('foo', 'bar')` returns a list {'foo', 'bar'}]], from = task:get_from(0) end if ((from or E)[1] or E).addr then - return from[1],'table' + return from[1], 'table' end return nil end, @@ -97,7 +101,7 @@ uses any type by default)]], rcpts = task:get_recipients(0) end if ((rcpts or E)[1] or E).addr then - return rcpts,'table_list' + return rcpts, 'table_list' end return nil end, @@ -111,7 +115,7 @@ uses any type by default)]], if not country then return nil else - return country,'string' + return country, 'string' end end, ['description'] = [[Get country (ASN module must be executed first)]], @@ -124,7 +128,7 @@ uses any type by default)]], if not asn then return nil else - return asn,'string' + return asn, 'string' end end, ['description'] = [[Get AS number (ASN module must be executed first)]], @@ -136,7 +140,7 @@ uses any type by default)]], if not auser then return nil else - return auser,'string' + return auser, 'string' end end, ['description'] = 'Get authenticated user name', @@ -144,14 +148,14 @@ uses any type by default)]], -- Get principal recipient ['to'] = { ['get_value'] = function(task) - return task:get_principal_recipient(),'string' + return task:get_principal_recipient(), 'string' end, ['description'] = 'Get principal recipient', }, -- Get content digest ['digest'] = { ['get_value'] = function(task) - return task:get_digest(),'string' + return task:get_digest(), 'string' end, ['description'] = 'Get content digest', }, @@ -160,14 +164,14 @@ uses any type by default)]], ['get_value'] = function(task, args) local parts = task:get_parts() or E local digests = {} - for i,p in ipairs(parts) do + for i, p in ipairs(parts) do if p:is_attachment() then table.insert(digests, common.get_cached_or_raw_digest(task, i, p, args)) end end if #digests > 0 then - return digests,'string_list' + return digests, 'string_list' end return nil @@ -184,7 +188,7 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, local parts = task:get_parts() or E local files = {} - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do local fname = p:get_filename() if fname then table.insert(files, fname) @@ -192,7 +196,7 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, end if #files > 0 then - return files,'string_list' + return files, 'string_list' end return nil @@ -205,7 +209,7 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, local text_parts = task:get_text_parts() or E local languages = {} - for _,p in ipairs(text_parts) do + for _, p in ipairs(text_parts) do local lang = p:get_language() if lang then table.insert(languages, lang) @@ -213,7 +217,7 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, end if #languages > 0 then - return languages,'string_list' + return languages, 'string_list' end return nil @@ -223,7 +227,7 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, -- Get helo value ['helo'] = { ['get_value'] = function(task) - return task:get_helo(),'string' + return task:get_helo(), 'string' end, ['description'] = 'Get helo value', }, @@ -238,20 +242,20 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, end if args[2]:match('full') then - return task:get_header_full(args[1], strong),'table_list' + return task:get_header_full(args[1], strong), 'table_list' end - return task:get_header(args[1], strong),'string' + return task:get_header(args[1], strong), 'string' else - return task:get_header(args[1]),'string' + return task:get_header(args[1]), 'string' end end, ['description'] = [[Get header with the name that is expected as an argument. The optional second argument accepts list of flags: - `full`: returns all headers with this name with all data (like task:get_header_full()) - `strong`: use case sensitive match when matching header's name]], - ['args_schema'] = {ts.string, - (ts.pattern("strong") + ts.pattern("full")):is_optional()} + ['args_schema'] = { ts.string, + (ts.pattern("strong") + ts.pattern("full")):is_optional() } }, -- Get list of received headers (returns list of tables) ['received'] = { @@ -261,10 +265,12 @@ The optional second argument accepts list of flags: return nil end if args[1] then - return fun.map(function(r) return r[args[1]] end, rh), 'string_list' + return fun.map(function(r) + return r[args[1]] + end, rh), 'string_list' end - return rh,'table_list' + return rh, 'table_list' end, ['description'] = [[Get list of received headers. If no arguments specified, returns list of tables. Otherwise, selects a specific element, @@ -278,9 +284,11 @@ e.g. `by_hostname`]], return nil end if args[1] then - return fun.map(function(r) return r[args[1]](r) end, urls), 'string_list' + return fun.map(function(r) + return r[args[1]](r) + end, urls), 'string_list' end - return urls,'userdata_list' + return urls, 'userdata_list' end, ['description'] = [[Get list of all urls. If no arguments specified, returns list of url objects. Otherwise, calls a specific method, @@ -299,21 +307,21 @@ e.g. `get_tld`]], if not urls[1] then return nil end - return urls,'userdata_list' + return urls, 'userdata_list' end, ['description'] = [[Get most specific urls. Arguments are equal to the Lua API function]], - ['args_schema'] = {ts.shape{ + ['args_schema'] = { ts.shape { limit = ts.number + ts.string / tonumber, esld_limit = (ts.number + ts.string / tonumber):is_optional(), exclude_flags = url_flags_ts, flags = url_flags_ts, - flags_mode = ts.one_of{'explicit'}:is_optional(), + flags_mode = ts.one_of { 'explicit' }:is_optional(), prefix = ts.string:is_optional(), need_content = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), need_emails = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), need_images = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), ignore_redirected = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), - }} + } } }, ['specific_urls_filter_map'] = { ['get_value'] = function(task, args) @@ -331,21 +339,23 @@ e.g. `get_tld`]], if not urls[1] then return nil end - return fun.filter(function(u) return map:get_key(tostring(u)) end, urls),'userdata_list' + return fun.filter(function(u) + return map:get_key(tostring(u)) + end, urls), 'userdata_list' end, ['description'] = [[Get most specific urls, filtered by some map. Arguments are equal to the Lua API function]], - ['args_schema'] = {ts.string, ts.shape{ + ['args_schema'] = { ts.string, ts.shape { limit = ts.number + ts.string / tonumber, esld_limit = (ts.number + ts.string / tonumber):is_optional(), exclude_flags = url_flags_ts, flags = url_flags_ts, - flags_mode = ts.one_of{'explicit'}:is_optional(), + flags_mode = ts.one_of { 'explicit' }:is_optional(), prefix = ts.string:is_optional(), need_content = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), need_emails = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), need_images = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), ignore_redirected = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), - }} + } } }, -- URLs filtered by flags ['urls_filtered'] = { @@ -354,13 +364,13 @@ e.g. `get_tld`]], if not urls[1] then return nil end - return urls,'userdata_list' + return urls, 'userdata_list' end, ['description'] = [[Get list of all urls filtered by flags_include/exclude (see rspamd_task:get_urls_filtered for description)]], - ['args_schema'] = {ts.array_of{ + ['args_schema'] = { ts.array_of { url_flags_ts:is_optional(), url_flags_ts:is_optional() - }} + } } }, -- Get all emails ['emails'] = { @@ -370,9 +380,11 @@ e.g. `get_tld`]], return nil end if args[1] then - return fun.map(function(r) return r[args[1]](r) end, urls), 'string_list' + return fun.map(function(r) + return r[args[1]](r) + end, urls), 'string_list' end - return urls,'userdata_list' + return urls, 'userdata_list' end, ['description'] = [[Get list of all emails. If no arguments specified, returns list of url objects. Otherwise, calls a specific method, @@ -383,11 +395,11 @@ e.g. `get_user`]], ['pool_var'] = { ['get_value'] = function(task, args) local type = args[2] or 'string' - return task:get_mempool():get_variable(args[1], type),(type) + return task:get_mempool():get_variable(args[1], type), (type) end, ['description'] = [[Get specific pool var. The first argument must be variable name, the second argument is optional and defines the type (string by default)]], - ['args_schema'] = {ts.string, ts.string:is_optional()} + ['args_schema'] = { ts.string, ts.string:is_optional() } }, -- Get value of specific key from task cache ['task_cache'] = { @@ -406,35 +418,35 @@ the second argument is optional and defines the type (string by default)]], end, ['description'] = [[Get value of specific key from task cache. The first argument must be the key name]], - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Get specific HTTP request header. The first argument must be header name. ['request_header'] = { ['get_value'] = function(task, args) local hdr = task:get_request_header(args[1]) if hdr then - return hdr,'string' + return hdr, 'string' end return nil end, ['description'] = [[Get specific HTTP request header. The first argument must be header name.]], - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Get task date, optionally formatted ['time'] = { ['get_value'] = function(task, args) local what = args[1] or 'message' - local dt = task:get_date{format = what, gmt = true} + local dt = task:get_date { format = what, gmt = true } if dt then if args[2] then -- Should be in format !xxx, as dt is in GMT - return os.date(args[2], dt),'string' + return os.date(args[2], dt), 'string' end - return tostring(dt),'string' + return tostring(dt), 'string' end return nil @@ -444,8 +456,8 @@ The first argument must be header name.]], - `message`: timestamp as defined by `Date` header The second argument is optional time format, see [os.date](http://pgl.yoyo.org/luai/i/os.date) description]], - ['args_schema'] = {ts.one_of{'connect', 'message'}:is_optional(), - ts.string:is_optional()} + ['args_schema'] = { ts.one_of { 'connect', 'message' }:is_optional(), + ts.string:is_optional() } }, -- Get text words from a message ['words'] = { @@ -473,13 +485,15 @@ The first argument must be header name.]], - `norm`: normalised words (lowercased) - `full`: list of tables ]], - ['args_schema'] = { ts.one_of { 'stem', 'raw', 'norm', 'full' }:is_optional()}, + ['args_schema'] = { ts.one_of { 'stem', 'raw', 'norm', 'full' }:is_optional() }, }, -- Get queue ID ['queueid'] = { ['get_value'] = function(task) local queueid = task:get_queue_id() - if queueid then return queueid,'string' end + if queueid then + return queueid, 'string' + end return nil end, ['description'] = [[Get queue ID]], @@ -488,7 +502,9 @@ The first argument must be header name.]], ['uid'] = { ['get_value'] = function(task) local uid = task:get_uid() - if uid then return uid,'string' end + if uid then + return uid, 'string' + end return nil end, ['description'] = [[Get ID of the task being processed]], @@ -497,7 +513,9 @@ The first argument must be header name.]], ['messageid'] = { ['get_value'] = function(task) local mid = task:get_message_id() - if mid then return mid,'string' end + if mid then + return mid, 'string' + end return nil end, ['description'] = [[Get message ID]], @@ -507,25 +525,25 @@ The first argument must be header name.]], ['get_value'] = function(task, args) local symbol = task:get_symbol(args[1], args[2]) if symbol then - return symbol[1],'table' + return symbol[1], 'table' end end, ['description'] = 'Get specific symbol. The first argument must be the symbol name. ' .. - 'The second argument is an optional shadow result name. ' .. - 'Returns the symbol table. See task:get_symbol()', - ['args_schema'] = {ts.string, ts.string:is_optional()} + 'The second argument is an optional shadow result name. ' .. + 'Returns the symbol table. See task:get_symbol()', + ['args_schema'] = { ts.string, ts.string:is_optional() } }, -- Get full scan result ['scan_result'] = { ['get_value'] = function(task, args) local res = task:get_metric_result(args[1]) if res then - return res,'table' + return res, 'table' end end, ['description'] = 'Get full scan result (either default or shadow if shadow result name is specified)' .. 'Returns the result table. See task:get_metric_result()', - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, -- Get list of metatokens as strings ['metatokens'] = { diff --git a/lualib/lua_selectors/init.lua b/lualib/lua_selectors/init.lua index 5823fef96..6cdd11584 100644 --- a/lualib/lua_selectors/init.lua +++ b/lualib/lua_selectors/init.lua @@ -51,21 +51,21 @@ local function implicit_tostring(t, ud_or_table) if t == 'table' then -- Table (very special) if ud_or_table.value then - return ud_or_table.value,'string' + return ud_or_table.value, 'string' elseif ud_or_table.addr then - return ud_or_table.addr,'string' + return ud_or_table.addr, 'string' end - return logger.slog("%s", ud_or_table),'string' + return logger.slog("%s", ud_or_table), 'string' elseif (t == 'string' or t == 'text') and type(ud_or_table) == 'userdata' then if ud_or_table.cookie and ud_or_table.cookie == text_cookie then -- Preserve opaque - return ud_or_table,'string' + return ud_or_table, 'string' else - return tostring(ud_or_table),'string' + return tostring(ud_or_table), 'string' end elseif t ~= 'nil' then - return tostring(ud_or_table),'string' + return tostring(ud_or_table), 'string' end return nil @@ -84,7 +84,7 @@ local function process_selector(task, sel) return pure_type(t) end - local input,etype = sel.selector.get_value(task, sel.selector.args) + local input, etype = sel.selector.get_value(task, sel.selector.args) if not input then lua_util.debugm(M, task, 'no value extracted for %s', sel.selector.name) @@ -98,14 +98,16 @@ local function process_selector(task, sel) local first_elt = pipe[1] if first_elt and (first_elt.method or - fun.any(function(t) return t == 'userdata' or t == 'table' end, first_elt.types)) then + fun.any(function(t) + return t == 'userdata' or t == 'table' + end, first_elt.types)) then -- Explicit conversion local meth = first_elt if meth.types[etype] then lua_util.debugm(M, task, 'apply method `%s` to %s', meth.name, etype) - input,etype = meth.process(input, etype, meth.args) + input, etype = meth.process(input, etype, meth.args) else local pt = pure_type(etype) @@ -114,7 +116,9 @@ local function process_selector(task, sel) meth.name, pt) -- Map method to a list of inputs, excluding empty elements -- We need to fold it down here to get a proper type resolution - input = fun.totable(fun.filter(function(map_elt, _) return map_elt end, + input = fun.totable(fun.filter(function(map_elt, _) + return map_elt + end, fun.map(function(list_elt) local ret, ty = meth.process(list_elt, pt, meth.args) etype = ty @@ -139,7 +143,9 @@ local function process_selector(task, sel) etype = 'string' else lua_util.debugm(M, task, 'apply implicit map %s->string', pt) - input = fun.filter(function(map_elt) return map_elt end, + input = fun.filter(function(map_elt) + return map_elt + end, fun.map(function(list_elt) local ret = implicit_tostring(pt, list_elt) return ret @@ -166,16 +172,20 @@ local function process_selector(task, sel) if pt and x.types['list'] then -- Generic list processor lua_util.debugm(M, task, 'apply list function `%s` to %s', x.name, t) - return {x.process(value, t, x.args)} + return { x.process(value, t, x.args) } elseif pt and x.map_type and x.types[pt] then local map_type = x.map_type .. '_list' lua_util.debugm(M, task, 'map `%s` to list of %s resulting %s', x.name, pt, map_type) -- Apply map, filtering empty values return { - fun.filter(function(map_elt) return map_elt end, + fun.filter(function(map_elt) + return map_elt + end, fun.map(function(list_elt) - if not list_elt then return nil end + if not list_elt then + return nil + end local ret, _ = x.process(list_elt, pt, x.args) return ret end, value)), @@ -187,14 +197,16 @@ local function process_selector(task, sel) end lua_util.debugm(M, task, 'apply %s to %s', x.name, t) - return {x.process(value, t, x.args)} + return { x.process(value, t, x.args) } end local res = fun.foldl(fold_function, - {input, etype}, + { input, etype }, pipe) - if not res or not res[1] then return nil end -- Pipeline failed + if not res or not res[1] then + return nil + end -- Pipeline failed if not allowed_type(res[2]) then -- Search for implicit conversion @@ -202,7 +214,9 @@ local function process_selector(task, sel) if pt then lua_util.debugm(M, task, 'apply implicit map %s->string_list', pt) - res[1] = fun.map(function(e) return implicit_tostring(pt, e) end, res[1]) + res[1] = fun.map(function(e) + return implicit_tostring(pt, e) + end, res[1]) res[2] = 'string_list' else res[1] = implicit_tostring(res[2], res[1]) @@ -222,14 +236,14 @@ end local function make_grammar() local l = require "lpeg" - local spc = l.S(" \t\n")^0 + local spc = l.S(" \t\n") ^ 0 local cont = l.R("\128\191") -- continuation byte local utf8_high = l.R("\194\223") * cont + l.R("\224\239") * cont * cont + l.R("\240\244") * cont * cont * cont - local atom = l.C((l.R("az") + l.R("AZ") + l.R("09") + l.S("_-") + utf8_high)^1) - local singlequoted_string = l.P "'" * l.C(((1 - l.S "'\r\n\f\\") + (l.P'\\' * 1))^0) * "'" - local doublequoted_string = l.P '"' * l.C(((1 - l.S'"\r\n\f\\') + (l.P'\\' * 1))^0) * '"' + local atom = l.C((l.R("az") + l.R("AZ") + l.R("09") + l.S("_-") + utf8_high) ^ 1) + local singlequoted_string = l.P "'" * l.C(((1 - l.S "'\r\n\f\\") + (l.P '\\' * 1)) ^ 0) * "'" + local doublequoted_string = l.P '"' * l.C(((1 - l.S '"\r\n\f\\') + (l.P '\\' * 1)) ^ 0) * '"' local argument = atom + singlequoted_string + doublequoted_string local dot = l.P(".") local semicolon = l.P(":") @@ -239,20 +253,22 @@ local function make_grammar() local tbl_ebrace = spc * "}" local ebrace = spc * ")" local comma = spc * "," * spc - local sel_separator = spc * l.S";*" * spc + local sel_separator = spc * l.S ";*" * spc - return l.P{ + return l.P { "LIST"; - LIST = l.Ct(l.V("EXPR")) * (sel_separator * l.Ct(l.V("EXPR")))^0, - EXPR = l.V("FUNCTION") * (semicolon * l.V("METHOD"))^-1 * (dot * l.V("PROCESSOR"))^0, - PROCESSOR = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace)^0), - FUNCTION = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace)^0), - METHOD = l.Ct(atom / function(e) return '__' .. e end * spc * (obrace * l.V("ARG_LIST") * ebrace)^0), - ARG_LIST = l.Ct((l.V("ARG") * comma^0)^0), + LIST = l.Ct(l.V("EXPR")) * (sel_separator * l.Ct(l.V("EXPR"))) ^ 0, + EXPR = l.V("FUNCTION") * (semicolon * l.V("METHOD")) ^ -1 * (dot * l.V("PROCESSOR")) ^ 0, + PROCESSOR = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0), + FUNCTION = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0), + METHOD = l.Ct(atom / function(e) + return '__' .. e + end * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0), + ARG_LIST = l.Ct((l.V("ARG") * comma ^ 0) ^ 0), ARG = l.Cf(tbl_obrace * l.V("NAMED_ARG") * tbl_ebrace, rawset) + argument + l.V("LIST_ARGS"), - NAMED_ARG = (l.Ct("") * l.Cg(argument * eqsign * (argument + l.V("LIST_ARGS")) * comma^0)^0), + NAMED_ARG = (l.Ct("") * l.Cg(argument * eqsign * (argument + l.V("LIST_ARGS")) * comma ^ 0) ^ 0), LIST_ARGS = l.Ct(tbl_obrace * l.V("LIST_ARG") * tbl_ebrace), - LIST_ARG = l.Cg(argument * comma^0)^0, + LIST_ARG = l.Cg(argument * comma ^ 0) ^ 0, } end @@ -262,27 +278,29 @@ local parser = make_grammar() -- @function lua_selectors.parse_selector(cfg, str) --]] exports.parse_selector = function(cfg, str) - local parsed = {parser:match(str)} + local parsed = { parser:match(str) } local output = {} - if not parsed or not parsed[1] then return nil end + if not parsed or not parsed[1] then + return nil + end local function check_args(name, schema, args) if schema then if getmetatable(schema) then -- Schema covers all arguments - local res,err = schema:transform(args) + local res, err = schema:transform(args) if not res then logger.errx(rspamd_config, 'invalid arguments for %s: %s', name, err) return false else - for i,elt in ipairs(res) do + for i, elt in ipairs(res) do args[i] = elt end end else - for i,selt in ipairs(schema) do - local res,err = selt:transform(args[i]) + for i, selt in ipairs(schema) do + local res, err = selt:transform(args[i]) if err then logger.errx(rspamd_config, 'invalid arguments for %s: argument number: %s, error: %s', name, i, err) @@ -301,7 +319,7 @@ exports.parse_selector = function(cfg, str) -- table of individual selectors -- each selector: list of functions -- each function: function name + optional list of arguments - for _,sel in ipairs(parsed) do + for _, sel in ipairs(parsed) do local res = { selector = {}, processor_pipe = {}, @@ -364,13 +382,15 @@ exports.parse_selector = function(cfg, str) local ret_type = type(ret) - if ret_type == 'nil' then return nil end + if ret_type == 'nil' then + return nil + end -- Now apply types heuristic if ret_type == 'string' then - return ret,'string' + return ret, 'string' elseif ret_type == 'table' then -- TODO: we need to ensure that 1) table is numeric 2) table has merely strings - return ret,'string_list' + return ret, 'string_list' else return implicit_tostring(ret_type, ret) end @@ -452,11 +472,13 @@ end exports.process_selectors = function(task, selectors_pipe) local ret = {} - for _,sel in ipairs(selectors_pipe) do + for _, sel in ipairs(selectors_pipe) do local r = process_selector(task, sel) -- If any element is nil, then the whole selector is nil - if not r then return nil end + if not r then + return nil + end table.insert(ret, r) end @@ -467,13 +489,17 @@ end -- @function lua_selectors.combine_selectors(task, selectors, delimiter) --]] exports.combine_selectors = function(_, selectors, delimiter) - if not delimiter then delimiter = '' end + if not delimiter then + delimiter = '' + end - if not selectors then return nil end + if not selectors then + return nil + end local have_tables, have_userdata - for _,s in ipairs(selectors) do + for _, s in ipairs(selectors) do if type(s) == 'table' then have_tables = true elseif type(s) == 'userdata' then @@ -493,7 +519,7 @@ exports.combine_selectors = function(_, selectors, delimiter) local tbl = {} local res = {} - for i,s in ipairs(selectors) do + for i, s in ipairs(selectors) do if type(s) == 'string' then rawset(tbl, i, fun.duplicate(s)) elseif type(s) == 'userdata' then @@ -505,7 +531,7 @@ exports.combine_selectors = function(_, selectors, delimiter) end fun.each(function(...) - table.insert(res, table.concat({...}, delimiter)) + table.insert(res, table.concat({ ... }, delimiter)) end, fun.zip(lua_util.unpack(tbl))) return res @@ -520,7 +546,7 @@ exports.flatten_selectors = function(_, selectors, _) local res = {} local function fill(tbl) - for _,s in ipairs(tbl) do + for _, s in ipairs(tbl) do if type(s) == 'string' then rawset(res, #res + 1, s) elseif type(s) == 'userdata' then @@ -553,7 +579,7 @@ exports.kv_table_from_pairs = function(log_obj, selectors, _) tbl_len) return end - for i=1,tbl_len,2 do + for i = 1, tbl_len, 2 do local k = tostring(tbl[i]) local v = tbl[i + 1] if type(v) == 'string' then @@ -561,7 +587,9 @@ exports.kv_table_from_pairs = function(log_obj, selectors, _) elseif type(v) == 'userdata' then res[k] = tostring(v) else - res[k] = fun.totable(fun.map(function(elt) return tostring(elt) end, v)) + res[k] = fun.totable(fun.map(function(elt) + return tostring(elt) + end, v)) end end end @@ -605,7 +633,7 @@ exports.create_selector_closure = function(cfg, selector_str, delimiter, flatten end local function display_selectors(tbl) - return fun.tomap(fun.map(function(k,v) + return fun.tomap(fun.map(function(k, v) return k, fun.tomap(fun.filter(function(kk, vv) return type(vv) ~= 'function' end, v)) diff --git a/lualib/lua_selectors/transforms.lua b/lualib/lua_selectors/transforms.lua index 55e1bffb7..4b839de28 100644 --- a/lualib/lua_selectors/transforms.lua +++ b/lualib/lua_selectors/transforms.lua @@ -36,7 +36,7 @@ local transform_function = { }, ['map_type'] = 'string', ['process'] = function(inp, _) - return inp:lower(),'string' + return inp:lower(), 'string' end, ['description'] = 'Returns the lowercased string', }, @@ -57,7 +57,7 @@ local transform_function = { ['list'] = true, }, ['process'] = function(inp, t) - return fun.head(inp),pure_type(t) + return fun.head(inp), pure_type(t) end, ['description'] = 'Returns the first element', }, @@ -67,7 +67,7 @@ local transform_function = { ['list'] = true, }, ['process'] = function(inp, t) - return fun.nth(fun.length(inp), inp),pure_type(t) + return fun.nth(fun.length(inp), inp), pure_type(t) end, ['description'] = 'Returns the last element', }, @@ -77,30 +77,30 @@ local transform_function = { ['list'] = true, }, ['process'] = function(inp, t, args) - return fun.nth(args[1] or 1, inp),pure_type(t) + return fun.nth(args[1] or 1, inp), pure_type(t) end, ['description'] = 'Returns the nth element', - ['args_schema'] = {ts.number + ts.string / tonumber} + ['args_schema'] = { ts.number + ts.string / tonumber } }, ['take_n'] = { ['types'] = { ['list'] = true, }, ['process'] = function(inp, t, args) - return fun.take_n(args[1] or 1, inp),t + return fun.take_n(args[1] or 1, inp), t end, ['description'] = 'Returns the n first elements', - ['args_schema'] = {ts.number + ts.string / tonumber} + ['args_schema'] = { ts.number + ts.string / tonumber } }, ['drop_n'] = { ['types'] = { ['list'] = true, }, ['process'] = function(inp, t, args) - return fun.drop_n(args[1] or 1, inp),t + return fun.drop_n(args[1] or 1, inp), t end, ['description'] = 'Returns list without the first n elements', - ['args_schema'] = {ts.number + ts.string / tonumber} + ['args_schema'] = { ts.number + ts.string / tonumber } }, -- Joins strings into a single string using separator in the argument ['join'] = { @@ -111,7 +111,7 @@ local transform_function = { return table.concat(fun.totable(inp), args[1] or ''), 'string' end, ['description'] = 'Joins strings into a single string using separator in the argument', - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, -- Joins strings into a set of strings using N elements and a separator in the argument ['join_nth'] = { @@ -124,13 +124,13 @@ local transform_function = { local inp_t = fun.totable(inp) local res = {} - for i=1, #inp_t, step do + for i = 1, #inp_t, step do table.insert(res, table.concat(inp_t, sep, i, i + step)) end - return res,'string_list' + return res, 'string_list' end, ['description'] = 'Joins strings into a set of strings using N elements and a separator in the argument', - ['args_schema'] = {ts.number + ts.string / tonumber, ts.string:is_optional()} + ['args_schema'] = { ts.number + ts.string / tonumber, ts.string:is_optional() } }, -- Joins tables into a table of strings ['join_tables'] = { @@ -139,10 +139,12 @@ local transform_function = { }, ['process'] = function(inp, _, args) local sep = args[1] or '' - return fun.map(function(t) return table.concat(t, sep) end, inp), 'string_list' + return fun.map(function(t) + return table.concat(t, sep) + end, inp), 'string_list' end, ['description'] = 'Joins tables into a table of strings', - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, -- Sort strings ['sort'] = { @@ -166,7 +168,9 @@ local transform_function = { tmp[val] = true end, inp) - return fun.map(function(k, _) return k end, tmp), t + return fun.map(function(k, _) + return k + end, tmp), t end, ['description'] = 'Returns a list of unique elements (using a hash table)', }, @@ -177,7 +181,7 @@ local transform_function = { }, ['map_type'] = 'string', ['process'] = function(inp, _, args) - return common.create_digest(inp, args),'string' + return common.create_digest(inp, args), 'string' end, ['description'] = [[Create a digest from a string. The first argument is encoding (`hex`, `base32` (and forms `bleach32`, `rbase32`), `base64`), @@ -197,8 +201,8 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, return inp:sub(start_pos, end_pos), 'string' end, ['description'] = 'Extracts substring; the first argument is start, the second is the last (like in Lua)', - ['args_schema'] = {(ts.number + ts.string / tonumber):is_optional(), - (ts.number + ts.string / tonumber):is_optional()} + ['args_schema'] = { (ts.number + ts.string / tonumber):is_optional(), + (ts.number + ts.string / tonumber):is_optional() } }, -- Prepends a string or a strings list ['prepend'] = { @@ -257,13 +261,13 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, end end flatten_table(res) - return flattened_table,'string_list' + return flattened_table, 'string_list' end return nil end, ['description'] = 'Regexp matching, returns all matches flattened in a single list', - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Returns a value if it exists in some map (or acts like a `filter` function) ['filter_map'] = { @@ -282,13 +286,13 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, local res = map:get_key(inp) if res then - return inp,t + return inp, t end return nil end, ['description'] = 'Returns a value if it exists in some map (or acts like a `filter` function)', - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Returns a value if it exists in some map (or acts like a `filter` function) ['except_map'] = { @@ -307,13 +311,13 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, local res = map:get_key(inp) if not res then - return inp,t + return inp, t end return nil end, ['description'] = 'Returns a value if it does not exists in some map (or acts like a `except` function)', - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Returns a value from some map corresponding to some key (or acts like a `map` function) ['apply_map'] = { @@ -332,13 +336,13 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, local res = map:get_key(inp) if res then - return res,t + return res, t end return nil end, ['description'] = 'Returns a value from some map corresponding to some key (or acts like a `map` function)', - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Drops input value and return values from function's arguments or an empty string ['id'] = { @@ -349,12 +353,12 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, ['map_type'] = 'string', ['process'] = function(_, _, args) if args[1] and args[2] then - return fun.map(tostring, args),'string_list' + return fun.map(tostring, args), 'string_list' elseif args[1] then - return args[1],'string' + return args[1], 'string' end - return '','string' + return '', 'string' end, ['description'] = 'Drops input value and return values from function\'s arguments or an empty string', ['args_schema'] = (ts.string + ts.array_of(ts.string)):is_optional() @@ -366,14 +370,14 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, ['map_type'] = 'string', ['process'] = function(inp, _, args) if inp == args[1] then - return inp,'string' + return inp, 'string' end return nil end, ['description'] = [[Boolean function equal. Returns either nil or its argument if input is equal to argument]], - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Boolean function in, returns either nil or its input if input is in args list ['in'] = { @@ -382,7 +386,11 @@ Returns either nil or its argument if input is equal to argument]], }, ['map_type'] = 'string', ['process'] = function(inp, t, args) - for _,a in ipairs(args) do if a == inp then return inp,t end end + for _, a in ipairs(args) do + if a == inp then + return inp, t + end + end return nil end, ['description'] = [[Boolean function in. @@ -395,8 +403,12 @@ Returns either nil or its input if input is in args list]], }, ['map_type'] = 'string', ['process'] = function(inp, t, args) - for _,a in ipairs(args) do if a == inp then return nil end end - return inp,t + for _, a in ipairs(args) do + if a == inp then + return nil + end + end + return inp, t end, ['description'] = [[Boolean function not in. Returns either nil or its input if input is not in args list]], @@ -411,12 +423,12 @@ Returns either nil or its input if input is not in args list]], if inp then return nil else - return (args[1] or 'true'),'string' + return (args[1] or 'true'), 'string' end end, ['description'] = [[Inverses input. Empty string comes the first argument or 'true', non-empty string comes nil]], - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, ['ipmask'] = { ['types'] = { @@ -436,19 +448,19 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], if ip:get_version() == 4 then local mask = tonumber(args[1]) - return ip:apply_mask(mask):to_string(),'string' + return ip:apply_mask(mask):to_string(), 'string' else -- IPv6 takes the second argument or the first one... local mask_str = args[2] or args[1] local mask = tonumber(mask_str) - return ip:apply_mask(mask):to_string(),'string' + return ip:apply_mask(mask):to_string(), 'string' end end, ['description'] = 'Applies mask to IP address.' .. - ' The first argument is the mask for IPv4 addresses, the second is the mask for IPv6 addresses.', - ['args_schema'] = {(ts.number + ts.string / tonumber), - (ts.number + ts.string / tonumber):is_optional()} + ' The first argument is the mask for IPv4 addresses, the second is the mask for IPv6 addresses.', + ['args_schema'] = { (ts.number + ts.string / tonumber), + (ts.number + ts.string / tonumber):is_optional() } }, -- Returns the string(s) with all non ascii chars replaced ['to_ascii'] = { @@ -460,16 +472,16 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], ['process'] = function(inp, _, args) if type(inp) == 'table' then return fun.map( - function(s) - return string.gsub(tostring(s), '[\128-\255]', args[1] or '?') - end, inp), 'string_list' + function(s) + return string.gsub(tostring(s), '[\128-\255]', args[1] or '?') + end, inp), 'string_list' else return string.gsub(tostring(inp), '[\128-\255]', '?'), 'string' end end, ['description'] = 'Returns the string with all non-ascii bytes replaced with the character ' .. - 'given as second argument or `?`', - ['args_schema'] = {ts.string:is_optional()} + 'given as second argument or `?`', + ['args_schema'] = { ts.string:is_optional() } }, -- Extracts tld from a hostname ['get_tld'] = { @@ -478,7 +490,7 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], }, ['map_type'] = 'string', ['process'] = function(inp, _, _) - return rspamd_util.get_tld(inp),'string' + return rspamd_util.get_tld(inp), 'string' end, ['description'] = 'Extracts tld from a hostname represented as a string', ['args_schema'] = {} @@ -498,7 +510,7 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], return rspamd_util.pack(string.rep(fmt, #res), lua_util.unpack(res)), 'string' end, ['description'] = 'Converts a list of strings to numbers & returns a packed string', - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, -- Filter nils from a list ['filter_string_nils'] = { @@ -506,7 +518,9 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], ['string_list'] = true }, ['process'] = function(inp, _, _) - return fun.filter(function(val) return type(val) == 'string' and val ~= 'nil' end, inp), 'string_list' + return fun.filter(function(val) + return type(val) == 'string' and val ~= 'nil' + end, inp), 'string_list' end, ['description'] = 'Removes all nils from a list of strings (when converted implicitly)', ['args_schema'] = {} @@ -518,12 +532,14 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], }, ['process'] = function(inp, _, args) local res = {} - for _,arg in ipairs(args) do + for _, arg in ipairs(args) do local meth = inp[arg] local ret = meth(inp) - if ret then table.insert(res, tostring(ret)) end + if ret then + table.insert(res, tostring(ret)) + end end - return res,'string_list' + return res, 'string_list' end, ['description'] = 'Apply a list of method calls to the userdata object', } diff --git a/lualib/lua_settings.lua b/lualib/lua_settings.lua index feeb03d28..d6d24d67e 100644 --- a/lualib/lua_settings.lua +++ b/lualib/lua_settings.lua @@ -37,7 +37,7 @@ local function register_settings_cb(from_postload) default_symbols = fun.totable(fun.filter(function(_, v) return not v.allowed_ids or #v.allowed_ids == 0 or v.flags.explicit_disable - end,all_symbols)) + end, all_symbols)) local explicit_symbols = lua_util.keys(fun.filter(function(k, v) return v.flags.explicit_disable @@ -45,7 +45,7 @@ local function register_settings_cb(from_postload) local symnames = lua_util.list_to_hash(lua_util.keys(all_symbols)) - for _,set in pairs(known_ids) do + for _, set in pairs(known_ids) do local s = set.settings.apply or {} set.symbols = lua_util.shallowcopy(symnames) local enabled_symbols = {} @@ -58,18 +58,18 @@ local function register_settings_cb(from_postload) -- Remove all symbols from set.symbols aside of explicit_disable symbols set.symbols = lua_util.list_to_hash(explicit_symbols) seen_enabled = true - for _,sym in ipairs(s.symbols_enabled) do + for _, sym in ipairs(s.symbols_enabled) do enabled_symbols[sym] = true set.symbols[sym] = true end end if s.groups_enabled then seen_enabled = true - for _,gr in ipairs(s.groups_enabled) do + for _, gr in ipairs(s.groups_enabled) do local syms = rspamd_config:get_group_symbols(gr) if syms then - for _,sym in ipairs(syms) do + for _, sym in ipairs(syms) do enabled_symbols[sym] = true set.symbols[sym] = true end @@ -80,18 +80,18 @@ local function register_settings_cb(from_postload) -- Disabled map if s.symbols_disabled then seen_disabled = true - for _,sym in ipairs(s.symbols_disabled) do + for _, sym in ipairs(s.symbols_disabled) do disabled_symbols[sym] = true set.symbols[sym] = false end end if s.groups_disabled then seen_disabled = true - for _,gr in ipairs(s.groups_disabled) do + for _, gr in ipairs(s.groups_disabled) do local syms = rspamd_config:get_group_symbols(gr) if syms then - for _,sym in ipairs(syms) do + for _, sym in ipairs(syms) do disabled_symbols[sym] = true set.symbols[sym] = false end @@ -100,8 +100,12 @@ local function register_settings_cb(from_postload) end -- Deal with complexity to avoid mess in C - if not seen_enabled then enabled_symbols = nil end - if not seen_disabled then disabled_symbols = nil end + if not seen_enabled then + enabled_symbols = nil + end + if not seen_disabled then + disabled_symbols = nil + end if enabled_symbols or disabled_symbols then -- Specify what symbols are really enabled for this settings id @@ -118,9 +122,9 @@ local function register_settings_cb(from_postload) end -- We now iterate over all symbols and check for allowed_ids/forbidden_ids - for k,v in pairs(all_symbols) do + for k, v in pairs(all_symbols) do if v.allowed_ids and not v.flags.explicit_disable then - for _,id in ipairs(v.allowed_ids) do + for _, id in ipairs(v.allowed_ids) do if known_ids[id] then local set = known_ids[id] if not set.has_specific_symbols then @@ -134,7 +138,7 @@ local function register_settings_cb(from_postload) end end if v.forbidden_ids then - for _,id in ipairs(v.forbidden_ids) do + for _, id in ipairs(v.forbidden_ids) do if known_ids[id] then local set = known_ids[id] if not set.has_specific_symbols then @@ -150,8 +154,10 @@ local function register_settings_cb(from_postload) end -- Now we create lists of symbols for each settings and digest - for _,set in pairs(known_ids) do - set.symbols = lua_util.keys(fun.filter(function(_, v) return v end, set.symbols)) + for _, set in pairs(known_ids) do + set.symbols = lua_util.keys(fun.filter(function(_, v) + return v + end, set.symbols)) table.sort(set.symbols) set.digest = lua_util.table_digest(set.symbols) end @@ -188,7 +194,7 @@ local function transform_settings_maybe(settings, name) if not apply.scores then apply.scores = {} end - for k,v in pairs(senabled) do + for k, v in pairs(senabled) do if tonumber(v) then -- Move to symbols as well apply.scores[k] = tonumber(v) @@ -205,7 +211,7 @@ local function transform_settings_maybe(settings, name) if apply.symbols then -- Check if added symbols are enabled - for k,v in pairs(apply.symbols) do + for k, v in pairs(apply.symbols) do local s -- Check if we have ["sym1", "sym2" ...] or {"sym1": xx, "sym2": yy} if type(k) == 'string' then @@ -251,7 +257,9 @@ local function register_settings_id(str, settings, from_postload) if not from_postload and not post_init_added then -- Use high priority to ensure that settings are initialised early but not before all -- plugins are loaded - rspamd_config:add_post_init(function () register_settings_cb(true) end, 150) + rspamd_config:add_post_init(function() + register_settings_cb(true) + end, 150) rspamd_config:add_config_unload(function() if post_init_added then known_ids = {} @@ -268,7 +276,6 @@ end exports.register_settings_id = register_settings_id - local function settings_by_id(id) if not post_init_performed then register_settings_cb(false) @@ -276,7 +283,6 @@ local function settings_by_id(id) return known_ids[id] end - exports.settings_by_id = settings_by_id exports.all_settings = function() if not post_init_performed then diff --git a/lualib/lua_smtp.lua b/lualib/lua_smtp.lua index 33c69159f..3c403497c 100644 --- a/lualib/lua_smtp.lua +++ b/lualib/lua_smtp.lua @@ -47,7 +47,7 @@ local function sendmail(opts, message, callback) wantcode = wantcode or '2' if merr then callback(false, string.format('error on stage %s: %s', - stage, merr)) + stage, merr)) if conn then conn:close() end @@ -108,9 +108,9 @@ local function sendmail(opts, message, callback) local function data_done_cb(merr, mdata) if no_error_read(merr, mdata, '3') then if type(message) == 'string' or type(message) == 'userdata' then - conn:add_write(pre_quit_cb, {message, CRLF.. '.' .. CRLF}) + conn:add_write(pre_quit_cb, { message, CRLF .. '.' .. CRLF }) else - table.insert(message, CRLF.. '.' .. CRLF) + table.insert(message, CRLF .. '.' .. CRLF) conn:add_write(pre_quit_cb, message) end end @@ -124,7 +124,7 @@ local function sendmail(opts, message, callback) -- RCPT phase local next_recipient local function rcpt_done_cb_gen(i) - return function (merr, mdata) + return function(merr, mdata) if no_error_read(merr, mdata) then if i == #opts.recipients then conn:add_write(data_cb, 'DATA' .. CRLF) @@ -136,7 +136,7 @@ local function sendmail(opts, message, callback) end local function rcpt_cb_gen(i) - return function (merr, _) + return function(merr, _) if no_error_write(merr, '2') then conn:add_read(rcpt_done_cb_gen(i), CRLF) end @@ -178,12 +178,12 @@ local function sendmail(opts, message, callback) if no_error_read(err, data) then stage = 'helo' conn:add_write(hello_cb, string.format('HELO %s%s', - opts.helo, CRLF)) + opts.helo, CRLF)) end end if type(opts.recipients) == 'string' then - opts.recipients = {opts.recipients} + opts.recipients = { opts.recipients } end local tcp_opts = lua_util.shallowcopy(opts) diff --git a/lualib/lua_stat.lua b/lualib/lua_stat.lua index e7111735a..a0f3303ad 100644 --- a/lualib/lua_stat.lua +++ b/lualib/lua_stat.lua @@ -29,7 +29,7 @@ local exports = {} local N = "stat_tools" -- luacheck: ignore (maybe unused) -- Performs synchronous conversion of redis schema -local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expire) +local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expire) -- Old schema is the following one: -- Keys are named <symbol>[<user>] @@ -40,7 +40,7 @@ local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expi -- So we can expire individual records, measure most popular elements by zranges, -- add new fields, such as tokens etc - local res,conn = lua_redis.redis_connect_sync(redis_params, true) + local res, conn = lua_redis.redis_connect_sync(redis_params, true) if not res then logger.errx("cannot connect to redis server") @@ -79,7 +79,7 @@ end return nconverted ]] - conn:add_cmd('EVAL', {lua_script, '3', symbol_spam, 'S', tostring(expire)}) + conn:add_cmd('EVAL', { lua_script, '3', symbol_spam, 'S', tostring(expire) }) local ret ret, res = conn:exec() @@ -90,7 +90,7 @@ return nconverted logger.messagex('converted %s elements from symbol %s', res, symbol_spam) end - conn:add_cmd('EVAL', {lua_script, '3', symbol_ham, 'H', tostring(expire)}) + conn:add_cmd('EVAL', { lua_script, '3', symbol_ham, 'H', tostring(expire) }) ret, res = conn:exec() if not ret then @@ -118,15 +118,15 @@ for _,k in ipairs(keys) do end ]] - conn:add_cmd('EVAL', {lua_script, '2', symbol_spam, 'learns_spam'}) - ret,res = conn:exec() + conn:add_cmd('EVAL', { lua_script, '2', symbol_spam, 'learns_spam' }) + ret, res = conn:exec() if not ret then logger.errx('error converting metadata for symbol %s: %s', symbol_spam, res) return false end - conn:add_cmd('EVAL', {lua_script, '2', symbol_ham, 'learns_ham'}) + conn:add_cmd('EVAL', { lua_script, '2', symbol_ham, 'learns_ham' }) ret, res = conn:exec() if not ret then @@ -150,8 +150,8 @@ exports.convert_bayes_schema = convert_bayes_schema -- learn_cache_ham - name for sqlite database with ham learn cache -- reset_previous - if true, then the old database is flushed (slow) local function convert_sqlite_to_redis(redis_params, - sqlite_db_spam, sqlite_db_ham, symbol_spam, symbol_ham, - learn_cache_db, expire, reset_previous) + sqlite_db_spam, sqlite_db_ham, symbol_spam, symbol_ham, + learn_cache_db, expire, reset_previous) local nusers = 0 local lim = 1000 -- Update each 1000 tokens local users_map = {} @@ -168,7 +168,7 @@ local function convert_sqlite_to_redis(redis_params, return false end - local res,conn = lua_redis.redis_connect_sync(redis_params, true) + local res, conn = lua_redis.redis_connect_sync(redis_params, true) if not res then logger.errx("cannot connect to redis server") @@ -187,19 +187,19 @@ for _,prefix in ipairs(members) do end ]] -- Common keys - for _,sym in ipairs({symbol_spam, symbol_ham}) do + for _, sym in ipairs({ symbol_spam, symbol_ham }) do logger.messagex('Cleaning up old data for %s', sym) - conn:add_cmd('EVAL', {script, '1', sym}) + conn:add_cmd('EVAL', { script, '1', sym }) conn:exec() - conn:add_cmd('DEL', {sym .. "_version"}) - conn:add_cmd('DEL', {sym .. "_keys"}) + conn:add_cmd('DEL', { sym .. "_version" }) + conn:add_cmd('DEL', { sym .. "_keys" }) conn:exec() end if learn_cache_db then -- Cleanup learned_cache logger.messagex('Cleaning up old data learned cache') - conn:add_cmd('DEL', {"learned_ids"}) + conn:add_cmd('DEL', { "learned_ids" }) conn:exec() end end @@ -240,16 +240,16 @@ end if is_spam then hash_key = 'S' end - for _,tok in ipairs(tokens) do + for _, tok in ipairs(tokens) do -- tok schema: -- tok[1] = token_id (uint64 represented as a string) -- tok[2] = token value (number) -- tok[3] = user_map[user_id] or '' local rkey = string.format('%s%s_%s', prefix, tok[3], tok[1]) - conn:add_cmd('HINCRBYFLOAT', {rkey, hash_key, tostring(tok[2])}) + conn:add_cmd('HINCRBYFLOAT', { rkey, hash_key, tostring(tok[2]) }) if expire and expire ~= 0 then - conn:add_cmd('EXPIRE', {rkey, tostring(expire)}) + conn:add_cmd('EXPIRE', { rkey, tostring(expire) }) end end @@ -268,13 +268,13 @@ end user = users_map[row.user] end - table.insert(tokens, {row.token, row.value, user}) + table.insert(tokens, { row.token, row.value, user }) num = num + 1 total = total + 1 if num > lim then -- TODO: we use the default 'RS' prefix, it can be false in case of -- classifiers with labels - local ret,err_str = send_batch(tokens, 'RS') + local ret, err_str = send_batch(tokens, 'RS') if not ret then logger.errx('Cannot send tokens to the redis server: ' .. err_str) db:sql('COMMIT;') @@ -289,7 +289,7 @@ end end -- Last batch if #tokens > 0 then - local ret,err_str = send_batch(tokens, 'RS') + local ret, err_str = send_batch(tokens, 'RS') if not ret then logger.errx('Cannot send tokens to the redis server: ' .. err_str) db:sql('COMMIT;') @@ -312,19 +312,19 @@ end learns_elt = "learns_spam" end - for id,learned in pairs(learns) do + for id, learned in pairs(learns) do local user = users_map[id] - if not conn:add_cmd('HSET', {'RS' .. user, learns_elt, learned}) then + if not conn:add_cmd('HSET', { 'RS' .. user, learns_elt, learned }) then logger.errx('Cannot update learns for user: ' .. user) return false end - if not conn:add_cmd('SADD', {symbol .. '_keys', 'RS' .. user}) then + if not conn:add_cmd('SADD', { symbol .. '_keys', 'RS' .. user }) then logger.errx('Cannot update learns for user: ' .. user) return false end end -- Set version - conn:add_cmd('SET', {symbol..'_version', '2'}) + conn:add_cmd('SET', { symbol .. '_version', '2' }) return conn:exec() end @@ -361,7 +361,7 @@ end is_spam = '1' end - if not conn:add_cmd('HSET', {'learned_ids', digest, is_spam}) then + if not conn:add_cmd('HSET', { 'learned_ids', digest, is_spam }) then logger.errx('Cannot add hash: ' .. digest) ret = false else @@ -376,7 +376,7 @@ end if ret then logger.messagex('Converted %s cached items from sqlite3 learned cache to redis', - total) + total) else logger.errx('Error occurred during sending data to redis') end @@ -422,7 +422,7 @@ local function load_sqlite_config(cfg) end local statfiles = cls.statfile - for _,stf in ipairs(statfiles) do + for _, stf in ipairs(statfiles) do local path = (stf.file or stf.path or stf.db or stf.dbname) local symbol = stf.symbol or 'undefined' @@ -460,8 +460,10 @@ local function load_sqlite_config(cfg) if classifier then if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.bayes then cls = cls.bayes end + for _, cls in ipairs(classifier) do + if cls.bayes then + cls = cls.bayes + end if cls.backend and cls.backend == 'sqlite3' then parse_classifier(cls) end @@ -470,7 +472,7 @@ local function load_sqlite_config(cfg) if classifier.bayes then classifier = classifier.bayes if classifier[1] then - for _,cls in ipairs(classifier) do + for _, cls in ipairs(classifier) do if cls.backend and cls.backend == 'sqlite3' then parse_classifier(cls) end @@ -512,7 +514,7 @@ local function redis_classifier_from_sqlite(sqlite_classifier, expire) result.expire = expire end - return {classifier = {bayes = result}} + return { classifier = { bayes = result } } end exports.redis_classifier_from_sqlite = redis_classifier_from_sqlite @@ -548,7 +550,7 @@ local function process_stat_config(cfg) -- Postprocess classify_headers local classify_headers_parsed = {} - for _,v in ipairs(res_config.classify_headers) do + for _, v in ipairs(res_config.classify_headers) do local s1, s2 = v:match("^([A-Z])[^%-]+%-([A-Z]).*$") local hname @@ -567,7 +569,7 @@ local function process_stat_config(cfg) if classify_headers_parsed[hname] then table.insert(classify_headers_parsed[hname], v) else - classify_headers_parsed[hname] = {v} + classify_headers_parsed[hname] = { v } end end @@ -585,7 +587,7 @@ local function get_mime_stat_tokens(task, res, i) local empty_html = false local online_text = false - for _,part in ipairs(parts) do + for _, part in ipairs(parts) do local fname = part:get_filename() local sz = part:get_length() @@ -692,8 +694,8 @@ local function get_headers_stat_tokens(task, cf, res, i) end ]]-- - for k,hdrs in pairs(cf.classify_headers_parsed) do - for _,hname in ipairs(hdrs) do + for k, hdrs in pairs(cf.classify_headers_parsed) do + for _, hname in ipairs(hdrs) do local value = task:get_header(hname) if value then @@ -719,7 +721,7 @@ end local function get_meta_stat_tokens(task, res, i) local day_and_hour = os.date('%u:%H', - task:get_date{format = 'message', gmt = true}) + task:get_date { format = 'message', gmt = true }) rawset(res, i, string.format("#dt:%s", day_and_hour)) lua_util.debugm("bayes", task, "added day_of_week token: %s", res[i]) @@ -737,7 +739,7 @@ local function get_meta_stat_tokens(task, res, i) local trace = task:get_symbol('DKIM_TRACE') local dkim_opts = trace[1]['options'] if dkim_opts then - for _,o in ipairs(dkim_opts) do + for _, o in ipairs(dkim_opts) do local check_res = string.sub(o, -1) local domain = string.sub(o, 1, -3) @@ -752,8 +754,7 @@ local function get_meta_stat_tokens(task, res, i) if aur then local spf = aur:match('spf=([a-z]+)') - local dkim,dkim_domain = aur:match('dkim=([a-z]+) header.d=([a-z.%-]+)') - + local dkim, dkim_domain = aur:match('dkim=([a-z]+) header.d=([a-z.%-]+)') if spf then table.insert(pol, 's=' .. spf) @@ -807,7 +808,7 @@ local function get_stat_tokens(task, cf) if cf.classify_images then local images = task:get_images() or E - for _,img in ipairs(images) do + for _, img in ipairs(images) do rawset(res, i, "image") i = i + 1 rawset(res, i, tostring(img:get_height())) @@ -838,10 +839,10 @@ local function get_stat_tokens(task, cf) end if cf.classify_urls then - local urls = lua_util.extract_specific_urls{task = task, limit = 5, esld_limit = 1} + local urls = lua_util.extract_specific_urls { task = task, limit = 5, esld_limit = 1 } if urls then - for _,u in ipairs(urls) do + for _, u in ipairs(urls) do rawset(res, i, string.format("#u:%s", u:get_tld())) lua_util.debugm("bayes", task, "added url token: %s", res[i]) diff --git a/lualib/lua_tcp_sync.lua b/lualib/lua_tcp_sync.lua index a85a3f974..f8e60446f 100644 --- a/lualib/lua_tcp_sync.lua +++ b/lualib/lua_tcp_sync.lua @@ -4,9 +4,9 @@ local lua_util = require "lua_util" local exports = {} local N = 'tcp_sync' -local tcp_sync = {_conn = nil, _data = '', _eof = false, _addr = ''} +local tcp_sync = { _conn = nil, _data = '', _eof = false, _addr = '' } local metatable = { - __tostring = function (self) + __tostring = function(self) return "class {tcp_sync connect to: " .. self._addr .. "}" end } @@ -66,7 +66,7 @@ end -- --]] function tcp_sync:read_until(pattern) - repeat + repeat local pos_start, pos_end = self._data:find(pattern, 1, true) if pos_start then local data = self._data:sub(1, pos_start - 1) @@ -196,7 +196,7 @@ function tcp_sync:shutdown() return self._conn:shutdown() end -exports.connect = function (args) +exports.connect = function(args) local is_ok, connection = rspamd_tcp.connect_sync(args) if not is_ok then return is_ok, connection diff --git a/lualib/lua_urls_compose.lua b/lualib/lua_urls_compose.lua index 5216721b9..11134215a 100644 --- a/lualib/lua_urls_compose.lua +++ b/lualib/lua_urls_compose.lua @@ -39,8 +39,8 @@ local function process_url(self, log_obj, url_tld, url_host) lua_util.debugm(N, log_obj, 'found compose tld for %s (host = %s)', url_tld, url_host) - for _,excl in ipairs(tld_elt.except_rules) do - local matched,ret = excl[2](url_tld, url_host) + for _, excl in ipairs(tld_elt.except_rules) do + local matched, ret = excl[2](url_tld, url_host) if matched then lua_util.debugm(N, log_obj, 'found compose exclusion for %s (%s) -> %s', url_host, excl[1], ret) @@ -55,7 +55,7 @@ local function process_url(self, log_obj, url_tld, url_host) if matches then local lua_pat_idx = math.huge - for m,_ in pairs(matches) do + for m, _ in pairs(matches) do if m < lua_pat_idx then lua_pat_idx = m end @@ -63,7 +63,7 @@ local function process_url(self, log_obj, url_tld, url_host) if #tld_elt.compose_rules >= lua_pat_idx then local lua_pat = tld_elt.compose_rules[lua_pat_idx] - local matched,ret = lua_pat[2](url_tld, url_host) + local matched, ret = lua_pat[2](url_tld, url_host) if not matched then lua_util.debugm(N, log_obj, 'NOT found compose inclusion for %s (%s) -> %s', @@ -85,8 +85,8 @@ local function process_url(self, log_obj, url_tld, url_host) end else -- Match one by one - for _,lua_pat in ipairs(tld_elt.compose_rules) do - local matched,ret = lua_pat[2](url_tld, url_host) + for _, lua_pat in ipairs(tld_elt.compose_rules) do + local matched, ret = lua_pat[2](url_tld, url_host) if matched then lua_util.debugm(N, log_obj, 'found compose inclusion for %s (%s) -> %s', url_host, lua_pat[1], ret) @@ -128,7 +128,7 @@ local function include_elt_gen(pat) return function(_, host) local matches = pat:search(host, false, true) if matches then - return true,matches[1][2] + return true, matches[1][2] end return false @@ -139,7 +139,7 @@ local function exclude_elt_gen(pat) pat = rspamd_regexp.create(tld_pattern_transform(pat)) return function(tld, host) if pat:search(host) then - return true,tld + return true, tld end return false @@ -150,22 +150,26 @@ local function compose_map_cb(self, map_text) local lpeg = require "lpeg" local singleline_comment = lpeg.P '#' * (1 - lpeg.S '\r\n\f') ^ 0 - local comments_strip_grammar = lpeg.C((1 - lpeg.P '#') ^ 1) * lpeg.S(' \t')^0 * singleline_comment^0 + local comments_strip_grammar = lpeg.C((1 - lpeg.P '#') ^ 1) * lpeg.S(' \t') ^ 0 * singleline_comment ^ 0 local function process_tld_rule(tld_elt, l) if l:sub(1, 1) == '!' then -- Exclusion elt - table.insert(tld_elt.except_rules, {l, exclude_elt_gen(l:sub(2))}) + table.insert(tld_elt.except_rules, { l, exclude_elt_gen(l:sub(2)) }) else - table.insert(tld_elt.compose_rules, {l, include_elt_gen(l)}) + table.insert(tld_elt.compose_rules, { l, include_elt_gen(l) }) end end local function process_map_line(l) -- Skip empty lines and comments - if #l == 0 then return end + if #l == 0 then + return + end l = comments_strip_grammar:match(l) - if not l or #l == 0 then return end + if not l or #l == 0 then + return + end -- Get TLD local tld = rspamd_util.get_tld(l) @@ -195,12 +199,12 @@ local function compose_map_cb(self, map_text) end local multipattern_threshold = 1 - for tld,tld_elt in pairs(self.tlds) do + for tld, tld_elt in pairs(self.tlds) do -- Sort patterns to have longest labels before shortest ones, -- so we can ensure that they match before table.sort(tld_elt.compose_rules, function(e1, e2) - local _,ndots1 = string.gsub(e1[1], '(%.)', '') - local _,ndots2 = string.gsub(e2[1], '(%.)', '') + local _, ndots1 = string.gsub(e1[1], '(%.)', '') + local _, ndots2 = string.gsub(e2[1], '(%.)', '') return ndots1 > ndots2 end) @@ -237,11 +241,13 @@ exports.add_composition_map = function(cfg, map_obj) tlds = {}, } - map = cfg:add_map{ + map = cfg:add_map { type = 'callback', description = 'URL compose map', url = map_obj, - callback = function(input) compose_map_cb(ret, input) end, + callback = function(input) + compose_map_cb(ret, input) + end, opaque_data = true, } diff --git a/lualib/lua_verdict.lua b/lualib/lua_verdict.lua index 8f3f23ea9..6ce99e660 100644 --- a/lualib/lua_verdict.lua +++ b/lualib/lua_verdict.lua @@ -32,7 +32,7 @@ local function default_verdict_function(task) if result then if result.passthrough then - return 'passthrough',nil + return 'passthrough', nil end local score = result.score @@ -40,21 +40,21 @@ local function default_verdict_function(task) local action = result.action if action == 'reject' and result.npositive > 1 then - return 'spam',score + return 'spam', score elseif action == 'no action' then if score < 0 or result.nnegative > 3 then - return 'ham',score + return 'ham', score end else -- All colors of junk if action == 'add header' or action == 'rewrite subject' then if result.npositive > 2 then - return 'junk',score + return 'junk', score end end end - return 'uncertain',score + return 'uncertain', score end end @@ -120,8 +120,8 @@ end exports.set_verdict_table = function(verdict_tbl, what) assert(type(verdict_tbl) == 'table' and - type(verdict_tbl.callback) == 'function' and - type(verdict_tbl.possible_verdicts) == 'table') + type(verdict_tbl.callback) == 'function' and + type(verdict_tbl.possible_verdicts) == 'table') if not what then -- Default verdict @@ -195,7 +195,7 @@ end exports.adjust_passthrough_action = function(task) local action = task:get_metric_action() if action == 'soft reject' then - local has_pr,_,_,module = task:has_pre_result() + local has_pr, _, _, module = task:has_pre_result() if has_pr and module then action = module diff --git a/lualib/plugins/dmarc.lua b/lualib/plugins/dmarc.lua index 24efb5078..6acf982c3 100644 --- a/lualib/plugins/dmarc.lua +++ b/lualib/plugins/dmarc.lua @@ -73,7 +73,7 @@ exports.default_settings = { -- Returns a key used to be inserted into dmarc report sample -exports.dmarc_report = function (task, settings, data) +exports.dmarc_report = function(task, settings, data) local rspamd_lua_utils = require "lua_util" local E = {} @@ -100,16 +100,15 @@ exports.dmarc_report = function (task, settings, data) local res = table.concat({ ip, data.spf_ok, data.dkim_ok, disposition_to_return, (data.sampled_out and 'sampled_out' or ''), data.domain, - dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, data.spf_domain, data.spf_result}, ',') + dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, data.spf_domain, data.spf_result }, ',') return res end - exports.gen_munging_callback = function(munging_opts, settings) local rspamd_util = require "rspamd_util" local lua_mime = require "lua_mime" - return function (task) + return function(task) if munging_opts.mitigate_allow_only then if not task:has_symbol(settings.symbols.allow) then lua_util.debugm(N, task, 'skip munging, no %s symbol', @@ -131,11 +130,11 @@ exports.gen_munging_callback = function(munging_opts, settings) end end if munging_opts.mitigate_strict_only then - local s = task:get_symbol(settings.symbols.allow) or {[1] = {}} + local s = task:get_symbol(settings.symbols.allow) or { [1] = {} } local sopts = s[1].options or {} local seen_strict - for _,o in ipairs(sopts) do + for _, o in ipairs(sopts) do if o == 'reject' or o == 'quarantine' then seen_strict = true break @@ -150,7 +149,7 @@ exports.gen_munging_callback = function(munging_opts, settings) end end if munging_opts.munge_map_condition then - local accepted,trace = munging_opts.munge_map_condition:process(task) + local accepted, trace = munging_opts.munge_map_condition:process(task) if not accepted then lua_util.debugm(task, 'skip munging, maps condition not satisfied: (%s)', trace) @@ -159,10 +158,10 @@ exports.gen_munging_callback = function(munging_opts, settings) end end -- Now, look for domain for munging - local mr = task:get_recipients({ 'mime', 'orig'}) + local mr = task:get_recipients({ 'mime', 'orig' }) local rcpt_found if mr then - for _,r in ipairs(mr) do + for _, r in ipairs(mr) do if r.domain and munging_opts.list_map:get_key(r.addr) then rcpt_found = r break @@ -176,7 +175,7 @@ exports.gen_munging_callback = function(munging_opts, settings) return end - local from = task:get_from({ 'mime', 'orig'}) + local from = task:get_from({ 'mime', 'orig' }) if not from or not from[1] then lua_util.debugm(task, 'skip munging, from is bad') @@ -204,7 +203,7 @@ exports.gen_munging_callback = function(munging_opts, settings) local add_hdrs = { ['From'] = { order = 1, value = hdr_encoded }, } - local remove_hdrs = {['From'] = 0} + local remove_hdrs = { ['From'] = 0 } local nreply = from.addr if munging_opts.reply_goes_to_list then @@ -222,9 +221,9 @@ exports.gen_munging_callback = function(munging_opts, settings) remove_hdrs['Reply-To'] = 1 end - add_hdrs['Reply-To'] = {order = 0, value = nreply} + add_hdrs['Reply-To'] = { order = 0, value = nreply } - add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded} + add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded } lua_mime.modify_headers(task, { remove = remove_hdrs, add = add_hdrs @@ -239,12 +238,12 @@ end local function gen_dmarc_grammar() local lpeg = require "lpeg" lpeg.locale(lpeg) - local space = lpeg.space^0 - local name = lpeg.C(lpeg.alpha^1) * space - local sep = space * (lpeg.S("\\;") * space) + (lpeg.space^1) - local value = lpeg.C(lpeg.P(lpeg.graph - sep)^1) - local pair = lpeg.Cg(name * "=" * space * value) * sep^-1 - local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset) + local space = lpeg.space ^ 0 + local name = lpeg.C(lpeg.alpha ^ 1) * space + local sep = space * (lpeg.S("\\;") * space) + (lpeg.space ^ 1) + local value = lpeg.C(lpeg.P(lpeg.graph - sep) ^ 1) + local pair = lpeg.Cg(name * "=" * space * value) * sep ^ -1 + local list = lpeg.Cf(lpeg.Ct("") * pair ^ 0, rawset) local version = lpeg.P("v") * space * lpeg.P("=") * space * lpeg.P("DMARC1") local record = version * sep * list @@ -297,7 +296,7 @@ local function dmarc_check_record(log_obj, record, is_tld) result.strict_dkim = true elseif dkim_pol ~= 'r' then failed_policy = 'adkim tag has invalid value: ' .. dkim_pol - return false,failed_policy + return false, failed_policy end end @@ -307,7 +306,7 @@ local function dmarc_check_record(log_obj, record, is_tld) result.strict_spf = true elseif spf_pol ~= 'r' then failed_policy = 'aspf tag has invalid value: ' .. spf_pol - return false,failed_policy + return false, failed_policy end end @@ -319,7 +318,7 @@ local function dmarc_check_record(log_obj, record, is_tld) result.dmarc_policy = 'quarantine' elseif (policy ~= 'none') then failed_policy = 'p tag has invalid value: ' .. policy - return false,failed_policy + return false, failed_policy end end @@ -336,7 +335,7 @@ local function dmarc_check_record(log_obj, record, is_tld) result.dmarc_policy = 'none' elseif (subdomain_policy ~= 'none') then failed_policy = 'sp tag has invalid value: ' .. subdomain_policy - return false,failed_policy + return false, failed_policy end end result.pct = elts['pct'] @@ -349,7 +348,7 @@ local function dmarc_check_record(log_obj, record, is_tld) end result.raw_elts = elts else - return false,false -- Ignore garbage + return false, false -- Ignore garbage end return true, result diff --git a/lualib/plugins/neural.lua b/lualib/plugins/neural.lua index 05dace489..6e88ef21c 100644 --- a/lualib/plugins/neural.lua +++ b/lualib/plugins/neural.lua @@ -96,7 +96,6 @@ local module_config = rspamd_config:get_all_opt(N) settings = lua_util.override_defaults(settings, module_config) local redis_params = lua_redis.parse_redis_server('neural') - local redis_lua_script_vectors_len = "neural_train_size.lua" local redis_lua_script_maybe_invalidate = "neural_maybe_invalidate.lua" local redis_lua_script_maybe_lock = "neural_maybe_lock.lua" @@ -106,17 +105,17 @@ local redis_script_id = {} local function load_scripts() redis_script_id.vectors_len = lua_redis.load_redis_script_from_file(redis_lua_script_vectors_len, - redis_params) + redis_params) redis_script_id.maybe_invalidate = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_invalidate, - redis_params) + redis_params) redis_script_id.maybe_lock = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_lock, - redis_params) + redis_params) redis_script_id.save_unlock = lua_redis.load_redis_script_from_file(redis_lua_script_save_unlock, - redis_params) + redis_params) end local function create_ann(n, nlayers, rule) - -- We ignore number of layers so far when using kann + -- We ignore number of layers so far when using kann local nhidden = math.floor(n * (rule.hidden_layer_mult or 1.0) + 1.0) local t = rspamd_kann.layer.input(n) t = rspamd_kann.transform.relu(t) @@ -146,7 +145,7 @@ local function learn_pca(inputs, max_inputs) -- scatter matrix is not filled with eigenvectors lua_util.debugm(N, 'eigenvalues: %s', eigenvals) local w = rspamd_tensor.new(2, max_inputs, #scatter_matrix[1]) - for i=1,max_inputs do + for i = 1, max_inputs do w[i] = scatter_matrix[#scatter_matrix - i + 1] end @@ -172,15 +171,19 @@ local function get_roc_thresholds(ann, inputs, outputs, alpha, beta) local a = {} local b = {} - for i=1,n do + for i = 1, n do r[i] = i end - local cmp = function(p, q) return p < q end + local cmp = function(p, q) + return p < q + end - table.sort(r, function(p, q) return cmp(x[p], x[q]) end) + table.sort(r, function(p, q) + return cmp(x[p], x[q]) + end) - for i=1,n do + for i = 1, n do a[i] = x[r[i]] b[i] = y[r[i]] end @@ -190,89 +193,89 @@ local function get_roc_thresholds(ann, inputs, outputs, alpha, beta) local function get_scores(nn, input_vectors) local scores = {} - for i=1,#inputs do + for i = 1, #inputs do local score = nn:apply1(input_vectors[i], nn.pca)[1] - scores[#scores+1] = score + scores[#scores + 1] = score end return scores end local fpr = {} - local fnr = {} - local scores = get_scores(ann, inputs) - - scores, outputs = sort_relative(scores, outputs) - - local n_samples = #outputs - local n_spam = 0 - local n_ham = 0 - local ham_count_ahead = {} - local spam_count_ahead = {} - local ham_count_behind = {} - local spam_count_behind = {} - - ham_count_ahead[n_samples + 1] = 0 - spam_count_ahead[n_samples + 1] = 0 - - for i=n_samples,1,-1 do - - if outputs[i][1] == 0 then - n_ham = n_ham + 1 - ham_count_ahead[i] = 1 - spam_count_ahead[i] = 0 - else - n_spam = n_spam + 1 - ham_count_ahead[i] = 0 - spam_count_ahead[i] = 1 - end - - ham_count_ahead[i] = ham_count_ahead[i] + ham_count_ahead[i + 1] - spam_count_ahead[i] = spam_count_ahead[i] + spam_count_ahead[i + 1] - end - - for i=1,n_samples do + local fnr = {} + local scores = get_scores(ann, inputs) + + scores, outputs = sort_relative(scores, outputs) + + local n_samples = #outputs + local n_spam = 0 + local n_ham = 0 + local ham_count_ahead = {} + local spam_count_ahead = {} + local ham_count_behind = {} + local spam_count_behind = {} + + ham_count_ahead[n_samples + 1] = 0 + spam_count_ahead[n_samples + 1] = 0 + + for i = n_samples, 1, -1 do + + if outputs[i][1] == 0 then + n_ham = n_ham + 1 + ham_count_ahead[i] = 1 + spam_count_ahead[i] = 0 + else + n_spam = n_spam + 1 + ham_count_ahead[i] = 0 + spam_count_ahead[i] = 1 + end + + ham_count_ahead[i] = ham_count_ahead[i] + ham_count_ahead[i + 1] + spam_count_ahead[i] = spam_count_ahead[i] + spam_count_ahead[i + 1] + end + + for i = 1, n_samples do if outputs[i][1] == 0 then - ham_count_behind[i] = 1 - spam_count_behind[i] = 0 - else - ham_count_behind[i] = 0 - spam_count_behind[i] = 1 - end - - if i ~= 1 then - ham_count_behind[i] = ham_count_behind[i] + ham_count_behind[i - 1] - spam_count_behind[i] = spam_count_behind[i] + spam_count_behind[i - 1] - end - end - - for i=1,n_samples do - fpr[i] = 0 - fnr[i] = 0 - - if (ham_count_ahead[i + 1] + ham_count_behind[i]) ~= 0 then - fpr[i] = ham_count_ahead[i + 1] / (ham_count_ahead[i + 1] + ham_count_behind[i]) - end - - if (spam_count_behind[i] + spam_count_ahead[i + 1]) ~= 0 then - fnr[i] = spam_count_behind[i] / (spam_count_behind[i] + spam_count_ahead[i + 1]) - end - end - - local p = n_spam / (n_spam + n_ham) - - local cost = {} - local min_cost_idx = 0 - local min_cost = math.huge - for i=1,n_samples do - cost[i] = ((1 - p) * alpha * fpr[i]) + (p * beta * fnr[i]) - if min_cost >= cost[i] then - min_cost = cost[i] - min_cost_idx = i - end - end - - return scores[min_cost_idx] + ham_count_behind[i] = 1 + spam_count_behind[i] = 0 + else + ham_count_behind[i] = 0 + spam_count_behind[i] = 1 + end + + if i ~= 1 then + ham_count_behind[i] = ham_count_behind[i] + ham_count_behind[i - 1] + spam_count_behind[i] = spam_count_behind[i] + spam_count_behind[i - 1] + end + end + + for i = 1, n_samples do + fpr[i] = 0 + fnr[i] = 0 + + if (ham_count_ahead[i + 1] + ham_count_behind[i]) ~= 0 then + fpr[i] = ham_count_ahead[i + 1] / (ham_count_ahead[i + 1] + ham_count_behind[i]) + end + + if (spam_count_behind[i] + spam_count_ahead[i + 1]) ~= 0 then + fnr[i] = spam_count_behind[i] / (spam_count_behind[i] + spam_count_ahead[i + 1]) + end + end + + local p = n_spam / (n_spam + n_ham) + + local cost = {} + local min_cost_idx = 0 + local min_cost = math.huge + for i = 1, n_samples do + cost[i] = ((1 - p) * alpha * fpr[i]) + (p * beta * fnr[i]) + if min_cost >= cost[i] then + min_cost = cost[i] + min_cost_idx = i + end + end + + return scores[min_cost_idx] end -- This function is intended to extend lock for ANN during training @@ -299,7 +302,7 @@ local function register_lock_extender(rule, set, ev_base, ann_key) true, -- is write redis_lock_extend_cb, --callback 'HINCRBY', -- command - {ann_key, 'lock', '30'} + { ann_key, 'lock', '30' } ) else lua_util.debugm(N, rspamd_config, "stop lock extension as learning_spawned is false") @@ -337,7 +340,8 @@ local function can_push_train_vector(rule, task, learn_type, nspam, nham) end end return true - else -- Enough learns + else + -- Enough learns rspamd_logger.infox(task, 'skip %s sample to keep spam/ham balance; too many spam samples: %s', learn_type, nspam) @@ -403,7 +407,7 @@ end -- Closure generator for unlock function local function gen_unlock_cb(rule, set, ann_key) - return function (err) + return function(err) if err then rspamd_logger.errx(rspamd_config, 'cannot unlock ANN %s:%s at %s from redis: %s', rule.prefix, set.name, ann_key, err) @@ -426,7 +430,7 @@ local function redis_ann_prefix(rule, settings_name) -- We also need to count metatokens: local n = meta_functions.version return string.format('%s%d_%s_%d_%s', - settings.prefix, plugin_ver, rule.prefix, n, settings_name) + settings.prefix, plugin_ver, rule.prefix, n, settings_name) end -- This function receives training vectors, checks them, spawn learning and saves ANN in Redis @@ -449,7 +453,7 @@ local function spawn_train(params) -- Used to show parsed vectors in a convenient format (for debugging only) local function debug_vec(t) local ret = {} - for i,v in ipairs(t) do + for i, v in ipairs(t) do if v ~= 0 then ret[#ret + 1] = string.format('%d=%.2f', i, v) end @@ -462,14 +466,14 @@ local function spawn_train(params) -- KANN automatically shuffles those samples -- 1.0 is used for spam and -1.0 is used for ham -- It implies that output layer can express that (e.g. tanh output) - for _,e in ipairs(params.spam_vec) do + for _, e in ipairs(params.spam_vec) do inputs[#inputs + 1] = e - outputs[#outputs + 1] = {1.0} + outputs[#outputs + 1] = { 1.0 } --rspamd_logger.debugm(N, rspamd_config, 'spam vector: %s', debug_vec(e)) end - for _,e in ipairs(params.ham_vec) do + for _, e in ipairs(params.ham_vec) do inputs[#inputs + 1] = e - outputs[#outputs + 1] = {-1.0} + outputs[#outputs + 1] = { -1.0 } --rspamd_logger.debugm(N, rspamd_config, 'ham vector: %s', debug_vec(e)) end @@ -486,7 +490,7 @@ local function spawn_train(params) rspamd_logger.errx(rspamd_config, 'ANN %s:%s: train error: observed nan in error cost!; value cost = %s', params.rule.prefix, params.set.name, value_cost) - for i,e in ipairs(inputs) do + for i, e in ipairs(inputs) do lua_util.debugm(N, rspamd_config, 'train vector %s -> %s', debug_vec(e), outputs[i][1]) end @@ -515,7 +519,7 @@ local function spawn_train(params) lua_util.debugm(N, rspamd_config, "start neural train for ANN %s:%s", params.rule.prefix, params.set.name) - local ret,err = pcall(train_ann.train1, train_ann, + local ret, err = pcall(train_ann.train1, train_ann, inputs, outputs, { lr = params.rule.train.learning_rate, max_epoch = params.rule.train.max_iterations, @@ -536,19 +540,19 @@ local function spawn_train(params) local roc_thresholds = {} if params.rule.roc_enabled then local spam_threshold = get_roc_thresholds(train_ann, - inputs, - outputs, - 1 - params.rule.roc_misclassification_cost, - params.rule.roc_misclassification_cost) + inputs, + outputs, + 1 - params.rule.roc_misclassification_cost, + params.rule.roc_misclassification_cost) local ham_threshold = get_roc_thresholds(train_ann, - inputs, - outputs, - params.rule.roc_misclassification_cost, - 1 - params.rule.roc_misclassification_cost) - roc_thresholds = {spam_threshold, ham_threshold} + inputs, + outputs, + params.rule.roc_misclassification_cost, + 1 - params.rule.roc_misclassification_cost) + roc_thresholds = { spam_threshold, ham_threshold } rspamd_logger.messagex("ROC thresholds: (spam_threshold: %s, ham_threshold: %s)", - roc_thresholds[1], roc_thresholds[2]) + roc_thresholds[1], roc_thresholds[2]) end if not seen_nan then @@ -585,7 +589,7 @@ local function spawn_train(params) false, -- is write gen_unlock_cb(params.rule, params.set, params.ann_key), --callback 'HDEL', -- command - {params.ann_key, 'lock'} + { params.ann_key, 'lock' } ) else rspamd_logger.infox(rspamd_config, 'saved ANN %s:%s to redis: %s', @@ -605,7 +609,7 @@ local function spawn_train(params) true, -- is write gen_unlock_cb(params.rule, params.set, params.ann_key), --callback 'HDEL', -- command - {params.ann_key, 'lock'} + { params.ann_key, 'lock' } ) else local parser = ucl.parser() @@ -653,17 +657,17 @@ local function spawn_train(params) params.set.ann.redis_key, params.ann_key) lua_redis.exec_redis_script(redis_script_id.save_unlock, - {ev_base = params.ev_base, is_write = true}, + { ev_base = params.ev_base, is_write = true }, redis_save_cb, - {profile.redis_key, - redis_ann_prefix(params.rule, params.set.name), - ann_data, - profile_serialized, - tostring(params.rule.ann_expire), - tostring(os.time()), - params.ann_key, -- old key to unlock... - roc_thresholds_serialized, - pca_data, + { profile.redis_key, + redis_ann_prefix(params.rule, params.set.name), + ann_data, + profile_serialized, + tostring(params.rule.ann_expire), + tostring(os.time()), + params.ann_key, -- old key to unlock... + roc_thresholds_serialized, + pca_data, }) end end @@ -672,7 +676,7 @@ local function spawn_train(params) fill_set_ann(params.set, params.ann_key) end - params.worker:spawn_process{ + params.worker:spawn_process { func = train, on_complete = ann_trained, proctitle = string.format("ANN train for %s/%s", params.rule.prefix, params.set.name), @@ -695,7 +699,9 @@ local function process_rules_settings() -- Ensure that we have an array... lua_util.debugm(N, rspamd_config, "use static profile for %s (%s): %s", rule.prefix, selt.name, profile) - if not profile[1] then profile = lua_util.keys(profile) end + if not profile[1] then + profile = lua_util.keys(profile) + end selt.symbols = profile else lua_util.debugm(N, rspamd_config, "use dynamic cfg based profile for %s (%s)", @@ -758,7 +764,7 @@ local function process_rules_settings() }) end - for k,rule in pairs(settings.rules) do + for k, rule in pairs(settings.rules) do if not rule.allowed_settings then rule.allowed_settings = {} elseif rule.allowed_settings == 'all' then @@ -788,7 +794,7 @@ local function process_rules_settings() -- Now, for each allowed settings, we store sorted symbols + digest -- We set table rule.settings[id] -> { name = name, symbols = symbols, digest = digest } - for s,_ in pairs(rule.allowed_settings) do + for s, _ in pairs(rule.allowed_settings) do -- Here, we have a name, set of symbols and local settings_id = s if type(settings_id) ~= 'number' then @@ -802,7 +808,7 @@ local function process_rules_settings() } process_settings_elt(rule, nelt) - for id,ex in pairs(rule.settings) do + for id, ex in pairs(rule.settings) do if type(ex) == 'table' then if nelt and lua_util.distance_sorted(ex.symbols, nelt.symbols) == 0 then -- Equal symbols, add reference @@ -829,7 +835,9 @@ local function get_rule_settings(task, rule) local sid = task:get_settings_id() or -1 local set = rule.settings[sid] - if not set then return nil end + if not set then + return nil + end while type(set) == 'number' do -- Reference to another settings! @@ -843,10 +851,10 @@ local function result_to_vector(task, profile) if not profile.zeros then -- Fill zeros vector local zeros = {} - for i=1,meta_functions.count_metatokens() do + for i = 1, meta_functions.count_metatokens() do zeros[i] = 0.0 end - for _,_ in ipairs(profile.symbols) do + for _, _ in ipairs(profile.symbols) do zeros[#zeros + 1] = 0.0 end profile.zeros = zeros @@ -855,7 +863,7 @@ local function result_to_vector(task, profile) local vec = lua_util.shallowcopy(profile.zeros) local mt = meta_functions.rspamd_gen_metatokens(task) - for i,v in ipairs(mt) do + for i, v in ipairs(mt) do vec[i] = v end diff --git a/lualib/plugins/rbl.lua b/lualib/plugins/rbl.lua index baa019f41..bff53f9ba 100644 --- a/lualib/plugins/rbl.lua +++ b/lualib/plugins/rbl.lua @@ -21,24 +21,24 @@ local lua_util = require "lua_util" -- Common RBL plugin definitions local check_types = { - from = { - connfilter = true, - }, - received = {}, - helo = { - connfilter = true, - }, - urls = {}, - content_urls = {}, - emails = {}, - replyto = {}, - dkim = {}, - rdns = { - connfilter = true, - }, - selector = { - require_argument = true, - }, + from = { + connfilter = true, + }, + received = {}, + helo = { + connfilter = true, + }, + urls = {}, + content_urls = {}, + emails = {}, + replyto = {}, + dkim = {}, + rdns = { + connfilter = true, + }, + selector = { + require_argument = true, + }, } local default_options = { @@ -93,8 +93,8 @@ local rule_schema_tbl = { exclude_private_ips = ts.boolean:is_optional(), exclude_users = ts.boolean:is_optional(), from = ts.boolean:is_optional(), - hash = ts.one_of{"sha1", "sha256", "sha384", "sha512", "md5", "blake2"}:is_optional(), - hash_format = ts.one_of{"hex", "base32", "base64"}:is_optional(), + hash = ts.one_of { "sha1", "sha256", "sha384", "sha512", "md5", "blake2" }:is_optional(), + hash_format = ts.one_of { "hex", "base32", "base64" }:is_optional(), hash_len = (ts.integer + ts.string / tonumber):is_optional(), helo = ts.boolean:is_optional(), ignore_default = ts.boolean:is_optional(), -- alias @@ -120,14 +120,16 @@ local rule_schema_tbl = { replyto = ts.boolean:is_optional(), requests_limit = (ts.integer + ts.string / tonumber):is_optional(), require_symbols = ( - ts.array_of(ts.string) + (ts.string / function(s) return {s} end) + ts.array_of(ts.string) + (ts.string / function(s) + return { s } + end) ):is_optional(), resolve_ip = ts.boolean:is_optional(), return_bits = return_bits_schema:is_optional(), return_codes = return_codes_schema:is_optional(), returnbits = return_bits_schema:is_optional(), returncodes = return_codes_schema:is_optional(), - selector = ts.one_of{ts.string, ts.table}:is_optional(), + selector = ts.one_of { ts.string, ts.table }:is_optional(), selector_flatten = ts.boolean:is_optional(), symbol = ts.string:is_optional(), symbols_prefixes = ts.map_of(ts.string, ts.string):is_optional(), @@ -137,7 +139,9 @@ local rule_schema_tbl = { urls = ts.boolean:is_optional(), whitelist = lua_maps.map_schema:is_optional(), whitelist_exception = ( - ts.array_of(ts.string) + (ts.string / function(s) return {s} end) + ts.array_of(ts.string) + (ts.string / function(s) + return { s } + end) ):is_optional(), checks = ts.array_of(ts.one_of(lua_util.keys(check_types))):is_optional(), exclude_checks = ts.array_of(ts.one_of(lua_util.keys(check_types))):is_optional(), @@ -148,13 +152,13 @@ local function convert_checks(rule) if rule.checks then local all_connfilter = true local exclude_checks = lua_util.list_to_hash(rule.exclude_checks or {}) - for _,check in ipairs(rule.checks) do + for _, check in ipairs(rule.checks) do if not exclude_checks[check] then local check_type = check_types[check] if check_type.require_argument then if not rule[check] then rspamd_logger.errx(rspamd_config, 'rbl rule %s has check %s which requires an argument', - rule.symbol, check) + rule.symbol, check) return nil end end @@ -167,12 +171,12 @@ local function convert_checks(rule) if not check_type then rspamd_logger.errx(rspamd_config, 'rbl rule %s has invalid check type: %s', - rule.symbol, check) + rule.symbol, check) return nil end else rspamd_logger.infox(rspamd_config, 'disable check %s in %s: excluded explicitly', - check, rule.symbol) + check, rule.symbol) end end rule.connfilter = all_connfilter @@ -180,7 +184,7 @@ local function convert_checks(rule) -- Now check if we have any check enabled at all local check_found = false - for k,_ in pairs(check_types) do + for k, _ in pairs(check_types) do if type(rule[k]) ~= 'nil' then check_found = true break @@ -199,7 +203,7 @@ end -- Add default boolean flags to the schema -for def_k,_ in pairs(default_options) do +for def_k, _ in pairs(default_options) do rule_schema_tbl[def_k:sub(#('default_') + 1)] = ts.boolean:is_optional() end diff --git a/lualib/redis_scripts/neural_maybe_invalidate.lua b/lualib/redis_scripts/neural_maybe_invalidate.lua index c54871717..517fa019d 100644 --- a/lualib/redis_scripts/neural_maybe_invalidate.lua +++ b/lualib/redis_scripts/neural_maybe_invalidate.lua @@ -8,7 +8,7 @@ local lim = tonumber(KEYS[2]) if card > lim then local to_delete = redis.call('ZRANGE', KEYS[1], 0, card - lim - 1) if to_delete then - for _,k in ipairs(to_delete) do + for _, k in ipairs(to_delete) do local tb = cjson.decode(k) if type(tb) == 'table' and type(tb.redis_key) == 'string' then redis.call('DEL', tb.redis_key) diff --git a/lualib/redis_scripts/neural_maybe_lock.lua b/lualib/redis_scripts/neural_maybe_lock.lua index 7b5c6a60f..f705115b0 100644 --- a/lualib/redis_scripts/neural_maybe_lock.lua +++ b/lualib/redis_scripts/neural_maybe_lock.lua @@ -11,7 +11,7 @@ if locked then locked = tonumber(locked) local expire = tonumber(KEYS[3]) if now > locked and (now - locked) < expire then - return {tostring(locked), redis.call('HGET', KEYS[1], 'hostname') or 'unknown'} + return { tostring(locked), redis.call('HGET', KEYS[1], 'hostname') or 'unknown' } end end redis.call('HSET', KEYS[1], 'lock', tostring(now)) diff --git a/lualib/redis_scripts/neural_train_size.lua b/lualib/redis_scripts/neural_train_size.lua index 5a00ae3fc..45ad6a931 100644 --- a/lualib/redis_scripts/neural_train_size.lua +++ b/lualib/redis_scripts/neural_train_size.lua @@ -13,8 +13,12 @@ local nspam = 0 local nham = 0 local ret = redis.call('SCARD', prefix .. '_spam_set') -if ret then nspam = tonumber(ret) end +if ret then + nspam = tonumber(ret) +end ret = redis.call('SCARD', prefix .. '_ham_set') -if ret then nham = tonumber(ret) end +if ret then + nham = tonumber(ret) +end -return {nspam,nham}
\ No newline at end of file +return { nspam, nham }
\ No newline at end of file diff --git a/lualib/redis_scripts/ratelimit_check.lua b/lualib/redis_scripts/ratelimit_check.lua index 1c2b32a69..d39cdf148 100644 --- a/lualib/redis_scripts/ratelimit_check.lua +++ b/lualib/redis_scripts/ratelimit_check.lua @@ -34,36 +34,46 @@ if not last then -- New bucket redis.call('HMSET', prefix, 'l', tostring(now), 'b', '0', 'dr', '10000', 'db', '10000', 'p', tostring(nrcpt)) redis.call('EXPIRE', prefix, KEYS[5]) - return {0, '0', '1', '1', '0'} + return { 0, '0', '1', '1', '0' } end last = tonumber(last) -local burst,pending = unpack(redis.call('HMGET', prefix, 'b', 'p')) -burst,pending = tonumber(burst or '0'),tonumber(pending or '0') +local burst, pending = unpack(redis.call('HMGET', prefix, 'b', 'p')) +burst, pending = tonumber(burst or '0'), tonumber(pending or '0') -- Sanity to avoid races -if burst < 0 then burst = 0 end -if pending < 0 then pending = 0 end +if burst < 0 then + burst = 0 +end +if pending < 0 then + pending = 0 +end pending = pending + nrcpt -- this message -- Perform leak if burst + pending > 0 then -- If we have any time passed if burst > 0 and last < now then dynr = tonumber(redis.call('HGET', prefix, 'dr')) / 10000.0 - if dynr == 0 then dynr = 0.0001 end + if dynr == 0 then + dynr = 0.0001 + end leak_rate = leak_rate * dynr leaked = ((now - last) * leak_rate) - if leaked > burst then leaked = burst end + if leaked > burst then + leaked = burst + end burst = burst - leaked redis.call('HINCRBYFLOAT', prefix, 'b', -(leaked)) redis.call('HSET', prefix, 'l', tostring(now)) end dynb = tonumber(redis.call('HGET', prefix, 'db')) / 10000.0 - if dynb == 0 then dynb = 0.0001 end + if dynb == 0 then + dynb = 0.0001 + end burst = burst + pending if burst > 0 and burst > max_burst * dynb then - return {1, tostring(burst - pending), tostring(dynr), tostring(dynb), tostring(leaked)} + return { 1, tostring(burst - pending), tostring(dynr), tostring(dynb), tostring(leaked) } end -- Increase pending if we allow ratelimit redis.call('HINCRBY', prefix, 'p', nrcpt) @@ -72,4 +82,4 @@ else redis.call('HMSET', prefix, 'b', '0', 'p', tostring(nrcpt)) end -return {0, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked)}
\ No newline at end of file +return { 0, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked) }
\ No newline at end of file diff --git a/lualib/redis_scripts/ratelimit_cleanup_pending.lua b/lualib/redis_scripts/ratelimit_cleanup_pending.lua index 67ae634f1..698a3ec19 100644 --- a/lualib/redis_scripts/ratelimit_cleanup_pending.lua +++ b/lualib/redis_scripts/ratelimit_cleanup_pending.lua @@ -19,7 +19,11 @@ end -- 2. Update the pending values based on the number of recipients (requests) local pending = redis.call('HGET', prefix, 'p') pending = tonumber(pending or '0') -if pending < nrcpt then pending = 0 else pending = pending - nrcpt end +if pending < nrcpt then + pending = 0 +else + pending = pending - nrcpt +end -- 3. Set the updated values back to Redis and update the expiration time for the bucket redis.call('HMSET', prefix, 'p', tostring(pending), 'l', KEYS[2]) diff --git a/lualib/redis_scripts/ratelimit_update.lua b/lualib/redis_scripts/ratelimit_update.lua index f08a250ea..caee8fb31 100644 --- a/lualib/redis_scripts/ratelimit_update.lua +++ b/lualib/redis_scripts/ratelimit_update.lua @@ -19,7 +19,7 @@ if not last then -- 2. Initialize a new bucket if the last hit time is not found (must not happen) redis.call('HMSET', prefix, 'l', tostring(now), 'b', tostring(nrcpt), 'dr', '10000', 'db', '10000', 'p', '0') redis.call('EXPIRE', prefix, KEYS[7]) - return {1, 1, 1} + return { 1, 1, 1 } end -- 3. Update the dynamic rate multiplier based on input parameters @@ -72,14 +72,22 @@ if max_db > 1 then end -- 5. Update the burst and pending values based on the number of recipients (requests) -local burst,pending = unpack(redis.call('HMGET', prefix, 'b', 'p')) -burst,pending = tonumber(burst or '0'),tonumber(pending or '0') -if burst < 0 then burst = nrcpt else burst = burst + nrcpt end -if pending < nrcpt then pending = 0 else pending = pending - nrcpt end +local burst, pending = unpack(redis.call('HMGET', prefix, 'b', 'p')) +burst, pending = tonumber(burst or '0'), tonumber(pending or '0') +if burst < 0 then + burst = nrcpt +else + burst = burst + nrcpt +end +if pending < nrcpt then + pending = 0 +else + pending = pending - nrcpt +end -- 6. Set the updated values back to Redis and update the expiration time for the bucket redis.call('HMSET', prefix, 'b', tostring(burst), 'p', tostring(pending), 'l', KEYS[2]) redis.call('EXPIRE', prefix, KEYS[7]) -- 7. Return the updated burst value, dynamic rate multiplier, and dynamic burst multiplier -return {tostring(burst), tostring(dr), tostring(db)}
\ No newline at end of file +return { tostring(burst), tostring(dr), tostring(db) }
\ No newline at end of file diff --git a/lualib/rescore_utility.lua b/lualib/rescore_utility.lua index 195ae4364..01e5aabcb 100644 --- a/lualib/rescore_utility.lua +++ b/lualib/rescore_utility.lua @@ -11,7 +11,7 @@ function utility.get_all_symbols(logs, ignore_symbols) for _, line in pairs(logs) do line = lua_util.rspamd_str_split(line, " ") - for i=4,(#line-1) do + for i = 4, (#line - 1) do line[i] = line[i]:gsub("%s+", "") if not symbols_set[line[i]] then symbols_set[line[i]] = true @@ -41,57 +41,57 @@ function utility.read_log_file(file) local fname = string.gsub(file, "(.*/)(.*)", "%2") for line in fd:lines() do - local start,stop = string.find(line, fname .. ':') + local start, stop = string.find(line, fname .. ':') if start and stop then table.insert(lines, string.sub(line, 1, start)) table.insert(messages, string.sub(line, stop + 1, -1)) end -end + end io.close(fd) - return lines,messages + return lines, messages end function utility.get_all_logs(dirs) -- Reads all log files in the directory and returns a list of logs. if type(dirs) == 'string' then - dirs = {dirs} + dirs = { dirs } end local all_logs = {} local all_messages = {} - for _,dir in ipairs(dirs) do + for _, dir in ipairs(dirs) do if dir:sub(-1, -1) == "/" then dir = dir:sub(1, -2) local files = rspamd_util.glob(dir .. "/*.log") for _, file in pairs(files) do - local logs,messages = utility.read_log_file(file) - for i=1,#logs do + local logs, messages = utility.read_log_file(file) + for i = 1, #logs do table.insert(all_logs, logs[i]) table.insert(all_messages, messages[i]) end end else - local logs,messages = utility.read_log_file(dir) - for i=1,#logs do + local logs, messages = utility.read_log_file(dir) + for i = 1, #logs do table.insert(all_logs, logs[i]) table.insert(all_messages, messages[i]) end end end - return all_logs,all_messages + return all_logs, all_messages end function utility.get_all_symbol_scores(conf, ignore_symbols) local symbols = conf:get_symbols_scores() return fun.tomap(fun.map(function(name, elt) - return name,elt['score'] + return name, elt['score'] end, fun.filter(function(name, elt) return not ignore_symbols[name] end, symbols))) @@ -158,7 +158,7 @@ function utility.generate_statistics_from_logs(logs, messages, threshold) true_negatives = true_negatives + 1 end - for j=4, (#log-1) do + for j = 4, (#log - 1) do if all_symbols_stats[log[j]] == nil then all_symbols_stats[log[j]] = { name = message, @@ -180,8 +180,8 @@ function utility.generate_statistics_from_logs(logs, messages, threshold) -- Find slowest message if ((tonumber(log[#log]) or 0) > file_stats.slowest) then - file_stats.slowest = tonumber(log[#log]) - file_stats.slowest_file = message + file_stats.slowest = tonumber(log[#log]) + file_stats.slowest_file = message end end end diff --git a/lualib/rspamadm/clickhouse.lua b/lualib/rspamadm/clickhouse.lua index 6bdc612d8..b22d8007c 100644 --- a/lualib/rspamadm/clickhouse.lua +++ b/lualib/rspamadm/clickhouse.lua @@ -66,62 +66,62 @@ parser:flag '--use-https' :argname('use_https') local neural_profile = parser:command 'neural_profile' - :description 'Generate symbols profile using data from Clickhouse' + :description 'Generate symbols profile using data from Clickhouse' neural_profile:option '-w --where' - :description 'WHERE clause for Clickhouse query' - :argname('where') + :description 'WHERE clause for Clickhouse query' + :argname('where') neural_profile:flag '-j --json' - :description 'Write output as JSON' - :argname('json') + :description 'Write output as JSON' + :argname('json') neural_profile:option '--days' - :description 'Number of days to collect stats for' - :argname('days') - :default('7') + :description 'Number of days to collect stats for' + :argname('days') + :default('7') neural_profile:option '--limit -l' - :description 'Maximum rows to fetch per day' - :argname('limit') + :description 'Maximum rows to fetch per day' + :argname('limit') neural_profile:option '--settings-id' - :description 'Settings ID to query' - :argname('settings_id') - :default('') + :description 'Settings ID to query' + :argname('settings_id') + :default('') local neural_train = parser:command 'neural_train' - :description 'Train neural using data from Clickhouse' + :description 'Train neural using data from Clickhouse' neural_train:option '--days' - :description 'Number of days to query data for' - :argname('days') - :default('7') + :description 'Number of days to query data for' + :argname('days') + :default('7') neural_train:option '--column-name-digest' - :description 'Name of neural profile digest column in Clickhouse' - :argname('column_name_digest') - :default('NeuralDigest') + :description 'Name of neural profile digest column in Clickhouse' + :argname('column_name_digest') + :default('NeuralDigest') neural_train:option '--column-name-vector' - :description 'Name of neural training vector column in Clickhouse' - :argname('column_name_vector') - :default('NeuralMpack') + :description 'Name of neural training vector column in Clickhouse' + :argname('column_name_vector') + :default('NeuralMpack') neural_train:option '--limit -l' - :description 'Maximum rows to fetch per day' - :argname('limit') + :description 'Maximum rows to fetch per day' + :argname('limit') neural_train:option '--profile -p' - :description 'Profile to use for training' - :argname('profile') - :default('default') + :description 'Profile to use for training' + :argname('profile') + :default('default') neural_train:option '--rule -r' - :description 'Rule to train' - :argname('rule') - :default('default') + :description 'Rule to train' + :argname('rule') + :default('default') neural_train:option '--spam -s' - :description 'WHERE clause to use for spam' - :argname('spam') - :default("Action == 'reject'") + :description 'WHERE clause to use for spam' + :argname('spam') + :default("Action == 'reject'") neural_train:option '--ham -h' - :description 'WHERE clause to use for ham' - :argname('ham') - :default('Score < 0') + :description 'WHERE clause to use for ham' + :argname('ham') + :default('Score < 0') neural_train:option '--url -u' - :description 'URL to use for training' - :argname('url') - :default('http://127.0.0.1:11334/plugins/neural/learn') + :description 'URL to use for training' + :argname('url') + :default('http://127.0.0.1:11334/plugins/neural/learn') local http_params = { config = rspamd_config, @@ -131,14 +131,14 @@ local http_params = { } local function load_config(config_file) - local _r,err = rspamd_config:load_ucl(config_file) + local _r, err = rspamd_config:load_ucl(config_file) if not _r then rspamd_logger.errx('cannot load %s: %s', config_file, err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', config_file, err) os.exit(1) @@ -196,7 +196,7 @@ local function get_excluded_symbols(known_symbols, correlations, seen_total) elseif not all_symbols[k] then remove[k] = 'nonexistent symbol' else - for fl,_ in pairs(all_symbols[k].flags or {}) do + for fl, _ in pairs(all_symbols[k].flags or {}) do if skip_flags[fl] then remove[k] = fl .. ' symbol' break @@ -238,7 +238,7 @@ local function handle_neural_profile(args) local nsym = #r['Symbols.Names'] - for i = 1,nsym do + for i = 1, nsym do local sym = r['Symbols.Names'][i] local t = known_symbols[sym] if not t then @@ -266,8 +266,8 @@ local function handle_neural_profile(args) end -- Fill correlations - for i = 1,nsym do - for j = 1,nsym do + for i = 1, nsym do + for j = 1, nsym do if i ~= j then local sym = r['Symbols.Names'][i] local inner_sym_name = r['Symbols.Names'][j] @@ -342,11 +342,11 @@ end local function post_neural_training(url, rule, spam_rows, ham_rows) -- Prepare JSON payload local payload = ucl.to_format( - { - ham_vec = ham_rows, - rule = rule, - spam_vec = spam_rows, - }, 'json') + { + ham_vec = ham_rows, + rule = rule, + spam_vec = spam_rows, + }, 'json') -- POST the payload local err, response = rspamd_http.request({ @@ -423,11 +423,11 @@ local function handle_neural_train(args) limit = string.format(' LIMIT %d', num_limit) -- Contains leading space end -- Prepare query elements - local conditions = {string.format("%s = '%s'", args.column_name_digest, symbols_digest)} + local conditions = { string.format("%s = '%s'", args.column_name_digest, symbols_digest) } local query_fmt = 'SELECT %s FROM rspamd WHERE %s%s' -- Run queries - for _, the_where in ipairs({args.ham, args.spam}) do + for _, the_where in ipairs({ args.ham, args.spam }) do -- Inform callback which group of vectors we're collecting this_where = the_where table.insert(conditions, the_where) -- should be 2nd from last condition @@ -437,7 +437,7 @@ local function handle_neural_train(args) if this_where == args.ham then if not want_ham then break - end + end else if not want_spam then break diff --git a/lualib/rspamadm/configgraph.lua b/lualib/rspamadm/configgraph.lua index 75eb2d940..07f14a9ca 100644 --- a/lualib/rspamadm/configgraph.lua +++ b/lualib/rspamadm/configgraph.lua @@ -31,7 +31,6 @@ parser:option "-c --config" parser:flag "-a --all" :description('Show all nodes, not just existing ones') - local function process_filename(fname) local cdir = rspamd_paths['CONFDIR'] .. '/' fname = fname:gsub(cdir, '') @@ -40,8 +39,8 @@ end local function output_dot(opts, nodes, adjacency) rspamd_logger.messagex("digraph rspamd {") - for k,node in pairs(nodes) do - local attrs = {"shape=box"} + for k, node in pairs(nodes) do + local attrs = { "shape=box" } local skip = false if node.exists then if node.priority >= 10 then @@ -62,7 +61,7 @@ local function output_dot(opts, nodes, adjacency) table.concat(attrs, ',')) end end - for _,adj in ipairs(adjacency) do + for _, adj in ipairs(adjacency) do local attrs = {} local skip = false @@ -95,7 +94,7 @@ local function load_config_traced(opts) local nodes = {} local function maybe_match_glob(file) - for _,gl in ipairs(glob_traces) do + for _, gl in ipairs(glob_traces) do if gl.re:match(file) then return gl end @@ -151,7 +150,7 @@ local function load_config_traced(opts) end end - local _r,err = rspamd_config:load_ucl(opts['config'], trace_func) + local _r, err = rspamd_config:load_ucl(opts['config'], trace_func) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) @@ -160,7 +159,6 @@ local function load_config_traced(opts) output_dot(opts, nodes, adjacency) end - local function handler(args) local res = parser:parse(args) diff --git a/lualib/rspamadm/confighelp.lua b/lualib/rspamadm/confighelp.lua index 1eecb9238..38b26b6fc 100644 --- a/lualib/rspamadm/confighelp.lua +++ b/lualib/rspamadm/confighelp.lua @@ -16,11 +16,11 @@ local parser = argparse() parser:argument "path":args "*" :description('Optional config paths') parser:flag "--no-color" - :description "Disable coloured output" + :description "Disable coloured output" parser:flag "--short" - :description "Show only option names" + :description "Show only option names" parser:flag "--no-examples" - :description "Do not show examples (implied by --short)" + :description "Do not show examples (implied by --short)" local function maybe_print_color(key) if not opts['no-color'] then @@ -84,10 +84,10 @@ local function print_help(key, value, tabs) if type(value['required']) == 'boolean' then if value['required'] then print(string.format('%s\tRequired: %s', tabs, - maybe_print_color(tostring(value['required'])))) + maybe_print_color(tostring(value['required'])))) else print(string.format('%s\tRequired: %s', tabs, - tostring(value['required']))) + tostring(value['required']))) end end if value['default'] then @@ -116,7 +116,7 @@ return function(args, res) local sorted = sort_values(res) - for _,v in ipairs(sorted) do + for _, v in ipairs(sorted) do print_help(v['key'], v['value'], '') print('') end diff --git a/lualib/rspamadm/configwizard.lua b/lualib/rspamadm/configwizard.lua index c79d73b97..27c358e3b 100644 --- a/lualib/rspamadm/configwizard.lua +++ b/lualib/rspamadm/configwizard.lua @@ -74,19 +74,29 @@ local function ask_yes_no(greet, default) local reply = rspamd_util.readline(greet) - if not reply then os.exit(0) end - if #reply == 0 then reply = def_str end + if not reply then + os.exit(0) + end + if #reply == 0 then + reply = def_str + end reply = reply:lower() - if reply == 'y' or reply == 'yes' then return true end + if reply == 'y' or reply == 'yes' then + return true + end return false end local function readline_default(greet, def_value) local reply = rspamd_util.readline(greet) - if not reply then os.exit(0) end + if not reply then + os.exit(0) + end - if #reply == 0 then return def_value end + if #reply == 0 then + return def_value + end return reply end @@ -95,7 +105,7 @@ local function readline_expire() local expire = '100d' repeat expire = readline_default("Expire time for new tokens [" .. expire .. "]: ", - expire) + expire) expire = lua_util.parse_time_interval(expire) if not expire then @@ -118,9 +128,9 @@ end local function print_changes(changes) local function print_change(k, c, where) printf('File: %s, changes list:', highlight(local_conf .. '/' - .. where .. '/'.. k)) + .. where .. '/' .. k)) - for ek,ev in pairs(c) do + for ek, ev in pairs(c) do printf("%s => %s", highlight(ek), rspamd_logger.slog("%s", ev)) end end @@ -144,7 +154,7 @@ local function apply_changes(changes) end local function apply_change(k, c, where) - local fname = local_conf .. '/' .. where .. '/'.. k + local fname = local_conf .. '/' .. where .. '/' .. k if not rspamd_util.file_exists(fname) then printf("Create file %s", highlight(fname)) @@ -181,7 +191,6 @@ local function apply_changes(changes) end end - local function setup_controller(controller, changes) printf("Setup %s and controller worker:", highlight("WebUI")) @@ -208,13 +217,13 @@ local function setup_redis(cfg, changes) printf("%s servers are not set:", highlight("Redis")) printf("The following modules will be enabled if you add Redis servers:") - for k,_ in pairs(rspamd_plugins_state.disabled_redis) do + for k, _ in pairs(rspamd_plugins_state.disabled_redis) do printf("\t* %s", highlight(k)) end if ask_yes_no("Do you wish to set Redis servers?", true) then local read_servers = readline_default("Input read only servers separated by `,` [default: localhost]: ", - "localhost") + "localhost") local rs = parse_servers(read_servers) if rs and #rs > 0 then @@ -263,7 +272,9 @@ end local function setup_dkim_signing(cfg, changes) -- Remove the trailing slash of a pathname, if present. local function remove_trailing_slash(path) - if string.sub(path, -1) ~= "/" then return path end + if string.sub(path, -1) ~= "/" then + return path + end return string.sub(path, 1, string.len(path) - 1) end @@ -281,7 +292,7 @@ local function setup_dkim_signing(cfg, changes) local use_esld local sign_domain = 'pet luacheck' - local defined_auth_types = {'header', 'envelope', 'auth', 'recipient'} + local defined_auth_types = { 'header', 'envelope', 'auth', 'recipient' } if sign_type == '4' then repeat @@ -318,7 +329,9 @@ local function setup_dkim_signing(cfg, changes) sign_authenticated = true end - if fun.any(function(s) return s == sign_domain end, defined_auth_types) then + if fun.any(function(s) + return s == sign_domain + end, defined_auth_types) then -- Allow mismatch allow_mismatch = ask_yes_no( string.format('Allow data %s, e.g. if mime from domain is not equal to authenticated user domain? ', @@ -337,7 +350,7 @@ local function setup_dkim_signing(cfg, changes) local dkim_keys_dir = rspamd_paths["DBDIR"] .. "/dkim/" local prompt = string.format("Enter output directory for the keys [default: %s]: ", - highlight(dkim_keys_dir)) + highlight(dkim_keys_dir)) dkim_keys_dir = remove_trailing_slash(readline_default(prompt, dkim_keys_dir)) local ret, err = rspamd_util.mkdir(dkim_keys_dir, true) @@ -349,7 +362,7 @@ local function setup_dkim_signing(cfg, changes) local function print_domains() printf("Domains configured:") - for k,v in pairs(domains) do + for k, v in pairs(domains) do printf("Domain: %s, selector: %s, privkey: %s", highlight(k), v.selector, v.privkey) end @@ -370,13 +383,15 @@ local function setup_dkim_signing(cfg, changes) until #domain ~= 0 local selector = readline_default("Enter selector [default: dkim]: ", 'dkim') - if not selector then selector = 'dkim' end + if not selector then + selector = 'dkim' + end local privkey_file = string.format("%s/%s.%s.key", dkim_keys_dir, domain, selector) if not rspamd_util.file_exists(privkey_file) then if ask_yes_no("Do you want to create privkey " .. highlight(privkey_file), - true) then + true) then local pubkey_file = privkey_file .. ".pub" local rspamd_cryptobox = require "rspamd_cryptobox" local sk, pk = rspamd_cryptobox.generate_keypair("rsa", 2048) @@ -402,7 +417,7 @@ local function setup_dkim_signing(cfg, changes) } until not ask_yes_no("Do you wish to add another DKIM domain?") - changes.l['dkim_signing.conf'] = {domain = domains} + changes.l['dkim_signing.conf'] = { domain = domains } local res_tbl = changes.l['dkim_signing.conf'] if sign_networks then @@ -426,7 +441,7 @@ local function check_redis_classifier(cls, changes) local symbol_spam, symbol_ham -- Load symbols from statfiles local statfiles = cls.statfile - for _,stf in ipairs(statfiles) do + for _, stf in ipairs(statfiles) do local symbol = stf.symbol or 'undefined' local spam @@ -484,12 +499,14 @@ local function check_redis_classifier(cls, changes) end local function get_version(conn) - conn:add_cmd("SMEMBERS", {"RS_keys"}) + conn:add_cmd("SMEMBERS", { "RS_keys" }) - local ret,members = conn:exec() + local ret, members = conn:exec() -- Empty db - if not ret or #members == 0 then return false,0 end + if not ret or #members == 0 then + return false, 0 + end -- We still need to check versions local lua_script = [[ @@ -502,10 +519,10 @@ end return ver ]] - conn:add_cmd('EVAL', {lua_script, '1', 'RS'}) - local _,ver = conn:exec() + conn:add_cmd('EVAL', { lua_script, '1', 'RS' }) + local _, ver = conn:exec() - return true,tonumber(ver) + return true, tonumber(ver) end local function check_expire(conn) @@ -522,21 +539,23 @@ end return ttl ]] - conn:add_cmd('EVAL', {lua_script, '0'}) - local _,ttl = conn:exec() + conn:add_cmd('EVAL', { lua_script, '0' }) + local _, ttl = conn:exec() return tonumber(ttl) end - local res,conn = lua_redis.redis_connect_sync(parsed_redis, true) + local res, conn = lua_redis.redis_connect_sync(parsed_redis, true) if not res then printf("Cannot connect to Redis server") return false end if not cls.new_schema then - local r,ver = get_version(conn) - if not r then return false end + local r, ver = get_version(conn) + if not r then + return false + end if ver ~= 2 then if not ver then printf('Key "RS_version" has not been found in Redis for %s/%s', @@ -562,17 +581,19 @@ return ttl end end else - local r,ver = get_version(conn) - if not r then return false end + local r, ver = get_version(conn) + if not r then + return false + end if ver ~= 2 then printf("You have configured new schema for %s/%s but your DB has old version: %s", - symbol_spam, symbol_ham, ver) + symbol_spam, symbol_ham, ver) try_convert(false) else printf( 'You have configured new schema for %s/%s and your DB already has new layout (v. %s).' .. ' DB conversion is not needed.', - symbol_spam, symbol_ham, ver) + symbol_spam, symbol_ham, ver) end end end @@ -584,7 +605,7 @@ local function setup_statistic(cfg, changes) if not redis_params then printf('You have %d sqlite classifiers, but you have no Redis servers being set', - #sqlite_configs) + #sqlite_configs) return false end @@ -596,7 +617,7 @@ local function setup_statistic(cfg, changes) local reset_previous = ask_yes_no("Reset previous data?") if ask_yes_no('Do you wish to convert them to Redis?', true) then - for _,cls in ipairs(sqlite_configs) do + for _, cls in ipairs(sqlite_configs) do if rspamd_util.file_exists(cls.db_spam) and rspamd_util.file_exists(cls.db_ham) then if not lua_stat_tools.convert_sqlite_to_redis(parsed_redis, cls.db_spam, cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, expire, @@ -634,8 +655,10 @@ local function setup_statistic(cfg, changes) if classifier then if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.bayes then cls = cls.bayes end + for _, cls in ipairs(classifier) do + if cls.bayes then + cls = cls.bayes + end if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, changes) end @@ -645,7 +668,7 @@ local function setup_statistic(cfg, changes) classifier = classifier.bayes if classifier[1] then - for _,cls in ipairs(classifier) do + for _, cls in ipairs(classifier) do if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, changes) end @@ -663,22 +686,24 @@ end local function find_worker(cfg, wtype) if cfg.worker then - for k,s in pairs(cfg.worker) do + for k, s in pairs(cfg.worker) do if type(k) == 'number' and type(s) == 'table' then - if s[wtype] then return s[wtype] end + if s[wtype] then + return s[wtype] + end end if type(s) == 'table' and s.type and s.type == wtype then return s end - if type(k) == 'string' and k == wtype then return s end + if type(k) == 'string' and k == wtype then + return s + end end end return nil end - - return { handler = function(cmd_args) local changes = { @@ -698,14 +723,14 @@ return { local opts = parser:parse(cmd_args) local args = opts['checks'] or {} - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -721,13 +746,13 @@ return { if #args > 0 then interactive_start = false - for _,arg in ipairs(args) do + for _, arg in ipairs(args) do if arg == 'all' then checks = all_checks elseif arg == 'list' then printf(highlight(rspamd_logo)) printf('Available modules') - for _,c in ipairs(all_checks) do + for _, c in ipairs(all_checks) do printf('- %s', c) end return @@ -740,7 +765,7 @@ return { end local function has_check(check) - for _,c in ipairs(checks) do + for _, c in ipairs(checks) do if c == check then return true end @@ -791,8 +816,12 @@ return { end local nchanges = 0 - for _,_ in pairs(changes.l) do nchanges = nchanges + 1 end - for _,_ in pairs(changes.o) do nchanges = nchanges + 1 end + for _, _ in pairs(changes.l) do + nchanges = nchanges + 1 + end + for _, _ in pairs(changes.o) do + nchanges = nchanges + 1 + end if nchanges > 0 then print_changes(changes) diff --git a/lualib/rspamadm/cookie.lua b/lualib/rspamadm/cookie.lua index bc4df9de2..7e0526a7e 100644 --- a/lualib/rspamadm/cookie.lua +++ b/lualib/rspamadm/cookie.lua @@ -44,7 +44,9 @@ parser:argument "cookie":args "?" local function gen_cookie(args, key) local cr = require "rspamd_cryptobox" - if not args.cookie then return end + if not args.cookie then + return + end local function encrypt() if #args.cookie > 31 then @@ -67,7 +69,7 @@ local function gen_cookie(args, key) extracted_cookie = args.cookie end - local dec_cookie,ts = cr.decrypt_cookie(key, extracted_cookie) + local dec_cookie, ts = cr.decrypt_cookie(key, extracted_cookie) if dec_cookie then if args.timestamp then @@ -79,7 +81,7 @@ local function gen_cookie(args, key) print('cannot decrypt cookie') os.exit(1) end - end + end if args.decrypt then decrypt() @@ -96,8 +98,10 @@ local function handler(args) end if res.key then - local pattern = {'^'} - for i=1,32 do pattern[i + 1] = '[a-zA-Z0-9]' end + local pattern = { '^' } + for i = 1, 32 do + pattern[i + 1] = '[a-zA-Z0-9]' + end pattern[34] = '$' if not res.key:match(table.concat(pattern, '')) then diff --git a/lualib/rspamadm/corpus_test.lua b/lualib/rspamadm/corpus_test.lua index 939c3c0fa..0e63f9f6f 100644 --- a/lualib/rspamadm/corpus_test.lua +++ b/lualib/rspamadm/corpus_test.lua @@ -177,10 +177,9 @@ local function handler(args) rspamd_logger.messagex("Messages/sec: %s", (total_msgs / elapsed_time)) end - return { name = 'corpustest', - aliases = {'corpus_test', 'corpus'}, + aliases = { 'corpus_test', 'corpus' }, handler = handler, description = parser._description } diff --git a/lualib/rspamadm/dkim_keygen.lua b/lualib/rspamadm/dkim_keygen.lua index 0d0ebe5a6..3d51903dc 100644 --- a/lualib/rspamadm/dkim_keygen.lua +++ b/lualib/rspamadm/dkim_keygen.lua @@ -32,39 +32,39 @@ parser:option '-k --privkey' :description 'Save private key to file instead of printing it to stdout' parser:option '-b --bits' :convert(function(input) - local n = tonumber(input) - if not n or n < 512 or n > 4096 then - return nil - end - return n - end) + local n = tonumber(input) + if not n or n < 512 or n > 4096 then + return nil + end + return n +end) :description 'Generate an RSA key with the specified number of bits (512 to 4096)' :default(1024) parser:option '-t --type' :description 'Key type: RSA, ED25519 or ED25119-seed' :convert { - ['rsa'] = 'rsa', - ['RSA'] = 'rsa', - ['ed25519'] = 'ed25519', - ['ED25519'] = 'ed25519', - ['ed25519-seed'] = 'ed25519-seed', - ['ED25519-seed'] = 'ed25519-seed', - } + ['rsa'] = 'rsa', + ['RSA'] = 'rsa', + ['ed25519'] = 'ed25519', + ['ED25519'] = 'ed25519', + ['ed25519-seed'] = 'ed25519-seed', + ['ED25519-seed'] = 'ed25519-seed', +} :default 'rsa' parser:option '-o --output' :description 'Output public key in the following format: dns, dnskey or plain' :convert { - ['dns'] = 'dns', - ['plain'] = 'plain', - ['dnskey'] = 'dnskey', - } + ['dns'] = 'dns', + ['plain'] = 'plain', + ['dnskey'] = 'dnskey', +} :default 'dns' parser:option '--priv-output' :description 'Output private key in the following format: PEM or DER (for RSA)' :convert { - ['pem'] = 'pem', - ['der'] = 'der', - } + ['pem'] = 'pem', + ['der'] = 'der', +} :default 'pem' parser:flag '-f --force' :description 'Force overwrite of existing files' @@ -83,7 +83,6 @@ local function split_string(input, max_length) return pieces end - local function print_public_key_dns(opts, base64_pk) local key_type = opts.type == 'rsa' and 'rsa' or 'ed25519' if #base64_pk < 255 - 2 then @@ -93,7 +92,7 @@ local function print_public_key_dns(opts, base64_pk) -- Split it by parts local parts = split_string(base64_pk) io.write(string.format('%s._domainkey IN TXT ( "v=DKIM1; k=%s; "\n', opts.selector, key_type)) - for i,part in ipairs(parts) do + for i, part in ipairs(parts) do if i == 1 then io.write(string.format('\t"p=%s"\n', part)) else @@ -121,7 +120,7 @@ end local function gen_rsa_key(opts) local rsa = require "rspamd_rsa" - local sk,pk = rsa.keypair(opts.bits or 1024) + local sk, pk = rsa.keypair(opts.bits or 1024) if opts.privkey then if opts.force then os.remove(opts.privkey) @@ -135,7 +134,7 @@ local function gen_rsa_key(opts) end local function gen_eddsa_key(opts) - local sk,pk = rspamd_cryptobox.gen_dkim_keypair(opts.type) + local sk, pk = rspamd_cryptobox.gen_dkim_keypair(opts.type) if opts.privkey and opts.force then os.remove(opts.privkey) @@ -156,7 +155,9 @@ end local function handler(args) local opts = parser:parse(args) - if not opts then os.exit(1) end + if not opts then + os.exit(1) + end if opts.type == 'rsa' then gen_rsa_key(opts) @@ -167,7 +168,7 @@ end return { name = 'dkim_keygen', - aliases = {'dkimkeygen'}, + aliases = { 'dkimkeygen' }, handler = handler, description = parser._description } diff --git a/lualib/rspamadm/dmarc_report.lua b/lualib/rspamadm/dmarc_report.lua index c81be13cf..65b1da158 100644 --- a/lualib/rspamadm/dmarc_report.lua +++ b/lualib/rspamadm/dmarc_report.lua @@ -46,9 +46,9 @@ parser:flag "-n --no-opt" :description "Do not reset reporting data/send reports" parser:argument "date" - :description "Date to process (today by default)" - :argname "<YYYYMMDD>" - :args "*" + :description "Date to process (today by default)" + :argname "<YYYYMMDD>" + :args "*" parser:option "-b --batch-size" :description "Send reports in batches up to <batch-size> messages" :argname "<number>" @@ -102,14 +102,14 @@ local redis_attrs = { local pool local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -118,11 +118,9 @@ end -- Concat elements using redis_keys.join_char local function redis_prefix(...) - return table.concat({...}, dmarc_settings.reporting.redis_keys.join_char) + return table.concat({ ... }, dmarc_settings.reporting.redis_keys.join_char) end - - local function get_rua(rep_key) local parts = lua_util.str_split(rep_key, dmarc_settings.reporting.redis_keys.join_char) @@ -144,8 +142,8 @@ local function get_domain(rep_key) end local function gen_uuid() - local template ='xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' - return string.gsub(template, '[xy]', function (c) + local template = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' + return string.gsub(template, '[xy]', function(c) local v = (c == 'x') and math.random(0, 0xf) or math.random(8, 0xb) return string.format('%x', v) end) @@ -159,7 +157,7 @@ local function gen_xml_grammar() local quot = lpeg.P('"') / '"' local apos = lpeg.P("'") / ''' local special = lt + gt + amp + quot + apos - local grammar = lpeg.Cs((special + 1)^0) + local grammar = lpeg.Cs((special + 1) ^ 0) return grammar end @@ -250,7 +248,7 @@ local function entry_to_xml(data) </auth_results> </record> ]] - return lua_util.jinja_template(xml_template, {data = data}, true) + return lua_util.jinja_template(xml_template, { data = data }, true) end -- Process a report entry stored in Redis splitting it to a lua table @@ -273,7 +271,7 @@ local function process_report_entry(data, score) if dkim_data and dkim_data ~= '' then local dkim_elts = lua_util.str_split(dkim_data, '|') for _, d in ipairs(dkim_elts) do - table.insert(row.dkim_results, {domain = d, result = result}) + table.insert(row.dkim_results, { domain = d, result = result }) end end end @@ -313,7 +311,7 @@ local function process_rua(dmarc_domain, rua) logger.errx('cannot resolve %s: %s; exclude %s', resolve_str, results, rua_part) else local found = false - for _,t in ipairs(results) do + for _, t in ipairs(results) do if string.match(t, 'v=DMARC1') then found = true break @@ -350,7 +348,7 @@ local function validate_reporting_domain(reporting_domain) config = rspamd_config, session = rspamadm_session, type = 'txt', - name = '_dmarc.' .. dmarc_domain , + name = '_dmarc.' .. dmarc_domain, }) if not is_ok or not results then @@ -358,8 +356,8 @@ local function validate_reporting_domain(reporting_domain) return nil end - for _,r in ipairs(results) do - local processed,rec = dmarc_common.dmarc_check_record(rspamd_config, r, false) + for _, r in ipairs(results) do + local processed, rec = dmarc_common.dmarc_check_record(rspamd_config, r, false) if processed and rec.rua then -- We need to check or alter rua if needed local processed_rua = process_rua(dmarc_domain, rec.rua) @@ -385,7 +383,7 @@ end -- Returns a list of recipients from a table as a string processing elements if needed local function rcpt_list(tbl, func) local res = {} - for _,r in ipairs(tbl) do + for _, r in ipairs(tbl) do if func then table.insert(res, func(r)) else @@ -431,14 +429,14 @@ local function send_reports_by_smtp(opts, reports, finish_cb) local nreports = math.min(#reports - cur_batch + 1, opts.batch_size) local next_start = cur_batch + nreports lua_util.debugm(N, 'send data for %s domains (from %s to %s)', - nreports, cur_batch, next_start-1) + nreports, cur_batch, next_start - 1) -- Shared across all closures local gen_args = { cont_func = send_data_in_batches, nreports = nreports, next_start = next_start } - for i=cur_batch,next_start-1 do + for i = cur_batch, next_start - 1 do local report = reports[i] local send_opts = { ev_base = rspamadm_ev_base, @@ -475,7 +473,7 @@ local function prepare_report(opts, start_time, end_time, rep_key) end local ret, results = lua_redis.request(redis_params, redis_attrs, - {'EXISTS', rep_key}) + { 'EXISTS', rep_key }) if not ret or not results or results == 0 then return nil @@ -484,7 +482,7 @@ local function prepare_report(opts, start_time, end_time, rep_key) -- Rename report key to avoid races if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'RENAME', rep_key, rep_key .. '_processing'}) + { 'RENAME', rep_key, rep_key .. '_processing' }) rep_key = rep_key .. '_processing' end @@ -494,7 +492,7 @@ local function prepare_report(opts, start_time, end_time, rep_key) if not dmarc_record then if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', rep_key}) + { 'DEL', rep_key }) end logger.messagex('Cannot process reports for domain %s; invalid dmarc record', reporting_domain) return nil @@ -502,11 +500,11 @@ local function prepare_report(opts, start_time, end_time, rep_key) -- Get all reports for a domain ret, results = lua_redis.request(redis_params, redis_attrs, - {'ZRANGE', rep_key, '0', '-1', 'WITHSCORES'}) + { 'ZRANGE', rep_key, '0', '-1', 'WITHSCORES' }) local report_entries = {} table.insert(report_entries, report_header(reporting_domain, start_time, end_time, dmarc_record)) - for i=1,#results,2 do + for i = 1, #results, 2 do local xml_record = entry_to_xml(process_report_entry(results[i], results[i + 1])) table.insert(report_entries, xml_record) end @@ -542,24 +540,23 @@ local function prepare_report(opts, start_time, end_time, rep_key) local rfooter = lua_util.jinja_template(report_footer, { uuid = uuid, }, true) - local message = rspamd_text.fromtable{ + local message = rspamd_text.fromtable { (rhead:gsub("\n", "\r\n")), rspamd_util.encode_base64(rspamd_util.gzip_compress(xml_to_compress), 73), rfooter:gsub("\n", "\r\n"), } - lua_util.debugm(N, 'got final message: %s', message) if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', rep_key}) + { 'DEL', rep_key }) end local report_rcpts = lua_util.str_split(rcpt_string, ',') if report_settings.bcc_addrs then - for _,b in ipairs(report_settings.bcc_addrs) do + for _, b in ipairs(report_settings.bcc_addrs) do table.insert(report_rcpts, b) end end @@ -574,7 +571,7 @@ end local function process_report_date(opts, start_time, end_time, date) local idx_key = redis_prefix(dmarc_settings.reporting.redis_keys.index_prefix, date) local ret, results = lua_redis.request(redis_params, redis_attrs, - {'EXISTS', idx_key}) + { 'EXISTS', idx_key }) if not ret or not results or results == 0 then logger.messagex('No reports for %s', date) @@ -584,24 +581,24 @@ local function process_report_date(opts, start_time, end_time, date) -- Rename index key to avoid races if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'RENAME', idx_key, idx_key .. '_processing'}) + { 'RENAME', idx_key, idx_key .. '_processing' }) idx_key = idx_key .. '_processing' end ret, results = lua_redis.request(redis_params, redis_attrs, - {'SMEMBERS', idx_key}) + { 'SMEMBERS', idx_key }) if not ret or not results then -- Remove bad key if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', idx_key}) + { 'DEL', idx_key }) end logger.messagex('Cannot get reports for %s', date) return {} end local reports = {} - for _,rep in ipairs(results) do + for _, rep in ipairs(results) do local report = prepare_report(opts, start_time, end_time, rep) if report then @@ -614,7 +611,7 @@ local function process_report_date(opts, start_time, end_time, date) -- Remove processed key if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', idx_key}) + { 'DEL', idx_key }) end return reports @@ -669,14 +666,14 @@ local function handler(args) os.exit(1) end - for _, e in ipairs({'email', 'domain', 'org_name'}) do + for _, e in ipairs({ 'email', 'domain', 'org_name' }) do if not dmarc_settings.reporting[e] then logger.errx('Missing required setting: dmarc.reporting.%s', e) return end end - local ret,results = lua_redis.request(redis_params, redis_attrs, { + local ret, results = lua_redis.request(redis_params, redis_attrs, { 'GET', 'rspamd_dmarc_last_collection' }) @@ -696,14 +693,14 @@ local function handler(args) local ndates = 0 local nreports = 0 local all_reports = {} - for _,date in ipairs(opts.date) do + for _, date in ipairs(opts.date) do lua_util.debugm(N, 'Process date %s', date) local reports_for_date = process_report_date(opts, start_time, start_collection, date) if #reports_for_date > 0 then ndates = ndates + 1 nreports = nreports + #reports_for_date - for _,r in ipairs(reports_for_date) do + for _, r in ipairs(reports_for_date) do table.insert(all_reports, r) end end @@ -718,8 +715,8 @@ local function handler(args) ndates, nreports, nsuccess, nfail) end lua_redis.request(redis_params, redis_attrs, - {'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2, - tostring(start_collection)}) + { 'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2, + tostring(start_collection) }) else logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed', ndates, nreports, nsuccess, nfail) @@ -736,7 +733,7 @@ end return { name = 'dmarc_report', - aliases = {'dmarc_reporting'}, + aliases = { 'dmarc_reporting' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/dns_tool.lua b/lualib/rspamadm/dns_tool.lua index e3ffb29dd..3eb09a82b 100644 --- a/lualib/rspamadm/dns_tool.lua +++ b/lualib/rspamadm/dns_tool.lua @@ -33,7 +33,7 @@ parser:option "-c --config" :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf") local spf = parser:command "spf" - :description "Extracts spf records" + :description "Extracts spf records" spf:mutex( spf:option "-d --domain" :description "Domain to use" @@ -69,14 +69,14 @@ local function red(str) end local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -109,7 +109,7 @@ local function spf_handler(opts) os.exit(1) end elseif opts.domain then - task:set_from('smtp', {user = 'user', domain = opts.domain}) + task:set_from('smtp', { user = 'user', domain = opts.domain }) else io.stderr:write('Neither domain nor from specified\n') os.exit(1) @@ -128,7 +128,9 @@ local function spf_handler(opts) end local function display_spf_results(elt, colored) - local dec = function(e) return e end + local dec = function(e) + return e + end local policy_decode = function(e) if e == rspamd_spf.policy.fail then return 'reject' @@ -144,12 +146,18 @@ local function spf_handler(opts) end if colored then - dec = function(e) return highlight(e) end + dec = function(e) + return highlight(e) + end - if elt.result == rspamd_spf.policy.pass then - dec = function(e) return green(e) end - elseif elt.result == rspamd_spf.policy.fail then - dec = function(e) return red(e) end + if elt.result == rspamd_spf.policy.pass then + dec = function(e) + return green(e) + end + elseif elt.result == rspamd_spf.policy.fail then + dec = function(e) + return red(e) + end end end @@ -182,7 +190,7 @@ local function spf_handler(opts) if result then printf('SPF record for %s; digest: %s', highlight(opts.domain or opts.from), highlight(record:get_digest())) - for _,elt in ipairs(record:get_elts()) do + for _, elt in ipairs(record:get_elts()) do if result and error_or_addr and elt.str and elt.str == error_or_addr.str then printf("%s", highlight('*** Matched ***')) display_spf_results(elt, true) @@ -218,7 +226,7 @@ end return { name = 'dnstool', - aliases = {'dns', 'dns_tool'}, + aliases = { 'dns', 'dns_tool' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/fuzzy_convert.lua b/lualib/rspamadm/fuzzy_convert.lua index 2d473ca46..a31baa4e2 100644 --- a/lualib/rspamadm/fuzzy_convert.lua +++ b/lualib/rspamadm/fuzzy_convert.lua @@ -13,13 +13,13 @@ local function connect_redis(server, password, db) end if password then - ret = conn:add_cmd('AUTH', {password}) + ret = conn:add_cmd('AUTH', { password }) if not ret then return nil, 'Cannot queue command' end end if db then - ret = conn:add_cmd('SELECT', {db}) + ret = conn:add_cmd('SELECT', { db }) if not ret then return nil, 'Cannot queue command' end @@ -126,7 +126,7 @@ local function update_counters(total, redis_host, redis_password, redis_db) return true end -return function (_, res) +return function(_, res) local db = sqlite3.open(res['source_db']) local shingles = {} local digests = {} @@ -152,11 +152,11 @@ return function (_, res) local expire_in = math.floor(now - row.time + res['expiry']) if expire_in >= 1 then - table.insert(digests, {row.digest, row.flag, row.value, expire_in}) + table.insert(digests, { row.digest, row.flag, row.value, expire_in }) num_batch_digests = num_batch_digests + 1 total_digests = total_digests + 1 for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do - table.insert(shingles, {srow.value, srow.number, expire_in, row.digest}) + table.insert(shingles, { srow.value, srow.number, expire_in, row.digest }) total_shingles = total_shingles + 1 num_batch_shingles = num_batch_shingles + 1 end @@ -188,8 +188,8 @@ return function (_, res) end local message = string.format( - 'Migrated %d digests and %d shingles', - total_digests, total_shingles + 'Migrated %d digests and %d shingles', + total_digests, total_shingles ) if not update_counters(total_digests, res['redis_host'], redis_password, redis_db) then message = message .. ' but failed to update counters' diff --git a/lualib/rspamadm/fuzzy_stat.lua b/lualib/rspamadm/fuzzy_stat.lua index d0c14d213..ef8a5de08 100644 --- a/lualib/rspamadm/fuzzy_stat.lua +++ b/lualib/rspamadm/fuzzy_stat.lua @@ -18,14 +18,14 @@ parser:flag "-n --number" parser:option "--sort" :description "Sort order" :convert { - checked = "checked", - matched = "matched", - errors = "errors", - name = "name" - } + checked = "checked", + matched = "matched", + errors = "errors", + name = "name" +} local function add_data(target, src) - for k,v in pairs(src) do + for k, v in pairs(src) do if type(v) == 'number' then if target[k] then target[k] = target[k] + v @@ -33,17 +33,25 @@ local function add_data(target, src) target[k] = v end elseif k == 'ips' then - if not target['ips'] then target['ips'] = {} end + if not target['ips'] then + target['ips'] = {} + end -- Iterate over IPs - for ip,st in pairs(v) do - if not target['ips'][ip] then target['ips'][ip] = {} end + for ip, st in pairs(v) do + if not target['ips'][ip] then + target['ips'][ip] = {} + end add_data(target['ips'][ip], st) end elseif k == 'flags' then - if not target['flags'] then target['flags'] = {} end + if not target['flags'] then + target['flags'] = {} + end -- Iterate over Flags - for flag,st in pairs(v) do - if not target['flags'][flag] then target['flags'][flag] = {} end + for flag, st in pairs(v) do + if not target['flags'][flag] then + target['flags'][flag] = {} + end add_data(target['flags'][flag], st) end elseif k == 'keypair' then @@ -109,8 +117,8 @@ end -- Sort by checked local function sort_hash_table(tbl, sort_opts, key_key) local res = {} - for k,v in pairs(tbl) do - table.insert(res, {[key_key] = k, data = v}) + for k, v in pairs(tbl) do + table.insert(res, { [key_key] = k, data = v }) end local function sort_order(elt) @@ -145,12 +153,12 @@ local function add_result(dst, src, k) if type(src) == 'table' then if type(dst) == 'number' then -- Convert dst to table - dst = {dst} + dst = { dst } elseif type(dst) == 'nil' then dst = {} end - for i,v in ipairs(src) do + for i, v in ipairs(src) do if dst[i] and k ~= 'fuzzy_stored' then dst[i] = dst[i] + v else @@ -193,7 +201,7 @@ local function print_result(r) end if type(r) == 'table' then local res = {} - for i,num in ipairs(r) do + for i, num in ipairs(r) do res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num)) end @@ -210,7 +218,7 @@ return function(args, res) opts = parser:parse(args) if wrk then - for _,pr in pairs(wrk) do + for _, pr in pairs(wrk) do -- processes cycle if pr['data'] then local id = pr['id'] @@ -225,7 +233,7 @@ return function(args, res) end -- General stats - for k,v in pairs(pr['data']) do + for k, v in pairs(pr['data']) do if k ~= 'keys' and k ~= 'errors_ips' then res_db[k] = add_result(res_db[k], v, k) elseif k == 'errors_ips' then @@ -233,7 +241,7 @@ return function(args, res) if not res_db['errors_ips'] then res_db['errors_ips'] = {} end - for ip,nerrors in pairs(v) do + for ip, nerrors in pairs(v) do if not res_db['errors_ips'][ip] then res_db['errors_ips'][ip] = nerrors else @@ -250,7 +258,7 @@ return function(args, res) res_db['keys'] = res_keys end -- Go through keys in input - for k,elts in pairs(pr['data']['keys']) do + for k, elts in pairs(pr['data']['keys']) do -- keys cycle if not res_keys[k] then res_keys[k] = {} @@ -259,7 +267,7 @@ return function(args, res) add_data(res_keys[k], elts) if elts['ips'] then - for ip,v in pairs(elts['ips']) do + for ip, v in pairs(elts['ips']) do if not res_ips[ip] then res_ips[ip] = {} end @@ -274,10 +282,10 @@ return function(args, res) end -- General stats - for db,st in pairs(res_databases) do + for db, st in pairs(res_databases) do print(string.format('Statistics for storage %s', db)) - for k,v in pairs(st) do + for k, v in pairs(st) do if k ~= 'keys' and k ~= 'errors_ips' then print(string.format('%s: %s', k, print_result(v))) end @@ -305,7 +313,7 @@ return function(args, res) print('\tIPs stat:') local sorted_ips = sort_hash_table(key_stat['ips'], opts, 'ip') - for _,v in ipairs(sorted_ips) do + for _, v in ipairs(sorted_ips) do print(string.format('\t%s', v['ip'])) print_stat(v['data'], '\t\t') print('') @@ -315,7 +323,7 @@ return function(args, res) if key_stat.flags then print('') print('\tFlags stat:') - for flag,v in pairs(key_stat.flags) do + for flag, v in pairs(key_stat.flags) do print(string.format('\t[%s]:', flag)) -- Remove irrelevant fields v.checked = nil diff --git a/lualib/rspamadm/grep.lua b/lualib/rspamadm/grep.lua index 426b93c7a..6ed05691e 100644 --- a/lualib/rspamadm/grep.lua +++ b/lualib/rspamadm/grep.lua @@ -80,7 +80,7 @@ local function handler(args) if search_str and not sensitive then search_str = string.lower(search_str) end - local inputs = res['input'] or {'stdin'} + local inputs = res['input'] or { 'stdin' } for _, n in ipairs(inputs) do local h, err @@ -115,7 +115,7 @@ local function handler(args) if buffer[hash] then table.insert(buffer[hash], line) else - buffer[hash] = {line} + buffer[hash] = { line } end end end diff --git a/lualib/rspamadm/keypair.lua b/lualib/rspamadm/keypair.lua index 27dfd59a3..f0716a22f 100644 --- a/lualib/rspamadm/keypair.lua +++ b/lualib/rspamadm/keypair.lua @@ -84,9 +84,9 @@ verify:mutex( :argname "<file>" ) verify:argument "file" - :description "File to verify" - :argname "<file>" - :args "*" + :description "File to verify" + :argname "<file>" + :args "*" verify:flag "-n --nist" :description "Uses nistp curves (P256)" verify:option "-s --suffix" @@ -143,15 +143,15 @@ decrypt:flag "-r --rm" -- Default command is generate, so duplicate options to be compatible parser:flag "-s --sign" - :description "Generates a sign keypair instead of the encryption one" + :description "Generates a sign keypair instead of the encryption one" parser:flag "-n --nist" - :description "Uses nistp curves (P256)" + :description "Uses nistp curves (P256)" parser:mutex( parser:flag "-j --json" - :description "Output JSON instead of UCL", + :description "Output JSON instead of UCL", parser:flag "-u --ucl" - :description "Output UCL" - :default(true) + :description "Output UCL" + :default(true) ) parser:option "-o --output" :description "Write keypair to file" @@ -174,10 +174,16 @@ local function ask_yes_no(greet, default) local reply = rspamd_util.readline(greet) - if not reply then os.exit(0) end - if #reply == 0 then reply = def_str end + if not reply then + os.exit(0) + end + if #reply == 0 then + reply = def_str + end reply = reply:lower() - if reply == 'y' or reply == 'yes' then return true end + if reply == 'y' or reply == 'yes' then + return true + end return false end @@ -221,7 +227,7 @@ end local function sign_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to sign') @@ -231,7 +237,7 @@ local function sign_handler(opts) end local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -243,7 +249,7 @@ local function sign_handler(opts) fatal("cannot load keypair: " .. opts.keypair) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local sig = rspamd_crypto.sign_file(kp, fname) if not sig then @@ -264,7 +270,7 @@ end local function verify_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to verify') @@ -275,7 +281,7 @@ local function verify_handler(opts) if opts.keypair then local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -290,10 +296,14 @@ local function verify_handler(opts) pk = kp:pk() alg = kp:alg() elseif opts.pubkey then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg) elseif opts.pubstr then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg) end @@ -303,7 +313,7 @@ local function verify_handler(opts) local valid = true - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local sig_fname = string.format('%s.%s', fname, opts.suffix or 'sig') local sig = rspamd_signature.load(sig_fname, alg) @@ -330,7 +340,7 @@ end local function encrypt_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to sign') @@ -341,7 +351,7 @@ local function encrypt_handler(opts) if opts.keypair then local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -356,10 +366,14 @@ local function encrypt_handler(opts) pk = kp:pk() alg = kp:alg() elseif opts.pubkey then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg) elseif opts.pubstr then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg) end @@ -367,7 +381,7 @@ local function encrypt_handler(opts) fatal("cannot load keypair: " .. opts.keypair) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local enc = rspamd_crypto.encrypt_file(pk, fname, alg) if not enc then @@ -404,7 +418,7 @@ end local function decrypt_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to decrypt') @@ -414,7 +428,7 @@ local function decrypt_handler(opts) end local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -426,7 +440,7 @@ local function decrypt_handler(opts) fatal("cannot load keypair: " .. opts.keypair) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local decrypted = rspamd_crypto.decrypt_file(kp, fname) if not decrypted then @@ -488,7 +502,7 @@ end return { name = 'keypair', - aliases = {'kp', 'key'}, + aliases = { 'kp', 'key' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 5c1cebac5..0b0605582 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -44,7 +44,7 @@ parser:mutex( parser:flag "-U --ucl" :description "UCL output", parser:flag "-M --messagepack" - :description "MessagePack output" + :description "MessagePack output" ) parser:flag "-C --compact" :description "Use compact format" @@ -67,12 +67,12 @@ extract:option "-o --output" :description "Output format ('raw', 'content', 'oneline', 'decoded', 'decoded_utf')" :argname("<type>") :convert { - raw = "raw", - content = "content", - oneline = "content_oneline", - decoded = "raw_parsed", - decoded_utf = "raw_utf" - } + raw = "raw", + content = "content", + oneline = "content_oneline", + decoded = "raw_parsed", + decoded_utf = "raw_utf" +} :default "content" extract:flag "-w --words" :description "Extracts words" @@ -86,16 +86,15 @@ extract:option "-F --words-format" :description "Words format ('stem', 'norm', 'raw', 'full')" :argname("<type>") :convert { - stem = "stem", - norm = "norm", - raw = "raw", - full = "full", - } + stem = "stem", + norm = "norm", + raw = "raw", + full = "full", +} :default "stem" - local stat = parser:command "stat st s" - :description "Extracts statistical data from MIME messages" + :description "Extracts statistical data from MIME messages" stat:argument "file" :description "File to process" :argname "<file>" @@ -123,7 +122,7 @@ urls:mutex( urls:flag "-H --host" :description "Get hosts only", urls:flag "-f --full" - :description "Show piecewise urls as processed by Rspamd" + :description "Show piecewise urls as processed by Rspamd" ) urls:flag "-u --unique" @@ -136,7 +135,7 @@ urls:flag "-r --reverse" :description "Reverse sort order" local modify = parser:command "modify mod m" - :description "Modifies MIME message" + :description "Modifies MIME message" modify:argument "file" :description "File to process" :argname "<file>" @@ -162,11 +161,11 @@ modify:option "-H --html-footer" :argname "<file>" local sign = parser:command "sign" - :description "Performs DKIM signing" + :description "Performs DKIM signing" sign:argument "file" - :description "File to process" - :argname "<file>" - :args "+" + :description "File to process" + :argname "<file>" + :args "+" sign:option "-d --domain" :description "Use specific domain" @@ -184,17 +183,17 @@ sign:option "-t type" :description "ARC or DKIM signing" :argname("<arc|dkim>") :convert { - ['arc'] = 'arc', - ['dkim'] = 'dkim', - } + ['arc'] = 'arc', + ['dkim'] = 'dkim', +} :default 'dkim' sign:option "-o --output" :description "Output format" :argname("<message|signature>") :convert { - ['message'] = 'message', - ['signature'] = 'signature', - } + ['message'] = 'message', + ['signature'] = 'signature', +} :default 'message' local dump = parser:command "dump" @@ -213,21 +212,21 @@ dump:mutex( :description "MessagePack output" ) dump:flag "-s --split" - :description "Split the output file contents such that no content is embedded" + :description "Split the output file contents such that no content is embedded" dump:option "-o --outdir" - :description "Output directory" - :argname("<directory>") + :description "Output directory" + :argname("<directory>") local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -239,7 +238,7 @@ local function load_task(opts, fname) fname = '-' end - local res,task = rspamd_task.load_from_file(fname, rspamd_config) + local res, task = rspamd_task.load_from_file(fname, rspamd_config) if not res then parser:error(string.format('cannot read message from %s: %s', fname, @@ -266,9 +265,15 @@ end local function output_fmt(opts) local fmt = 'json' - if opts.compact then fmt = 'json-compact' end - if opts.ucl then fmt = 'ucl' end - if opts.messagepack then fmt = 'msgpack' end + if opts.compact then + fmt = 'json-compact' + end + if opts.ucl then + fmt = 'ucl' + end + if opts.messagepack then + fmt = 'msgpack' + end return fmt end @@ -320,20 +325,20 @@ local function extract_handler(opts) if not opts.json and not opts.ucl then table.insert(out, rspamd_logger.slog('Part: %s: %s, language: %s, size: %s (%s raw), words: %s', - part:get_mimepart():get_digest():sub(1,8), - t, - part:get_language(), - part:get_length(), part:get_raw_length(), - part:get_words_count())) + part:get_mimepart():get_digest():sub(1, 8), + t, + part:get_language(), + part:get_length(), part:get_raw_length(), + part:get_words_count())) table.insert(out, rspamd_logger.slog('Stats: %s', - fun.foldl(function(acc, k, v) - if acc ~= '' then - return string.format('%s, %s:%s', acc, k, v) - else - return string.format('%s:%s', k,v) - end - end, '', part:get_stats()))) + fun.foldl(function(acc, k, v) + if acc ~= '' then + return string.format('%s, %s:%s', acc, k, v) + else + return string.format('%s:%s', k, v) + end + end, '', part:get_stats()))) end end end @@ -342,11 +347,11 @@ local function extract_handler(opts) if opts.part then if not opts.json and not opts.ucl then - local mtype,msubtype = part:get_type() - local det_mtype,det_msubtype = part:get_detected_type() + local mtype, msubtype = part:get_type() + local det_mtype, det_msubtype = part:get_detected_type() table.insert(out, rspamd_logger.slog('Mime Part: %s: %s/%s (%s/%s detected), filename: %s (%s detected ext), size: %s', - part:get_digest():sub(1,8), + part:get_digest():sub(1, 8), mtype, msubtype, det_mtype, det_msubtype, part:get_filename(), @@ -378,7 +383,7 @@ local function extract_handler(opts) end end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) out_elts[fname] = {} @@ -396,10 +401,12 @@ local function extract_handler(opts) if opts.text or opts.html then local mp = task:get_parts() or {} - for _,mime_part in ipairs(mp) do + for _, mime_part in ipairs(mp) do local how = opts.output local part - if mime_part:is_text() then part = mime_part:get_text() end + if mime_part:is_text() then + part = mime_part:get_text() + end if part and opts.text and not part:is_html() then maybe_print_text_part_info(part, out_elts[fname]) @@ -445,24 +452,25 @@ local function extract_handler(opts) end hc:foreach_tag('any', function(tag) - local elt = {} - local ex = tag:get_extra() - elt.tag = tag:get_type() - if ex then - elt.extra = ex - end - local content = tag:get_content() - if content then - elt.content = tostring(content) - end - local style = tag:get_style() - if style then - elt.style = style - end - table.insert(res, elt) + local elt = {} + local ex = tag:get_extra() + elt.tag = tag:get_type() + if ex then + elt.extra = ex + end + local content = tag:get_content() + if content then + elt.content = tostring(content) + end + local style = tag:get_style() + if style then + elt.style = style + end + table.insert(res, elt) end) table.insert(out_elts[fname], res) - else -- opts.structure + else + -- opts.structure table.insert(out_elts[fname], tostring(part:get_content(how))) end if opts.invisible then @@ -485,7 +493,9 @@ local function extract_handler(opts) print_elts(out_elts, opts, process_func) -- To avoid use after free we postpone tasks destruction - for _,task in ipairs(tasks) do task:destroy() end + for _, task in ipairs(tasks) do + task:destroy() + end end local function stat_handler(opts) @@ -498,7 +508,7 @@ local function stat_handler(opts) local process_func - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) out_elts[fname] = {} @@ -514,7 +524,9 @@ local function stat_handler(opts) process_func = function(e) return string.format('%s (%d): "%s"+"%s", [%s]', e.data, e.win, e.t1 or "", e.t2 or "", table.concat(fun.totable( - fun.map(function(k) return k end, e.flags)), ",")) + fun.map(function(k) + return k + end, e.flags)), ",")) end elseif opts.fuzzy then local parts = task:get_parts() or {} @@ -523,7 +535,7 @@ local function stat_handler(opts) local ret = string.format('part: %s(%s): %s', e.type, e.file or "", e.digest) if opts.shingles and e.shingles then local sgl = {} - for _,s in ipairs(e.shingles) do + for _, s in ipairs(e.shingles) do table.insert(sgl, string.format('%s: %s+%s+%s', s[1], s[2], s[3], s[4])) end @@ -531,26 +543,26 @@ local function stat_handler(opts) end return ret end - for _,part in ipairs(parts) do + for _, part in ipairs(parts) do if not part:is_multipart() then local text = part:get_text() if text then - local digest,shingles = text:get_fuzzy_hashes(task:get_mempool()) + local digest, shingles = text:get_fuzzy_hashes(task:get_mempool()) table.insert(out_elts[fname], { digest = digest, shingles = shingles, type = string.format('%s/%s', - ({part:get_type()})[1], - ({part:get_type()})[2]) + ({ part:get_type() })[1], + ({ part:get_type() })[2]) }) else table.insert(out_elts[fname], { digest = part:get_digest(), file = part:get_filename(), type = string.format('%s/%s', - ({part:get_type()})[1], - ({part:get_type()})[2]) + ({ part:get_type() })[1], + ({ part:get_type() })[2]) }) end end @@ -568,9 +580,11 @@ local function urls_handler(opts) rspamd_url.init(rspamd_config:get_tld_path()) local out_elts = {} - if opts.json then rspamd_logger.messagex('[') end + if opts.json then + rspamd_logger.messagex('[') + end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do out_elts[fname] = {} local task = load_task(opts, fname) local elts = {} @@ -605,7 +619,7 @@ local function urls_handler(opts) end end - for _,u in ipairs(task:get_urls(true)) do + for _, u in ipairs(task:get_urls(true)) do process_url(u) end @@ -672,12 +686,11 @@ local function urls_handler(opts) end end - - for s,u in lua_util.spairs(elts, sfunc) do + for s, u in lua_util.spairs(elts, sfunc) do process_elt(s, u) end else - for s,u in pairs(elts) do + for s, u in pairs(elts) do process_elt(s, u) end end @@ -723,7 +736,7 @@ local function modify_handler(opts) html_footer = read_file(opts['html_footer']) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) local newline_s = newline(task) local seen_cte @@ -732,14 +745,14 @@ local function modify_handler(opts) local out = {} -- Start with headers local function process_headers_cb(name, hdr) - for _,h in ipairs(opts['remove_header']) do + for _, h in ipairs(opts['remove_header']) do if name:match(h) then return end end - for _,h in ipairs(opts['rewrite_header']) do - local hname,hpattern = h:match('^([^=]+)=(.+)$') + for _, h in ipairs(opts['rewrite_header']) do + local hname, hpattern = h:match('^([^=]+)=(.+)$') if hname == name then local new_value = string.format(hpattern, hdr.decoded) new_value = string.format('%s:%s%s', @@ -769,10 +782,10 @@ local function modify_handler(opts) out[#out + 1] = hdr.raw:gsub('\r?\n?$', '') end - task:headers_foreach(process_headers_cb, {full = true}) + task:headers_foreach(process_headers_cb, { full = true }) - for _,h in ipairs(opts['add_header']) do - local hname,hvalue = h:match('^([^=]+)=(.+)$') + for _, h in ipairs(opts['add_header']) do + local hname, hvalue = h:match('^([^=]+)=(.+)$') if hname and hvalue then out[#out + 1] = string.format('%s: %s', hname, @@ -789,14 +802,14 @@ local function modify_handler(opts) out[#out + 1] = '' if rewrite.out then - for _,o in ipairs(rewrite.out) do + for _, o in ipairs(rewrite.out) do out[#out + 1] = o end else - out[#out + 1] = {task:get_rawbody(), false} + out[#out + 1] = { task:get_rawbody(), false } end - for _,o in ipairs(out) do + for _, o in ipairs(out) do if type(o) == 'string' then io.write(o) io.write(newline_s) @@ -844,7 +857,7 @@ local function sign_handler(opts) os.exit(1) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) local ctx = lua_dkim.create_sign_context(task, sign_key, nil, opts.algorithm) @@ -917,7 +930,7 @@ local function write_dump_content(dump_content, fname, extension, outdir) end if dump_content:save_in_file(outpath) then wrote_filepath = outpath - io.write(wrote_filepath.."\n") + io.write(wrote_filepath .. "\n") else io.stderr:write(string.format("Unable to save dump content to file: %s\n", outpath)) end @@ -954,7 +967,7 @@ local function dump_handler(opts) load_config(opts) rspamd_url.init(rspamd_config:get_tld_path()) - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) local data, extension = get_dump_content(task, opts, fname) write_dump_content(data, fname, extension, opts.outdir) @@ -969,7 +982,7 @@ local function handler(args) local command = opts.command if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } elseif type(opts.file) == 'none' then opts.file = {} end @@ -993,7 +1006,7 @@ end return { name = 'mime', - aliases = {'mime_tool'}, + aliases = { 'mime_tool' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/neural_test.lua b/lualib/rspamadm/neural_test.lua index cb056b97a..31d21a990 100644 --- a/lualib/rspamadm/neural_test.lua +++ b/lualib/rspamadm/neural_test.lua @@ -4,9 +4,9 @@ local lua_util = require "lua_util" local ucl = require "ucl" local parser = argparse() - :name "rspamadm neural_test" - :description "Test the neural network with labelled dataset" - :help_description_margin(32) + :name "rspamadm neural_test" + :description "Test the neural network with labelled dataset" + :help_description_margin(32) parser:option "-c --config" :description "Path to config file" @@ -40,26 +40,24 @@ parser:option '--rule' :description 'Rule to test' :argname('<rule>') - local HAM = "HAM" local SPAM = "SPAM" local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) end end - local function scan_email(rspamc_path, host, n_parallel, path, timeout) local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s", @@ -128,13 +126,13 @@ end local function get_stats_from_scan_results(results, rules) local rule_stats = {} - for rule,_ in pairs(rules) do - rule_stats[rule] = {tp = 0, tn = 0, fp = 0, fn = 0} + for rule, _ in pairs(rules) do + rule_stats[rule] = { tp = 0, tn = 0, fp = 0, fn = 0 } end - for _,result in ipairs(results) do - for _,symbol in ipairs(result["symbols"]) do - for name,rule in pairs(rules) do + for _, result in ipairs(results) do + for _, symbol in ipairs(result["symbols"]) do + for name, rule in pairs(rules) do if rule.symbol_spam and rule.symbol_spam == symbol then if result.type == HAM then rule_stats[name].fp = rule_stats[name].fp + 1 @@ -152,7 +150,7 @@ local function get_stats_from_scan_results(results, rules) end end - for rule,_ in pairs(rules) do + for rule, _ in pairs(rules) do rule_stats[rule].fpr = rule_stats[rule].fp / (rule_stats[rule].fp + rule_stats[rule].tn) rule_stats[rule].fnr = rule_stats[rule].fn / (rule_stats[rule].fn + rule_stats[rule].tp) end @@ -222,10 +220,9 @@ local function handler(args) end - return { name = "neuraltest", - aliases = {"neural_test"}, + aliases = { "neural_test" }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/publicsuffix.lua b/lualib/rspamadm/publicsuffix.lua index f8e1bac93..96bf0699e 100644 --- a/lualib/rspamadm/publicsuffix.lua +++ b/lualib/rspamadm/publicsuffix.lua @@ -31,17 +31,17 @@ parser:option '-c --config' :default(rspamd_paths['CONFDIR'] .. '/rspamd.conf') parser:command 'compile' - :description 'Compile publicsuffix list if needed' + :description 'Compile publicsuffix list if needed' local function load_config(config_file) - local _r,err = rspamd_config:load_ucl(config_file) + local _r, err = rspamd_config:load_ucl(config_file) if not _r then rspamd_logger.errx('cannot load %s: %s', config_file, err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', config_file, err) os.exit(1) @@ -67,7 +67,6 @@ local function handler(args) load_config(cmd_opts.config_file) - if cmd_opts.command == 'compile' then compile_handler(cmd_opts) else diff --git a/lualib/rspamadm/stat_convert.lua b/lualib/rspamadm/stat_convert.lua index 3b54826aa..62a19a221 100644 --- a/lualib/rspamadm/stat_convert.lua +++ b/lualib/rspamadm/stat_convert.lua @@ -4,7 +4,7 @@ local ucl = require "ucl" local logger = require "rspamd_logger" local lua_util = require "lua_util" -return function (_, res) +return function(_, res) local redis_params = lua_redis.try_load_redis_servers(res.redis, nil) if res.expire then res.expire = lua_util.parse_time_interval(res.expire) @@ -22,7 +22,7 @@ return function (_, res) return false end - for _,cls in ipairs(sqlite_params) do + for _, cls in ipairs(sqlite_params) do if not stat_tools.convert_sqlite_to_redis(redis_params, cls.db_spam, cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, res.expire, res.reset_previous) then @@ -33,6 +33,6 @@ return function (_, res) logger.messagex('Converted classifier to the from sqlite to redis') logger.messagex('Suggested configuration:') logger.messagex(ucl.to_format(stat_tools.redis_classifier_from_sqlite(cls, res.expire), - 'config')) + 'config')) end end diff --git a/lualib/rspamadm/statistics_dump.lua b/lualib/rspamadm/statistics_dump.lua index 0ed921f6a..6bc045850 100644 --- a/lualib/rspamadm/statistics_dump.lua +++ b/lualib/rspamadm/statistics_dump.lua @@ -68,31 +68,31 @@ restore:argument "file" :argname "<file>" :args "*" restore:option "-b --batch-size" - :description "Number of entires to process at once" - :argname("<elts>") - :convert(tonumber) - :default(1000) + :description "Number of entires to process at once" + :argname("<elts>") + :convert(tonumber) + :default(1000) restore:option "-m --mode" :description "Number of entires to process at once" :argname("<append|subtract|replace>") :convert { - ['append'] = 'append', - ['subtract'] = 'subtract', - ['replace'] = 'replace', - } + ['append'] = 'append', + ['subtract'] = 'subtract', + ['replace'] = 'replace', +} :default 'append' restore:flag "-n --no-operation" - :description "Only show redis commands to be issued" + :description "Only show redis commands to be issued" local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -128,9 +128,9 @@ local function check_redis_classifier(cls, cfg) local statfiles = cls.statfile if statfiles[1] then - for _,stf in ipairs(statfiles) do + for _, stf in ipairs(statfiles) do if not stf.symbol then - for k,v in pairs(stf) do + for k, v in pairs(stf) do check_statfile_table(v, k) end else @@ -138,7 +138,7 @@ local function check_redis_classifier(cls, cfg) end end else - for stn,stf in pairs(statfiles) do + for stn, stf in pairs(statfiles) do check_statfile_table(stf, stn) end end @@ -168,7 +168,7 @@ end local function redis_map_zip(ar) local data = {} - for j=1,#ar,2 do + for j = 1, #ar, 2 do data[ar[j]] = ar[j + 1] end @@ -178,7 +178,7 @@ end -- Used to clear tables local clear_fcn = table.clear or function(tbl) local keys = lua_util.keys(tbl) - for _,k in ipairs(keys) do + for _, k in ipairs(keys) do tbl[k] = nil end end @@ -197,7 +197,7 @@ local function dump_out(out, opts, last) compress_ctx:stream(rspamd_text.fromtable(out), 'flush'):write() end else - for _,o in ipairs(out) do + for _, o in ipairs(out) do io.write(o) end end @@ -213,7 +213,7 @@ local function dump_cdb(out, opts, last, pattern) out.cdb_builder:add('_lrnham_', rspamd_i64.fromstring(results.learns_ham or '0')) end - for _,o in ipairs(results.elts) do + for _, o in ipairs(results.elts) do out.cdb_builder:add(o.key, o.value) end @@ -227,9 +227,9 @@ local function dump_pattern(conn, pattern, opts, out, key) local cursor = 0 repeat - conn:add_cmd('SCAN', {tostring(cursor), - 'MATCH', pattern, - 'COUNT', tostring(opts.batch_size)}) + conn:add_cmd('SCAN', { tostring(cursor), + 'MATCH', pattern, + 'COUNT', tostring(opts.batch_size) }) local ret, results = conn:exec() if not ret then @@ -242,26 +242,26 @@ local function dump_pattern(conn, pattern, opts, out, key) local elts = results[2] local tokens = {} - for _,e in ipairs(elts) do - conn:add_cmd('HGETALL', {e}) + for _, e in ipairs(elts) do + conn:add_cmd('HGETALL', { e }) end -- This function returns many results, each for each command -- So if we have batch 1000, then we would have 1000 tables in form -- [result, {hash_content}] - local all_results = {conn:exec()} + local all_results = { conn:exec() } - for i=1,#all_results,2 do + for i = 1, #all_results, 2 do local r, hash_content = all_results[i], all_results[i + 1] if r then -- List to a hash map local data = redis_map_zip(hash_content) - tokens[#tokens + 1] = {key = elts[(i + 1)/2], data = data} + tokens[#tokens + 1] = { key = elts[(i + 1) / 2], data = data } end end -- Output keeping track of the commas - for i,d in ipairs(tokens) do + for i, d in ipairs(tokens) do if cursor == 0 and i == #tokens or not opts.json then if opts.cdb then table.insert(out[key].elts, { @@ -302,8 +302,8 @@ end local function dump_handler(opts) local patterns_seen = {} - for _,cls in ipairs(classifiers) do - local res,conn = lua_redis.redis_connect_sync(cls.redis_params, false) + for _, cls in ipairs(classifiers) do + local res, conn = lua_redis.redis_connect_sync(cls.redis_params, false) if not res then rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params) @@ -314,7 +314,7 @@ local function dump_handler(opts) local function check_keys(sym) local sym_keys_pattern = string.format("%s_keys", sym) conn:add_cmd('SMEMBERS', { sym_keys_pattern }) - local ret,keys = conn:exec() + local ret, keys = conn:exec() if not ret then rspamd_logger.errx("cannot execute command to get keys: %s", keys) @@ -325,11 +325,11 @@ local function dump_handler(opts) out[#out + 1] = string.format('"%s": %s\n', sym_keys_pattern, ucl.to_format(keys, 'json-compact')) end - for _,k in ipairs(keys) do + for _, k in ipairs(keys) do local pat = string.format('%s*_*', k) if not patterns_seen[pat] then - conn:add_cmd('HGETALL', {k}) - local _ret,additional_keys = conn:exec() + conn:add_cmd('HGETALL', { k }) + local _ret, additional_keys = conn:exec() if _ret then if opts.json then @@ -359,19 +359,19 @@ local function dump_handler(opts) end local function obj_to_redis_arguments(obj, opts, cmd_pipe) - local key,value = next(obj) + local key, value = next(obj) if type(key) == 'string' then if type(value) == 'table' then if not value[1] then if opts.mode == 'replace' then local cmd = 'HMSET' - local params = {key} - for k,v in pairs(value) do + local params = { key } + for k, v in pairs(value) do table.insert(params, k) table.insert(params, v) end - table.insert(cmd_pipe, {cmd, params}) + table.insert(cmd_pipe, { cmd, params }) else local cmd = 'HINCRBYFLOAT' local mult = 1.0 @@ -379,19 +379,19 @@ local function obj_to_redis_arguments(obj, opts, cmd_pipe) mult = (-mult) end - for k,v in pairs(value) do + for k, v in pairs(value) do if tonumber(v) then v = tonumber(v) - table.insert(cmd_pipe, {cmd, {key, k, tostring(v * mult)}}) + table.insert(cmd_pipe, { cmd, { key, k, tostring(v * mult) } }) else - table.insert(cmd_pipe, {'HSET', {key, k, v}}) + table.insert(cmd_pipe, { 'HSET', { key, k, v } }) end end end else -- Numeric table of elements (e.g. _keys) - it is actually a set in Redis - for _,elt in ipairs(value) do - table.insert(cmd_pipe, {'SADD', {key, elt}}) + for _, elt in ipairs(value) do + table.insert(cmd_pipe, { 'SADD', { key, elt } }) end end end @@ -403,17 +403,17 @@ end local function execute_batch(batch, conns, opts) local cmd_pipe = {} - for _,cmd in ipairs(batch) do + for _, cmd in ipairs(batch) do obj_to_redis_arguments(cmd, opts, cmd_pipe) end if opts.no_operation then - for _,cmd in ipairs(cmd_pipe) do + for _, cmd in ipairs(cmd_pipe) do rspamd_logger.messagex('%s %s', cmd[1], table.concat(cmd[2], ' ')) end else for _, conn in ipairs(conns) do - for _,cmd in ipairs(cmd_pipe) do + for _, cmd in ipairs(cmd_pipe) do local is_ok, err = conn:add_cmd(cmd[1], cmd[2]) if not is_ok then @@ -427,11 +427,11 @@ local function execute_batch(batch, conns, opts) end local function restore_handler(opts) - local files = opts.file or {'-'} + local files = opts.file or { '-' } local conns = {} - for _,cls in ipairs(classifiers) do - local res,conn = lua_redis.redis_connect_sync(cls.redis_params, true) + for _, cls in ipairs(classifiers) do + local res, conn = lua_redis.redis_connect_sync(cls.redis_params, true) if not res then rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params) @@ -443,7 +443,7 @@ local function restore_handler(opts) local batch = {} - for _,f in ipairs(files) do + for _, f in ipairs(files) do local fd if f ~= '-' then fd = io.open(f, 'r') @@ -454,7 +454,7 @@ local function restore_handler(opts) for line in io.lines() do local ucl_parser = ucl.parser() local res, err - res,err = ucl_parser:parse_string(line) + res, err = ucl_parser:parse_string(line) if not res then rspamd_logger.errx("%s: cannot read line %s: %s", f, cur_line, err) @@ -470,7 +470,9 @@ local function restore_handler(opts) end end - if fd then fd:close() end + if fd then + fd:close() + end end if #batch > 0 then @@ -492,8 +494,10 @@ local function handler(args) if classifier then if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.bayes then cls = cls.bayes end + for _, cls in ipairs(classifier) do + if cls.bayes then + cls = cls.bayes + end if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, obj) end @@ -503,7 +507,7 @@ local function handler(args) classifier = classifier.bayes if classifier[1] then - for _,cls in ipairs(classifier) do + for _, cls in ipairs(classifier) do if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, obj) end @@ -518,7 +522,7 @@ local function handler(args) end if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } elseif type(opts.file) == 'none' then opts.file = {} end @@ -534,7 +538,7 @@ end return { name = 'statistics_dump', - aliases = {'stat_dump', 'bayes_dump'}, + aliases = { 'stat_dump', 'bayes_dump' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/template.lua b/lualib/rspamadm/template.lua index bb799f730..ca1779aa8 100644 --- a/lualib/rspamadm/template.lua +++ b/lualib/rspamadm/template.lua @@ -42,17 +42,17 @@ parser:mutex( :description "Store files with the new suffix" :argname "<suffix>", parser:flag "-i --inplace" - :description "Replace input file(s)" + :description "Replace input file(s)" ) local lua_util = require "lua_util" local function set_env(opts, env) if opts.env then - for _,fname in ipairs(opts.env) do + for _, fname in ipairs(opts.env) do for kv in assert(io.open(fname)):lines() do if not kv:match('%s*#.*') then - local k,v = kv:match('([^=%s]+)%s*=%s*(.+)') + local k, v = kv:match('([^=%s]+)%s*=%s*(.+)') if k and v then env[k] = v @@ -65,14 +65,14 @@ local function set_env(opts, env) end if opts.lua_env then - for _,fname in ipairs(opts.env) do - local ret,res_or_err = pcall(loadfile(fname)) + for _, fname in ipairs(opts.env) do + local ret, res_or_err = pcall(loadfile(fname)) if not ret then io.write(string.format('cannot load %s: %s\n', fname, res_or_err)) else if type(res_or_err) == 'table' then - for k,v in pairs(res_or_err) do + for k, v in pairs(res_or_err) do env[k] = lua_util.deepcopy(v) end else @@ -100,8 +100,10 @@ local function handler(args) local env = {} set_env(opts, env) - if not opts.file or #opts.file == 0 then opts.file = {'-'} end - for _,fname in ipairs(opts.file) do + if not opts.file or #opts.file == 0 then + opts.file = { '-' } + end + for _, fname in ipairs(opts.file) do local content = read_file(fname) local res = lua_util.jinja_template(content, env, opts.no_vars) diff --git a/lualib/rspamadm/vault.lua b/lualib/rspamadm/vault.lua index 2c7d5abfe..840e504e0 100644 --- a/lualib/rspamadm/vault.lua +++ b/lualib/rspamadm/vault.lua @@ -43,28 +43,27 @@ parser:option "-o --output" :description "Output format ('ucl', 'json', 'json-compact', 'yaml')" :argname("<type>") :convert { - ucl = "ucl", - json = "json", - ['json-compact'] = "json-compact", - yaml = "yaml", - } - :default "ucl" + ucl = "ucl", + json = "json", + ['json-compact'] = "json-compact", + yaml = "yaml", +} + :default "ucl" parser:command "list ls l" - :description "List elements in the vault" + :description "List elements in the vault" local show = parser:command "show get" - :description "Extract element from the vault" + :description "Extract element from the vault" show:argument "domain" - :description "Domain to create key for" - :args "+" + :description "Domain to create key for" + :args "+" local delete = parser:command "delete del rm remove" - :description "Delete element from the vault" + :description "Delete element from the vault" delete:argument "domain" - :description "Domain to create delete key(s) for" - :args "+" - + :description "Domain to create delete key(s) for" + :args "+" local newkey = parser:command "newkey new create" :description "Add new key to the vault" @@ -77,10 +76,10 @@ newkey:option "-s --selector" newkey:option "-A --algorithm" :argname("<type>") :convert { - rsa = "rsa", - ed25519 = "ed25519", - eddsa = "ed25519", - } + rsa = "rsa", + ed25519 = "ed25519", + eddsa = "ed25519", +} :default "rsa" newkey:option "-b --bits" :argname("<nbits>") @@ -137,18 +136,18 @@ end local function parse_vault_reply(data) local p = ucl.parser() - local res,parser_err = p:parse_string(data) + local res, parser_err = p:parse_string(data) if not res then - return nil,parser_err + return nil, parser_err else - return p:get_object(),nil + return p:get_object(), nil end end local function maybe_print_vault_data(opts, data, func) if data then - local res,parser_err = parse_vault_reply(data) + local res, parser_err = parse_vault_reply(data) if not res then printf('vault reply for cannot be parsed: %s', parser_err) @@ -169,9 +168,9 @@ local function print_dkim_txt_record(b64, selector, alg) local prefix = string.format("v=DKIM1; k=%s; p=", alg) b64 = prefix .. b64 if #b64 < 255 then - labels = {'"' .. b64 .. '"'} + labels = { '"' .. b64 .. '"' } else - for sl=1,#b64,256 do + for sl = 1, #b64, 256 do table.insert(labels, '"' .. b64:sub(sl, sl + 255) .. '"') end end @@ -182,7 +181,7 @@ end local function show_handler(opts, domain) local uri = vault_url(opts, domain) - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -206,7 +205,7 @@ end local function delete_handler(opts, domain) local uri = vault_url(opts, domain) - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -229,7 +228,7 @@ end local function list_handler(opts) local uri = vault_url(opts) - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -258,7 +257,7 @@ end local function create_and_push_key(opts, domain, existing) local uri = vault_url(opts, domain) - local sk,pk = genkey(opts) + local sk, pk = genkey(opts) local res = { selectors = { @@ -274,7 +273,7 @@ local function create_and_push_key(opts, domain, existing) } } - for _,sel in ipairs(existing) do + for _, sel in ipairs(existing) do res.selectors[#res.selectors + 1] = sel end @@ -282,7 +281,7 @@ local function create_and_push_key(opts, domain, existing) res.selectors[1].valid_end = os.time() + opts.expire * 3600 * 24 end - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -303,7 +302,7 @@ local function create_and_push_key(opts, domain, existing) maybe_print_vault_data(opts, data.content) os.exit(1) else - maybe_printf(opts,'stored key for: %s, selector: %s', domain, opts.selector) + maybe_printf(opts, 'stored key for: %s, selector: %s', domain, opts.selector) maybe_printf(opts, 'please place the corresponding public key as following:') if opts.silent then @@ -322,7 +321,7 @@ local function newkey_handler(opts, domain) os.date("!%Y%m%d")) end - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -335,7 +334,7 @@ local function newkey_handler(opts, domain) } if is_http_error(err, data) or not data.content then - create_and_push_key(opts, domain,{}) + create_and_push_key(opts, domain, {}) else -- Key exists local rep = parse_vault_reply(data.content) @@ -348,11 +347,11 @@ local function newkey_handler(opts, domain) local elts = rep.data.selectors if not elts then - create_and_push_key(opts, domain,{}) + create_and_push_key(opts, domain, {}) os.exit(0) end - for _,sel in ipairs(elts) do + for _, sel in ipairs(elts) do if sel.alg == opts.algorithm then printf('key with the specific algorithm %s is already presented at %s selector for %s domain', opts.algorithm, sel.selector, domain) @@ -370,7 +369,7 @@ local function roll_handler(opts, domain) selectors = {} } - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -414,7 +413,7 @@ local function roll_handler(opts, domain) table.insert(nkeys[sel.alg], sel) end - for _,sel in ipairs(elts) do + for _, sel in ipairs(elts) do if sel.valid_end and sel.valid_end < os.time() then if not opts.remove_expired then insert_key(sel, false) @@ -428,7 +427,7 @@ local function roll_handler(opts, domain) end -- Now we need to ensure that all but one selectors have either expired or just a single key - for alg,keys in pairs(nkeys) do + for alg, keys in pairs(nkeys) do table.sort(keys, function(k1, k2) if k1.valid_end and k2.valid_end then return k1.valid_end > k2.valid_end @@ -441,8 +440,8 @@ local function roll_handler(opts, domain) end) -- Exclude the key with the highest expiration date and examine the rest if not (#keys == 1 or fun.all(function(k) - return k.valid_end and k.valid_end < os.time() - end, fun.tail(keys))) then + return k.valid_end and k.valid_end < os.time() + end, fun.tail(keys))) then printf('bad keys list for %s and %s algorithm', domain, alg) fun.each(function(k) if not k.valid_end then @@ -459,7 +458,7 @@ local function roll_handler(opts, domain) if not opts.remove_expired then -- OK to process -- Insert keys for each algorithm in pairs <old_key(s)>, <new_key> - local sk,pk = genkey({algorithm = alg, bits = keys[1].bits}) + local sk, pk = genkey({ algorithm = alg, bits = keys[1].bits }) local selector = string.format('%s-%s', alg, os.date("!%Y%m%d")) @@ -482,14 +481,14 @@ local function roll_handler(opts, domain) table.insert(res.selectors, nelt) end - for _,k in ipairs(keys) do + for _, k in ipairs(keys) do table.insert(res.selectors, k) end end end -- We can now store res in the vault - err,data = rspamd_http.request{ + err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -510,9 +509,9 @@ local function roll_handler(opts, domain) maybe_print_vault_data(opts, data.content) os.exit(1) else - for _,key in ipairs(res.selectors) do - if not key.valid_end or key.valid_end > os.time() + opts.ttl * 3600 * 24 then - maybe_printf(opts,'rolled key for: %s, new selector: %s', domain, key.selector) + for _, key in ipairs(res.selectors) do + if not key.valid_end or key.valid_end > os.time() + opts.ttl * 3600 * 24 then + maybe_printf(opts, 'rolled key for: %s, new selector: %s', domain, key.selector) maybe_printf(opts, 'please place the corresponding public key as following:') if opts.silent then @@ -553,13 +552,21 @@ local function handler(args) if command == 'list' then list_handler(opts) elseif command == 'show' then - fun.each(function(d) show_handler(opts, d) end, opts.domain) + fun.each(function(d) + show_handler(opts, d) + end, opts.domain) elseif command == 'newkey' then - fun.each(function(d) newkey_handler(opts, d) end, opts.domain) + fun.each(function(d) + newkey_handler(opts, d) + end, opts.domain) elseif command == 'roll' then - fun.each(function(d) roll_handler(opts, d) end, opts.domain) + fun.each(function(d) + roll_handler(opts, d) + end, opts.domain) elseif command == 'delete' then - fun.each(function(d) delete_handler(opts, d) end, opts.domain) + fun.each(function(d) + delete_handler(opts, d) + end, opts.domain) else parser:error(string.format('command %s is not implemented', command)) end diff --git a/rules/bitcoin.lua b/rules/bitcoin.lua index 1c902e93e..6a70721f8 100644 --- a/rules/bitcoin.lua +++ b/rules/bitcoin.lua @@ -26,7 +26,7 @@ local off = 0 local base58_dec = fun.tomap(fun.map( function(c) off = off + 1 - return c,(off - 1) + return c, (off - 1) end, "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")) @@ -34,11 +34,13 @@ local function is_traditional_btc_address(word) local hash = require "rspamd_cryptobox_hash" local bytes = {} - for i=1,25 do bytes[i] = 0 end + for i = 1, 25 do + bytes[i] = 0 + end -- Base58 decode loop fun.each(function(ch) local acc = base58_dec[ch] or 0 - for i=25,1,-1 do + for i = 25, 1, -1 do acc = acc + (58 * bytes[i]); bytes[i] = acc % 256 acc = math.floor(acc / 256); @@ -46,14 +48,14 @@ local function is_traditional_btc_address(word) end, word) -- Now create a validation tag local sha256 = hash.create_specific('sha256') - for i=1,21 do + for i = 1, 21 do sha256:update(string.char(bytes[i])) end sha256 = hash.create_specific('sha256', sha256:bin()):bin() -- Compare tags local valid = true - for i=1,4 do + for i = 1, 4 do if string.sub(sha256, i, i) ~= string.char(bytes[21 + i]) then valid = false end @@ -65,13 +67,13 @@ end -- Beach32 checksum combiner local function polymod(...) local chk = 1; - local gen = {0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3}; - for _,t in ipairs({...}) do - for _,v in ipairs(t) do + local gen = { 0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3 }; + for _, t in ipairs({ ... }) do + for _, v in ipairs(t) do local top = bit.rshift(chk, 25) chk = bit.bxor(bit.lshift(bit.band(chk, 0x1ffffff), 5), v) - for i=1,5 do + for i = 1, 5 do if bit.band(bit.rshift(top, i - 1), 0x1) ~= 0 then chk = bit.bxor(chk, gen[i]) end @@ -100,7 +102,6 @@ local function verify_beach32_cksum(hrp, elts) return polymod(hrpExpand(hrp), elts) == 1 end - local function gen_bleach32_table(input) local d = {} local i = 1 @@ -167,7 +168,9 @@ local function is_segwit_bech32_address(task, word) -- For semicolon table.insert(polymod_tbl, 0) - fun.each(function(byte) table.insert(polymod_tbl, byte) end, decoded) + fun.each(function(byte) + table.insert(polymod_tbl, byte) + end, decoded) lua_util.debugm(N, task, 'final polymod table: %s', polymod_tbl) return rspamd_util.btc_polymod(polymod_tbl) diff --git a/rules/bounce.lua b/rules/bounce.lua index 08e1faef4..fb74b9797 100644 --- a/rules/bounce.lua +++ b/rules/bounce.lua @@ -57,12 +57,11 @@ rspamd_config.BOUNCE = { return false end - local parts = task:get_parts() local top_type, top_subtype, params = parts[1]:get_type_full() -- RFC 3464, RFC 8098 if top_type == 'multipart' and top_subtype == 'report' and params and - (params['report-type'] == 'delivery-status' or params['report-type'] == 'disposition-notification') then + (params['report-type'] == 'delivery-status' or params['report-type'] == 'disposition-notification') then -- Assume that inner parts are OK, don't check them to save time return true, 1.0, 'DSN' end @@ -75,9 +74,9 @@ rspamd_config.BOUNCE = { -- Check common bounce senders if (from_user == 'postmaster' or from_user == 'mailer-daemon') then bounce_sender = from_user - -- MDaemon >= 14.5 sends multipart/report (RFC 3464) DSN covered above, - -- but older versions send non-standard bounces with localized subjects and they - -- are still around + -- MDaemon >= 14.5 sends multipart/report (RFC 3464) DSN covered above, + -- but older versions send non-standard bounces with localized subjects and they + -- are still around elseif from_user == 'mdaemon' and task:has_header('X-MDDSN-Message') then return true, 1.0, 'MDaemon' end diff --git a/rules/content.lua b/rules/content.lua index 0936f5898..667b7ec74 100644 --- a/rules/content.lua +++ b/rules/content.lua @@ -34,7 +34,9 @@ local function process_pdf_specific(task, part, specific) end if suspicious_factor > 0.5 then - if suspicious_factor > 1.0 then suspicious_factor = 1.0 end + if suspicious_factor > 1.0 then + suspicious_factor = 1.0 + end task:insert_result('PDF_SUSPICIOUS', suspicious_factor, part:get_filename() or 'unknown') end @@ -59,7 +61,7 @@ local tags_processors = { local function process_specific_cb(task) local parts = task:get_parts() or {} - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do if p:is_specific() then local data = p:get_specific() @@ -72,45 +74,45 @@ local function process_specific_cb(task) end end -local id = rspamd_config:register_symbol{ +local id = rspamd_config:register_symbol { type = 'callback', name = 'SPECIFIC_CONTENT_CHECK', callback = process_specific_cb } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', name = 'PDF_ENCRYPTED', parent = id, - groups = {"content", "pdf"}, + groups = { "content", "pdf" }, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', name = 'PDF_JAVASCRIPT', parent = id, - groups = {"content", "pdf"}, + groups = { "content", "pdf" }, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', name = 'PDF_SUSPICIOUS', parent = id, - groups = {"content", "pdf"}, + groups = { "content", "pdf" }, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', name = 'PDF_LONG_TRAILER', parent = id, - groups = {"content", "pdf"}, + groups = { "content", "pdf" }, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', name = 'PDF_MANY_OBJECTS', parent = id, - groups = {"content", "pdf"}, + groups = { "content", "pdf" }, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', name = 'PDF_TIMEOUT', parent = id, - groups = {"content", "pdf"}, + groups = { "content", "pdf" }, } diff --git a/rules/controller/fuzzy.lua b/rules/controller/fuzzy.lua index 7e4c96fe1..193e6fd4c 100644 --- a/rules/controller/fuzzy.lua +++ b/rules/controller/fuzzy.lua @@ -19,16 +19,16 @@ local function handle_gen_fuzzy(task, conn, req_params) local ret, hashes task:process_message() if req_params.rule then - ret,hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, req_params.rule) + ret, hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, req_params.rule) elseif req_params.flag then - ret,hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, tonumber(req_params.flag)) + ret, hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, tonumber(req_params.flag)) else conn:send_error(404, 'missing rule or flag') return end if ret then - conn:send_ucl({success = true, hashes = hashes}) + conn:send_ucl({ success = true, hashes = hashes }) else conn:send_error(500, 'cannot generate hashes') end diff --git a/rules/controller/init.lua b/rules/controller/init.lua index 9d60200f8..17fbbfc43 100644 --- a/rules/controller/init.lua +++ b/rules/controller/init.lua @@ -39,7 +39,7 @@ if rspamd_util.file_exists(local_conf .. '/controller.lua') then end end -for plug,paths in pairs(controller_plugin_paths) do +for plug, paths in pairs(controller_plugin_paths) do if not rspamd_plugins[plug] then rspamd_plugins[plug] = {} end @@ -49,7 +49,7 @@ for plug,paths in pairs(controller_plugin_paths) do local webui = rspamd_plugins[plug].webui - for path,attrs in pairs(paths) do + for path, attrs in pairs(paths) do if type(attrs) == 'table' then if type(attrs.handler) ~= 'function' then rspamd_logger.infox(rspamd_config, 'controller plugin %s; webui path %s has invalid handler: %s; ignore it', @@ -61,7 +61,7 @@ for plug,paths in pairs(controller_plugin_paths) do end else rspamd_logger.infox(rspamd_config, 'controller plugin %s; webui path %s has invalid type: %s; ignore it', - plug, path, type(attrs)) + plug, path, type(attrs)) end end end diff --git a/rules/controller/maps.lua b/rules/controller/maps.lua index a45e3d306..718e292f6 100644 --- a/rules/controller/maps.lua +++ b/rules/controller/maps.lua @@ -26,7 +26,7 @@ local function maybe_fill_maps_cache() maps_cache = {} maps_aliases = {} local maps = rspamd_config:get_maps() - for _,m in ipairs(maps) do + for _, m in ipairs(maps) do -- We get the first url here and that's it local url = m:get_uri() if url ~= 'static' then @@ -81,12 +81,12 @@ local function handle_query_map(_, conn, req_params) end local results = {} - for _,key in ipairs(keys_to_check) do - for uri,m in pairs(maps_cache) do + for _, key in ipairs(keys_to_check) do + for uri, m in pairs(maps_cache) do check_specific_map(key, uri, m, results, req_params.report_misses) end end - conn:send_ucl{ + conn:send_ucl { success = (#results > 0), results = results } @@ -106,7 +106,7 @@ local function handle_query_specific_map(_, conn, req_params) if req_params.maps then local map_names = lua_util.str_split(req_params.maps, ',') maps_to_check = {} - for _,mn in ipairs(map_names) do + for _, mn in ipairs(map_names) do if maps_cache[mn] then maps_to_check[mn] = maps_cache[mn] else @@ -122,13 +122,13 @@ local function handle_query_specific_map(_, conn, req_params) end local results = {} - for _,key in ipairs(keys_to_check) do - for uri,m in pairs(maps_to_check) do + for _, key in ipairs(keys_to_check) do + for uri, m in pairs(maps_to_check) do check_specific_map(key, uri, m, results, req_params.report_misses) end end - conn:send_ucl{ + conn:send_ucl { success = (#results > 0), results = results } @@ -136,13 +136,13 @@ end local function handle_list_maps(_, conn, _) maybe_fill_maps_cache() - conn:send_ucl{ + conn:send_ucl { maps = lua_util.keys(maps_cache), aliases = maps_aliases } end -local query_json_schema = ts.shape{ +local query_json_schema = ts.shape { maps = ts.array_of(ts.string):is_optional(), report_misses = ts.boolean:is_optional(), values = ts.array_of(ts.string), @@ -170,7 +170,7 @@ local function handle_query_json(task, conn) local results = {} if obj.maps then - for _,mn in ipairs(obj.maps) do + for _, mn in ipairs(obj.maps) do if maps_cache[mn] then maps_to_check[mn] = maps_cache[mn] else @@ -188,12 +188,12 @@ local function handle_query_json(task, conn) maps_to_check = maps_cache end - for _,key in ipairs(obj.values) do - for uri,m in pairs(maps_to_check) do + for _, key in ipairs(obj.values) do + for uri, m in pairs(maps_to_check) do check_specific_map(key, uri, m, results, report_misses) end end - conn:send_ucl{ + conn:send_ucl { success = (#results > 0), results = results } diff --git a/rules/controller/neural.lua b/rules/controller/neural.lua index db95cb7e9..aef104247 100644 --- a/rules/controller/neural.lua +++ b/rules/controller/neural.lua @@ -22,7 +22,7 @@ local E = {} -- Controller neural plugin -local learn_request_schema = ts.shape{ +local learn_request_schema = ts.shape { ham_vec = ts.array_of(ts.array_of(ts.number)), rule = ts.string:is_optional(), spam_vec = ts.array_of(ts.array_of(ts.number)), @@ -48,7 +48,7 @@ local function handle_learn(task, conn) local set = neural_common.get_rule_settings(task, rule) local version = ((set.ann or E).version or 0) + 1 - neural_common.spawn_train{ + neural_common.spawn_train { ev_base = task:get_ev_base(), ann_key = neural_common.new_ann_key(rule, set, version), set = set, diff --git a/rules/controller/selectors.lua b/rules/controller/selectors.lua index 8f3dc0931..7fc2894d7 100644 --- a/rules/controller/selectors.lua +++ b/rules/controller/selectors.lua @@ -30,7 +30,7 @@ local function handle_check_selector(_, conn, req_params) if req_params.selector and req_params.selector ~= '' then local selector = lua_selectors.create_selector_closure(rspamd_config, req_params.selector, '', true) - conn:send_ucl({success = selector and true}) + conn:send_ucl({ success = selector and true }) else conn:send_error(404, 'missing selector') end @@ -45,7 +45,7 @@ local function handle_check_message(task, conn, req_params) else task:process_message() local elts = selector(task) - conn:send_ucl({success = true, data = elts}) + conn:send_ucl({ success = true, data = elts }) end else conn:send_error(404, 'missing selector') diff --git a/rules/forwarding.lua b/rules/forwarding.lua index 7d79a0c31..a008c587d 100644 --- a/rules/forwarding.lua +++ b/rules/forwarding.lua @@ -19,22 +19,24 @@ limitations under the License. local rspamd_util = require "rspamd_util" rspamd_config.FWD_GOOGLE = { - callback = function (task) + callback = function(task) if not (task:has_from(1) and task:has_recipients(1)) then return false end - local envfrom = task:get_from{'smtp', 'orig'} + local envfrom = task:get_from { 'smtp', 'orig' } local envrcpts = task:get_recipients(1) -- Forwarding will only be to a single recipient - if #envrcpts > 1 then return false end + if #envrcpts > 1 then + return false + end -- Get recipient and compute VERP address local rcpt = envrcpts[1].addr:lower() - local verp = rcpt:gsub('@','=') + local verp = rcpt:gsub('@', '=') -- Get the user portion of the envfrom local ef_user = envfrom[1].user:lower() -- Check for a match if ef_user:find('+caf_=' .. verp, 1, true) then - local _,_,user = ef_user:find('^(.+)+caf_=') + local _, _, user = ef_user:find('^(.+)+caf_=') if user then user = user .. '@' .. envfrom[1].domain return true, user @@ -48,7 +50,7 @@ rspamd_config.FWD_GOOGLE = { } rspamd_config.FWD_YANDEX = { - callback = function (task) + callback = function(task) if not (task:has_from(1) and task:has_recipients(1)) then return false end @@ -64,7 +66,7 @@ rspamd_config.FWD_YANDEX = { } rspamd_config.FWD_MAILRU = { - callback = function (task) + callback = function(task) if not (task:has_from(1) and task:has_recipients(1)) then return false end @@ -80,14 +82,16 @@ rspamd_config.FWD_MAILRU = { } rspamd_config.FWD_SRS = { - callback = function (task) + callback = function(task) if not (task:has_from(1) and task:has_recipients(1)) then return false end local envfrom = task:get_from(1) local envrcpts = task:get_recipients(1) -- Forwarding is only to a single recipient - if #envrcpts > 1 then return false end + if #envrcpts > 1 then + return false + end -- Get recipient and compute rewritten SRS address local srs = '=' .. envrcpts[1].domain:lower() .. '=' .. envrcpts[1].user:lower() @@ -104,10 +108,10 @@ rspamd_config.FWD_SRS = { } rspamd_config.FORWARDED = { - callback = function (task) + callback = function(task) local function normalize_addr(addr) addr = string.match(addr, '^<?([^>]*)>?$') or addr - local cap, _,domain = string.match(addr, '^([^%+][^%+]*)(%+[^@]*)@(.*)$') + local cap, _, domain = string.match(addr, '^([^%+][^%+]*)(%+[^@]*)@(.*)$') if cap then addr = string.format('%s@%s', cap, domain) end @@ -115,10 +119,14 @@ rspamd_config.FORWARDED = { return addr end - if not task:has_recipients(1) or not task:has_recipients(2) then return false end + if not task:has_recipients(1) or not task:has_recipients(2) then + return false + end local envrcpts = task:get_recipients(1) -- Forwarding will only be for single recipient messages - if #envrcpts > 1 then return false end + if #envrcpts > 1 then + return false + end -- Get any other headers we might need local has_list_unsub = task:has_header('List-Unsubscribe') local to = task:get_recipients(2) diff --git a/rules/headers_checks.lua b/rules/headers_checks.lua index f3d93efe7..f28b0bc7a 100644 --- a/rules/headers_checks.lua +++ b/rules/headers_checks.lua @@ -23,7 +23,7 @@ local tonumber = tonumber local fun = require "fun" local E = {} -local rcvd_cb_id = rspamd_config:register_symbol{ +local rcvd_cb_id = rspamd_config:register_symbol { name = 'CHECK_RECEIVED', type = 'callback', score = 0.0, @@ -40,12 +40,12 @@ local rcvd_cb_id = rspamd_config:register_symbol{ local def = 'ZERO' local received = task:get_received_headers() local nreceived = fun.reduce(function(acc, rcvd) - return acc + 1 - end, 0, fun.filter(function(h) - return not h['flags']['artificial'] - end, received)) + return acc + 1 + end, 0, fun.filter(function(h) + return not h['flags']['artificial'] + end, received)) - for k,v in pairs(cnts) do + for k, v in pairs(cnts) do if nreceived >= tonumber(k) then def = v end @@ -55,7 +55,7 @@ local rcvd_cb_id = rspamd_config:register_symbol{ end } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCVD_COUNT_ZERO', score = 0.0, parent = rcvd_cb_id, @@ -63,7 +63,7 @@ rspamd_config:register_symbol{ description = 'Message has no Received headers', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCVD_COUNT_ONE', score = 0.0, parent = rcvd_cb_id, @@ -71,7 +71,7 @@ rspamd_config:register_symbol{ description = 'Message has one Received header', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCVD_COUNT_TWO', score = 0.0, parent = rcvd_cb_id, @@ -79,7 +79,7 @@ rspamd_config:register_symbol{ description = 'Message has two Received headers', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCVD_COUNT_THREE', score = 0.0, parent = rcvd_cb_id, @@ -87,7 +87,7 @@ rspamd_config:register_symbol{ description = 'Message has 3-5 Received headers', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCVD_COUNT_FIVE', score = 0.0, parent = rcvd_cb_id, @@ -95,7 +95,7 @@ rspamd_config:register_symbol{ description = 'Message has 5-7 Received headers', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCVD_COUNT_SEVEN', score = 0.0, parent = rcvd_cb_id, @@ -103,7 +103,7 @@ rspamd_config:register_symbol{ description = 'Message has 7-11 Received headers', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCVD_COUNT_TWELVE', score = 0.0, parent = rcvd_cb_id, @@ -118,8 +118,8 @@ local prio_cb_id = rspamd_config:register_symbol { description = 'X-Priority check callback rule', score = 0.0, group = 'headers', - callback = function (task) - local cnts = { + callback = function(task) + local cnts = { [1] = 'ONE', [2] = 'TWO', [3] = 'THREE', @@ -127,11 +127,13 @@ local prio_cb_id = rspamd_config:register_symbol { } local def = 'ZERO' local xprio = task:get_header('X-Priority'); - if not xprio then return false end - local _,_,x = xprio:find('^%s?(%d+)'); + if not xprio then + return false + end + local _, _, x = xprio:find('^%s?(%d+)'); if (x) then x = tonumber(x) - for k,v in pairs(cnts) do + for k, v in pairs(cnts) do if x >= tonumber(k) then def = v end @@ -140,7 +142,7 @@ local prio_cb_id = rspamd_config:register_symbol { end end } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'HAS_X_PRIO_ZERO', score = 0.0, parent = prio_cb_id, @@ -148,7 +150,7 @@ rspamd_config:register_symbol{ description = 'Message has X-Priority header set to 0', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'HAS_X_PRIO_ONE', score = 0.0, parent = prio_cb_id, @@ -156,7 +158,7 @@ rspamd_config:register_symbol{ description = 'Message has X-Priority header set to 1', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'HAS_X_PRIO_TWO', score = 0.0, parent = prio_cb_id, @@ -164,7 +166,7 @@ rspamd_config:register_symbol{ description = 'Message has X-Priority header set to 2', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'HAS_X_PRIO_THREE', score = 0.0, parent = prio_cb_id, @@ -172,7 +174,7 @@ rspamd_config:register_symbol{ description = 'Message has X-Priority header set to 3 or 4', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'HAS_X_PRIO_FIVE', score = 0.0, parent = prio_cb_id, @@ -214,7 +216,7 @@ local check_replyto_id = rspamd_config:register_symbol({ end -- See if Reply-To matches From in some way - local from = task:get_from{'mime', 'orig'} + local from = task:get_from { 'mime', 'orig' } local from_h = get_raw_header(task, 'From') if not (from and from[1]) then return false @@ -257,7 +259,7 @@ local check_replyto_id = rspamd_config:register_symbol({ end }) -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'REPLYTO_UNPARSEABLE', score = 1.0, parent = check_replyto_id, @@ -265,7 +267,7 @@ rspamd_config:register_symbol{ description = 'Reply-To header could not be parsed', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'HAS_REPLYTO', score = 0.0, parent = check_replyto_id, @@ -273,7 +275,7 @@ rspamd_config:register_symbol{ description = 'Has Reply-To header', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'REPLYTO_EQ_FROM', score = 0.0, parent = check_replyto_id, @@ -281,7 +283,7 @@ rspamd_config:register_symbol{ description = 'Reply-To header is identical to From header', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'REPLYTO_ADDR_EQ_FROM', score = 0.0, parent = check_replyto_id, @@ -289,7 +291,7 @@ rspamd_config:register_symbol{ description = 'Reply-To header is identical to SMTP From', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'REPLYTO_DOM_EQ_FROM_DOM', score = 0.0, parent = check_replyto_id, @@ -297,7 +299,7 @@ rspamd_config:register_symbol{ description = 'Reply-To domain matches the From domain', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'REPLYTO_DOM_NEQ_FROM_DOM', score = 0.0, parent = check_replyto_id, @@ -305,7 +307,7 @@ rspamd_config:register_symbol{ description = 'Reply-To domain does not match the From domain', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'REPLYTO_DN_EQ_FROM_DN', score = 0.0, parent = check_replyto_id, @@ -313,7 +315,7 @@ rspamd_config:register_symbol{ description = 'Reply-To display name matches From', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'REPLYTO_EMAIL_HAS_TITLE', score = 2.0, parent = check_replyto_id, @@ -321,7 +323,7 @@ rspamd_config:register_symbol{ description = 'Reply-To header has title', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'REPLYTO_EQ_TO_ADDR', score = 5.0, parent = check_replyto_id, @@ -332,7 +334,7 @@ rspamd_config:register_symbol{ rspamd_config:register_dependency('CHECK_REPLYTO', 'CHECK_FROM') -local check_mime_id = rspamd_config:register_symbol{ +local check_mime_id = rspamd_config:register_symbol { name = 'CHECK_MIME', type = 'callback', group = 'headers', @@ -383,7 +385,7 @@ local check_mime_id = rspamd_config:register_symbol{ end } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'MISSING_MIME_VERSION', score = 2.0, parent = check_mime_id, @@ -391,7 +393,7 @@ rspamd_config:register_symbol{ description = 'MIME-Version header is missing in MIME message', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'MIME_MA_MISSING_TEXT', score = 2.0, parent = check_mime_id, @@ -399,7 +401,7 @@ rspamd_config:register_symbol{ description = 'MIME multipart/alternative missing text/plain part', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'MIME_MA_MISSING_HTML', score = 1.0, parent = check_mime_id, @@ -411,12 +413,16 @@ rspamd_config:register_symbol{ -- Used to be called IS_LIST rspamd_config.PREVIOUSLY_DELIVERED = { callback = function(task) - if not task:has_recipients(2) then return false end + if not task:has_recipients(2) then + return false + end local to = task:get_recipients(2) local rcvds = task:get_header_full('Received') - if not rcvds then return false end + if not rcvds then + return false + end for _, rcvd in ipairs(rcvds) do - local _,_,addr = rcvd['decoded']:lower():find("%sfor%s<(.-)>") + local _, _, addr = rcvd['decoded']:lower():find("%sfor%s<(.-)>") if addr then for _, toa in ipairs(to) do if toa and toa.addr:lower() == addr then @@ -442,8 +448,10 @@ rspamd_config.BROKEN_HEADERS = { rspamd_config.BROKEN_CONTENT_TYPE = { callback = function(task) - return fun.any(function(p) return p:is_broken() end, - task:get_parts()) + return fun.any(function(p) + return p:is_broken() + end, + task:get_parts()) end, score = 1.5, group = 'headers', @@ -451,18 +459,20 @@ rspamd_config.BROKEN_CONTENT_TYPE = { } rspamd_config.HEADER_RCONFIRM_MISMATCH = { - callback = function (task) + callback = function(task) local header_from = nil local cread = task:get_header('X-Confirm-Reading-To') if task:has_from('mime') then - header_from = task:get_from('mime')[1] + header_from = task:get_from('mime')[1] end local header_cread = nil if cread then local headers_cread = util.parse_mail_address(cread, task:get_mempool()) - if headers_cread then header_cread = headers_cread[1] end + if headers_cread then + header_cread = headers_cread[1] + end end if header_from and header_cread then @@ -480,9 +490,11 @@ rspamd_config.HEADER_RCONFIRM_MISMATCH = { } rspamd_config.HEADER_FORGED_MDN = { - callback = function (task) + callback = function(task) local mdn = task:get_header('Disposition-Notification-To') - if not mdn then return false end + if not mdn then + return false + end local header_rp = nil if task:has_from('smtp') then @@ -492,9 +504,15 @@ rspamd_config.HEADER_FORGED_MDN = { -- Parse mail addr local headers_mdn = util.parse_mail_address(mdn, task:get_mempool()) - if headers_mdn and not header_rp then return true end - if header_rp and not headers_mdn then return false end - if not headers_mdn and not header_rp then return false end + if headers_mdn and not header_rp then + return true + end + if header_rp and not headers_mdn then + return false + end + if not headers_mdn and not header_rp then + return false + end local found_match = false for _, h in ipairs(headers_mdn) do @@ -535,7 +553,7 @@ rspamd_config.MULTIPLE_UNIQUE_HEADERS = { local max_mult = 0.0 local res_tbl = {} - for hdr,mult in pairs(headers_unique) do + for hdr, mult in pairs(headers_unique) do local hc = task:get_header_count(hdr) if hc > 1 then @@ -548,7 +566,7 @@ rspamd_config.MULTIPLE_UNIQUE_HEADERS = { end if res > 0 then - return true,max_mult,table.concat(res_tbl, ',') + return true, max_mult, table.concat(res_tbl, ',') end return false @@ -577,7 +595,9 @@ rspamd_config.MULTIPLE_FROM = { callback = function(task) local from = task:get_from('mime') if from and from[2] then - return true, 1.0, fun.totable(fun.map(function(a) return a.raw end, from)) + return true, 1.0, fun.totable(fun.map(function(a) + return a.raw + end, from)) end return false end, @@ -587,7 +607,7 @@ rspamd_config.MULTIPLE_FROM = { } rspamd_config.MV_CASE = { - callback = function (task) + callback = function(task) return task:has_header('Mime-Version', true) end, description = 'Mime-Version .vs. MIME-Version', @@ -595,7 +615,7 @@ rspamd_config.MV_CASE = { group = 'headers' } -local check_from_id = rspamd_config:register_symbol{ +local check_from_id = rspamd_config:register_symbol { name = 'CHECK_FROM', type = 'callback', score = 0.0, @@ -610,10 +630,10 @@ local check_from_id = rspamd_config:register_symbol{ if not (from[1]["flags"]["valid"]) then task:insert_result('FROM_INVALID', 1.0) end - if (from[1].name == nil or from[1].name == '' ) then + if (from[1].name == nil or from[1].name == '') then task:insert_result('FROM_NO_DN', 1.0) elseif (from[1].name and - util.strequal_caseless(from[1].name, from[1].addr)) then + util.strequal_caseless(from[1].name, from[1].addr)) then task:insert_result('FROM_DN_EQ_ADDR', 1.0) elseif (from[1].name and from[1].name ~= '') then task:insert_result('FROM_HAS_DN', 1.0) @@ -622,11 +642,11 @@ local check_from_id = rspamd_config:register_symbol{ local match, match_end match, match_end = n:find('^mrs?[%.%s]') if match then - task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end-1)) + task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end - 1)) end match, match_end = n:find('^dr[%.%s]') if match then - task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end-1)) + task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end - 1)) end -- Check for excess spaces if n:find('%s%s') then @@ -644,19 +664,21 @@ local check_from_id = rspamd_config:register_symbol{ end local to = task:get_recipients(2) - if not (to and to[1] and #to == 1 and from and from[1]) then return false end + if not (to and to[1] and #to == 1 and from and from[1]) then + return false + end -- Check if FROM == TO if (util.strequal_caseless(to[1].addr, from[1].addr)) then task:insert_result('TO_EQ_FROM', 1.0) elseif (to[1].domain and from[1].domain and - util.strequal_caseless(to[1].domain, from[1].domain)) + util.strequal_caseless(to[1].domain, from[1].domain)) then task:insert_result('TO_DOM_EQ_FROM_DOM', 1.0) end end } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'ENVFROM_INVALID', score = 2.0, group = 'headers', @@ -664,7 +686,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'Envelope from does not have a valid format', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'FROM_INVALID', score = 2.0, group = 'headers', @@ -672,7 +694,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'From header does not have a valid format', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'FROM_NO_DN', score = 0.0, group = 'headers', @@ -680,7 +702,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'From header does not have a display name', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'FROM_DN_EQ_ADDR', score = 1.0, group = 'headers', @@ -688,7 +710,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'From header display name is the same as the address', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'FROM_HAS_DN', score = 0.0, group = 'headers', @@ -696,7 +718,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'From header has a display name', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'FROM_NAME_EXCESS_SPACE', score = 1.0, group = 'headers', @@ -704,7 +726,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'From header display name contains excess whitespace', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'FROM_NAME_HAS_TITLE', score = 1.0, group = 'headers', @@ -712,7 +734,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'From header display name has a title (Mr/Mrs/Dr)', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'FROM_EQ_ENVFROM', score = 0.0, group = 'headers', @@ -720,7 +742,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'From address is the same as the envelope', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'FROM_NEQ_ENVFROM', score = 0.0, group = 'headers', @@ -728,7 +750,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'From address is different to the envelope', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'TO_EQ_FROM', score = 0.0, group = 'headers', @@ -736,7 +758,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'To address matches the From address', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'TO_DOM_EQ_FROM_DOM', score = 0.0, group = 'headers', @@ -745,7 +767,7 @@ rspamd_config:register_symbol{ description = 'To domain is the same as the From domain', } -local check_to_cc_id = rspamd_config:register_symbol{ +local check_to_cc_id = rspamd_config:register_symbol { name = 'CHECK_TO_CC', type = 'callback', score = 0.0, @@ -764,10 +786,12 @@ local check_to_cc_id = rspamd_config:register_symbol{ [50] = 'GT_50' } local def = 'ZERO' - if (not to) then return false end + if (not to) then + return false + end -- Add symbol for recipient count local nrcpt = #to - for k,v in pairs(cnts) do + for k, v in pairs(cnts) do if nrcpt >= tonumber(k) then def = v end @@ -820,7 +844,7 @@ local check_to_cc_id = rspamd_config:register_symbol{ end } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCPT_COUNT_ZERO', score = 0.0, parent = check_to_cc_id, @@ -828,7 +852,7 @@ rspamd_config:register_symbol{ description = 'No recipients', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCPT_COUNT_ONE', score = 0.0, parent = check_to_cc_id, @@ -836,7 +860,7 @@ rspamd_config:register_symbol{ description = 'One recipient', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCPT_COUNT_TWO', score = 0.0, parent = check_to_cc_id, @@ -844,7 +868,7 @@ rspamd_config:register_symbol{ description = 'Two recipients', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCPT_COUNT_THREE', score = 0.0, parent = check_to_cc_id, @@ -852,7 +876,7 @@ rspamd_config:register_symbol{ description = '3-5 recipients', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCPT_COUNT_FIVE', score = 0.0, parent = check_to_cc_id, @@ -860,7 +884,7 @@ rspamd_config:register_symbol{ description = '5-7 recipients', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCPT_COUNT_SEVEN', score = 0.0, parent = check_to_cc_id, @@ -868,7 +892,7 @@ rspamd_config:register_symbol{ description = '7-11 recipients', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCPT_COUNT_TWELVE', score = 0.0, parent = check_to_cc_id, @@ -876,7 +900,7 @@ rspamd_config:register_symbol{ description = '12-50 recipients', group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'RCPT_COUNT_GT_50', score = 0.0, parent = check_to_cc_id, @@ -885,7 +909,7 @@ rspamd_config:register_symbol{ group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'TO_DN_RECIPIENTS', score = 2.0, group = 'headers', @@ -893,7 +917,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'To header display name is "Recipients"', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'TO_DN_NONE', score = 0.0, group = 'headers', @@ -901,7 +925,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'None of the recipients have display names', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'TO_DN_ALL', score = 0.0, group = 'headers', @@ -909,7 +933,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'All the recipients have display names', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'TO_DN_SOME', score = 0.0, group = 'headers', @@ -917,7 +941,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'Some of the recipients have display names', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'TO_DN_EQ_ADDR_ALL', score = 0.0, group = 'headers', @@ -925,7 +949,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'All of the recipients have display names that are the same as their address', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'TO_DN_EQ_ADDR_SOME', score = 0.0, group = 'headers', @@ -933,7 +957,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'Some of the recipients have display names that are the same as their address', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'TO_MATCH_ENVRCPT_ALL', score = 0.0, group = 'headers', @@ -941,7 +965,7 @@ rspamd_config:register_symbol{ type = 'virtual', description = 'All of the recipients match the envelope', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { name = 'TO_MATCH_ENVRCPT_SOME', score = 0.0, group = 'headers', @@ -954,8 +978,10 @@ rspamd_config:register_symbol{ rspamd_config.CTYPE_MISSING_DISPOSITION = { callback = function(task) local parts = task:get_parts() - if (not parts) or (parts and #parts < 1) then return false end - for _,p in ipairs(parts) do + if (not parts) or (parts and #parts < 1) then + return false + end + for _, p in ipairs(parts) do local ct = p:get_header('Content-Type') if (ct and ct:lower():match('^application/octet%-stream') ~= nil) then local cd = p:get_header('Content-Disposition') @@ -969,7 +995,7 @@ rspamd_config.CTYPE_MISSING_DISPOSITION = { local parent = p:get_parent() if parent then - local t,st = parent:get_type() + local t, st = parent:get_type() if t == 'multipart' and st == 'encrypted' then -- Special case @@ -991,15 +1017,21 @@ rspamd_config.CTYPE_MISSING_DISPOSITION = { rspamd_config.CTYPE_MIXED_BOGUS = { callback = function(task) local ct = task:get_header('Content-Type') - if (not ct) then return false end + if (not ct) then + return false + end local parts = task:get_parts() - if (not parts) then return false end - if (not ct:lower():match('^multipart/mixed')) then return false end + if (not parts) then + return false + end + if (not ct:lower():match('^multipart/mixed')) then + return false + end local found = false -- Check each part and look for a part that isn't multipart/* or text/plain or text/html local ntext_parts = 0 - for _,p in ipairs(parts) do - local mtype,_ = p:get_type() + for _, p in ipairs(parts) do + local mtype, _ = p:get_type() if mtype then if mtype == 'text' and not p:is_attachment() then ntext_parts = ntext_parts + 1 @@ -1013,7 +1045,9 @@ rspamd_config.CTYPE_MIXED_BOGUS = { end end end - if (not found) then return true end + if (not found) then + return true + end return false end, description = 'multipart/mixed without non-textual part', @@ -1023,7 +1057,9 @@ rspamd_config.CTYPE_MIXED_BOGUS = { local function check_for_base64_text(part) local ct = part:get_header('Content-Type') - if (not ct) then return false end + if (not ct) then + return false + end ct = ct:lower() if (ct:match('^text')) then -- Check encoding @@ -1042,7 +1078,9 @@ rspamd_config.MIME_BASE64_TEXT = { return true else local parts = task:get_parts() - if (not parts) then return false end + if (not parts) then + return false + end -- Check each part and look for base64 encoded text parts for _, part in ipairs(parts) do if (check_for_base64_text(part)) then @@ -1060,7 +1098,9 @@ rspamd_config.MIME_BASE64_TEXT = { rspamd_config.MIME_BASE64_TEXT_BOGUS = { callback = function(task) local parts = task:get_text_parts() - if (not parts) then return false end + if (not parts) then + return false + end -- Check each part and look for base64 encoded text parts -- where the part does not have any 8bit characters within it for _, part in ipairs(parts) do @@ -1113,7 +1153,7 @@ rspamd_config.INVALID_RCPT_8BIT = { } rspamd_config.XM_CASE = { - callback = function (task) + callback = function(task) return task:has_header('X-mailer', true) end, description = 'X-mailer .vs. X-Mailer', diff --git a/rules/html.lua b/rules/html.lua index df42f2f0e..7c352c2e1 100644 --- a/rules/html.lua +++ b/rules/html.lua @@ -42,16 +42,15 @@ end local function check_html_image(task, min, max) local tp = task:get_text_parts() - for _,p in ipairs(tp) do + for _, p in ipairs(tp) do if p:is_html() then local hc = p:get_html() local len = p:get_length() - if hc and len >= min and len < max then local images = hc:get_images() if images then - for _,i in ipairs(images) do + for _, i in ipairs(images) do local tag = i['tag'] if tag then if has_anchor_parent(tag) then @@ -99,16 +98,22 @@ rspamd_config.R_EMPTY_IMAGE = { callback = function(task) local tp = task:get_text_parts() -- get text parts in a message - for _,p in ipairs(tp) do -- iterate over text parts array using `ipairs` - if p:is_html() then -- if the current part is html part + for _, p in ipairs(tp) do + -- iterate over text parts array using `ipairs` + if p:is_html() then + -- if the current part is html part local hc = p:get_html() -- we get HTML context local len = p:get_length() -- and part's length - if hc and len < 50 then -- if we have a part that has less than 50 bytes of text + if hc and len < 50 then + -- if we have a part that has less than 50 bytes of text local images = hc:get_images() -- then we check for HTML images - if images then -- if there are images - for _,i in ipairs(images) do -- then iterate over images in the part - if i['height'] + i['width'] >= 400 then -- if we have a large image + if images then + -- if there are images + for _, i in ipairs(images) do + -- then iterate over images in the part + if i['height'] + i['width'] >= 400 then + -- if we have a large image local tag = i['tag'] if tag then if not has_anchor_parent(tag) then @@ -174,7 +179,7 @@ rspamd_config.R_SUSPICIOUS_IMAGES = { description = 'Message contains many suspicious messages' } -local vis_check_id = rspamd_config:register_symbol{ +local vis_check_id = rspamd_config:register_symbol { name = 'HTML_VISIBLE_CHECKS', type = 'callback', group = 'html', @@ -190,12 +195,14 @@ local vis_check_id = rspamd_config:register_symbol{ local normal_len = 0 local transp_len = 0 - for _,p in ipairs(tp) do -- iterate over text parts array using `ipairs` + for _, p in ipairs(tp) do + -- iterate over text parts array using `ipairs` normal_len = normal_len + p:get_length() - if p:is_html() and p:get_html() then -- if the current part is html part + if p:is_html() and p:get_html() then + -- if the current part is html part local hc = p:get_html() -- we get HTML context - hc:foreach_tag({'font', 'span', 'div', 'p', 'td'}, function(tag, clen, is_leaf) + hc:foreach_tag({ 'font', 'span', 'div', 'p', 'td' }, function(tag, clen, is_leaf) local bl = tag:get_style() if bl then if not bl.visible and clen > 0 and is_leaf then @@ -214,8 +221,12 @@ local vis_check_id = rspamd_config:register_symbol{ local tr = transp_len / (normal_len + transp_len) if tr > transp_rate then transp_rate = tr - if not bl.color then bl.color = {0, 0, 0} end - if not bl.bgcolor then bl.bgcolor = {0, 0, 0} end + if not bl.color then + bl.color = { 0, 0, 0 } + end + if not bl.bgcolor then + bl.bgcolor = { 0, 0, 0 } + end arg = string.format('%s color #%x%x%x bgcolor #%x%x%x', tag:get_type(), bl.color[1], bl.color[2], bl.color[3], @@ -288,7 +299,7 @@ local vis_check_id = rspamd_config:register_symbol{ end, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = vis_check_id, name = 'R_WHITE_ON_WHITE', @@ -298,7 +309,7 @@ rspamd_config:register_symbol{ one_shot = true, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = vis_check_id, name = 'ZERO_FONT', @@ -308,7 +319,7 @@ rspamd_config:register_symbol{ group = 'html' } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = vis_check_id, name = 'MANY_INVISIBLE_PARTS', @@ -324,10 +335,12 @@ rspamd_config.EXT_CSS = { local re = regexp_lib.create_cached('/^.*\\.css(?:[?#].*)?$/i') local tp = task:get_text_parts() -- get text parts in a message local ret = false - for _,p in ipairs(tp) do -- iterate over text parts array using `ipairs` - if p:is_html() and p:get_html() then -- if the current part is html part + for _, p in ipairs(tp) do + -- iterate over text parts array using `ipairs` + if p:is_html() and p:get_html() then + -- if the current part is html part local hc = p:get_html() -- we get HTML context - hc:foreach_tag({'link'}, function(tag) + hc:foreach_tag({ 'link' }, function(tag) local bl = tag:get_extra() if bl then local s = tostring(bl) @@ -357,26 +370,36 @@ rspamd_config.HTTP_TO_HTTPS = { local found_opts local tp = task:get_text_parts() or {} - for _,p in ipairs(tp) do + for _, p in ipairs(tp) do if p:is_html() then local hc = p:get_html() - if (not hc) then return false end + if (not hc) then + return false + end local found = false - hc:foreach_tag('a', function (tag, _) + hc:foreach_tag('a', function(tag, _) -- Skip this loop if we already have a match - if (found) then return true end + if (found) then + return true + end local c = tag:get_content() if (c) then - if (not https_re:match(c)) then return false end + if (not https_re:match(c)) then + return false + end local u = tag:get_extra() - if (not u) then return false end + if (not u) then + return false + end local url_proto = u:get_protocol() - if url_proto ~= 'http' then return false end + if url_proto ~= 'http' then + return false + end -- Capture matches for http in href to https in visible part only found = true found_opts = u:get_host() @@ -387,7 +410,7 @@ rspamd_config.HTTP_TO_HTTPS = { end) if (found) then - return true,1.0,found_opts + return true, 1.0, found_opts end return false @@ -403,14 +426,20 @@ rspamd_config.HTTP_TO_HTTPS = { rspamd_config.HTTP_TO_IP = { callback = function(task) local tp = task:get_text_parts() - if (not tp) then return false end - for _,p in ipairs(tp) do + if (not tp) then + return false + end + for _, p in ipairs(tp) do if p:is_html() then local hc = p:get_html() - if (not hc) then return false end + if (not hc) then + return false + end local found = false - hc:foreach_tag('a', function (tag, length) - if (found) then return true end + hc:foreach_tag('a', function(tag, length) + if (found) then + return true + end local u = tag:get_extra() if (u) then u = tostring(u):lower() @@ -420,7 +449,9 @@ rspamd_config.HTTP_TO_IP = { end return false end) - if found then return true end + if found then + return true + end return false end end diff --git a/rules/mid.lua b/rules/mid.lua index 14c701cf6..1bac26c61 100644 --- a/rules/mid.lua +++ b/rules/mid.lua @@ -17,7 +17,9 @@ limitations under the License. local rspamd_util = require "rspamd_util" local function mid_check_func(task) local mid = task:get_header('Message-ID') - if not mid then return false end + if not mid then + return false + end -- Check for 'bare' IP addresses in RHS if mid:find("@%d+%.%d+%.%d+%.%d+>$") then task:insert_result('MID_BARE_IP', 1.0) @@ -39,7 +41,7 @@ local function mid_check_func(task) local fd if (from and from[1] and from[1].domain and from[1].domain ~= '') then fd = from[1].domain:lower() - local _,_,md = mid:find("@([^>]+)>?$") + local _, _, md = mid:find("@([^>]+)>?$") -- See if all or part of the From address -- can be found in the Message-ID -- extract tld @@ -49,7 +51,7 @@ local function mid_check_func(task) fdtld = rspamd_util.get_tld(fd) mdtld = rspamd_util.get_tld(md) end - if (mid:lower():find(from[1].addr:lower(),1,true)) then + if (mid:lower():find(from[1].addr:lower(), 1, true)) then task:insert_result('MID_CONTAINS_FROM', 1.0) elseif (md and fd == md:lower()) then task:insert_result('MID_RHS_MATCH_FROM', 1.0) @@ -61,12 +63,12 @@ local function mid_check_func(task) local to = task:get_recipients(2) if (to and to[1] and to[1].domain and to[1].domain ~= '') then local td = to[1].domain:lower() - local _,_,md = mid:find("@([^>]+)>?$") + local _, _, md = mid:find("@([^>]+)>?$") -- Skip if from domain == to domain if ((fd and fd ~= td) or not fd) then -- See if all or part of the To address -- can be found in the Message-ID - if (mid:lower():find(to[1].addr:lower(),1,true)) then + if (mid:lower():find(to[1].addr:lower(), 1, true)) then task:insert_result('MID_CONTAINS_TO', 1.0) elseif (md and td == md:lower()) then task:insert_result('MID_RHS_MATCH_TO', 1.0) @@ -115,9 +117,11 @@ rspamd_config:register_symbol { callback = function(task) local mid = task:get_header('Message-ID') - if not mid then return end + if not mid then + return + end local mime_from = task:get_from('mime') - local _,_,mid_realm = mid:find("@([a-z]+)>?$") + local _, _, mid_realm = mid:find("@([a-z]+)>?$") if mid_realm and mime_from and mime_from[1] and mime_from[1].user then if (mid_realm == mime_from[1].user) then return true diff --git a/rules/misc.lua b/rules/misc.lua index b27a1bc53..17e3b8ac7 100644 --- a/rules/misc.lua +++ b/rules/misc.lua @@ -47,7 +47,7 @@ rspamd_config.R_PARTS_DIFFER = { score = (nd - 0.5) end task:insert_result('R_PARTS_DIFFER', score, - string.format('%.1f%%', tostring(100.0 * nd))) + string.format('%.1f%%', tostring(100.0 * nd))) end end end @@ -75,15 +75,15 @@ local date_id = rspamd_config:register_symbol({ return end - local dt = task:get_date({format = 'connect', gmt = true}) + local dt = task:get_date({ format = 'connect', gmt = true }) local date_diff = dt - dm if date_diff > 86400 then -- Older than a day - task:insert_result('DATE_IN_PAST', 1.0, tostring(math.floor(date_diff/3600))) + task:insert_result('DATE_IN_PAST', 1.0, tostring(math.floor(date_diff / 3600))) elseif -date_diff > 7200 then -- More than 2 hours in the future - task:insert_result('DATE_IN_FUTURE', 1.0, tostring(math.floor(-date_diff/3600))) + task:insert_result('DATE_IN_FUTURE', 1.0, tostring(math.floor(-date_diff / 3600))) end end }) @@ -124,15 +124,15 @@ rspamd_config:register_symbol({ parent = date_id, }) -local obscured_id = rspamd_config:register_symbol{ +local obscured_id = rspamd_config:register_symbol { callback = function(task) - local susp_urls = task:get_urls_filtered({ 'obscured', 'zw_spaces'}) + local susp_urls = task:get_urls_filtered({ 'obscured', 'zw_spaces' }) if susp_urls and susp_urls[1] then local obs_flag = url_flags_tab.obscured local zw_flag = url_flags_tab.zw_spaces - for _,u in ipairs(susp_urls) do + for _, u in ipairs(susp_urls) do local fl = u:get_flags_num() if bit.band(fl, obs_flag) ~= 0 then task:insert_result('R_SUSPICIOUS_URL', 1.0, u:get_host()) @@ -152,7 +152,7 @@ local obscured_id = rspamd_config:register_symbol{ group = 'url' } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', name = 'ZERO_WIDTH_SPACE_URL', score = 7.0, @@ -162,9 +162,8 @@ rspamd_config:register_symbol{ parent = obscured_id, } - rspamd_config.ENVFROM_PRVS = { - callback = function (task) + callback = function(task) --[[ Detect PRVS/BATV addresses to avoid FORGED_SENDER https://en.wikipedia.org/wiki/Bounce_Address_Tag_Validation @@ -183,7 +182,9 @@ rspamd_config.ENVFROM_PRVS = { local re_text = '^(?:(prvs|msprvs1)=([^=]+)=|btv1==[^=]+==)(.+@(.+))$' local re = rspamd_regexp.create_cached(re_text) local c = re:search(envfrom[1].addr:lower(), false, true) - if not c then return false end + if not c then + return false + end local ef = c[1][4] -- See if it matches the From header local from = task:get_from(2) @@ -207,23 +208,25 @@ rspamd_config.ENVFROM_PRVS = { } rspamd_config.ENVFROM_VERP = { - callback = function (task) + callback = function(task) if not (task:has_from(1) and task:has_recipients(1)) then return false end local envfrom = task:get_from(1) local envrcpts = task:get_recipients(1) -- VERP only works for single recipient messages - if #envrcpts > 1 then return false end + if #envrcpts > 1 then + return false + end -- Get recipient and compute VERP address local rcpt = envrcpts[1].addr:lower() - local verp = rcpt:gsub('@','=') + local verp = rcpt:gsub('@', '=') -- Get the user portion of the envfrom local ef_user = envfrom[1].user:lower() -- See if the VERP representation of the recipient appears in it if ef_user:find(verp, 1, true) - and not ef_user:find('+caf_=' .. verp, 1, true) -- Google Forwarding - and not ef_user:find('^srs[01]=') -- SRS + and not ef_user:find('+caf_=' .. verp, 1, true) -- Google Forwarding + and not ef_user:find('^srs[01]=') -- SRS then return true end @@ -235,12 +238,14 @@ rspamd_config.ENVFROM_VERP = { type = 'mime', } -local check_rcvd = rspamd_config:register_symbol{ +local check_rcvd = rspamd_config:register_symbol { name = 'CHECK_RCVD', group = 'headers', - callback = function (task) + callback = function(task) local rcvds = task:get_received_headers() - if not rcvds or #rcvds == 0 then return false end + if not rcvds or #rcvds == 0 then + return false + end local all_tls = fun.all(function(rc) return rc.flags and rc.flags['ssl'] @@ -275,7 +280,7 @@ local check_rcvd = rspamd_config:register_symbol{ type = 'callback,mime', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = check_rcvd, name = 'RCVD_TLS_ALL', @@ -284,7 +289,7 @@ rspamd_config:register_symbol{ group = 'headers' } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = check_rcvd, name = 'RCVD_TLS_LAST', @@ -293,7 +298,7 @@ rspamd_config:register_symbol{ group = 'headers' } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = check_rcvd, name = 'RCVD_NO_TLS_LAST', @@ -302,7 +307,7 @@ rspamd_config:register_symbol{ group = 'headers' } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = check_rcvd, name = 'RCVD_VIA_SMTP_AUTH', @@ -313,7 +318,7 @@ rspamd_config:register_symbol{ } rspamd_config.RCVD_HELO_USER = { - callback = function (task) + callback = function(task) -- Check HELO argument from MTA local helo = task:get_helo() if (helo and helo:lower():find('^user$')) then @@ -321,11 +326,17 @@ rspamd_config.RCVD_HELO_USER = { end -- Check Received headers local rcvds = task:get_header_full('Received') - if not rcvds then return false end + if not rcvds then + return false + end for _, rcvd in ipairs(rcvds) do local r = rcvd['decoded']:lower() - if (r:find("^%s*from%suser%s")) then return true end - if (r:find("helo[%s=]user[%s%)]")) then return true end + if (r:find("^%s*from%suser%s")) then + return true + end + if (r:find("helo[%s=]user[%s%)]")) then + return true + end end end, description = 'HELO User spam pattern', @@ -335,11 +346,13 @@ rspamd_config.RCVD_HELO_USER = { } rspamd_config.URI_COUNT_ODD = { - callback = function (task) + callback = function(task) local ct = task:get_header('Content-Type') if (ct and ct:lower():find('^multipart/alternative')) then - local urls = task:get_urls_filtered(nil, {'subject', 'html_displayed', 'special'}) or {} - local nurls = fun.foldl(function(acc, val) return acc + val:get_count() end, 0, urls) + local urls = task:get_urls_filtered(nil, { 'subject', 'html_displayed', 'special' }) or {} + local nurls = fun.foldl(function(acc, val) + return acc + val:get_count() + end, 0, urls) if nurls % 2 == 1 then return true, 1.0, tostring(nurls) @@ -352,7 +365,7 @@ rspamd_config.URI_COUNT_ODD = { } rspamd_config.HAS_ATTACHMENT = { - callback = function (task) + callback = function(task) local parts = task:get_parts() if parts and #parts > 1 then for _, p in ipairs(parts) do @@ -376,7 +389,7 @@ local function freemail_reply_neq_from(task) local ff = task:get_symbol('FREEMAIL_FROM') local frt_opts = frt[1]['options'] local ff_opts = ff[1]['options'] - return ( frt_opts and ff_opts and frt_opts[1] ~= ff_opts[1] ) + return (frt_opts and ff_opts and frt_opts[1] ~= ff_opts[1]) end rspamd_config:register_symbol({ @@ -404,7 +417,8 @@ rspamd_config.OMOGRAPH_URL = { local h1 = u:get_host() local h2 = u:get_phished() - if h2 then -- Due to changes of the phished flag in 2.8 + if h2 then + -- Due to changes of the phished flag in 2.8 h2 = h2:get_host() end if h1 and h2 then @@ -448,20 +462,20 @@ rspamd_config.URL_IN_SUBJECT = { local urls = task:get_urls() if urls then - for _,u in ipairs(urls) do + for _, u in ipairs(urls) do local flags = u:get_flags() if flags.subject then if flags.schemaless then - return true,0.1,u:get_host() + return true, 0.1, u:get_host() end local subject = task:get_subject() if subject then if tostring(u) == subject then - return true,1.0,u:get_host() + return true, 1.0, u:get_host() end end - return true,0.25,u:get_host() + return true, 0.25, u:get_host() end end end @@ -474,18 +488,20 @@ rspamd_config.URL_IN_SUBJECT = { description = 'Subject contains URL' } -local aliases_id = rspamd_config:register_symbol{ +local aliases_id = rspamd_config:register_symbol { type = 'prefilter', name = 'EMAIL_PLUS_ALIASES', callback = function(task) local function check_from(type) if task:has_from(type) then local addr = task:get_from(type)[1] - local na,tags = lua_util.remove_email_aliases(addr) + local na, tags = lua_util.remove_email_aliases(addr) if na then task:set_from(type, addr, 'alias') task:insert_result('TAGGED_FROM', 1.0, fun.totable( - fun.filter(function(t) return t and #t > 0 end, tags))) + fun.filter(function(t) + return t and #t > 0 + end, tags))) end end end @@ -500,11 +516,15 @@ local aliases_id = rspamd_config:register_symbol{ local addrs = task:get_recipients(type) for _, addr in ipairs(addrs) do - local na,tags = lua_util.remove_email_aliases(addr) + local na, tags = lua_util.remove_email_aliases(addr) if na then modified = true - fun.each(function(t) table.insert(all_tags, t) end, - fun.filter(function(t) return t and #t > 0 end, tags)) + fun.each(function(t) + table.insert(all_tags, t) + end, + fun.filter(function(t) + return t and #t > 0 + end, tags)) end end @@ -523,7 +543,7 @@ local aliases_id = rspamd_config:register_symbol{ group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = aliases_id, name = 'TAGGED_RCPT', @@ -531,7 +551,7 @@ rspamd_config:register_symbol{ group = 'headers', score = 0.0, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = aliases_id, name = 'TAGGED_FROM', @@ -540,18 +560,26 @@ rspamd_config:register_symbol{ score = 0.0, } -local check_from_display_name = rspamd_config:register_symbol{ +local check_from_display_name = rspamd_config:register_symbol { type = 'callback,mime', name = 'FROM_DISPLAY_CALLBACK', - callback = function (task) + callback = function(task) local from = task:get_from(2) - if not (from and from[1] and from[1].name) then return false end + if not (from and from[1] and from[1].name) then + return false + end -- See if we can parse an email address from the name local parsed = rspamd_parsers.parse_mail_address(from[1].name, task:get_mempool()) - if not parsed then return false end - if not (parsed[1] and parsed[1]['addr']) then return false end + if not parsed then + return false + end + if not (parsed[1] and parsed[1]['addr']) then + return false + end -- Make sure we did not mistake e.g. <something>@<name> for an email address - if not parsed[1]['domain'] or not parsed[1]['domain']:find('%.') then return false end + if not parsed[1]['domain'] or not parsed[1]['domain']:find('%.') then + return false + end -- See if the parsed domains differ if not rspamd_util.strequal_caseless(from[1]['domain'], parsed[1]['domain']) then -- See if the destination domain is the same as the spoof @@ -580,7 +608,7 @@ local check_from_display_name = rspamd_config:register_symbol{ group = 'headers', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = check_from_display_name, name = 'SPOOF_DISPLAY_NAME', @@ -589,7 +617,7 @@ rspamd_config:register_symbol{ score = 8.0, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = check_from_display_name, name = 'FROM_NEQ_DISPLAY_NAME', @@ -599,15 +627,19 @@ rspamd_config:register_symbol{ } rspamd_config.SPOOF_REPLYTO = { - callback = function (task) + callback = function(task) -- First check for a Reply-To header local rt = task:get_header_full('Reply-To') - if not rt or not rt[1] then return false end + if not rt or not rt[1] then + return false + end -- Get From and To headers rt = rt[1]['value'] local from = task:get_from(2) local to = task:get_recipients(2) - if not (from and from[1] and from[1].addr) then return false end + if not (from and from[1] and from[1].addr) then + return false + end if (to and to[1] and to[1].addr) then -- Handle common case for Web Contact forms of From = To if rspamd_util.strequal_caseless(from[1].addr, to[1].addr) then @@ -616,9 +648,13 @@ rspamd_config.SPOOF_REPLYTO = { end -- SMTP recipients must contain From domain to = task:get_recipients(1) - if not to then return false end + if not to then + return false + end -- Try mitigate some possible FPs on mailing list posts - if #to == 1 and rspamd_util.strequal_caseless(to[1].addr, from[1].addr) then return false end + if #to == 1 and rspamd_util.strequal_caseless(to[1].addr, from[1].addr) then + return false + end local found_fromdom = false for _, t in ipairs(to) do if rspamd_util.strequal_caseless(t.domain, from[1].domain) then @@ -626,10 +662,14 @@ rspamd_config.SPOOF_REPLYTO = { break end end - if not found_fromdom then return false end + if not found_fromdom then + return false + end -- Parse Reply-To header local parsed = ((rspamd_parsers.parse_mail_address(rt, task:get_mempool()) or E)[1] or E).domain - if not parsed then return false end + if not parsed then + return false + end -- Reply-To domain must be different to From domain if not rspamd_util.strequal_caseless(parsed, from[1].domain) then return true, from[1].domain, parsed @@ -652,14 +692,18 @@ rspamd_config.INFO_TO_INFO_LU = { return false end local to = task:get_recipients('smtp') - if not to then return false end + if not to then + return false + end local found = false - for _,r in ipairs(to) do + for _, r in ipairs(to) do if rspamd_util.strequal_caseless(r['user'], 'info') then found = true end end - if found then return true end + if found then + return true + end return false end, description = 'info@ From/To address with List-Unsubscribe headers', @@ -674,12 +718,12 @@ rspamd_config.R_BAD_CTE_7BIT = { callback = function(task) local tp = task:get_text_parts() or {} - for _,p in ipairs(tp) do + for _, p in ipairs(tp) do local cte = p:get_mimepart():get_cte() or '' if cte ~= '8bit' and p:has_8bit_raw() then - local _,_,attrs = p:get_mimepart():get_type_full() + local _, _, attrs = p:get_mimepart():get_type_full() local mul = 1.0 - local params = {cte} + local params = { cte } if attrs then if attrs.charset and attrs.charset:lower() == "utf-8" then -- Penalise rule as people don't know that utf8 is surprisingly @@ -689,7 +733,7 @@ rspamd_config.R_BAD_CTE_7BIT = { end end - return true,mul,params + return true, mul, params end end @@ -701,8 +745,7 @@ rspamd_config.R_BAD_CTE_7BIT = { type = 'mime', } - -local check_encrypted_name = rspamd_config:register_symbol{ +local check_encrypted_name = rspamd_config:register_symbol { name = 'BOGUS_ENCRYPTED_AND_TEXT', callback = function(task) local parts = task:get_parts() or {} @@ -714,14 +757,14 @@ local check_encrypted_name = rspamd_config:register_symbol{ local children = part:get_children() or {} local text_kids = {} - for _,cld in ipairs(children) do + for _, cld in ipairs(children) do if cld:is_multipart() then check_part(cld) elseif cld:is_text() then seen_text = true text_kids[#text_kids + 1] = cld else - local type,subtype,_ = cld:get_type_full() + local type, subtype, _ = cld:get_type_full() if type:lower() == 'application' then if string.find(subtype:lower(), 'pkcs7%-mime') then @@ -743,8 +786,8 @@ local check_encrypted_name = rspamd_config:register_symbol{ end if seen_text and seen_encrypted then -- Ensure that our seen text is not really part of pgp #3205 - for _,tp in ipairs(text_kids) do - local t,_ = tp:get_type() + for _, tp in ipairs(text_kids) do + local t, _ = tp:get_type() seen_text = false -- reset temporary if t and t == 'text' then seen_text = true @@ -756,7 +799,7 @@ local check_encrypted_name = rspamd_config:register_symbol{ end end - for _,part in ipairs(parts) do + for _, part in ipairs(parts) do check_part(part) end @@ -771,7 +814,7 @@ local check_encrypted_name = rspamd_config:register_symbol{ group = 'mime_types', } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = check_encrypted_name, name = 'ENCRYPTED_PGP', @@ -781,7 +824,7 @@ rspamd_config:register_symbol{ one_shot = true } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = check_encrypted_name, name = 'ENCRYPTED_SMIME', @@ -791,7 +834,7 @@ rspamd_config:register_symbol{ one_shot = true } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = check_encrypted_name, name = 'SIGNED_PGP', @@ -801,7 +844,7 @@ rspamd_config:register_symbol{ one_shot = true } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'virtual', parent = check_encrypted_name, name = 'SIGNED_SMIME', diff --git a/rules/regexp/compromised_hosts.lua b/rules/regexp/compromised_hosts.lua index cfd560bc2..e120b181b 100644 --- a/rules/regexp/compromised_hosts.lua +++ b/rules/regexp/compromised_hosts.lua @@ -44,10 +44,12 @@ reconf['HAS_X_SOURCE'] = { -- X-Authenticated-Sender: accord.host-care.com: sales@cortaflex.si rspamd_config.HAS_X_AS = { - callback = function (task) + callback = function(task) local xas = task:get_header('X-Authenticated-Sender') - if not xas then return false end - local _,_,auth = xas:find('[^:]+:%s(.+)$') + if not xas then + return false + end + local _, _, auth = xas:find('[^:]+:%s(.+)$') if auth then -- TODO: see if we can parse an e-mail address from auth -- and see if it matches the from address or not @@ -63,10 +65,12 @@ rspamd_config.HAS_X_AS = { -- X-Get-Message-Sender-Via: accord.host-care.com: authenticated_id: sales@cortaflex.si rspamd_config.HAS_X_GMSV = { - callback = function (task) + callback = function(task) local xgmsv = task:get_header('X-Get-Message-Sender-Via') - if not xgmsv then return false end - local _,_,auth = xgmsv:find('authenticated_id: (.+)$') + if not xgmsv then + return false + end + local _, _, auth = xgmsv:find('authenticated_id: (.+)$') if auth then -- TODO: see if we can parse an e-mail address from auth -- and see if it matches the from address or not. @@ -146,21 +150,21 @@ reconf['HIDDEN_SOURCE_OBJ'] = { group = "compromised_hosts" } -local hidden_uri_re = rspamd_regexp.create_cached('/(?!\\/\\.well[-_]known\\/)(?:^\\.[A-Za-z0-9]|\\/'.. +local hidden_uri_re = rspamd_regexp.create_cached('/(?!\\/\\.well[-_]known\\/)(?:^\\.[A-Za-z0-9]|\\/' .. '\\.[A-Za-z0-9]|\\/\\.\\.\\/)/i') rspamd_config.URI_HIDDEN_PATH = { - callback = function (task) + callback = function(task) local urls = task:get_urls(false) if (urls) then - for _, url in ipairs(urls) do - if (not (url:is_subject() and url:is_html_displayed())) then - local path = url:get_path() - if (hidden_uri_re:match(path)) then - -- TODO: need url:is_schemeless() to improve this - return true, 1.0, url:get_text() - end - end + for _, url in ipairs(urls) do + if (not (url:is_subject() and url:is_html_displayed())) then + local path = url:get_path() + if (hidden_uri_re:match(path)) then + -- TODO: need url:is_schemeless() to improve this + return true, 1.0, url:get_text() + end end + end end end, description = 'Message contains URI with a hidden path', @@ -176,19 +180,23 @@ reconf['MID_RHS_WWW'] = { } rspamd_config.FROM_SERVICE_ACCT = { - callback = function (task) + callback = function(task) local re = rspamd_regexp.create_cached('/^(?:www-data|anonymous|ftp|apache|nobody|guest|nginx|web|www)@/i'); -- From local from = task:get_from(2) if (from and from[1]) then - if (re:match(from[1].addr)) then return true end + if (re:match(from[1].addr)) then + return true + end end -- Sender local sender = task:get_header('Sender') if sender then local s = util.parse_mail_address(sender, task:get_mempool()) if (s and s[1]) then - if (re:match(s[1].addr)) then return true end + if (re:match(s[1].addr)) then + return true + end end end -- Reply-To @@ -196,7 +204,9 @@ rspamd_config.FROM_SERVICE_ACCT = { if replyto then local rt = util.parse_mail_address(replyto, task:get_mempool()) if (rt and rt[1]) then - if (re:match(rt[1].addr)) then return true end + if (re:match(rt[1].addr)) then + return true + end end end end, diff --git a/rules/regexp/headers.lua b/rules/regexp/headers.lua index 42c08ca3f..b634dd909 100644 --- a/rules/regexp/headers.lua +++ b/rules/regexp/headers.lua @@ -380,8 +380,8 @@ reconf['SUSPICIOUS_BOUNDARY3'] = { group = 'mua' } -- Forged OE/MSO boundary -local suspicious_boundary_01C4 = 'Content-Type=/^\\s*multipart.+boundary="----=_NextPart_000_[A-Z\\d]{4}_01C4[\\dA-F]{4}\\.[A-Z\\d]{8}"[\\r\\n]*$/siX' -local suspicious_boundary_01C4_date = 'Date=/^\\s*\\w\\w\\w,\\s+\\d+\\s+\\w\\w\\w 20(0[56789]|1\\d)/' +local suspicious_boundary_01C4 = 'Content-Type=/^\\s*multipart.+boundary="----=_NextPart_000_[A-Z\\d]{4}_01C4[\\dA-F]{4}\\.[A-Z\\d]{8}"[\\r\\n]*$/siX' +local suspicious_boundary_01C4_date = 'Date=/^\\s*\\w\\w\\w,\\s+\\d+\\s+\\w\\w\\w 20(0[56789]|1\\d)/' reconf['SUSPICIOUS_BOUNDARY4'] = { re = string.format('(%s) & (%s)', suspicious_boundary_01C4, suspicious_boundary_01C4_date), score = 4.0, @@ -439,24 +439,27 @@ reconf['FORGED_MUA_OPERA_MSGID'] = { -- Detect forged Mozilla Mail/Thunderbird/Seamonkey/Postbox headers -- Mozilla based X-Mailer -local user_agent_mozilla5 = 'User-Agent=/^\\s*Mozilla\\/5\\.0/H' -local user_agent_thunderbird = 'User-Agent=/^\\s*(Thunderbird|Mozilla Thunderbird|Mozilla\\/.*Gecko\\/.*(Thunderbird|Betterbird|Icedove)\\/)/H' -local user_agent_seamonkey = 'User-Agent=/^\\s*Mozilla\\/5\\.0\\s.+\\sSeaMonkey\\/\\d+\\.\\d+/H' -local user_agent_postbox = [[User-Agent=/^\s*Mozilla\/5\.0\s\([^)]+\)\sGecko\/\d+\sPostboxApp\/\d+(?:\.\d+){2,3}$/H]] -local user_agent_mozilla = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla5, user_agent_thunderbird, user_agent_seamonkey, user_agent_postbox) +local user_agent_mozilla5 = 'User-Agent=/^\\s*Mozilla\\/5\\.0/H' +local user_agent_thunderbird = 'User-Agent=/^\\s*(Thunderbird|Mozilla Thunderbird|Mozilla\\/.*Gecko\\/.*(Thunderbird|Betterbird|Icedove)\\/)/H' +local user_agent_seamonkey = 'User-Agent=/^\\s*Mozilla\\/5\\.0\\s.+\\sSeaMonkey\\/\\d+\\.\\d+/H' +local user_agent_postbox = [[User-Agent=/^\s*Mozilla\/5\.0\s\([^)]+\)\sGecko\/\d+\sPostboxApp\/\d+(?:\.\d+){2,3}$/H]] +local user_agent_mozilla = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla5, user_agent_thunderbird, + user_agent_seamonkey, user_agent_postbox) -- Mozilla based common Message-ID template -local mozilla_msgid_common = 'Message-ID=/^\\s*<[\\dA-F]{8}\\.\\d{1,7}\\@([^>\\.]+\\.)+[^>\\.]+>$/H' -local mozilla_msgid_common_sec = 'Message-ID=/^\\s*<[\\da-f]{8}-([\\da-f]{4}-){3}[\\da-f]{12}\\@([^>\\.]+\\.)+[^>\\.]+>$/H' -local mozilla_msgid = 'Message-ID=/^\\s*<(3[3-9A-F]|[4-9A-F][\\dA-F])[\\dA-F]{6}\\.(\\d0){1,4}\\d\\@([^>\\.]+\\.)+[^>\\.]+>$/H' +local mozilla_msgid_common = 'Message-ID=/^\\s*<[\\dA-F]{8}\\.\\d{1,7}\\@([^>\\.]+\\.)+[^>\\.]+>$/H' +local mozilla_msgid_common_sec = 'Message-ID=/^\\s*<[\\da-f]{8}-([\\da-f]{4}-){3}[\\da-f]{12}\\@([^>\\.]+\\.)+[^>\\.]+>$/H' +local mozilla_msgid = 'Message-ID=/^\\s*<(3[3-9A-F]|[4-9A-F][\\dA-F])[\\dA-F]{6}\\.(\\d0){1,4}\\d\\@([^>\\.]+\\.)+[^>\\.]+>$/H' -- Summary rule for forged Mozilla Mail Message-ID header reconf['FORGED_MUA_MOZILLA_MAIL_MSGID'] = { - re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_mozilla, mozilla_msgid_common, mozilla_msgid, unusable_msgid), + re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_mozilla, mozilla_msgid_common, mozilla_msgid, + unusable_msgid), score = 4.0, description = 'Message pretends to be send from Mozilla Mail but has forged Message-ID', group = 'mua' } reconf['FORGED_MUA_MOZILLA_MAIL_MSGID_UNKNOWN'] = { - re = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla, mozilla_msgid_common, mozilla_msgid, unusable_msgid), + re = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla, mozilla_msgid_common, mozilla_msgid, + unusable_msgid), score = 2.5, description = 'Message pretends to be send from Mozilla Mail but has forged Message-ID', group = 'mua' @@ -464,39 +467,45 @@ reconf['FORGED_MUA_MOZILLA_MAIL_MSGID_UNKNOWN'] = { -- Summary rule for forged Thunderbird Message-ID header reconf['FORGED_MUA_THUNDERBIRD_MSGID'] = { - re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_thunderbird, mozilla_msgid_common, mozilla_msgid, unusable_msgid), + re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_thunderbird, mozilla_msgid_common, mozilla_msgid, + unusable_msgid), score = 4.0, description = 'Forged mail pretending to be from Mozilla Thunderbird but has forged Message-ID', group = 'mua' } reconf['FORGED_MUA_THUNDERBIRD_MSGID_UNKNOWN'] = { - re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_thunderbird, mozilla_msgid_common, mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid), + re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_thunderbird, mozilla_msgid_common, + mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid), score = 2.5, description = 'Forged mail pretending to be from Mozilla Thunderbird but has forged Message-ID', group = 'mua' } -- Summary rule for forged Seamonkey Message-ID header reconf['FORGED_MUA_SEAMONKEY_MSGID'] = { - re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common, mozilla_msgid, unusable_msgid), + re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common, mozilla_msgid, + unusable_msgid), score = 4.0, description = 'Forged mail pretending to be from Mozilla Seamonkey but has forged Message-ID', group = 'mua' } reconf['FORGED_MUA_SEAMONKEY_MSGID_UNKNOWN'] = { - re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common, mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid), + re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common, + mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid), score = 2.5, description = 'Forged mail pretending to be from Mozilla Seamonkey but has forged Message-ID', group = 'mua' } -- Summary rule for forged Postbox Message-ID header reconf['FORGED_MUA_POSTBOX_MSGID'] = { - re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_postbox, mozilla_msgid_common, mozilla_msgid, unusable_msgid), + re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_postbox, mozilla_msgid_common, mozilla_msgid, + unusable_msgid), score = 4.0, description = 'Forged mail pretending to be from Postbox but has forged Message-ID', group = 'mua' } reconf['FORGED_MUA_POSTBOX_MSGID_UNKNOWN'] = { - re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_postbox, mozilla_msgid_common, mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid), + re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_postbox, mozilla_msgid_common, + mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid), score = 2.5, description = 'Forged mail pretending to be from Postbox but has forged Message-ID', group = 'mua' @@ -647,8 +656,10 @@ reconf['MISSING_MIMEOLE'] = { -- Empty delimiters between header names and header values local function gen_check_header_delimiter_empty(header_name) return function(task) - for _,rh in ipairs(task:get_header_full(header_name) or {}) do - if rh['empty_separator'] then return true end + for _, rh in ipairs(task:get_header_full(header_name) or {}) do + if rh['empty_separator'] then + return true + end end return false end @@ -707,10 +718,10 @@ reconf['RCVD_ILLEGAL_CHARS'] = { group = 'headers' } -local MAIL_RU_Return_Path = 'Return-path=/^\\s*<.+\\@mail\\.ru>$/iX' -local MAIL_RU_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@mail\\.ru>$/iX' -local MAIL_RU_From = 'From=/\\@mail\\.ru>?$/iX' -local MAIL_RU_Received = 'Received=/from mail\\.ru \\(/mH' +local MAIL_RU_Return_Path = 'Return-path=/^\\s*<.+\\@mail\\.ru>$/iX' +local MAIL_RU_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@mail\\.ru>$/iX' +local MAIL_RU_From = 'From=/\\@mail\\.ru>?$/iX' +local MAIL_RU_Received = 'Received=/from mail\\.ru \\(/mH' reconf['FAKE_RECEIVED_mail_ru'] = { re = string.format('(%s) & !(((%s) | (%s)) & (%s))', @@ -720,26 +731,26 @@ reconf['FAKE_RECEIVED_mail_ru'] = { group = 'headers' } -local GMAIL_COM_Return_Path = 'Return-path=/^\\s*<.+\\@gmail\\.com>$/iX' -local GMAIL_COM_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@gmail\\.com>$/iX' -local GMAIL_COM_From = 'From=/\\@gmail\\.com>?$/iX' +local GMAIL_COM_Return_Path = 'Return-path=/^\\s*<.+\\@gmail\\.com>$/iX' +local GMAIL_COM_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@gmail\\.com>$/iX' +local GMAIL_COM_From = 'From=/\\@gmail\\.com>?$/iX' -local UKR_NET_Return_Path = 'Return-path=/^\\s*<.+\\@ukr\\.net>$/iX' -local UKR_NET_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@ukr\\.net>$/iX' -local UKR_NET_From = 'From=/\\@ukr\\.net>?$/iX' +local UKR_NET_Return_Path = 'Return-path=/^\\s*<.+\\@ukr\\.net>$/iX' +local UKR_NET_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@ukr\\.net>$/iX' +local UKR_NET_From = 'From=/\\@ukr\\.net>?$/iX' -local RECEIVED_smtp_yandex_ru_1 = 'Received=/from \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] \\((port=\\d+ )?helo=smtp\\.yandex\\.ru\\)/iX' -local RECEIVED_smtp_yandex_ru_2 = 'Received=/from \\[UNAVAILABLE\\] \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]:\\d+ helo=smtp\\.yandex\\.ru\\)/iX' -local RECEIVED_smtp_yandex_ru_3 = 'Received=/from \\S+ \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]:\\d+ helo=smtp\\.yandex\\.ru\\)/iX' -local RECEIVED_smtp_yandex_ru_4 = 'Received=/from \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] \\(account \\S+ HELO smtp\\.yandex\\.ru\\)/iX' -local RECEIVED_smtp_yandex_ru_5 = 'Received=/from smtp\\.yandex\\.ru \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]\\)/iX' -local RECEIVED_smtp_yandex_ru_6 = 'Received=/from smtp\\.yandex\\.ru \\(\\S+ \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]\\)/iX' -local RECEIVED_smtp_yandex_ru_7 = 'Received=/from \\S+ \\(HELO smtp\\.yandex\\.ru\\) \\(\\S+\\@\\d+\\.\\d+\\.\\d+\\.\\d+\\)/iX' -local RECEIVED_smtp_yandex_ru_8 = 'Received=/from \\S+ \\(HELO smtp\\.yandex\\.ru\\) \\(\\d+\\.\\d+\\.\\d+\\.\\d+\\)/iX' -local RECEIVED_smtp_yandex_ru_9 = 'Received=/from \\S+ \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] helo=smtp\\.yandex\\.ru\\)/iX' +local RECEIVED_smtp_yandex_ru_1 = 'Received=/from \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] \\((port=\\d+ )?helo=smtp\\.yandex\\.ru\\)/iX' +local RECEIVED_smtp_yandex_ru_2 = 'Received=/from \\[UNAVAILABLE\\] \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]:\\d+ helo=smtp\\.yandex\\.ru\\)/iX' +local RECEIVED_smtp_yandex_ru_3 = 'Received=/from \\S+ \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]:\\d+ helo=smtp\\.yandex\\.ru\\)/iX' +local RECEIVED_smtp_yandex_ru_4 = 'Received=/from \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] \\(account \\S+ HELO smtp\\.yandex\\.ru\\)/iX' +local RECEIVED_smtp_yandex_ru_5 = 'Received=/from smtp\\.yandex\\.ru \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]\\)/iX' +local RECEIVED_smtp_yandex_ru_6 = 'Received=/from smtp\\.yandex\\.ru \\(\\S+ \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]\\)/iX' +local RECEIVED_smtp_yandex_ru_7 = 'Received=/from \\S+ \\(HELO smtp\\.yandex\\.ru\\) \\(\\S+\\@\\d+\\.\\d+\\.\\d+\\.\\d+\\)/iX' +local RECEIVED_smtp_yandex_ru_8 = 'Received=/from \\S+ \\(HELO smtp\\.yandex\\.ru\\) \\(\\d+\\.\\d+\\.\\d+\\.\\d+\\)/iX' +local RECEIVED_smtp_yandex_ru_9 = 'Received=/from \\S+ \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] helo=smtp\\.yandex\\.ru\\)/iX' reconf['FAKE_RECEIVED_smtp_yandex_ru'] = { - re = string.format('(((%s) & ((%s) | (%s))) | ((%s) & ((%s) | (%s))) '.. + re = string.format('(((%s) & ((%s) | (%s))) | ((%s) & ((%s) | (%s))) ' .. ' | ((%s) & ((%s) | (%s)))) & (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s)', MAIL_RU_From, MAIL_RU_Return_Path, MAIL_RU_X_Envelope_From, GMAIL_COM_From, GMAIL_COM_Return_Path, GMAIL_COM_X_Envelope_From, UKR_NET_From, UKR_NET_Return_Path, diff --git a/rules/regexp/misc.lua b/rules/regexp/misc.lua index 4caa5da42..d723f2901 100644 --- a/rules/regexp/misc.lua +++ b/rules/regexp/misc.lua @@ -55,10 +55,10 @@ reconf['INTRODUCTION'] = { local onion_uri_v2 = '/[a-z0-9]{16}\\.onion?/{url}i' local onion_uri_v3 = '/[a-z0-9]{56}\\.onion?/{url}i' reconf['HAS_ONION_URI'] = { - re = string.format('(%s | %s)', onion_uri_v2, onion_uri_v3), - description = 'Contains .onion hidden service URI', - score = 0.0, - group = 'url' + re = string.format('(%s | %s)', onion_uri_v2, onion_uri_v3), + description = 'Contains .onion hidden service URI', + score = 0.0, + group = 'url' } local my_victim = [[/(?:victim|prey)/{words}]] @@ -82,7 +82,7 @@ reconf['LEAKED_PASSWORD_SCAM_RE'] = { check_data_images = function(task) local tp = task:get_text_parts() or {} - for _,p in ipairs(tp) do + for _, p in ipairs(tp) do if p:is_html() then local hc = p:get_html() diff --git a/rules/regexp/upstream_spam_filters.lua b/rules/regexp/upstream_spam_filters.lua index b88e85709..b92f473b5 100644 --- a/rules/regexp/upstream_spam_filters.lua +++ b/rules/regexp/upstream_spam_filters.lua @@ -52,8 +52,8 @@ reconf['SPAM_FLAG'] = { reconf['UNITEDINTERNET_SPAM'] = { re = string.format('%s || %s', - 'X-UI-Filterresults=/^junk:/H', - 'X-UI-Out-Filterresults=/^junk:/H'), + 'X-UI-Filterresults=/^junk:/H', + 'X-UI-Out-Filterresults=/^junk:/H'), score = 5.0, description = "United Internet says this message is spam", group = 'upstream_spam_filters' diff --git a/rules/rspamd.lua b/rules/rspamd.lua index 5d93dec00..39017f169 100644 --- a/rules/rspamd.lua +++ b/rules/rspamd.lua @@ -16,7 +16,7 @@ limitations under the License. -- This is main lua config file for rspamd -require "global_functions" () +require "global_functions"() config['regexp'] = {} rspamd_maps = {} -- Global maps @@ -53,11 +53,13 @@ if rspamd_util.file_exists(local_conf .. '/local.d/rspamd.lua') then dofile(local_conf .. '/local.d/rspamd.lua') end -local rmaps = rspamd_config:get_all_opt("lua_maps") +local rmaps = rspamd_config:get_all_opt("lua_maps") if rmaps and type(rmaps) == 'table' then local rspamd_logger = require "rspamd_logger" - for k,v in pairs(rmaps) do - local status,map_or_err = pcall(function () return rspamd_config:add_map(v) end) + for k, v in pairs(rmaps) do + local status, map_or_err = pcall(function() + return rspamd_config:add_map(v) + end) if not status then rspamd_logger.errx(rspamd_config, "cannot add map %s: %s", k, map_or_err) diff --git a/rules/subject_checks.lua b/rules/subject_checks.lua index a21dc86cc..f781e1d6d 100644 --- a/rules/subject_checks.lua +++ b/rules/subject_checks.lua @@ -22,8 +22,8 @@ local subject_re = rspamd_regexp.create('/^(?:(?:Re|Fwd|Fw|Aw|Antwort|Sv):\\s*)+ local function test_subject(task, check_function, rate) local function normalize_linear(a, x) - local f = a * x - return true, (( f < 1 ) and f or 1), tostring(x) + local f = a * x + return true, ((f < 1) and f or 1), tostring(x) end local sbj = task:get_header('Subject') @@ -48,7 +48,7 @@ rspamd_config.SUBJ_ALL_CAPS = { local caps_test = function(sbj) return util.is_uppercase(sbj) end - return test_subject(task, caps_test, 1.0/40.0) + return test_subject(task, caps_test, 1.0 / 40.0) end, score = 3.0, group = 'subject', @@ -61,7 +61,7 @@ rspamd_config.LONG_SUBJ = { local length_test = function(_, len) return len > 200 end - return test_subject(task, length_test, 1.0/400.0) + return test_subject(task, length_test, 1.0 / 400.0) end, score = 3.0, group = 'subject', diff --git a/src/plugins/lua/antivirus.lua b/src/plugins/lua/antivirus.lua index 1623d33e8..26b8509b8 100644 --- a/src/plugins/lua/antivirus.lua +++ b/src/plugins/lua/antivirus.lua @@ -27,49 +27,49 @@ local N = "antivirus" if confighelp then rspamd_config:add_example(nil, 'antivirus', - "Check messages for viruses", - [[ -antivirus { - # multiple scanners could be checked, for each we create a configuration block with an arbitrary name - clamav { - # If set force this action if any virus is found (default unset: no action is forced) - # action = "reject"; - # If set, then rejection message is set to this value (mention single quotes) - # message = '${SCANNER}: virus found: "${VIRUS}"'; - # Scan mime_parts separately - otherwise the complete mail will be transferred to AV Scanner - #scan_mime_parts = true; - # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity) - #scan_text_mime = false; - #scan_image_mime = false; - # If `max_size` is set, messages > n bytes in size are not scanned - max_size = 20000000; - # symbol to add (add it to metric if you want non-zero weight) - symbol = "CLAM_VIRUS"; - # type of scanner: "clamav", "fprot", "sophos" or "savapi" - type = "clamav"; - # For "savapi" you must also specify the following variable - product_id = 12345; - # You can enable logging for clean messages - log_clean = true; - # servers to query (if port is unspecified, scanner-specific default is used) - # can be specified multiple times to pool servers - # can be set to a path to a unix socket - # Enable this in local.d/antivirus.conf - servers = "127.0.0.1:3310"; - # if `patterns` is specified virus name will be matched against provided regexes and the related - # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. - patterns { - # symbol_name = "pattern"; - JUST_EICAR = "^Eicar-Test-Signature$"; + "Check messages for viruses", + [[ + antivirus { + # multiple scanners could be checked, for each we create a configuration block with an arbitrary name + clamav { + # If set force this action if any virus is found (default unset: no action is forced) + # action = "reject"; + # If set, then rejection message is set to this value (mention single quotes) + # message = '${SCANNER}: virus found: "${VIRUS}"'; + # Scan mime_parts separately - otherwise the complete mail will be transferred to AV Scanner + #scan_mime_parts = true; + # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity) + #scan_text_mime = false; + #scan_image_mime = false; + # If `max_size` is set, messages > n bytes in size are not scanned + max_size = 20000000; + # symbol to add (add it to metric if you want non-zero weight) + symbol = "CLAM_VIRUS"; + # type of scanner: "clamav", "fprot", "sophos" or "savapi" + type = "clamav"; + # For "savapi" you must also specify the following variable + product_id = 12345; + # You can enable logging for clean messages + log_clean = true; + # servers to query (if port is unspecified, scanner-specific default is used) + # can be specified multiple times to pool servers + # can be set to a path to a unix socket + # Enable this in local.d/antivirus.conf + servers = "127.0.0.1:3310"; + # if `patterns` is specified virus name will be matched against provided regexes and the related + # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. + patterns { + # symbol_name = "pattern"; + JUST_EICAR = "^Eicar-Test-Signature$"; + } + # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. + whitelist = "/etc/rspamd/antivirus.wl"; + # Replace content that exactly matches the following string to the EICAR pattern + # Useful for E2E testing when another party removes/blocks EICAR attachments + #eicar_fake_pattern = 'testpatterneicar'; } - # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. - whitelist = "/etc/rspamd/antivirus.wl"; - # Replace content that exactly matches the following string to the EICAR pattern - # Useful for E2E testing when another party removes/blocks EICAR attachments - #eicar_fake_pattern = 'testpatterneicar'; } -} -]]) + ]]) return end @@ -84,7 +84,9 @@ local function add_antivirus_rule(sym, opts) return nil end - if not opts.symbol then opts.symbol = sym:upper() end + if not opts.symbol then + opts.symbol = sym:upper() + end local cfg = lua_antivirus[opts.type] if not cfg then @@ -106,13 +108,15 @@ local function add_antivirus_rule(sym, opts) -- WORKAROUND for deprecated attachments_only if opts.attachments_only ~= nil then opts.scan_mime_parts = opts.attachments_only - rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. '.. + rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. ' .. 'Please use scan_mime_parts = %s instead', opts.symbol, opts.type, opts.attachments_only) end -- WORKAROUND for deprecated attachments_only local rule = cfg.configure(opts) - if not rule then return nil end + if not rule then + return nil + end rule.type = opts.type rule.symbol_fail = opts.symbol_fail @@ -169,7 +173,7 @@ local function add_antivirus_rule(sym, opts) if clen == #opts.eicar_fake_pattern and content == opts.eicar_fake_pattern then rspamd_logger.infox(task, 'found eicar fake replacement part in the part (filename="%s")', - p:get_filename()) + p:get_filename()) content = eicar_pattern end end @@ -190,8 +194,12 @@ if opts and type(opts) == 'table' then local has_valid = false for k, m in pairs(opts) do if type(m) == 'table' then - if not m.type then m.type = k end - if not m.name then m.name = k end + if not m.type then + m.type = k + end + if not m.name then + m.name = k + end local cb = add_antivirus_rule(k, m) if not cb then diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua index 697c38b48..ff19aef4c 100644 --- a/src/plugins/lua/arc.lua +++ b/src/plugins/lua/arc.lua @@ -86,13 +86,15 @@ local ar_settings = lua_auth_results.default_settings local function parse_arc_header(hdr, target, is_aar) -- Split elements by ';' and trim spaces local arr = fun.totable(fun.map( - function(val) - return fun.totable(fun.map(lua_util.rspamd_str_trim, - fun.filter(function(v) return v and #v > 0 end, - lua_util.rspamd_str_split(val.decoded, ';') - ) - )) - end, hdr + function(val) + return fun.totable(fun.map(lua_util.rspamd_str_trim, + fun.filter(function(v) + return v and #v > 0 + end, + lua_util.rspamd_str_split(val.decoded, ';') + ) + )) + end, hdr )) -- v[1] is the key and v[2] is the value @@ -106,8 +108,10 @@ local function parse_arc_header(hdr, target, is_aar) -- Now we have two tables in format: -- [arc_header] -> [{arc_header1_elts}, {arc_header2_elts}...] - for i,elts in ipairs(arr) do - if not target[i] then target[i] = {} end + for i, elts in ipairs(arr) do + if not target[i] then + target[i] = {} + end if not is_aar then -- For normal ARC headers we split by kv pair, like k=v fun.each(function(v) @@ -120,7 +124,7 @@ local function parse_arc_header(hdr, target, is_aar) else -- For AAR we check special case of i=%d and pass everything else to -- AAR specific parser - for _,elt in ipairs(elts) do + for _, elt in ipairs(elts) do if string.match(elt, "%s*i%s*=%s*%d+%s*") then local pair = lua_util.rspamd_str_split(elt, '=') fill_arc_header_table(pair, target[i]) @@ -149,26 +153,26 @@ end local function arc_validate_seals(task, seals, sigs, seal_headers, sig_headers) local fail_reason - for i = 1,#seals do + for i = 1, #seals do if (sigs[i].i or 0) ~= i then fail_reason = string.format('bad i for signature: %d, expected %d; d=%s', sigs[i].i, i, sigs[i].d) rspamd_logger.infox(task, fail_reason) task:insert_result(arc_symbols['invalid'], 1.0, fail_reason) - return false,fail_reason + return false, fail_reason end if (seals[i].i or 0) ~= i then fail_reason = string.format('bad i for seal: %d, expected %d; d=%s', seals[i].i, i, seals[i].d) rspamd_logger.infox(task, fail_reason) - task:insert_result(arc_symbols['invalid'], 1.0,fail_reason) - return false,fail_reason + task:insert_result(arc_symbols['invalid'], 1.0, fail_reason) + return false, fail_reason end if not seals[i].cv then fail_reason = string.format('no cv on i=%d', i) task:insert_result(arc_symbols['invalid'], 1.0, fail_reason) - return false,fail_reason + return false, fail_reason end if i == 1 then @@ -176,18 +180,18 @@ local function arc_validate_seals(task, seals, sigs, seal_headers, sig_headers) if seals[i].cv ~= 'none' then fail_reason = 'cv is not "none" for i=1' task:insert_result(arc_symbols['invalid'], 1.0, fail_reason) - return false,fail_reason + return false, fail_reason end else if seals[i].cv ~= 'pass' then - fail_reason = string.format('cv is %s on i=%d', seals[i].cv, i) + fail_reason = string.format('cv is %s on i=%d', seals[i].cv, i) task:insert_result(arc_symbols['reject'], 1.0, fail_reason) - return true,fail_reason + return true, fail_reason end end end - return true,nil + return true, nil end local function arc_callback(task) @@ -244,7 +248,7 @@ local function arc_callback(task) lua_util.debugm(N, task, 'got %s arc sections', #cbdata.seals) -- Now check sanity of what we have - local valid,validation_error = arc_validate_seals(task, cbdata.seals, cbdata.sigs, + local valid, validation_error = arc_validate_seals(task, cbdata.seals, cbdata.sigs, arc_seal_headers, arc_sig_headers) if not valid then task:cache_set('arc-failure', validation_error) @@ -261,7 +265,7 @@ local function arc_callback(task) end local function gen_arc_seal_cb(index, sig) - return function (_, res, err, domain) + return function(_, res, err, domain) lua_util.debugm(N, task, 'checked arc seal: %s(%s), %s processed', res, err, index) @@ -283,7 +287,7 @@ local function arc_callback(task) else task:cache_set(AR_TRUSTED_CACHE_KEY, cur_aar) local seen_dmarc - for _,ar in ipairs(cur_aar.ar) do + for _, ar in ipairs(cur_aar.ar) do if ar.dmarc then local dmarc_fwd = ar.dmarc seen_dmarc = true @@ -326,7 +330,7 @@ local function arc_callback(task) local function arc_signature_cb(_, res, err, domain) lua_util.debugm(N, task, 'checked arc signature %s: %s(%s)', - domain, res, err) + domain, res, err) if not res then cbdata.res = 'fail' @@ -336,7 +340,7 @@ local function arc_callback(task) end if cbdata.res == 'success' then -- Verify seals - for i,sig in ipairs(cbdata.seals) do + for i, sig in ipairs(cbdata.seals) do local ret, lerr = dkim_verify(task, sig.header, gen_arc_seal_cb(i, sig), 'arc-seal') if not ret then cbdata.res = 'fail' @@ -348,8 +352,8 @@ local function arc_callback(task) end else task:insert_result(arc_symbols['reject'], 1.0, - rspamd_logger.slog('signature check failed: %s, %s', cbdata.res, - cbdata.errors)) + rspamd_logger.slog('signature check failed: %s, %s', cbdata.res, + cbdata.errors)) end end @@ -397,7 +401,7 @@ local function arc_callback(task) local processed = 0 local sig = cbdata.sigs[#cbdata.sigs] -- last AMS - local ret,err = dkim_verify(task, sig.header, arc_signature_cb, 'arc-sign') + local ret, err = dkim_verify(task, sig.header, arc_signature_cb, 'arc-sign') if not ret then cbdata.res = 'fail' @@ -405,13 +409,13 @@ local function arc_callback(task) else processed = processed + 1 lua_util.debugm(N, task, 'processed arc signature %s[%s]: %s(%s), %s total', - sig.d, sig.i, ret, err, #cbdata.seals) + sig.d, sig.i, ret, err, #cbdata.seals) end if processed == 0 then task:insert_result(arc_symbols['reject'], 1.0, - rspamd_logger.slog('cannot verify %s of %s signatures: %s', - #arc_sig_headers - processed, #arc_sig_headers, cbdata.errors)) + rspamd_logger.slog('cannot verify %s of %s signatures: %s', + #arc_sig_headers - processed, #arc_sig_headers, cbdata.errors)) end end @@ -421,21 +425,20 @@ if not opts or type(opts) ~= 'table' then end if opts['symbols'] then - for k,_ in pairs(arc_symbols) do + for k, _ in pairs(arc_symbols) do if opts['symbols'][k] then arc_symbols[k] = opts['symbols'][k] end end end - local id = rspamd_config:register_symbol({ name = 'ARC_CHECK', type = 'callback', group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, callback = arc_callback, - augmentations = {lua_util.dns_timeout_augmentation(rspamd_config)}, + augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) }, }) rspamd_config:register_symbol({ name = 'ARC_CALLBACK', -- compatibility symbol @@ -449,7 +452,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = -1.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) rspamd_config:register_symbol({ name = arc_symbols['reject'], @@ -457,7 +460,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 2.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) rspamd_config:register_symbol({ name = arc_symbols['invalid'], @@ -465,7 +468,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 1.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) rspamd_config:register_symbol({ name = arc_symbols['dnsfail'], @@ -473,7 +476,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 0.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) rspamd_config:register_symbol({ name = arc_symbols['na'], @@ -481,7 +484,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 0.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) rspamd_config:register_dependency('ARC_CHECK', 'SPF_CHECK') @@ -535,13 +538,13 @@ local function arc_sign_seal(task, params, header) for i = 1, #arc_seals, 1 do if arc_auth_results[i] then local s = dkim_canonicalize('ARC-Authentication-Results', - arc_auth_results[i].raw_header) + arc_auth_results[i].raw_header) sha_ctx:update(s) lua_util.debugm(N, task, 'update signature with header: %s', s) end if arc_sigs[i] then local s = dkim_canonicalize('ARC-Message-Signature', - arc_sigs[i].raw_header) + arc_sigs[i].raw_header) sha_ctx:update(s) lua_util.debugm(N, task, 'update signature with header: %s', s) end @@ -554,8 +557,8 @@ local function arc_sign_seal(task, params, header) end header = lua_util.fold_header(task, - 'ARC-Message-Signature', - header) + 'ARC-Message-Signature', + header) cur_auth_results = string.format('i=%d; %s', cur_idx, cur_auth_results) cur_auth_results = lua_util.fold_header(task, @@ -563,7 +566,7 @@ local function arc_sign_seal(task, params, header) cur_auth_results, ';') local s = dkim_canonicalize('ARC-Authentication-Results', - cur_auth_results) + cur_auth_results) sha_ctx:update(s) lua_util.debugm(N, task, 'update signature with header: %s', s) s = dkim_canonicalize('ARC-Message-Signature', header) @@ -588,17 +591,17 @@ local function arc_sign_seal(task, params, header) local sig = rspamd_rsa.sign_memory(privkey, sha_ctx:bin()) cur_arc_seal = string.format('%s%s', cur_arc_seal, - sig:base64(70, nl_type)) + sig:base64(70, nl_type)) lua_mime.modify_headers(task, { add = { - ['ARC-Authentication-Results'] = {order = 1, value = cur_auth_results}, - ['ARC-Message-Signature'] = {order = 1, value = header}, - ['ARC-Seal'] = {order = 1, value = lua_util.fold_header(task, - 'ARC-Seal', cur_arc_seal) } + ['ARC-Authentication-Results'] = { order = 1, value = cur_auth_results }, + ['ARC-Message-Signature'] = { order = 1, value = header }, + ['ARC-Seal'] = { order = 1, value = lua_util.fold_header(task, + 'ARC-Seal', cur_arc_seal) } }, -- RFC requires a strict order for these headers to be inserted - order = {'ARC-Authentication-Results', 'ARC-Message-Signature', 'ARC-Seal'}, + order = { 'ARC-Authentication-Results', 'ARC-Message-Signature', 'ARC-Seal' }, }) task:insert_result(settings.sign_symbol, 1.0, string.format('%s:s=%s:i=%d', params.domain, params.selector, cur_idx)) @@ -733,7 +736,7 @@ local function arc_signing_cb(task) selector = cur_selector.selector }) - local exists,err = rspamd_util.file_exists(cur_selector.key) + local exists, err = rspamd_util.file_exists(cur_selector.key) if not exists then if err and err == 'No such file or directory' then lua_util.debugm(N, task, 'cannot read key from %s: %s', cur_selector.key, err) @@ -775,7 +778,7 @@ if settings.use_redis then redis_params = rspamd_parse_redis_server('arc') if not redis_params then - rspamd_logger.errx(rspamd_config, 'no servers are specified, '.. + rspamd_logger.errx(rspamd_config, 'no servers are specified, ' .. 'but module is configured to load keys from redis, disable arc signing') return end @@ -786,7 +789,7 @@ end local sym_reg_tbl = { name = settings['sign_symbol'], callback = arc_signing_cb, - groups = {"policies", "arc"}, + groups = { "policies", "arc" }, flags = 'ignore_passthrough', score = 0.0, } @@ -805,7 +808,7 @@ if settings.whitelisted_signers_map then type = 'virtual', score = -2.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) end @@ -827,7 +830,7 @@ if settings.adjust_dmarc and settings.whitelisted_signers_map then sym_to_adjust = ar_settings.dmarc_symbols.quarantine end if sym_to_adjust and trusted_arc_ar and trusted_arc_ar.ar then - for _,ar in ipairs(trusted_arc_ar.ar) do + for _, ar in ipairs(trusted_arc_ar.ar) do if ar.dmarc then local dmarc_fwd = ar.dmarc if dmarc_fwd == 'pass' then diff --git a/src/plugins/lua/asn.lua b/src/plugins/lua/asn.lua index e870fdb3b..24da19eb4 100644 --- a/src/plugins/lua/asn.lua +++ b/src/plugins/lua/asn.lua @@ -81,7 +81,7 @@ local function asn_check(task) end lua_util.debugm(N, task, 'got reply from %s when requesting %s: %s', - serv, req_name, results[1]) + serv, req_name, results[1]) local parts = rspamd_re:split(results[1]) -- "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8 @@ -106,9 +106,9 @@ end -- Configuration options local configure_asn_module = function() - local opts = rspamd_config:get_all_opt('asn') + local opts = rspamd_config:get_all_opt('asn') if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do options[k] = v end end @@ -145,7 +145,7 @@ if configure_asn_module() then callback = asn_check, priority = lua_util.symbols_priorities.high, flags = 'empty,nostat', - augmentations = {lua_util.dns_timeout_augmentation(rspamd_config)}, + augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) }, }) if options['symbol'] then rspamd_config:register_symbol({ @@ -156,7 +156,7 @@ if configure_asn_module() then score = 0, }) end - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = options['symbol_fail'], parent = id, type = 'virtual', diff --git a/src/plugins/lua/aws_s3.lua b/src/plugins/lua/aws_s3.lua index 60981df9e..30e88d2cd 100644 --- a/src/plugins/lua/aws_s3.lua +++ b/src/plugins/lua/aws_s3.lua @@ -35,7 +35,7 @@ local settings = { inline_content_limit = nil, } -local settings_schema = ts.shape{ +local settings_schema = ts.shape { s3_bucket = ts.string, s3_region = ts.string, s3_host = ts.string, @@ -103,11 +103,11 @@ local function structured_data(task, nonce, queue_id) local message_split = lua_mime.message_to_ucl(task) if settings.inline_content_limit and settings.inline_content_limit > 0 then - for i,part in ipairs(message_split.parts or {}) do + for i, part in ipairs(message_split.parts or {}) do if part.content and #part.content >= settings.inline_content_limit then local ref = convert_to_ref(task, nonce, queue_id, part, external_refs) lua_util.debugm(N, task, "convert part number %s to a reference %s", - i, ref) + i, ref) end end end @@ -137,7 +137,7 @@ local function s3_aws_callback(task) local aws_host = string.format('%s.%s', settings.s3_bucket, settings.s3_host) local function gen_s3_http_callback(path, what) - return function (http_err, code, body, headers) + return function(http_err, code, body, headers) if http_err then if settings.fail_action then @@ -201,7 +201,7 @@ local function s3_aws_callback(task) timeout = settings.s3_timeout, }) - for ref,part_content in pairs(external_refs) do + for ref, part_content in pairs(external_refs) do local part_hdrs = lua_aws.aws_request_enrich({ region = settings.s3_region, headers = { @@ -235,7 +235,7 @@ if not opts then end settings = lua_util.override_defaults(settings, opts) -local res,err = settings_schema:transform(settings) +local res, err = settings_schema:transform(settings) if not res then rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err) @@ -263,7 +263,7 @@ rspamd_config:register_symbol({ name = 'EXPORT_AWS_S3', type = is_postfilter and 'postfilter' or 'idempotent', callback = s3_aws_callback, - augmentations = {string.format("timeout=%f", settings.s3_timeout)}, + augmentations = { string.format("timeout=%f", settings.s3_timeout) }, priority = is_postfilter and lua_util.symbols_priorities.high or nil, flags = 'empty,explicit_disable,ignore_passthrough,nostat', })
\ No newline at end of file diff --git a/src/plugins/lua/bayes_expiry.lua b/src/plugins/lua/bayes_expiry.lua index 97d463f81..44ff9dafa 100644 --- a/src/plugins/lua/bayes_expiry.lua +++ b/src/plugins/lua/bayes_expiry.lua @@ -72,9 +72,9 @@ local function check_redis_classifier(cls, cfg) local statfiles = cls.statfile if statfiles[1] then - for _,stf in ipairs(statfiles) do + for _, stf in ipairs(statfiles) do if not stf.symbol then - for k,v in pairs(stf) do + for k, v in pairs(stf) do check_statfile_table(v, k) end else @@ -82,7 +82,7 @@ local function check_redis_classifier(cls, cfg) end end else - for stn,stf in pairs(statfiles) do + for stn, stf in pairs(statfiles) do check_statfile_table(stf, stn) end end @@ -132,8 +132,10 @@ local classifier = obj.classifier if classifier then if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.bayes then cls = cls.bayes end + for _, cls in ipairs(classifier) do + if cls.bayes then + cls = cls.bayes + end if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, obj) end @@ -143,7 +145,7 @@ if classifier then classifier = classifier.bayes if classifier[1] then - for _,cls in ipairs(classifier) do + for _, cls in ipairs(classifier) do if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, obj) end @@ -157,11 +159,10 @@ if classifier then end end - local opts = rspamd_config:get_all_opt(N) if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end end @@ -171,11 +172,13 @@ end if settings.cluster_nodes == 0 then local neighbours = obj.neighbours or {} local n_neighbours = 0 - for _,_ in pairs(neighbours) do n_neighbours = n_neighbours + 1 end + for _, _ in pairs(neighbours) do + n_neighbours = n_neighbours + 1 + end settings.cluster_nodes = n_neighbours end - -- Fill template +-- Fill template template.count = settings.count template.threshold = settings.threshold template.common_ttl = settings.common_ttl @@ -184,7 +187,7 @@ template.significant_factor = settings.significant_factor template.expire_step = settings.interval template.hostname = rspamd_util.get_hostname() -for k,v in pairs(template) do +for k, v in pairs(template) do template[k] = tostring(v) end @@ -438,12 +441,12 @@ local function expire_step(cls, ev_base, worker) data[5] } logger.infox(rspamd_config, - 'finished expiry %s: %s items checked, %s significant (%s %s), ' .. - '%s insignificant (%s %s), %s common (%s discriminated), ' .. - '%s infrequent (%s %s), %s mean, %s std', - lutil.unpack(d)) + 'finished expiry %s: %s items checked, %s significant (%s %s), ' .. + '%s insignificant (%s %s), %s common (%s discriminated), ' .. + '%s infrequent (%s %s), %s mean, %s std', + lutil.unpack(d)) if cycle then - for i,cl in ipairs({'in ham', 'in spam', 'total'}) do + for i, cl in ipairs({ 'in ham', 'in spam', 'total' }) do logger.infox(rspamd_config, 'tokens occurrences, %s: {%s}', cl, occ_distr[i]) end end @@ -457,29 +460,31 @@ local function expire_step(cls, ev_base, worker) end end lredis.exec_redis_script(cls.script, - {ev_base = ev_base, is_write = true}, + { ev_base = ev_base, is_write = true }, redis_step_cb, - {'RS*_*', cls.expiry} + { 'RS*_*', cls.expiry } ) end -rspamd_config:add_on_load(function (_, ev_base, worker) +rspamd_config:add_on_load(function(_, ev_base, worker) -- Exit unless we're the first 'controller' worker - if not worker:is_primary_controller() then return end + if not worker:is_primary_controller() then + return + end local unique_redis_params = {} -- Push redis script to all unique redis servers - for _,cls in ipairs(settings.classifiers) do + for _, cls in ipairs(settings.classifiers) do if not unique_redis_params[cls.redis_params.hash] then unique_redis_params[cls.redis_params.hash] = cls.redis_params end end - for h,rp in pairs(unique_redis_params) do + for h, rp in pairs(unique_redis_params) do local script_id = lredis.add_redis_script(lutil.template(expiry_script, template), rp) - for _,cls in ipairs(settings.classifiers) do + for _, cls in ipairs(settings.classifiers) do if cls.redis_params.hash == h then cls.script = script_id end @@ -487,10 +492,10 @@ rspamd_config:add_on_load(function (_, ev_base, worker) end -- Expire tokens at regular intervals - for _,cls in ipairs(settings.classifiers) do + for _, cls in ipairs(settings.classifiers) do rspamd_config:add_periodic(ev_base, settings['interval'], - function () + function() expire_step(cls, ev_base, worker) return true end, true) diff --git a/src/plugins/lua/bimi.lua b/src/plugins/lua/bimi.lua index e90a4e3b9..e3f555e48 100644 --- a/src/plugins/lua/bimi.lua +++ b/src/plugins/lua/bimi.lua @@ -73,12 +73,12 @@ end local function gen_bimi_grammar() local lpeg = require "lpeg" lpeg.locale(lpeg) - local space = lpeg.space^0 - local name = lpeg.C(lpeg.alpha^1) * space - local sep = (lpeg.S("\\;") * space) + (lpeg.space^1) - local value = lpeg.C(lpeg.P(lpeg.graph - sep)^1) - local pair = lpeg.Cg(name * "=" * space * value) * sep^-1 - local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset) + local space = lpeg.space ^ 0 + local name = lpeg.C(lpeg.alpha ^ 1) * space + local sep = (lpeg.S("\\;") * space) + (lpeg.space ^ 1) + local value = lpeg.C(lpeg.P(lpeg.graph - sep) ^ 1) + local pair = lpeg.Cg(name * "=" * space * value) * sep ^ -1 + local list = lpeg.Cf(lpeg.Ct("") * pair ^ 0, rawset) local version = lpeg.P("v") * space * lpeg.P("=") * space * lpeg.P("BIMI1") local record = version * sep * list @@ -114,7 +114,7 @@ local function insert_bimi_headers(task, domain, bimi_content) local content = rspamd_util.encode_base64(rspamd_util.decode_base64(bimi_content), 73, task:get_newlines_type()) lua_mime.modify_headers(task, { - remove = {[hdr_name] = 0}, + remove = { [hdr_name] = 0 }, add = { [hdr_name] = { order = 0, @@ -123,12 +123,12 @@ local function insert_bimi_headers(task, domain, bimi_content) } } }) - task:insert_result('BIMI_VALID', 1.0, {domain}) + task:insert_result('BIMI_VALID', 1.0, { domain }) end local function process_bimi_json(task, domain, redis_data) local parser = ucl.parser() - local _,err = parser:parse_string(redis_data) + local _, err = parser:parse_string(redis_data) if err then rspamd_logger.errx(task, "cannot parse BIMI result from Redis for %s: %s", @@ -163,7 +163,7 @@ local function make_helper_request(task, domain, record, redis_server) end if is_sync then local parser = ucl.parser() - local _,err = parser:parse_string(body) + local _, err = parser:parse_string(body) if err then rspamd_logger.errx(task, "cannot parse BIMI result from helper for %s: %s", @@ -185,18 +185,18 @@ local function make_helper_request(task, domain, record, redis_server) upstream:fail() else lua_util.debugm(N, task, 'stored bimi image in Redis for domain %s; key=%s', - domain, redis_key) + domain, redis_key) end end - ret,_,upstream = lua_redis.redis_make_request(task, + ret, _, upstream = lua_redis.redis_make_request(task, redis_params, -- connect params redis_key, -- hash key true, -- is write redis_set_cb, --callback 'PSETEX', -- command - {redis_key, tostring(settings.redis_min_expiry * 1000.0), - ucl.to_format(d, "json-compact")}) + { redis_key, tostring(settings.redis_min_expiry * 1000.0), + ucl.to_format(d, "json-compact") }) if not ret then rspamd_logger.warnx(task, 'cannot make request to Redis when storing image; domain %s', @@ -227,7 +227,7 @@ local function make_helper_request(task, domain, record, redis_server) local serialised = ucl.to_format(request_data, 'json-compact') lua_util.debugm(N, task, "send request to BIMI helper: %s", - serialised) + serialised) rspamd_http.request({ task = task, mime_type = 'application/json', @@ -241,7 +241,7 @@ end local function check_bimi_vmc(task, domain, record) local redis_key = string.format('%s%s', settings.redis_prefix, - domain) + domain) local ret, _, upstream local function redis_cached_cb(err, data) @@ -276,13 +276,13 @@ local function check_bimi_vmc(task, domain, record) end -- We first check Redis and then try to use helper - ret,_,upstream = lua_redis.redis_make_request(task, + ret, _, upstream = lua_redis.redis_make_request(task, redis_params, -- connect params redis_key, -- hash key false, -- is write redis_cached_cb, --callback 'GET', -- command - {redis_key}) + { redis_key }) if not ret then rspamd_logger.warnx(task, 'cannot make request to Redis; domain %s', domain) @@ -291,12 +291,12 @@ end local function check_bimi_dns(task, domain) local resolve_name = string.format('default._bimi.%s', domain) - local dns_cb = function (_, _, results, err) + local dns_cb = function(_, _, results, err) if err then lua_util.debugm(N, task, "cannot resolve bimi for %s: %s", domain, err) else - for _,rec in ipairs(results) do + for _, rec in ipairs(results) do local res = check_bimi_record(task, rec) if res then @@ -319,7 +319,7 @@ local function check_bimi_dns(task, domain) end end task:get_resolver():resolve_txt({ - task=task, + task = task, name = resolve_name, callback = dns_cb, forced = true @@ -329,7 +329,9 @@ end local function bimi_callback(task) local dmarc_domain_maybe = check_dmarc_policy(task) - if not dmarc_domain_maybe then return end + if not dmarc_domain_maybe then + return + end -- We can either check BIMI via DNS or check Redis cache @@ -347,7 +349,7 @@ if not opts then end settings = lua_util.override_defaults(settings, opts) -local res,err = settings_schema:transform(settings) +local res, err = settings_schema:transform(settings) if not res then rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err) @@ -365,10 +367,10 @@ if redis_params then name = 'BIMI_CHECK', type = 'normal', callback = bimi_callback, - augmentations = {string.format("timeout=%f", settings.helper_timeout or - redis_params.timeout or 0.0)} + augmentations = { string.format("timeout=%f", settings.helper_timeout or + redis_params.timeout or 0.0) } }) - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = 'BIMI_VALID', type = 'virtual', parent = id, diff --git a/src/plugins/lua/clickhouse.lua b/src/plugins/lua/clickhouse.lua index 6deaa3112..25eabc760 100644 --- a/src/plugins/lua/clickhouse.lua +++ b/src/plugins/lua/clickhouse.lua @@ -44,26 +44,26 @@ local settings = { collect_garbage = false, -- Perform GC collection after sending the data check_timeout = 10.0, -- Periodic timeout timeout = 5.0, - bayes_spam_symbols = {'BAYES_SPAM'}, - bayes_ham_symbols = {'BAYES_HAM'}, - ann_symbols_spam = {'NEURAL_SPAM'}, - ann_symbols_ham = {'NEURAL_HAM'}, - fuzzy_symbols = {'FUZZY_DENIED'}, - whitelist_symbols = {'WHITELIST_DKIM', 'WHITELIST_SPF_DKIM', 'WHITELIST_DMARC'}, - dkim_allow_symbols = {'R_DKIM_ALLOW'}, - dkim_reject_symbols = {'R_DKIM_REJECT'}, - dkim_dnsfail_symbols = {'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL'}, - dkim_na_symbols = {'R_DKIM_NA'}, - dmarc_allow_symbols = {'DMARC_POLICY_ALLOW'}, - dmarc_reject_symbols = {'DMARC_POLICY_REJECT'}, - dmarc_quarantine_symbols = {'DMARC_POLICY_QUARANTINE'}, - dmarc_softfail_symbols = {'DMARC_POLICY_SOFTFAIL'}, - dmarc_na_symbols = {'DMARC_NA'}, - spf_allow_symbols = {'R_SPF_ALLOW'}, - spf_reject_symbols = {'R_SPF_FAIL'}, - spf_dnsfail_symbols = {'R_SPF_DNSFAIL', 'R_SPF_PERMFAIL'}, - spf_neutral_symbols = {'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL'}, - spf_na_symbols = {'R_SPF_NA'}, + bayes_spam_symbols = { 'BAYES_SPAM' }, + bayes_ham_symbols = { 'BAYES_HAM' }, + ann_symbols_spam = { 'NEURAL_SPAM' }, + ann_symbols_ham = { 'NEURAL_HAM' }, + fuzzy_symbols = { 'FUZZY_DENIED' }, + whitelist_symbols = { 'WHITELIST_DKIM', 'WHITELIST_SPF_DKIM', 'WHITELIST_DMARC' }, + dkim_allow_symbols = { 'R_DKIM_ALLOW' }, + dkim_reject_symbols = { 'R_DKIM_REJECT' }, + dkim_dnsfail_symbols = { 'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL' }, + dkim_na_symbols = { 'R_DKIM_NA' }, + dmarc_allow_symbols = { 'DMARC_POLICY_ALLOW' }, + dmarc_reject_symbols = { 'DMARC_POLICY_REJECT' }, + dmarc_quarantine_symbols = { 'DMARC_POLICY_QUARANTINE' }, + dmarc_softfail_symbols = { 'DMARC_POLICY_SOFTFAIL' }, + dmarc_na_symbols = { 'DMARC_NA' }, + spf_allow_symbols = { 'R_SPF_ALLOW' }, + spf_reject_symbols = { 'R_SPF_FAIL' }, + spf_dnsfail_symbols = { 'R_SPF_DNSFAIL', 'R_SPF_PERMFAIL' }, + spf_neutral_symbols = { 'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL' }, + spf_na_symbols = { 'R_SPF_NA' }, stop_symbols = {}, ipmask = 19, ipmask6 = 48, @@ -96,7 +96,7 @@ local settings = { } --- @language SQL -local clickhouse_schema = {[[ +local clickhouse_schema = { [[ CREATE TABLE IF NOT EXISTS rspamd ( Date Date COMMENT 'Date (used for partitioning)', @@ -156,8 +156,8 @@ CREATE TABLE IF NOT EXISTS rspamd PARTITION BY toMonday(Date) ORDER BY TS ]], -[[CREATE TABLE IF NOT EXISTS rspamd_version ( Version UInt32) ENGINE = TinyLog]], -{[[INSERT INTO rspamd_version (Version) Values (${SCHEMA_VERSION})]], true}, + [[CREATE TABLE IF NOT EXISTS rspamd_version ( Version UInt32) ENGINE = TinyLog]], + { [[INSERT INTO rspamd_version (Version) Values (${SCHEMA_VERSION})]], true }, } -- This describes SQL queries to migrate between versions @@ -305,7 +305,9 @@ local function clickhouse_main_row(res) 'SettingsId', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_attachments_row(res) @@ -316,7 +318,9 @@ local function clickhouse_attachments_row(res) 'Attachments.Digest', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_urls_row(res) @@ -325,14 +329,18 @@ local function clickhouse_urls_row(res) 'Urls.Url', 'Urls.Flags', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_emails_row(res) local fields = { 'Emails', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_symbols_row(res) @@ -341,7 +349,9 @@ local function clickhouse_symbols_row(res) 'Symbols.Scores', 'Symbols.Options', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_groups_row(res) @@ -349,7 +359,9 @@ local function clickhouse_groups_row(res) 'Groups.Names', 'Groups.Scores', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_asn_row(res) @@ -358,11 +370,15 @@ local function clickhouse_asn_row(res) 'Country', 'IPNet', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_extra_columns(res) - for _,v in ipairs(settings.extra_columns) do table.insert(res, v.name) end + for _, v in ipairs(settings.extra_columns) do + table.insert(res, v.name) + end end local function today(ts) @@ -371,7 +387,7 @@ end local function clickhouse_check_symbol(task, settings_field_name, fields_table, field_name, value, value_negative) - for _,s in ipairs(settings[settings_field_name] or {}) do + for _, s in ipairs(settings[settings_field_name] or {}) do if task:has_symbol(s) then if value_negative then local sym = task:get_symbol(s)[1] @@ -399,7 +415,7 @@ local function clickhouse_send_data(task, ev_base, why, gen_rows, cust_rows) #gen_rows + #cust_rows, ip_addr, why) local function gen_success_cb(what, how_many) - return function (_, _) + return function(_, _) rspamd_logger.messagex(log_object, "sent %s rows of %s to clickhouse server %s; started as %s", how_many, what, ip_addr, why) upstream:ok() @@ -407,7 +423,7 @@ local function clickhouse_send_data(task, ev_base, why, gen_rows, cust_rows) end local function gen_fail_cb(what, how_many) - return function (_, err) + return function(_, err) rspamd_logger.errx(log_object, "cannot send %s rows of %s data to clickhouse server %s: %s; started as %s", how_many, what, ip_addr, err, why) upstream:fail() @@ -453,9 +469,9 @@ local function clickhouse_send_data(task, ev_base, why, gen_rows, cust_rows) string.format('INSERT INTO rspamd (%s)', table.concat(fields, ','))) - for k,crows in pairs(cust_rows) do + for k, crows in pairs(cust_rows) do if #crows > 1 then - send_data('custom data ('..k..')', crows, + send_data('custom data (' .. k .. ')', crows, settings.custom_rules[k].first_row()) end end @@ -470,7 +486,7 @@ local function clickhouse_collect(task) return end - for _,sym in ipairs(settings.stop_symbols) do + for _, sym in ipairs(settings.stop_symbols) do if task:has_symbol(sym) then rspamd_logger.infox(task, 'skip Clickhouse storage for message: symbol %s has fired', sym) return @@ -478,7 +494,7 @@ local function clickhouse_collect(task) end if settings.exceptions then - local excepted,trace = settings.exceptions:process(task) + local excepted, trace = settings.exceptions:process(task) if excepted then rspamd_logger.infox(task, 'skipped Clickhouse storage for message: excepted (%s)', trace) @@ -490,7 +506,7 @@ local function clickhouse_collect(task) local from_domain = '' local from_user = '' if task:has_from('smtp') then - local from = task:get_from({'smtp','orig'})[1] + local from = task:get_from({ 'smtp', 'orig' })[1] if from then from_domain = from['domain']:lower() @@ -501,7 +517,7 @@ local function clickhouse_collect(task) local mime_domain = '' local mime_user = '' if task:has_from('mime') then - local from = task:get_from({'mime','orig'})[1] + local from = task:get_from({ 'mime', 'orig' })[1] if from then mime_domain = from['domain']:lower() mime_user = from['user'] @@ -510,7 +526,7 @@ local function clickhouse_collect(task) local mime_recipients = {} if task:has_recipients('mime') then - local recipients = task:get_recipients({'mime','orig'}) + local recipients = task:get_recipients({ 'mime', 'orig' }) for _, rcpt in ipairs(recipients) do table.insert(mime_recipients, rcpt['user'] .. '@' .. rcpt['domain']:lower()) end @@ -561,79 +577,76 @@ local function clickhouse_collect(task) local ret - ret = clickhouse_check_symbol(task,'bayes_spam_symbols', fields, + ret = clickhouse_check_symbol(task, 'bayes_spam_symbols', fields, 'bayes', 'spam') if not ret then - clickhouse_check_symbol(task,'bayes_ham_symbols', fields, + clickhouse_check_symbol(task, 'bayes_ham_symbols', fields, 'bayes', 'ham') end - clickhouse_check_symbol(task,'ann_symbols_spam', fields, + clickhouse_check_symbol(task, 'ann_symbols_spam', fields, 'ann', 'spam') if not ret then - clickhouse_check_symbol(task,'ann_symbols_ham', fields, + clickhouse_check_symbol(task, 'ann_symbols_ham', fields, 'ann', 'ham') end - clickhouse_check_symbol(task,'whitelist_symbols', fields, + clickhouse_check_symbol(task, 'whitelist_symbols', fields, 'whitelist', 'blacklist', 'whitelist') - clickhouse_check_symbol(task,'fuzzy_symbols', fields, + clickhouse_check_symbol(task, 'fuzzy_symbols', fields, 'fuzzy', 'deny') - - ret = clickhouse_check_symbol(task,'dkim_allow_symbols', fields, + ret = clickhouse_check_symbol(task, 'dkim_allow_symbols', fields, 'dkim', 'allow') if not ret then - ret = clickhouse_check_symbol(task,'dkim_reject_symbols', fields, + ret = clickhouse_check_symbol(task, 'dkim_reject_symbols', fields, 'dkim', 'reject') end if not ret then - ret = clickhouse_check_symbol(task,'dkim_dnsfail_symbols', fields, + ret = clickhouse_check_symbol(task, 'dkim_dnsfail_symbols', fields, 'dkim', 'dnsfail') end if not ret then - clickhouse_check_symbol(task,'dkim_na_symbols', fields, + clickhouse_check_symbol(task, 'dkim_na_symbols', fields, 'dkim', 'na') end - - ret = clickhouse_check_symbol(task,'dmarc_allow_symbols', fields, + ret = clickhouse_check_symbol(task, 'dmarc_allow_symbols', fields, 'dmarc', 'allow') if not ret then - ret = clickhouse_check_symbol(task,'dmarc_reject_symbols', fields, + ret = clickhouse_check_symbol(task, 'dmarc_reject_symbols', fields, 'dmarc', 'reject') end if not ret then - ret = clickhouse_check_symbol(task,'dmarc_quarantine_symbols', fields, + ret = clickhouse_check_symbol(task, 'dmarc_quarantine_symbols', fields, 'dmarc', 'quarantine') end if not ret then - ret = clickhouse_check_symbol(task,'dmarc_softfail_symbols', fields, + ret = clickhouse_check_symbol(task, 'dmarc_softfail_symbols', fields, 'dmarc', 'softfail') end if not ret then - clickhouse_check_symbol(task,'dmarc_na_symbols', fields, + clickhouse_check_symbol(task, 'dmarc_na_symbols', fields, 'dmarc', 'na') end - - ret = clickhouse_check_symbol(task,'spf_allow_symbols', fields, + ret = clickhouse_check_symbol(task, 'spf_allow_symbols', fields, 'spf', 'allow') if not ret then - ret = clickhouse_check_symbol(task,'spf_reject_symbols', fields, + ret = clickhouse_check_symbol(task, 'spf_reject_symbols', fields, 'spf', 'reject') end if not ret then - ret = clickhouse_check_symbol(task,'spf_neutral_symbols', fields, + ret = clickhouse_check_symbol(task, 'spf_neutral_symbols', fields, 'spf', 'neutral') end if not ret then - ret = clickhouse_check_symbol(task,'spf_dnsfail_symbols', fields, + ret = clickhouse_check_symbol(task, 'spf_dnsfail_symbols', fields, 'spf', 'dnsfail') end if not ret then - clickhouse_check_symbol(task,'spf_na_symbols', fields, + clickhouse_check_symbol(task, 'spf_na_symbols', fields, 'spf', 'na') end @@ -644,10 +657,10 @@ local function clickhouse_collect(task) local nurls = 0 local task_urls = task:get_urls({ - content = true, - images = true, - emails = false, - sort = true, + content = true, + images = true, + emails = false, + sort = true, }) or {} nurls = #task_urls @@ -753,9 +766,9 @@ local function clickhouse_collect(task) if #attachments_fnames > 0 then table.insert(row, attachments_fnames) - table.insert(row, attachments_ctypes) - table.insert(row, attachments_lengths) - table.insert(row, attachments_digests) + table.insert(row, attachments_ctypes) + table.insert(row, attachments_lengths) + table.insert(row, attachments_digests) else table.insert(row, {}) table.insert(row, {}) @@ -769,7 +782,7 @@ local function clickhouse_collect(task) local urls_flags = {} if settings.full_urls then - for i,u in ipairs(task_urls) do + for i, u in ipairs(task_urls) do urls_urls[i] = u:get_text() urls_tlds[i] = u:get_tld() or u:get_host() urls_flags[i] = u:get_flags_num() @@ -781,7 +794,7 @@ local function clickhouse_collect(task) idx_tbl = {}, -- indexed by host + flags, reference to an index in ord_tbl __newindex = function(t, k, v) local idx = getmetatable(t).idx_tbl - local ord = getmetatable(t).ord_tbl + local ord = getmetatable(t).ord_tbl local key = k:get_host() .. tostring(k:get_flags_num()) if idx[key] then ord[idx[key]] = v -- replace @@ -806,7 +819,7 @@ local function clickhouse_collect(task) -- Extra index needed for making this unique local urls_idx = {} setmetatable(urls_idx, mt) - for _,u in ipairs(task_urls) do + for _, u in ipairs(task_urls) do if not urls_idx[u] then urls_idx[u] = u urls_urls[#urls_urls + 1] = u:get_host() @@ -828,7 +841,7 @@ local function clickhouse_collect(task) if task:has_urls(true) then local emails = task:get_emails() or {} local emails_formatted = {} - for i,u in ipairs(emails) do + for i, u in ipairs(emails) do emails_formatted[i] = string.format('%s@%s', u:get_user(), u:get_host()) end table.insert(row, emails_formatted) @@ -862,7 +875,7 @@ local function clickhouse_collect(task) local scores_tab = {} local options_tab = {} - for _,s in ipairs(symbols) do + for _, s in ipairs(symbols) do table.insert(syms_tab, s.name or '') table.insert(scores_tab, s.score) @@ -880,7 +893,7 @@ local function clickhouse_collect(task) local groups = task:get_groups() local groups_tab = {} local gr_scores_tab = {} - for gr,sc in pairs(groups) do + for gr, sc in pairs(groups) do table.insert(groups_tab, gr) table.insert(gr_scores_tab, sc) end @@ -890,7 +903,7 @@ local function clickhouse_collect(task) -- Extra columns if #settings.extra_columns > 0 then - for _,col in ipairs(settings.extra_columns) do + for _, col in ipairs(settings.extra_columns) do local elts = col.real_selector(task) if elts then @@ -902,8 +915,10 @@ local function clickhouse_collect(task) end -- Custom data - for k,rule in pairs(settings.custom_rules) do - if not custom_rows[k] then custom_rows[k] = {} end + for k, rule in pairs(settings.custom_rules) do + if not custom_rows[k] then + custom_rows[k] = {} + end table.insert(custom_rows[k], lua_clickhouse.row_to_tsv(rule.get_row(task))) end @@ -923,9 +938,9 @@ local function do_remove_partition(ev_base, cfg, table_name, partition) local remove_partition_sql = "ALTER TABLE ${table_name} ${remove_method} PARTITION '${partition}'" local remove_method = (settings.retention.method == 'drop') and 'DROP' or 'DETACH' local sql_params = { - ['table_name'] = table_name, - ['remove_method'] = remove_method, - ['partition'] = partition + ['table_name'] = table_name, + ['remove_method'] = remove_method, + ['partition'] = partition } local sql = lua_util.template(remove_partition_sql, sql_params) @@ -939,9 +954,9 @@ local function do_remove_partition(ev_base, cfg, table_name, partition) local err, _ = lua_clickhouse.generic_sync(upstream, settings, ch_params, sql) if err then rspamd_logger.errx(rspamd_config, - "cannot detach partition %s:%s from server %s: %s", - table_name, partition, - settings['server'], err) + "cannot detach partition %s:%s from server %s: %s", + table_name, partition, + settings['server'], err) return end @@ -1080,15 +1095,14 @@ local function clickhouse_remove_old_partitions(cfg, ev_base) "GROUP BY partition, table " .. "HAVING max(max_date) < toDate(now() - interval ${month} month)" - local table_names = {'rspamd'} + local table_names = { 'rspamd' } local tables = table.concat(table_names, "', '") local sql_params = { tables = tables, - month = settings.retention.period_months, + month = settings.retention.period_months, } local sql = lua_util.template(partition_to_remove_sql, sql_params) - local ch_params = { ev_base = ev_base, config = cfg, @@ -1096,8 +1110,8 @@ local function clickhouse_remove_old_partitions(cfg, ev_base) local err, rows = lua_clickhouse.select_sync(upstream, settings, ch_params, sql) if err then rspamd_logger.errx(rspamd_config, - "cannot send data to clickhouse server %s: %s", - settings['server'], err) + "cannot send data to clickhouse server %s: %s", + settings['server'], err) else fun.each(function(row) do_remove_partition(ev_base, cfg, row.table, row.partition) @@ -1139,11 +1153,11 @@ local function upload_clickhouse_schema(upstream, ev_base, cfg, initial) -- Process element and return nil if statement should be skipped local function preprocess_schema_elt(v) if type(v) == 'string' then - return lua_util.template(v, {SCHEMA_VERSION = tostring(schema_version)}) + return lua_util.template(v, { SCHEMA_VERSION = tostring(schema_version) }) elseif type(v) == 'table' then -- Pair of statement + boolean if initial == v[2] then - return lua_util.template(v[1], {SCHEMA_VERSION = tostring(schema_version)}) + return lua_util.template(v[1], { SCHEMA_VERSION = tostring(schema_version) }) else rspamd_logger.debugm(N, rspamd_config, 'skip clickhouse schema element %s: schema already exists', v) @@ -1155,12 +1169,14 @@ local function upload_clickhouse_schema(upstream, ev_base, cfg, initial) -- Apply schema elements sequentially, users additions are concatenated to the tail fun.each(upload_schema_elt, - -- Also template schema version - fun.filter(function(v) return v ~= nil end, - fun.map(preprocess_schema_elt, - fun.chain(clickhouse_schema, settings.schema_additions) + -- Also template schema version + fun.filter(function(v) + return v ~= nil + end, + fun.map(preprocess_schema_elt, + fun.chain(clickhouse_schema, settings.schema_additions) + ) ) - ) ) end @@ -1171,7 +1187,7 @@ local function maybe_apply_migrations(upstream, ev_base, cfg, version) } -- Apply migrations sequentially local function migration_recursor(i) - if i < schema_version then + if i < schema_version then if migrations[i] then -- We also need to apply statements sequentially local function sql_recursor(j) @@ -1189,7 +1205,7 @@ local function maybe_apply_migrations(upstream, ev_base, cfg, version) -- Apply the next statement sql_recursor(j + 1) end - end , + end, function(_, err) rspamd_logger.errx(rspamd_config, "cannot apply migration %s: '%s' on clickhouse server %s: %s", @@ -1221,7 +1237,7 @@ local function add_extra_columns(upstream, ev_base, cfg) } -- Apply migrations sequentially local function columns_recursor(i) - if i <= #settings.extra_columns then + if i <= #settings.extra_columns then local col = settings.extra_columns[i] local prev_column if i == 1 then @@ -1242,7 +1258,7 @@ local function add_extra_columns(upstream, ev_base, cfg) col.name, col.type, prev_column) -- Apply the next statement columns_recursor(i + 1) - end , + end, function(_, err) rspamd_logger.errx(rspamd_config, "cannot apply add column alter %s: '%s' on clickhouse server %s: %s", @@ -1268,7 +1284,7 @@ local function check_rspamd_table(upstream, ev_base, cfg) local err, rows = lua_clickhouse.select_sync(upstream, settings, ch_params, sql) if err then rspamd_logger.errx(rspamd_config, "cannot check rspamd table in clickhouse server %s: %s", - upstream:get_addr():to_string(true), err) + upstream:get_addr():to_string(true), err) return end @@ -1290,20 +1306,19 @@ local function check_rspamd_table(upstream, ev_base, cfg) end end - local function check_clickhouse_upstream(upstream, ev_base, cfg) local ch_params = { ev_base = ev_base, config = cfg, } -- If we have some custom rules, we just send its schema to the upstream - for k,rule in pairs(settings.custom_rules) do + for k, rule in pairs(settings.custom_rules) do if rule.schema then local sql = lua_util.template(rule.schema, settings) local err, _ = lua_clickhouse.generic_sync(upstream, settings, ch_params, sql) if err then rspamd_logger.errx(rspamd_config, 'cannot send custom schema %s to clickhouse server %s: ' .. - 'cannot make request (%s)', + 'cannot make request (%s)', k, upstream:get_addr():to_string(true), err) end end @@ -1320,7 +1335,7 @@ local function check_clickhouse_upstream(upstream, ev_base, cfg) else rspamd_logger.errx(rspamd_config, "cannot get version on clickhouse server %s: %s", - upstream:get_addr():to_string(true), err) + upstream:get_addr():to_string(true), err) end else upload_clickhouse_schema(upstream, ev_base, cfg, false) @@ -1339,13 +1354,13 @@ if opts then if opts.limit and not opts.limits then settings.limits.max_rows = opts.limit end - for k,v in pairs(opts) do + for k, v in pairs(opts) do if k == 'custom_rules' then if not v[1] then - v = {v} + v = { v } end - for i,rule in ipairs(v) do + for i, rule in ipairs(v) do if rule.schema and rule.first_row and rule.get_row then local first_row, get_row local loadstring = loadstring or load @@ -1418,9 +1433,11 @@ if opts then -- Select traverse function depending on what we have local iter_func = settings.extra_columns[1] and ipairs or pairs - for col_name,col_data in iter_func(settings.extra_columns) do + for col_name, col_data in iter_func(settings.extra_columns) do -- Array based extra columns - if col_data.name then col_name = col_data.name end + if col_data.name then + col_name = col_data.name + end if not col_data.selector or not col_data.type then rspamd_logger.errx(rspamd_config, 'cannot add clickhouse extra row %s: no type or no selector', col_name) @@ -1459,7 +1476,9 @@ if opts then -- preserve strict order when doing altering if need_sort then rspamd_logger.infox(rspamd_config, 'sort extra columns as they are not configured as an array') - table.sort(columns_transformed, function(c1, c2) return c1.name < c2.name end) + table.sort(columns_transformed, function(c1, c2) + return c1.name < c2.name + end) end settings.extra_columns = columns_transformed end @@ -1469,7 +1488,7 @@ if opts then type = 'idempotent', callback = clickhouse_collect, flags = 'empty,explicit_disable,ignore_passthrough', - augmentations = {string.format("timeout=%f", settings.timeout)}, + augmentations = { string.format("timeout=%f", settings.timeout) }, }) rspamd_config:register_finish_script(function(task) if nrows > 0 then @@ -1499,7 +1518,7 @@ if opts then if worker:is_primary_controller() then local upstreams = settings.upstream:all_upstreams() - for _,up in ipairs(upstreams) do + for _, up in ipairs(upstreams) do check_clickhouse_upstream(up, ev_base, cfg) end diff --git a/src/plugins/lua/clustering.lua b/src/plugins/lua/clustering.lua index 3a8a361b1..d97bdb97e 100644 --- a/src/plugins/lua/clustering.lua +++ b/src/plugins/lua/clustering.lua @@ -44,7 +44,7 @@ local default_rule = { score_mult = 0.1, } -local rule_schema = ts.shape{ +local rule_schema = ts.shape { max_elts = ts.number + ts.string / tonumber, expire = ts.number + ts.string / lua_util.parse_time_interval, expire_overflow = ts.number + ts.string / lua_util.parse_time_interval, @@ -157,9 +157,9 @@ local function clusterting_filter_cb(task, rule) 'processed rule %s, selectors: source="%s", cluster="%s"; data: %s elts, %s score, %s elt score', rule.name, source_selector, cluster_selector, cur_elts, total_score, element_score) if final_score > 0.1 then - task:insert_result(rule.symbol, final_score, {source_selector, - tostring(size_score), - tostring(cluster_score)}) + task:insert_result(rule.symbol, final_score, { source_selector, + tostring(size_score), + tostring(cluster_score) }) end end @@ -182,14 +182,18 @@ local function clusterting_filter_cb(task, rule) end lua_redis.exec_redis_script(query_cluster_id, - {task = task, is_write = false, key = source_selector}, + { task = task, is_write = false, key = source_selector }, redis_get_cb, - {source_selector, cluster_selector}) + { source_selector, cluster_selector }) end local function clusterting_idempotent_cb(task, rule) - if task:has_flag('skip') then return end - if not rule.allow_local and lua_util.is_rspamc_or_controller(task) then return end + if task:has_flag('skip') then + return + end + if not rule.allow_local and lua_util.is_rspamc_or_controller(task) then + return + end local verdict = lua_verdict.get_specific_verdict(N, task) local score @@ -230,7 +234,7 @@ local function clusterting_idempotent_cb(task, rule) end lua_redis.exec_redis_script(update_cluster_id, - {task = task, is_write = true, key = source_selector}, + { task = task, is_write = true, key = source_selector }, redis_set_cb, { source_selector, @@ -258,22 +262,26 @@ if not redis_params then end if opts['rules'] then - for k,v in pairs(opts['rules']) do + for k, v in pairs(opts['rules']) do local raw_rule = lua_util.override_defaults(default_rule, v) - local rule,err = rule_schema:transform(raw_rule) + local rule, err = rule_schema:transform(raw_rule) if not rule then rspamd_logger.errx(rspamd_config, 'invalid clustering rule %s: %s', k, err) else - if not rule.symbol then rule.symbol = k end - if not rule.prefix then rule.prefix = k .. "_" end + if not rule.symbol then + rule.symbol = k + end + if not rule.prefix then + rule.prefix = k .. "_" + end rule.source_selector = lua_selectors.create_selector_closure(rspamd_config, rule.source_selector, '') - rule.cluster_selector = lua_selectors.create_selector_closure(rspamd_config, + rule.cluster_selector = lua_selectors.create_selector_closure(rspamd_config, rule.cluster_selector, '') if rule.source_selector and rule.cluster_selector then rule.name = k @@ -287,21 +295,23 @@ if opts['rules'] then query_cluster_id = lua_redis.add_redis_script(query_cluster_script, redis_params) update_cluster_id = lua_redis.add_redis_script(update_cluster_script, redis_params) local function callback_gen(f, rule) - return function(task) return f(task, rule) end + return function(task) + return f(task, rule) + end end - for _,rule in ipairs(rules) do - rspamd_config:register_symbol{ + for _, rule in ipairs(rules) do + rspamd_config:register_symbol { name = rule.symbol, type = 'normal', callback = callback_gen(clusterting_filter_cb, rule), } - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = rule.symbol .. '_STORE', type = 'idempotent', flags = 'empty,explicit_disable,ignore_passthrough', callback = callback_gen(clusterting_idempotent_cb, rule), - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)} + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) } } end else diff --git a/src/plugins/lua/dcc.lua b/src/plugins/lua/dcc.lua index 4cbe301e0..850832057 100644 --- a/src/plugins/lua/dcc.lua +++ b/src/plugins/lua/dcc.lua @@ -25,20 +25,19 @@ local lua_util = require "lua_util" local rspamd_logger = require "rspamd_logger" local dcc = require("lua_scanners").filter('dcc').dcc - if confighelp then rspamd_config:add_example(nil, 'dcc', - "Check messages for 'bulkiness' using DCC", - [[ -dcc { - socket = "/var/dcc/dccifd"; # Unix socket - servers = "127.0.0.1:10045" # OR TCP upstreams - timeout = 2s; # Timeout to wait for checks - body_max = 999999; # Bulkness threshold for body - fuz1_max = 999999; # Bulkness threshold for fuz1 - fuz2_max = 999999; # Bulkness threshold for fuz2 -} -]]) + "Check messages for 'bulkiness' using DCC", + [[ + dcc { + socket = "/var/dcc/dccifd"; # Unix socket + servers = "127.0.0.1:10045" # OR TCP upstreams + timeout = 2s; # Timeout to wait for checks + body_max = 999999; # Bulkness threshold for body + fuz1_max = 999999; # Bulkness threshold for fuz1 + fuz2_max = 999999; # Bulkness threshold for fuz2 + } + ]]) return end @@ -53,18 +52,22 @@ end -- WORKAROUND for deprecated host and port settings if opts['host'] ~= nil and opts['port'] ~= nil then opts['servers'] = opts['host'] .. ':' .. opts['port'] - rspamd_logger.warnx(rspamd_config, 'Using host and port parameters is deprecated. '.. - 'Please use servers = "%s:%s"; instead', opts['host'], opts['port']) + rspamd_logger.warnx(rspamd_config, 'Using host and port parameters is deprecated. ' .. + 'Please use servers = "%s:%s"; instead', opts['host'], opts['port']) end if opts['host'] ~= nil and not opts['port'] then opts['socket'] = opts['host'] - rspamd_logger.warnx(rspamd_config, 'Using host parameters is deprecated. '.. - 'Please use socket = "%s"; instead', opts['host']) + rspamd_logger.warnx(rspamd_config, 'Using host parameters is deprecated. ' .. + 'Please use socket = "%s"; instead', opts['host']) end -- WORKAROUND for deprecated host and port settings -if not opts.symbol_bulk then opts.symbol_bulk = symbol_bulk end -if not opts.symbol then opts.symbol = symbol end +if not opts.symbol_bulk then + opts.symbol_bulk = symbol_bulk +end +if not opts.symbol then + opts.symbol = symbol +end rule = dcc.configure(opts) @@ -74,17 +77,17 @@ if rule then callback = check_dcc, type = 'callback', }) - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', parent = id, name = opts.symbol } - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', parent = id, name = opts.symbol_bulk } - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', parent = id, name = 'DCC_FAIL' diff --git a/src/plugins/lua/dkim_signing.lua b/src/plugins/lua/dkim_signing.lua index 355ce89b8..6c05520ce 100644 --- a/src/plugins/lua/dkim_signing.lua +++ b/src/plugins/lua/dkim_signing.lua @@ -58,7 +58,7 @@ local function insert_sign_results(task, ret, hdr, dkim_params) if settings.use_milter_headers then lua_mime.modify_headers(task, { add = { - ['DKIM-Signature'] = {order = 1, value = hdr}, + ['DKIM-Signature'] = { order = 1, value = hdr }, } }) end @@ -106,7 +106,7 @@ local function sign_error(task, msg) end local function dkim_signing_cb(task) - local ret,selectors = dkim_sign_tools.prepare_dkim_signing(N, task, settings) + local ret, selectors = dkim_sign_tools.prepare_dkim_signing(N, task, settings) if not ret then return @@ -140,8 +140,10 @@ local function dkim_signing_cb(task) end end -local opts = rspamd_config:get_all_opt('dkim_signing') -if not opts then return end +local opts = rspamd_config:get_all_opt('dkim_signing') +if not opts then + return +end dkim_sign_tools.process_signing_settings(N, settings, opts) @@ -167,7 +169,7 @@ end local sym_reg_tbl = { name = settings['symbol'], callback = dkim_signing_cb, - groups = {"policies", "dkim"}, + groups = { "policies", "dkim" }, flags = 'ignore_passthrough', score = 0.0, } diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua index 17ef5c274..bf1e338da 100644 --- a/src/plugins/lua/dmarc.lua +++ b/src/plugins/lua/dmarc.lua @@ -115,7 +115,6 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) table.insert(reason, "No valid SPF") end - local opts = ((task:get_symbol('DKIM_TRACE') or E)[1] or E).options local dkim_results = { pass = {}, @@ -124,12 +123,11 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) fail = {}, } - if opts then dkim_results.pass = {} local dkim_violated - for _,opt in ipairs(opts) do + for _, opt in ipairs(opts) do local check_res = string.sub(opt, -1) local domain = string.sub(opt, 1, -3):lower() @@ -247,7 +245,7 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) therefore cannot apply the advertised DMARC policy. ]]-- if spf_tmpfail or dkim_tmpfail then - task:insert_result(settings.symbols['dnsfail'], 1.0, policy.domain.. + task:insert_result(settings.symbols['dnsfail'], 1.0, policy.domain .. ' : ' .. 'SPF/DKIM temp error', policy.dmarc_policy) else -- We can now check the failed policy and maybe send report data elt @@ -305,34 +303,33 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) -- Prepare and send redis report element local period = os.date('%Y%m%d', - task:get_date({format = 'connect', gmt = false})) + task:get_date({ format = 'connect', gmt = false })) -- Dmarc domain key must include dmarc domain, rua and period local dmarc_domain_key = table.concat( - {settings.reporting.redis_keys.report_prefix, dmarc_esld, policy.rua, period}, + { settings.reporting.redis_keys.report_prefix, dmarc_esld, policy.rua, period }, settings.reporting.redis_keys.join_char) local report_data = dmarc_common.dmarc_report(task, settings, { - spf_ok = spf_ok and 'pass' or 'fail', - dkim_ok = dkim_ok and 'pass' or 'fail', - disposition = (disposition == "softfail") and "none" or disposition, - sampled_out = sampled_out, - domain = hdrfromdom, - spf_domain = spf_domain, - dkim_results = dkim_results, - spf_result = spf_result + spf_ok = spf_ok and 'pass' or 'fail', + dkim_ok = dkim_ok and 'pass' or 'fail', + disposition = (disposition == "softfail") and "none" or disposition, + sampled_out = sampled_out, + domain = hdrfromdom, + spf_domain = spf_domain, + dkim_results = dkim_results, + spf_result = spf_result }) - - local idx_key = table.concat({settings.reporting.redis_keys.index_prefix, period}, + local idx_key = table.concat({ settings.reporting.redis_keys.index_prefix, period }, settings.reporting.redis_keys.join_char) if report_data then lua_redis.exec_redis_script(take_report_id, - {task = task, is_write = true}, + { task = task, is_write = true }, dmarc_report_cb, - {idx_key, dmarc_domain_key, - tostring(settings.reporting.max_entries), tostring(settings.reporting.keys_expire)}, - {hdrfromdom, report_data}) + { idx_key, dmarc_domain_key, + tostring(settings.reporting.max_entries), tostring(settings.reporting.keys_expire) }, + { hdrfromdom, report_data }) end end end @@ -365,13 +362,12 @@ local function dmarc_callback(task) return maybe_force_action(task, 'na') elseif (from or E)[1] then task:insert_result(settings.symbols['na'], 1.0, 'No domain in From header') - return maybe_force_action(task,'na') + return maybe_force_action(task, 'na') else task:insert_result(settings.symbols['na'], 1.0, 'No From header') - return maybe_force_action(task,'na') + return maybe_force_action(task, 'na') end - local dns_checks_inflight = 0 local dmarc_domain_policy = {} local dmarc_tld_policy = {} @@ -403,7 +399,7 @@ local function dmarc_callback(task) policy_target = dmarc_tld_policy end - return function (_, _, results, err) + return function(_, _, results, err) dns_checks_inflight = dns_checks_inflight - 1 if not seen_invalid then @@ -421,8 +417,8 @@ local function dmarc_callback(task) else local has_valid_policy = false - for _,rec in ipairs(results) do - local ret,results_or_err = dmarc_common.dmarc_check_record(task, rec, is_tld) + for _, rec in ipairs(results) do + local ret, results_or_err = dmarc_common.dmarc_check_record(task, rec, is_tld) if not ret then if results_or_err then @@ -442,7 +438,7 @@ local function dmarc_callback(task) end has_valid_policy = true - for k,v in pairs(results_or_err) do + for k, v in pairs(results_or_err) do policy_target[k] = v end end @@ -471,7 +467,7 @@ local function dmarc_callback(task) local resolve_name = '_dmarc.' .. hfromdom task:get_resolver():resolve_txt({ - task=task, + task = task, name = resolve_name, callback = gen_dmarc_cb(hfromdom, false), forced = true @@ -482,7 +478,7 @@ local function dmarc_callback(task) resolve_name = '_dmarc.' .. dmarc_domain task:get_resolver():resolve_txt({ - task=task, + task = task, name = resolve_name, callback = gen_dmarc_cb(dmarc_domain, true), forced = true @@ -492,7 +488,6 @@ local function dmarc_callback(task) end end - local opts = rspamd_config:get_all_opt('dmarc') settings = lua_util.override_defaults(settings, opts) @@ -523,7 +518,6 @@ if type(settings.reporting) == 'table' then }) end - if settings.reporting == true then rspamd_logger.errx(rspamd_config, 'old style dmarc reporting is NO LONGER supported, please read the documentation') elseif settings.reporting.enabled then @@ -575,49 +569,49 @@ rspamd_config:register_symbol({ name = settings.symbols['allow'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['reject'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['quarantine'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['softfail'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['dnsfail'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['badpolicy'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['na'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) @@ -639,7 +633,7 @@ if settings.munging then local munging_opts = lua_util.override_defaults(munging_defaults, settings.munging) - if not munging_opts.list_map then + if not munging_opts.list_map then rspamd_logger.errx(rspamd_config, 'cannot enable DMARC munging with no list_map parameter') return @@ -648,7 +642,7 @@ if settings.munging then munging_opts.list_map = lua_maps.map_add_from_ucl(munging_opts.list_map, 'set', 'DMARC munging map of the recipients addresses to munge') - if not munging_opts.list_map then + if not munging_opts.list_map then rspamd_logger.errx(rspamd_config, 'cannot enable DMARC munging with invalid list_map (invalid map)') return @@ -656,19 +650,18 @@ if settings.munging then if munging_opts.munge_map_condition then munging_opts.munge_map_condition = lua_maps_expressions.create(rspamd_config, - munging_opts.munge_map_condition, N) + munging_opts.munge_map_condition, N) end - rspamd_config:register_symbol({ name = 'DMARC_MUNGED', type = 'normal', flags = 'nostat', score = 0, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, callback = dmarc_common.gen_munging_callback(munging_opts, settings), - augmentations = {lua_util.dns_timeout_augmentation(rspamd_config)}, + augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) }, }) rspamd_config:register_dependency('DMARC_MUNGED', 'DMARC_CHECK') diff --git a/src/plugins/lua/dynamic_conf.lua b/src/plugins/lua/dynamic_conf.lua index 8498b9b7c..5af26a933 100644 --- a/src/plugins/lua/dynamic_conf.lua +++ b/src/plugins/lua/dynamic_conf.lua @@ -52,7 +52,7 @@ end local function apply_dynamic_actions(_, acts) fun.each(function(k, v) - if type(v) == 'table' then + if type(v) == 'table' then v['name'] = k if not v['priority'] then v['priority'] = settings.priority @@ -145,7 +145,7 @@ local function update_dynamic_conf(cfg, ev_base, recv) settings.redis_key, true, redis_version_set_cb, - 'HINCRBY', {settings.redis_key, 'v', '1'}) + 'HINCRBY', { settings.redis_key, 'v', '1' }) end end @@ -158,12 +158,12 @@ local function update_dynamic_conf(cfg, ev_base, recv) fun.each(function(k, v) cur_settings.data.scores[k] = v end, - fun.filter(function(k) - if cur_settings.updates.symbols[k] then - return false - end - return true - end, recv['scores'])) + fun.filter(function(k) + if cur_settings.updates.symbols[k] then + return false + end + return true + end, recv['scores'])) end if recv['actions'] then if not cur_settings.data.actions then @@ -172,18 +172,18 @@ local function update_dynamic_conf(cfg, ev_base, recv) fun.each(function(k, v) cur_settings.data.actions[k] = v end, - fun.filter(function(k) - if cur_settings.updates.actions[k] then - return false - end - return true - end, recv['actions'])) + fun.filter(function(k) + if cur_settings.updates.actions[k] then + return false + end + return true + end, recv['actions'])) end end local newdata = ucl.to_format(cur_settings.data, 'json-compact') rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params, settings.redis_key, true, - redis_data_set_cb, 'HSET', {settings.redis_key, 'd', newdata}) + redis_data_set_cb, 'HSET', { settings.redis_key, 'd', newdata }) end local function check_dynamic_conf(cfg, ev_base) @@ -192,7 +192,7 @@ local function check_dynamic_conf(cfg, ev_base) rspamd_logger.errx(cfg, "cannot read dynamic conf from redis: %s", redis_err) elseif data and type(data) == 'string' then local parser = ucl.parser() - local _,err = parser:parse_string(data) + local _, err = parser:parse_string(data) if err then rspamd_logger.errx(cfg, "cannot load dynamic conf from redis: %s", err) @@ -214,11 +214,11 @@ local function check_dynamic_conf(cfg, ev_base) if not cur_settings.version or (rver and rver > cur_settings.version) then rspamd_logger.infox(cfg, "need to load fresh dynamic settings with version %s, local version is %s", - rver, cur_settings.version) + rver, cur_settings.version) cur_settings.version = rver rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params, settings.redis_key, false, - redis_load_cb, 'HGET', {settings.redis_key, 'd'}) + redis_load_cb, 'HGET', { settings.redis_key, 'd' }) elseif cur_settings.updates.has_updates then -- Need to send our updates to Redis update_dynamic_conf(cfg, ev_base) @@ -231,7 +231,7 @@ local function check_dynamic_conf(cfg, ev_base) rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params, settings.redis_key, false, - redis_check_cb, 'HGET', {settings.redis_key, 'v'}) + redis_check_cb, 'HGET', { settings.redis_key, 'v' }) end local section = rspamd_config:get_all_opt("dynamic_conf") @@ -242,7 +242,7 @@ if section then return end - for k,v in pairs(section) do + for k, v in pairs(section) do settings[k] = v end diff --git a/src/plugins/lua/elastic.lua b/src/plugins/lua/elastic.lua index ce2d01b80..ccbb7c198 100644 --- a/src/plugins/lua/elastic.lua +++ b/src/plugins/lua/elastic.lua @@ -43,7 +43,7 @@ local settings = { limit = 500, index_pattern = 'rspamd-%Y.%m.%d', template_file = rspamd_paths['SHAREDIR'] .. '/elastic/rspamd_template.json', - kibana_file = rspamd_paths['SHAREDIR'] ..'/elastic/kibana.json', + kibana_file = rspamd_paths['SHAREDIR'] .. '/elastic/kibana.json', key_prefix = 'elastic-', expire = 3600, timeout = 5.0, @@ -61,23 +61,25 @@ local settings = { } local function read_file(path) - local file = io.open(path, "rb") - if not file then return nil end - local content = file:read "*a" - file:close() - return content + local file = io.open(path, "rb") + if not file then + return nil + end + local content = file:read "*a" + file:close() + return content end local function elastic_send_data(task) local es_index = os.date(settings['index_pattern']) local tbl = {} - for _,value in pairs(rows) do + for _, value in pairs(rows) do if settings.elasticsearch_version >= 7 then - table.insert(tbl, '{ "index" : { "_index" : "'..es_index.. - '","pipeline": "rspamd-geoip"} }') + table.insert(tbl, '{ "index" : { "_index" : "' .. es_index .. + '","pipeline": "rspamd-geoip"} }') else - table.insert(tbl, '{ "index" : { "_index" : "'..es_index.. - '", "_type" : "_doc" ,"pipeline": "rspamd-geoip"} }') + table.insert(tbl, '{ "index" : { "_index" : "' .. es_index .. + '", "_type" : "_doc" ,"pipeline": "rspamd-geoip"} }') end table.insert(tbl, ucl.to_format(value, 'json-compact')) end @@ -87,7 +89,7 @@ local function elastic_send_data(task) local upstream = settings.upstream:get_upstream_round_robin() local ip_addr = upstream:get_addr():to_string(true) - local push_url = connect_prefix .. ip_addr .. '/'..es_index..'/_bulk' + local push_url = connect_prefix .. ip_addr .. '/' .. es_index .. '/_bulk' local bulk_json = table.concat(tbl, "\n") local function http_callback(err, code, _, _) @@ -127,7 +129,7 @@ local function get_general_metadata(task) local r = {} local ip_addr = task:get_ip() - if ip_addr and ip_addr:is_valid() then + if ip_addr and ip_addr:is_valid() then r.is_local = ip_addr:is_local() r.ip = tostring(ip_addr) else @@ -153,10 +155,10 @@ local function get_general_metadata(task) r.action = task:get_metric_action() r.rspamd_server = HOSTNAME if r.user ~= 'unknown' then - r.direction = "Outbound" + r.direction = "Outbound" end local s = task:get_metric_score()[1] - r.score = s + r.score = s local rcpt = task:get_recipients('smtp') if rcpt then @@ -169,14 +171,14 @@ local function get_general_metadata(task) r.rcpt = 'unknown' end - local from = task:get_from{'smtp', 'orig'} + local from = task:get_from { 'smtp', 'orig' } if ((from or E)[1] or E).addr then r.from = from[1].addr else r.from = 'unknown' end - local mime_from = task:get_from{'mime', 'orig'} + local mime_from = task:get_from { 'mime', 'orig' } if ((mime_from or E)[1] or E).addr then r.mime_from = mime_from[1].addr else @@ -188,7 +190,7 @@ local function get_general_metadata(task) r.asn = {} local pool = task:get_mempool() r.asn.country = pool:get_variable("country") or 'unknown' - r.asn.asn = pool:get_variable("asn") or 0 + r.asn.asn = pool:get_variable("asn") or 0 r.asn.ipnet = pool:get_variable("ipnet") or 'unknown' local function process_header(name) @@ -244,12 +246,18 @@ local function get_general_metadata(task) end local function elastic_collect(task) - if not enabled then return end - if task:has_flag('skip') then return end - if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then return end + if not enabled then + return + end + if task:has_flag('skip') then + return + end + if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then + return + end - local row = {['rspamd_meta'] = get_general_metadata(task), - ['@timestamp'] = tostring(util.get_time() * 1000)} + local row = { ['rspamd_meta'] = get_general_metadata(task), + ['@timestamp'] = tostring(util.get_time() * 1000) } table.insert(rows, row) nrows = nrows + 1 if nrows > settings['limit'] then @@ -272,7 +280,6 @@ local function elastic_collect(task) end end - local opts = rspamd_config:get_all_opt('elastic') local function check_elastic_server(cfg, ev_base, _) @@ -282,7 +289,7 @@ local function check_elastic_server(cfg, ev_base, _) local function http_callback(err, code, body, _) if code == 200 then local parser = ucl.parser() - local res,ucl_err = parser:parse_string(body) + local res, ucl_err = parser:parse_string(body) if not res then rspamd_logger.infox(rspamd_config, 'failed to parse reply from %s: %s', plugins_url, ucl_err) @@ -290,9 +297,9 @@ local function check_elastic_server(cfg, ev_base, _) return end local obj = parser:get_object() - for node,value in pairs(obj['nodes']) do + for node, value in pairs(obj['nodes']) do local plugin_found = false - for _,plugin in pairs(value['plugins']) do + for _, plugin in pairs(value['plugins']) do if plugin['name'] == 'ingest-geoip' then plugin_found = true lua_util.debugm(N, "ingest-geoip plugin has been found") @@ -326,7 +333,9 @@ end -- import ingest pipeline and kibana dashboard/visualization local function initial_setup(cfg, ev_base, worker) - if not worker:is_primary_controller() then return end + if not worker:is_primary_controller() then + return + end local upstream = settings.upstream:get_upstream_round_robin() local ip_addr = upstream:get_addr():to_string(true) @@ -337,7 +346,7 @@ local function initial_setup(cfg, ev_base, worker) local kibana_mappings = read_file(settings['kibana_file']) if kibana_mappings then local parser = ucl.parser() - local res,parser_err = parser:parse_string(kibana_mappings) + local res, parser_err = parser:parse_string(kibana_mappings) if not res then rspamd_logger.infox(rspamd_config, 'kibana template cannot be parsed: %s', parser_err) @@ -347,14 +356,14 @@ local function initial_setup(cfg, ev_base, worker) end local obj = parser:get_object() local tbl = {} - for _,item in ipairs(obj) do - table.insert(tbl, '{ "index" : { "_index" : ".kibana", "_type" : "doc" ,"_id": "'.. - item['_type'] .. ':' .. item["_id"]..'"} }') + for _, item in ipairs(obj) do + table.insert(tbl, '{ "index" : { "_index" : ".kibana", "_type" : "doc" ,"_id": "' .. + item['_type'] .. ':' .. item["_id"] .. '"} }') table.insert(tbl, ucl.to_format(item['_source'], 'json-compact')) end table.insert(tbl, '') -- For last \n - local kibana_url = connect_prefix .. ip_addr ..'/.kibana/_bulk' + local kibana_url = connect_prefix .. ip_addr .. '/.kibana/_bulk' local function kibana_template_callback(err, code, body, _) if code ~= 200 then rspamd_logger.errx('cannot put template to %s: %s(%s) (%s)', kibana_url, @@ -389,7 +398,7 @@ local function initial_setup(cfg, ev_base, worker) if enabled then -- create ingest pipeline - local geoip_url = connect_prefix .. ip_addr ..'/_ingest/pipeline/rspamd-geoip' + local geoip_url = connect_prefix .. ip_addr .. '/_ingest/pipeline/rspamd-geoip' local function geoip_cb(err, code, body, _) if code ~= 200 then rspamd_logger.errx('cannot get data from %s: %s(%s) (%s)', @@ -425,7 +434,7 @@ local function initial_setup(cfg, ev_base, worker) timeout = settings.timeout, }) -- create template mappings if not exist - local template_url = connect_prefix .. ip_addr ..'/_template/rspamd' + local template_url = connect_prefix .. ip_addr .. '/_template/rspamd' local function http_template_put_callback(err, code, body, _) if code ~= 200 then rspamd_logger.errx('cannot put template to %s: %s(%s) (%s)', @@ -477,7 +486,7 @@ end redis_params = rspamd_redis.parse_redis_server('elastic') if redis_params and opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end @@ -494,11 +503,11 @@ if redis_params and opts then end settings.upstream = upstream_list.create(rspamd_config, - settings['server'] or settings['servers'], 9200) + settings['server'] or settings['servers'], 9200) if not settings.upstream then rspamd_logger.errx('cannot parse elastic address: %s', - settings['server'] or settings['servers']) + settings['server'] or settings['servers']) lua_util.disable_module(N, "config") return end @@ -511,7 +520,7 @@ if redis_params and opts then elastic_template = read_file(settings['template_file']); if not elastic_template then rspamd_logger.infox(rspamd_config, 'elastic unable to read %s, disabling module', - settings['template_file']) + settings['template_file']) lua_util.disable_module(N, "config") return end @@ -521,10 +530,10 @@ if redis_params and opts then type = 'idempotent', callback = elastic_collect, flags = 'empty,explicit_disable,ignore_passthrough', - augmentations = {string.format("timeout=%f", settings.timeout)}, + augmentations = { string.format("timeout=%f", settings.timeout) }, }) - rspamd_config:add_on_load(function(cfg, ev_base,worker) + rspamd_config:add_on_load(function(cfg, ev_base, worker) if worker:is_scanner() then check_elastic_server(cfg, ev_base, worker) -- check for elasticsearch requirements initial_setup(cfg, ev_base, worker) -- import mappings pipeline and visualizations diff --git a/src/plugins/lua/external_relay.lua b/src/plugins/lua/external_relay.lua index 9c0f06ff8..3660f921e 100644 --- a/src/plugins/lua/external_relay.lua +++ b/src/plugins/lua/external_relay.lua @@ -34,40 +34,40 @@ local settings = { rules = {}, } -local config_schema = ts.shape{ +local config_schema = ts.shape { enabled = ts.boolean:is_optional(), rules = ts.map_of( - ts.string, ts.one_of{ - ts.shape{ - priority = ts.number:is_optional(), - strategy = 'authenticated', - symbol = ts.string:is_optional(), - user_map = lua_maps.map_schema:is_optional(), - }, - ts.shape{ - count = ts.number, - priority = ts.number:is_optional(), - strategy = 'count', - symbol = ts.string:is_optional(), - }, - ts.shape{ - priority = ts.number:is_optional(), - strategy = 'local', - symbol = ts.string:is_optional(), - }, - ts.shape{ - hostname_map = lua_maps.map_schema, - priority = ts.number:is_optional(), - strategy = 'hostname_map', - symbol = ts.string:is_optional(), - }, - ts.shape{ - ip_map = lua_maps.map_schema, - priority = ts.number:is_optional(), - strategy = 'ip_map', - symbol = ts.string:is_optional(), - }, - } + ts.string, ts.one_of { + ts.shape { + priority = ts.number:is_optional(), + strategy = 'authenticated', + symbol = ts.string:is_optional(), + user_map = lua_maps.map_schema:is_optional(), + }, + ts.shape { + count = ts.number, + priority = ts.number:is_optional(), + strategy = 'count', + symbol = ts.string:is_optional(), + }, + ts.shape { + priority = ts.number:is_optional(), + strategy = 'local', + symbol = ts.string:is_optional(), + }, + ts.shape { + hostname_map = lua_maps.map_schema, + priority = ts.number:is_optional(), + strategy = 'hostname_map', + symbol = ts.string:is_optional(), + }, + ts.shape { + ip_map = lua_maps.map_schema, + priority = ts.number:is_optional(), + strategy = 'ip_map', + symbol = ts.string:is_optional(), + }, + } ), } diff --git a/src/plugins/lua/external_services.lua b/src/plugins/lua/external_services.lua index d6fedeece..a68517c08 100644 --- a/src/plugins/lua/external_services.lua +++ b/src/plugins/lua/external_services.lua @@ -27,91 +27,90 @@ local N = "external_services" if confighelp then rspamd_config:add_example(nil, 'external_services', - "Check messages using external services (e.g. OEM AS engines, DCC, Pyzor etc)", - [[ -external_services { - # multiple scanners could be checked, for each we create a configuration block with an arbitrary name - - oletools { - # If set force this action if any virus is found (default unset: no action is forced) - # action = "reject"; - # If set, then rejection message is set to this value (mention single quotes) - # If `max_size` is set, messages > n bytes in size are not scanned - # max_size = 20000000; - # log_clean = true; - # servers = "127.0.0.1:10050"; - # cache_expire = 86400; - # scan_mime_parts = true; - # extended = false; - # if `patterns` is specified virus name will be matched against provided regexes and the related - # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. - patterns { - # symbol_name = "pattern"; - JUST_EICAR = "^Eicar-Test-Signature$"; + "Check messages using external services (e.g. OEM AS engines, DCC, Pyzor etc)", + [[ + external_services { + # multiple scanners could be checked, for each we create a configuration block with an arbitrary name + + oletools { + # If set force this action if any virus is found (default unset: no action is forced) + # action = "reject"; + # If set, then rejection message is set to this value (mention single quotes) + # If `max_size` is set, messages > n bytes in size are not scanned + # max_size = 20000000; + # log_clean = true; + # servers = "127.0.0.1:10050"; + # cache_expire = 86400; + # scan_mime_parts = true; + # extended = false; + # if `patterns` is specified virus name will be matched against provided regexes and the related + # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. + patterns { + # symbol_name = "pattern"; + JUST_EICAR = "^Eicar-Test-Signature$"; + } + # mime-part regex matching in content-type or filename + mime_parts_filter_regex { + #GEN1 = "application\/octet-stream"; + DOC2 = "application\/msword"; + DOC3 = "application\/vnd\.ms-word.*"; + XLS = "application\/vnd\.ms-excel.*"; + PPT = "application\/vnd\.ms-powerpoint.*"; + GEN2 = "application\/vnd\.openxmlformats-officedocument.*"; + } + # Mime-Part filename extension matching (no regex) + mime_parts_filter_ext { + doc = "doc"; + dot = "dot"; + docx = "docx"; + dotx = "dotx"; + docm = "docm"; + dotm = "dotm"; + xls = "xls"; + xlt = "xlt"; + xla = "xla"; + xlsx = "xlsx"; + xltx = "xltx"; + xlsm = "xlsm"; + xltm = "xltm"; + xlam = "xlam"; + xlsb = "xlsb"; + ppt = "ppt"; + pot = "pot"; + pps = "pps"; + ppa = "ppa"; + pptx = "pptx"; + potx = "potx"; + ppsx = "ppsx"; + ppam = "ppam"; + pptm = "pptm"; + potm = "potm"; + ppsm = "ppsm"; + } + # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. + whitelist = "/etc/rspamd/antivirus.wl"; } - # mime-part regex matching in content-type or filename - mime_parts_filter_regex { - #GEN1 = "application\/octet-stream"; - DOC2 = "application\/msword"; - DOC3 = "application\/vnd\.ms-word.*"; - XLS = "application\/vnd\.ms-excel.*"; - PPT = "application\/vnd\.ms-powerpoint.*"; - GEN2 = "application\/vnd\.openxmlformats-officedocument.*"; + dcc { + # If set force this action if any virus is found (default unset: no action is forced) + # action = "reject"; + # If set, then rejection message is set to this value (mention single quotes) + # If `max_size` is set, messages > n bytes in size are not scanned + max_size = 20000000; + #servers = "127.0.0.1:10045; + # if `patterns` is specified virus name will be matched against provided regexes and the related + # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. + patterns { + # symbol_name = "pattern"; + JUST_EICAR = "^Eicar-Test-Signature$"; + } + # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. + whitelist = "/etc/rspamd/antivirus.wl"; } - # Mime-Part filename extension matching (no regex) - mime_parts_filter_ext { - doc = "doc"; - dot = "dot"; - docx = "docx"; - dotx = "dotx"; - docm = "docm"; - dotm = "dotm"; - xls = "xls"; - xlt = "xlt"; - xla = "xla"; - xlsx = "xlsx"; - xltx = "xltx"; - xlsm = "xlsm"; - xltm = "xltm"; - xlam = "xlam"; - xlsb = "xlsb"; - ppt = "ppt"; - pot = "pot"; - pps = "pps"; - ppa = "ppa"; - pptx = "pptx"; - potx = "potx"; - ppsx = "ppsx"; - ppam = "ppam"; - pptm = "pptm"; - potm = "potm"; - ppsm = "ppsm"; - } - # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. - whitelist = "/etc/rspamd/antivirus.wl"; - } - dcc { - # If set force this action if any virus is found (default unset: no action is forced) - # action = "reject"; - # If set, then rejection message is set to this value (mention single quotes) - # If `max_size` is set, messages > n bytes in size are not scanned - max_size = 20000000; - #servers = "127.0.0.1:10045; - # if `patterns` is specified virus name will be matched against provided regexes and the related - # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. - patterns { - # symbol_name = "pattern"; - JUST_EICAR = "^Eicar-Test-Signature$"; - } - # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. - whitelist = "/etc/rspamd/antivirus.wl"; } -} -]]) + ]]) return end - local function add_scanner_rule(sym, opts) if not opts.type then rspamd_logger.errx(rspamd_config, 'unknown type for external scanner rule %s', sym) @@ -130,7 +129,7 @@ local function add_scanner_rule(sym, opts) if not rule then rspamd_logger.errx(rspamd_config, 'cannot configure %s for %s', - opts.type, rule.symbol or sym:upper()) + opts.type, rule.symbol or sym:upper()) return nil end @@ -204,8 +203,12 @@ if opts and type(opts) == 'table' then local has_valid = false for k, m in pairs(opts) do if type(m) == 'table' and m.servers then - if not m.type then m.type = k end - if not m.name then m.name = k end + if not m.type then + m.type = k + end + if not m.name then + m.name = k + end local cb, nrule = add_scanner_rule(k, m) if not cb then @@ -334,7 +337,7 @@ if opts and type(opts) == 'table' then end if m.symbols then local function reg_symbols(tbl) - for _,sym in pairs(tbl) do + for _, sym in pairs(tbl) do if type(sym) == 'string' then rspamd_config:register_symbol({ type = 'virtual', @@ -389,7 +392,7 @@ if opts and type(opts) == 'table' then -- Add preloads if a module requires that if type(m.preloads) == 'table' then - for _,preload in ipairs(m.preloads) do + for _, preload in ipairs(m.preloads) do rspamd_config:add_on_load(function(cfg, ev_base, worker) preload(m, cfg, ev_base, worker) end) diff --git a/src/plugins/lua/force_actions.lua b/src/plugins/lua/force_actions.lua index 5f4620e6d..7f203b066 100644 --- a/src/plugins/lua/force_actions.lua +++ b/src/plugins/lua/force_actions.lua @@ -59,7 +59,7 @@ local function gen_cb(params) return 0 end - local e, err = rspamd_expression.create(params.expr, {parse_atom, process_atom}, params.pool) + local e, err = rspamd_expression.create(params.expr, { parse_atom, process_atom }, params.pool) if err then rspamd_logger.errx(rspamd_config, 'Couldnt create expression [%1]: %2', params.expr, err) return @@ -112,9 +112,9 @@ local function gen_cb(params) if type(params.message) == 'string' then -- process selector expressions in the message local message = string.gsub(params.message, '(${(.-)})', process_message_selectors) - task:set_pre_result{action = params.act, message = message, module = N, flags = flags} + task:set_pre_result { action = params.act, message = message, module = N, flags = flags } else - task:set_pre_result{action = params.act, module = N, flags = flags} + task:set_pre_result { action = params.act, module = N, flags = flags } end return true, params.act end @@ -143,11 +143,11 @@ local function configure_module() end if type(expr) == 'string' then -- expr, act, pool, message, subject, raction, honor, limit, flags - local cb, atoms = gen_cb{expr = expr, - act = action, - pool = rspamd_config:get_mempool(), - message = message, - subject = subject} + local cb, atoms = gen_cb { expr = expr, + act = action, + pool = rspamd_config:get_mempool(), + message = message, + subject = subject } if cb and atoms then local h = rspamd_cryptobox_hash.create() h:update(expr) @@ -175,19 +175,23 @@ local function configure_module() if action and expr then local flags = {} - if sett.least then table.insert(flags, "least") end - if sett.process_all then table.insert(flags, "process_all") end + if sett.least then + table.insert(flags, "least") + end + if sett.process_all then + table.insert(flags, "process_all") + end local raction = lua_util.list_to_hash(sett.require_action) local honor = lua_util.list_to_hash(sett.honor_action) - local cb, atoms = gen_cb{expr = expr, - act = action, - pool = rspamd_config:get_mempool(), - message = sett.message, - subject = sett.subject, - raction = raction, - honor = honor, - limit = sett.limit, - flags = table.concat(flags, ',')} + local cb, atoms = gen_cb { expr = expr, + act = action, + pool = rspamd_config:get_mempool(), + message = sett.message, + subject = sett.subject, + raction = raction, + honor = honor, + limit = sett.limit, + flags = table.concat(flags, ',') } if cb and atoms then local t = {} if (raction or honor) then @@ -196,7 +200,7 @@ local function configure_module() else t.type = 'normal' if not sett.least then - t.augmentations = {'passthrough', 'important'} + t.augmentations = { 'passthrough', 'important' } end end t.name = 'FORCE_ACTION_' .. name diff --git a/src/plugins/lua/forged_recipients.lua b/src/plugins/lua/forged_recipients.lua index 4b8e720c3..0d51db392 100644 --- a/src/plugins/lua/forged_recipients.lua +++ b/src/plugins/lua/forged_recipients.lua @@ -40,10 +40,14 @@ local function check_forged_headers(task) local smtp_rcpts = task:get_recipients(1) local smtp_from = task:get_from(1) - if not smtp_rcpts then return end - if #smtp_rcpts == 0 then return end + if not smtp_rcpts then + return + end + if #smtp_rcpts == 0 then + return + end - local mime_rcpts = task:get_recipients({ 'mime', 'orig'}) + local mime_rcpts = task:get_recipients({ 'mime', 'orig' }) if not mime_rcpts then return @@ -88,10 +92,10 @@ local function check_forged_headers(task) end end - for _,mime_rcpt in ipairs(mime_rcpts) do + for _, mime_rcpt in ipairs(mime_rcpts) do if mime_rcpt.addr and mime_rcpt.addr ~= '' then local addr = string.lower(mime_rcpt.addr) - local dom = string.lower(mime_rcpt.domain) + local dom = string.lower(mime_rcpt.domain) local matched_smtp_addr = smtp_rcpt_map[addr] if matched_smtp_addr then -- Direct match, go forward @@ -119,13 +123,13 @@ local function check_forged_headers(task) local opts = {} local seen_mime_unmatched = false local seen_smtp_unmatched = false - for _,mime_rcpt in ipairs(mime_rcpts) do + for _, mime_rcpt in ipairs(mime_rcpts) do if not mime_rcpt.matched then seen_mime_unmatched = true table.insert(opts, 'm:' .. mime_rcpt.addr) end end - for _,smtp_rcpt in ipairs(smtp_rcpts) do + for _, smtp_rcpt in ipairs(smtp_rcpts) do if not smtp_rcpt.matched then if not smtp_rcpt_domain_map[smtp_rcpt.domain:lower()]._seen_mime_domain then seen_smtp_unmatched = true @@ -149,7 +153,7 @@ local function check_forged_headers(task) end -- Configuration -local opts = rspamd_config:get_all_opt('forged_recipients') +local opts = rspamd_config:get_all_opt('forged_recipients') if opts then if opts['symbol_rcpt'] or opts['symbol_sender'] then local id = rspamd_config:register_symbol({ @@ -169,7 +173,7 @@ if opts then end if opts['symbol_sender'] then symbol_sender = opts['symbol_sender'] - rspamd_config:register_symbol({ + rspamd_config:register_symbol({ name = symbol_sender, type = 'virtual', parent = id, diff --git a/src/plugins/lua/fuzzy_collect.lua b/src/plugins/lua/fuzzy_collect.lua index d64830c6b..132ace90c 100644 --- a/src/plugins/lua/fuzzy_collect.lua +++ b/src/plugins/lua/fuzzy_collect.lua @@ -39,7 +39,7 @@ local function send_data_mirror(m, cfg, ev_base, body) rspamd_logger.infox(cfg, 'saved data on %s(%s)', m.server, m.name) end end - rspamd_http.request{ + rspamd_http.request { url = string.format('http://%s//update_v1/%s', m.server, m.name), resolver = cfg:get_resolver(), config = cfg, @@ -58,8 +58,10 @@ local function collect_fuzzy_hashes(cfg, ev_base) rspamd_logger.errx(cfg, 'cannot load data: %s', err) else -- Here, we actually copy body once for each mirror - fun.each(function(_, v) send_data_mirror(v, cfg, ev_base, body) end, - settings.mirrors) + fun.each(function(_, v) + send_data_mirror(v, cfg, ev_base, body) + end, + settings.mirrors) end end @@ -70,13 +72,13 @@ local function collect_fuzzy_hashes(cfg, ev_base) if settings.saved_cookie ~= tostring(body) then settings.saved_cookie = tostring(body) rspamd_logger.infox(cfg, 'received collection cookie %s', - tostring(rspamd_util.encode_base32(settings.saved_cookie:sub(1, 6)))) + tostring(rspamd_util.encode_base32(settings.saved_cookie:sub(1, 6)))) local sig = rspamd_cryptolib.sign_memory(settings.sign_keypair, - settings.saved_cookie) + settings.saved_cookie) if not sig then rspamd_logger.info(cfg, 'cannot sign cookie') else - rspamd_http.request{ + rspamd_http.request { url = string.format('http://%s/data', settings.collect_server), resolver = cfg:get_resolver(), config = cfg, @@ -96,8 +98,8 @@ local function collect_fuzzy_hashes(cfg, ev_base) end end rspamd_logger.infox(cfg, 'start fuzzy collection, next sync in %s seconds', - settings.sync_time) - rspamd_http.request{ + settings.sync_time) + rspamd_http.request { url = string.format('http://%s/cookie', settings.collect_server), resolver = cfg:get_resolver(), config = cfg, @@ -141,7 +143,7 @@ end local opts = rspamd_config:get_all_opt('fuzzy_collect') if opts and type(opts) == 'table' then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end local sane_config = true @@ -180,9 +182,9 @@ if opts and type(opts) == 'table' then rspamd_config:add_on_load(function(_, ev_base, worker) if worker:is_primary_controller() then rspamd_config:add_periodic(ev_base, 0.0, - function(cfg, _) - return collect_fuzzy_hashes(cfg, ev_base) - end) + function(cfg, _) + return collect_fuzzy_hashes(cfg, ev_base) + end) end end) else diff --git a/src/plugins/lua/greylist.lua b/src/plugins/lua/greylist.lua index db1afeed7..16f498417 100644 --- a/src/plugins/lua/greylist.lua +++ b/src/plugins/lua/greylist.lua @@ -111,7 +111,9 @@ local function data_key(task) local body = task:get_rawbody() - if not body then return nil end + if not body then + return nil + end local len = body:len() if len > settings['max_data_len'] then @@ -175,16 +177,16 @@ local function check_time(task, tm, type, now) if not t then rspamd_logger.errx(task, 'not a valid number: %s', tm) - return false,false + return false, false end if now - t < settings['timeout'] then - return true,true + return true, true else -- We just set variable to pass when in post-filter stage task:get_mempool():set_variable("grey_whitelisted", type) - return true,false + return true, false end end @@ -197,7 +199,7 @@ local function greylist_message(task, end_time, why) if settings.message_func then task:set_pre_result(settings['action'], - settings.message_func(task, end_time), N) + settings.message_func(task, end_time), N) else local message = settings['message'] if settings.report_time then @@ -237,12 +239,12 @@ local function greylist_check(task) local greylisted_meta = false if data then - local end_time_body,end_time_meta + local end_time_body, end_time_meta local now = rspamd_util.get_time() if data[1] and type(data[1]) ~= 'userdata' then local tm = tonumber(data[1]) or now - ret_body,greylisted_body = check_time(task, data[1], 'body', now) + ret_body, greylisted_body = check_time(task, data[1], 'body', now) if greylisted_body then end_time_body = tm + settings['timeout'] task:get_mempool():set_variable("grey_greylisted_body", @@ -253,7 +255,7 @@ local function greylist_check(task) if data[2] and type(data[2]) ~= 'userdata' then if not ret_body or greylisted_body then local tm = tonumber(data[2]) or now - ret_meta,greylisted_meta = check_time(task, data[2], 'meta', now) + ret_meta, greylisted_meta = check_time(task, data[2], 'meta', now) if greylisted_meta then end_time_meta = tm + settings['timeout'] @@ -298,7 +300,7 @@ local function greylist_check(task) false, -- is write redis_get_cb, --callback 'MGET', -- command - {body_key, meta_key} -- arguments + { body_key, meta_key } -- arguments ) if not ret then rspamd_logger.errx(task, 'cannot make redis request to check results') @@ -310,10 +312,12 @@ local function greylist_set(task) local ip = task:get_ip() -- Don't do anything if pre-result has been already set - if task:has_pre_result() then return end + if task:has_pre_result() then + return + end -- Check whitelist_symbols - for _,sym in ipairs(settings.whitelist_symbols) do + for _, sym in ipairs(settings.whitelist_symbols) do if task:has_symbol(sym) then rspamd_logger.infox(task, 'skip greylisting as we have found symbol %s', sym) if action == 'greylist' then @@ -387,7 +391,7 @@ local function greylist_set(task) local function redis_set_cb(err) if err then rspamd_logger.errx(task, 'got error %s when setting greylisting record on server %s', - err, upstream:get_addr()) + err, upstream:get_addr()) end end @@ -402,18 +406,20 @@ local function greylist_set(task) task:insert_result(settings['symbol'], 0.0, 'pass', is_whitelisted) rspamd_logger.infox(task, 'greylisting pass (%s) until %s', - is_whitelisted, - rspamd_util.time_to_string(rspamd_util.get_time() + settings['expire'])) + is_whitelisted, + rspamd_util.time_to_string(rspamd_util.get_time() + settings['expire'])) - if not settings.check_local and is_rspamc then return end + if not settings.check_local and is_rspamc then + return + end - ret,conn,upstream = lua_redis.redis_make_request(task, - redis_params, -- connect params - hash_key, -- hash key - true, -- is write - redis_set_cb, --callback - 'EXPIRE', -- command - {body_key, tostring(toint(settings['expire']))} -- arguments + ret, conn, upstream = lua_redis.redis_make_request(task, + redis_params, -- connect params + hash_key, -- hash key + true, -- is write + redis_set_cb, --callback + 'EXPIRE', -- command + { body_key, tostring(toint(settings['expire'])) } -- arguments ) -- Update greylisting record expire if ret then @@ -424,19 +430,21 @@ local function greylist_set(task) rspamd_logger.errx(task, 'got error while connecting to redis') end elseif do_greylisting or do_greylisting_required then - if not settings.check_local and is_rspamc then return end + if not settings.check_local and is_rspamc then + return + end local t = tostring(toint(rspamd_util.get_time())) local end_time = rspamd_util.time_to_string(t + settings['timeout']) rspamd_logger.infox(task, 'greylisted until "%s", new record', end_time) greylist_message(task, end_time, 'new record') -- Create new record - ret,conn,upstream = lua_redis.redis_make_request(task, - redis_params, -- connect params - hash_key, -- hash key - true, -- is write - redis_set_cb, --callback - 'SETEX', -- command - {body_key, tostring(toint(settings['expire'])), t} -- arguments + ret, conn, upstream = lua_redis.redis_make_request(task, + redis_params, -- connect params + hash_key, -- hash key + true, -- is write + redis_set_cb, --callback + 'SETEX', -- command + { body_key, tostring(toint(settings['expire'])), t } -- arguments ) if ret then @@ -472,7 +480,7 @@ if opts then settings.message_func = assert(load(opts['message_func']))() end - for k,v in pairs(opts) do + for k, v in pairs(opts) do if k ~= 'message_func' then settings[k] = v end @@ -493,9 +501,9 @@ if opts then end whitelisted_ip = lua_map.rspamd_map_add(N, 'whitelisted_ip', 'radix', - 'Greylist whitelist ip map') + 'Greylist whitelist ip map') whitelist_domains_map = lua_map.rspamd_map_add(N, 'whitelist_domains_url', - 'map', 'Greylist whitelist domains map') + 'map', 'Greylist whitelist domains map') redis_params = lua_redis.parse_redis_server(N) if not redis_params then @@ -515,14 +523,14 @@ if opts then type = 'postfilter', callback = greylist_set, priority = lua_util.symbols_priorities.medium, - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}, + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }, }) local id = rspamd_config:register_symbol({ name = 'GREYLIST_CHECK', type = 'prefilter', callback = greylist_check, priority = lua_util.symbols_priorities.medium, - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)} + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) } }) rspamd_config:register_symbol({ name = settings.symbol, diff --git a/src/plugins/lua/hfilter.lua b/src/plugins/lua/hfilter.lua index f68d5b579..601786f33 100644 --- a/src/plugins/lua/hfilter.lua +++ b/src/plugins/lua/hfilter.lua @@ -200,7 +200,7 @@ local function check_regexp(str, regexp_text) end local function add_static_map(data) - return rspamd_config:add_map{ + return rspamd_config:add_map { type = 'regexp_multi', url = { upstreams = 'static', @@ -227,13 +227,13 @@ local function check_host(task, host, symbol_suffix, eq_ip, eq_host) local function check_host_cb_mx(_, to_resolve, results, err) if err and (err ~= 'requested record is not found' and err ~= 'no records with this name') then - lua_util.debugm(N, task, 'error looking up %s: %s', to_resolve, err) + lua_util.debugm(N, task, 'error looking up %s: %s', to_resolve, err) end if not results then task:insert_result('HFILTER_' .. symbol_suffix .. '_NORES_A_OR_MX', 1.0, - to_resolve) + to_resolve) else - for _,mx in pairs(results) do + for _, mx in pairs(results) do if mx['name'] then local failed_mx_address = 0 -- Capture failed_mx_address @@ -244,12 +244,12 @@ local function check_host(task, host, symbol_suffix, eq_ip, eq_host) if failed_mx_address >= 2 then task:insert_result('HFILTER_' .. symbol_suffix .. '_NORESOLVE_MX', - 1.0, mx['name']) + 1.0, mx['name']) end end task:get_resolver():resolve('a', { - task=task, + task = task, name = mx['name'], callback = check_host_cb_mx_a }) @@ -266,7 +266,7 @@ local function check_host(task, host, symbol_suffix, eq_ip, eq_host) if not results then failed_address = failed_address + 1 else - for _,result in pairs(results) do + for _, result in pairs(results) do table.insert(resolved_address, result:to_string()) end end @@ -274,7 +274,7 @@ local function check_host(task, host, symbol_suffix, eq_ip, eq_host) if failed_address >= 2 then -- No A or AAAA records if eq_ip and eq_ip ~= '' then - for _,result in pairs(resolved_address) do + for _, result in pairs(resolved_address) do if result == eq_ip then return true end @@ -303,7 +303,7 @@ local function check_host(task, host, symbol_suffix, eq_ip, eq_host) if check_fqdn(host) then if eq_host == '' or eq_host ~= host then task:get_resolver():resolve('a', { - task=task, + task = task, name = host, callback = check_host_cb_a }) @@ -329,7 +329,7 @@ local function hfilter_callback(task) if parts then local plain_text_part, html_text_part - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do if p:is_html() then html_text_part = p else @@ -345,7 +345,9 @@ local function hfilter_callback(task) local rel = url_len / plen if rel > 0.8 then local sc = (rel - 0.8) * 5.0 - if sc > 1.0 then sc = 1.0 end + if sc > 1.0 then + sc = 1.0 + end task:insert_result('HFILTER_URL_ONLY', sc, tostring(sc)) local lines = part:get_lines_count() if lines > 0 and lines < 2 then @@ -405,7 +407,7 @@ local function hfilter_callback(task) if not find_badip and not find_bareip then -- Regexp check HELO (checks_hello) local weights = checks_hello_map:get_key(helo) - for _,weight in ipairs(weights or {}) do + for _, weight in ipairs(weights or {}) do weight = tonumber(weight) or 0 if weight > weight_helo then weight_helo = weight @@ -413,7 +415,7 @@ local function hfilter_callback(task) end -- Regexp check HELO (checks_hellohost) weights = checks_hellohost_map:get_key(helo) - for _,weight in ipairs(weights or {}) do + for _, weight in ipairs(weights or {}) do weight = tonumber(weight) or 0 if weight > weight_helo then weight_helo = weight @@ -436,7 +438,7 @@ local function hfilter_callback(task) if hostname then -- Check regexp HOSTNAME local weights = checks_hellohost_map:get_key(hostname) - for _,weight in ipairs(weights or {}) do + for _, weight in ipairs(weights or {}) do weight = tonumber(weight) or 0 if weight > weight_hostname then weight_hostname = weight @@ -460,7 +462,7 @@ local function hfilter_callback(task) local from = task:get_from(1) if from then --FROM host check - for _,fr in ipairs(from) do + for _, fr in ipairs(from) do local fr_split = rspamd_str_split(fr['addr'], '@') if #fr_split == 2 then check_host(task, fr_split[2], 'FROMHOST', '', '') @@ -485,7 +487,7 @@ local function hfilter_callback(task) if frombounce then if count_rcpt > 1 then task:insert_result('HFILTER_RCPT_BOUNCEMOREONE', 1.00, - tostring(count_rcpt)) + tostring(count_rcpt)) end end end @@ -555,13 +557,15 @@ local timeout = 0.0 local opts = rspamd_config:get_all_opt('hfilter') if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do config[k] = v end end local function append_t(t, a) - for _,v in ipairs(a) do table.insert(t, v) end + for _, v in ipairs(a) do + table.insert(t, v) + end end if config['helo_enabled'] then checks_hello_bareip_map = add_static_map(checks_hello_bareip) @@ -594,14 +598,14 @@ end --dumper(symbols_enabled) if #symbols_enabled > 0 then - local id = rspamd_config:register_symbol{ + local id = rspamd_config:register_symbol { name = 'HFILTER_CHECK', callback = hfilter_callback, type = 'callback', - augmentations = {string.format("timeout=%f", timeout)}, + augmentations = { string.format("timeout=%f", timeout) }, } - for _,sym in ipairs(symbols_enabled) do - rspamd_config:register_symbol{ + for _, sym in ipairs(symbols_enabled) do + rspamd_config:register_symbol { type = 'virtual', score = 1.0, parent = id, diff --git a/src/plugins/lua/history_redis.lua b/src/plugins/lua/history_redis.lua index cfedfeac3..9cbe7caf7 100644 --- a/src/plugins/lua/history_redis.lua +++ b/src/plugins/lua/history_redis.lua @@ -94,8 +94,13 @@ local function normalise_results(tbl, task) return type(v) == 'table' and v.score end, metric) - fun.each(function(k, v) v.name = nil; tbl.symbols[k] = v; end, symbols) - fun.each(function(k, v) tbl[k] = v end, others) + fun.each(function(k, v) + v.name = nil; + tbl.symbols[k] = v; + end, symbols) + fun.each(function(k, v) + tbl[k] = v + end, others) -- Reset the original metric tbl.default = nil @@ -141,7 +146,7 @@ local function history_save(task) return end - local data = task:get_protocol_reply{'metrics', 'basic'} + local data = task:get_protocol_reply { 'metrics', 'basic' } local prefix = settings.key_prefix .. hostname if data then @@ -160,19 +165,19 @@ local function history_save(task) end local ret, conn, _ = lua_redis.rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - true, -- is write - redis_llen_cb, --callback - 'LPUSH', -- command - {prefix, json} -- arguments + redis_params, -- connect params + nil, -- hash key + true, -- is write + redis_llen_cb, --callback + 'LPUSH', -- command + { prefix, json } -- arguments ) if ret then - conn:add_cmd('LTRIM', {prefix, '0', string.format('%d', settings.nrows-1)}) + conn:add_cmd('LTRIM', { prefix, '0', string.format('%d', settings.nrows - 1) }) if settings.expire and settings.expire > 0 then - conn:add_cmd('EXPIRE', {prefix, string.format('%d', settings.expire)}) + conn:add_cmd('EXPIRE', { prefix, string.format('%d', settings.expire) }) end end end @@ -188,19 +193,19 @@ local function handle_history_request(task, conn, from, to, reset) local function redis_ltrim_cb(err, _) if err then rspamd_logger.errx(task, 'got error %s when resetting history: %s', - err) + err) conn:send_error(504, '{"error": "' .. err .. '"}') else conn:send_string('{"success":true}') end end lua_redis.rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - true, -- is write - redis_ltrim_cb, --callback - 'LTRIM', -- command - {prefix, '0', '0'} -- arguments + redis_params, -- connect params + nil, -- hash key + true, -- is write + redis_ltrim_cb, --callback + 'LTRIM', -- command + { prefix, '0', '0' } -- arguments ) else local function redis_lrange_cb(err, data) @@ -211,14 +216,16 @@ local function handle_history_request(task, conn, from, to, reset) if settings.compress then local t1 = rspamd_util:get_ticks() - data = fun.totable(fun.filter(function(e) return e ~= nil end, - fun.map(function(e) - local _,dec = rspamd_util.zstd_decompress(e) - if dec then - return dec - end - return nil - end, data))) + data = fun.totable(fun.filter(function(e) + return e ~= nil + end, + fun.map(function(e) + local _, dec = rspamd_util.zstd_decompress(e) + if dec then + return dec + end + return nil + end, data))) lua_util.debugm(N, task, 'decompress took %s ms', (rspamd_util:get_ticks() - t1) * 1000.0) collectgarbage() @@ -226,23 +233,25 @@ local function handle_history_request(task, conn, from, to, reset) -- Parse elements using ucl local t1 = rspamd_util:get_ticks() data = fun.totable( - fun.map(function (_, obj) return obj end, - fun.filter(function(res, obj) - if res then - return true - end - return false + fun.map(function(_, obj) + return obj end, - fun.map(function(elt) - local parser = ucl.parser() - local res,_ = parser:parse_text(elt) - - if res then - return true, parser:get_object() - else - return false, nil - end - end, data)))) + fun.filter(function(res, obj) + if res then + return true + end + return false + end, + fun.map(function(elt) + local parser = ucl.parser() + local res, _ = parser:parse_text(elt) + + if res then + return true, parser:get_object() + else + return false, nil + end + end, data)))) lua_util.debugm(N, task, 'parse took %s ms', (rspamd_util:get_ticks() - t1) * 1000.0) collectgarbage() @@ -254,26 +263,26 @@ local function handle_history_request(task, conn, from, to, reset) collectgarbage() else rspamd_logger.errx(task, 'got error %s when getting history: %s', - err) + err) conn:send_error(504, '{"error": "' .. err .. '"}') end end lua_redis.rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - false, -- is write - redis_lrange_cb, --callback - 'LRANGE', -- command - {prefix, string.format('%d', from), string.format('%d', to)}, -- arguments - {opaque_data = true} + redis_params, -- connect params + nil, -- hash key + false, -- is write + redis_lrange_cb, --callback + 'LRANGE', -- command + { prefix, string.format('%d', from), string.format('%d', to) }, -- arguments + { opaque_data = true } ) end end -local opts = rspamd_config:get_all_opt('history_redis') +local opts = rspamd_config:get_all_opt('history_redis') if opts then settings = lua_util.override_defaults(settings, opts) - local res,err = settings_schema:transform(settings) + local res, err = settings_schema:transform(settings) if not res then rspamd_logger.warnx(rspamd_config, '%s: plugin is misconfigured: %s', N, err) @@ -292,7 +301,7 @@ if opts then type = 'idempotent', callback = history_save, flags = 'empty,explicit_disable,ignore_passthrough', - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)} + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) } }) lua_redis.register_prefix(settings.key_prefix .. hostname, N, "Redis history", { diff --git a/src/plugins/lua/http_headers.lua b/src/plugins/lua/http_headers.lua index c437392e4..1c6494a1e 100644 --- a/src/plugins/lua/http_headers.lua +++ b/src/plugins/lua/http_headers.lua @@ -55,7 +55,7 @@ local dmarc_symbols = { local opts = rspamd_config:get_all_opt('dmarc') if opts and opts['symbols'] then - for k,_ in pairs(dmarc_symbols) do + for k, _ in pairs(dmarc_symbols) do if opts['symbols'][k] then dmarc_symbols[k] = opts['symbols'][k] end @@ -64,7 +64,7 @@ end opts = rspamd_config:get_all_opt('dkim') if opts then - for k,_ in pairs(dkim_symbols) do + for k, _ in pairs(dkim_symbols) do if opts[k] then dkim_symbols[k] = opts[k] end @@ -73,7 +73,7 @@ end opts = rspamd_config:get_all_opt('spf') if opts then - for k,_ in pairs(spf_symbols) do + for k, _ in pairs(spf_symbols) do if opts[k] then spf_symbols[k] = opts[k] end @@ -95,28 +95,28 @@ rspamd_config:add_condition("DKIM_CHECK", function(task) local p_obj = parser:get_object() local results = p_obj['results'] if not results and p_obj['result'] then - results = {{result = p_obj['result'], domain = 'unknown'}} + results = { { result = p_obj['result'], domain = 'unknown' } } end if results then for _, obj in ipairs(results) do - local dkim_domain = obj['domain'] or 'unknown' + local dkim_domain = obj['domain'] or 'unknown' if obj['result'] == 'pass' or obj['result'] == 'allow' then task:insert_result(dkim_symbols['symbol_allow'], 1.0, 'http header') task:insert_result(dkim_symbols['symbol_trace'], 1.0, - string.format('%s:%s', dkim_domain, dkim_trace.pass)) + string.format('%s:%s', dkim_domain, dkim_trace.pass)) elseif obj['result'] == 'fail' or obj['result'] == 'reject' then task:insert_result(dkim_symbols['symbol_deny'], 1.0, 'http header') task:insert_result(dkim_symbols['symbol_trace'], 1.0, - string.format('%s:%s', dkim_domain, dkim_trace.fail)) + string.format('%s:%s', dkim_domain, dkim_trace.fail)) elseif obj['result'] == 'tempfail' or obj['result'] == 'softfail' then task:insert_result(dkim_symbols['symbol_tempfail'], 1.0, 'http header') task:insert_result(dkim_symbols['symbol_trace'], 1.0, - string.format('%s:%s', dkim_domain, dkim_trace.temperror)) + string.format('%s:%s', dkim_domain, dkim_trace.temperror)) elseif obj['result'] == 'permfail' then task:insert_result(dkim_symbols['symbol_permfail'], 1.0, 'http header') task:insert_result(dkim_symbols['symbol_trace'], 1.0, - string.format('%s:%s', dkim_domain, dkim_trace.permerror)) + string.format('%s:%s', dkim_domain, dkim_trace.permerror)) elseif obj['result'] == 'na' then task:insert_result(dkim_symbols['symbol_na'], 1.0, 'http header') end diff --git a/src/plugins/lua/maillist.lua b/src/plugins/lua/maillist.lua index 023867a8d..be1401c32 100644 --- a/src/plugins/lua/maillist.lua +++ b/src/plugins/lua/maillist.lua @@ -104,7 +104,8 @@ local function check_ml_mailman(task) -- not much elase we can check, Subjects can be changed in settings return true end - else -- Mailman 3 + else + -- Mailman 3 -- XXX not Mailman3 admin messages have this headers, but one -- which don't usually have List-* headers examined below if task:has_header('List-Administrivia') then @@ -113,7 +114,7 @@ local function check_ml_mailman(task) end -- List-Archive and List-Post are optional, check other headers - for _, h in ipairs({'List-Help', 'List-Subscribe', 'List-Unsubscribe'}) do + for _, h in ipairs({ 'List-Help', 'List-Subscribe', 'List-Unsubscribe' }) do header = task:get_header(h) if not (header and header:find('<mailto:', 1, true)) then return false @@ -173,7 +174,7 @@ local function check_generic_list_headers(task) end, } - for hname,hscore in pairs(common_list_headers) do + for hname, hscore in pairs(common_list_headers) do if task:has_header(hname) then if type(hscore) == 'number' then score = score + hscore @@ -210,7 +211,9 @@ local function check_maillist(task) elseif check_ml_cgp(task) then task:insert_result(symbol, 1, 'cgp') else - if score > 2 then score = 2 end + if score > 2 then + score = 2 + end task:insert_result(symbol, 0.5 * score, 'generic') end end @@ -219,7 +222,7 @@ end -- Configuration -local opts = rspamd_config:get_all_opt('maillist') +local opts = rspamd_config:get_all_opt('maillist') if opts then if opts['symbol'] then symbol = opts['symbol'] diff --git a/src/plugins/lua/maps_stats.lua b/src/plugins/lua/maps_stats.lua index bf03f7801..d4188103a 100644 --- a/src/plugins/lua/maps_stats.lua +++ b/src/plugins/lua/maps_stats.lua @@ -66,7 +66,7 @@ local function process_map(map, ev_base, _) true, -- is write redis_zrange_cb, --callback 'ZREMRANGEBYRANK', -- command - {key, '0', tostring(-(settings.count) - 1)} -- arguments + { key, '0', tostring(-(settings.count) - 1) } -- arguments ) end end @@ -78,14 +78,14 @@ local function process_map(map, ev_base, _) true, -- is write redis_card_cb, --callback 'ZCARD', -- command - {key} -- arguments + { key } -- arguments ) if ret and conn then local stats = map:get_stats(true) - for k,s in pairs(stats) do + for k, s in pairs(stats) do if s > 0 then - conn:add_cmd('ZINCRBY', {key, tostring(s), k}) + conn:add_cmd('ZINCRBY', { key, tostring(s), k }) end end end @@ -99,7 +99,7 @@ end local opts = rspamd_config:get_all_opt(N) if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end end @@ -107,7 +107,7 @@ end redis_params = lua_redis.parse_redis_server(N, opts) -- XXX, this is a poor approach as not all maps are defined here... local tmaps = rspamd_config:get_maps() -for _,m in ipairs(tmaps) do +for _, m in ipairs(tmaps) do if m:get_uri() ~= 'static' then lua_redis.register_prefix(settings.prefix .. m:get_uri(), N, 'Maps stats data', { @@ -118,13 +118,13 @@ for _,m in ipairs(tmaps) do end if redis_params then - rspamd_config:add_on_load(function (_, ev_base, worker) + rspamd_config:add_on_load(function(_, ev_base, worker) local maps = rspamd_config:get_maps() - for _,m in ipairs(maps) do + for _, m in ipairs(maps) do rspamd_config:add_periodic(ev_base, settings['interval'], - function () + function() process_map(m, ev_base, worker) return true end, true) diff --git a/src/plugins/lua/metadata_exporter.lua b/src/plugins/lua/metadata_exporter.lua index 6c4cf0019..9d0826735 100644 --- a/src/plugins/lua/metadata_exporter.lua +++ b/src/plugins/lua/metadata_exporter.lua @@ -84,7 +84,7 @@ local function get_general_metadata(task, flatten, no_content) local fuzzy = task:get_mempool():get_variable("fuzzy_hashes", "fstrings") if fuzzy and #fuzzy > 0 then local fz = {} - for _,h in ipairs(fuzzy) do + for _, h in ipairs(fuzzy) do table.insert(fz, h) end if not flatten then @@ -224,7 +224,7 @@ local formatters = { meta.mail_to = table.concat(display_emails, ', ') meta.our_message_id = rspamd_util.random_hex(12) .. '@rspamd' meta.date = rspamd_util.time_to_string(rspamd_util.get_time()) - return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets} + return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets } end, json = function(task) return ucl.to_format(get_general_metadata(task), 'json-compact') @@ -276,7 +276,7 @@ end local pushers = { redis_pubsub = function(task, formatted, rule) - local _,ret,upstream + local _, ret, upstream local function redis_pub_cb(err) if err then rspamd_logger.errx(task, 'got error %s when publishing on server %s', @@ -285,13 +285,13 @@ local pushers = { end return true end - ret,_,upstream = rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - true, -- is write - redis_pub_cb, --callback - 'PUBLISH', -- command - {rule.channel, formatted} -- arguments + ret, _, upstream = rspamd_redis_make_request(task, + redis_params, -- connect params + nil, -- hash key + true, -- is write + redis_pub_cb, --callback + 'PUBLISH', -- command + { rule.channel, formatted } -- arguments ) if not ret then rspamd_logger.errx(task, 'error connecting to redis') @@ -300,7 +300,7 @@ local pushers = { end, http = function(task, formatted, rule) local function http_callback(err, code) - local valid_status = {200, 201, 202, 204} + local valid_status = { 200, 201, 202, 204 } if err then rspamd_logger.errx(task, 'got error %s in http callback', err) @@ -327,14 +327,14 @@ local pushers = { end end rspamd_http.request({ - task=task, - url=rule.url, - user=rule.user, - password=rule.password, - body=formatted, - callback=http_callback, - mime_type=rule.mime_type or settings.mime_type, - headers=hdrs, + task = task, + url = rule.url, + user = rule.user, + password = rule.password, + body = formatted, + callback = http_callback, + mime_type = rule.mime_type or settings.mime_type, + headers = hdrs, }) end, send_mail = function(task, formatted, rule, extra) @@ -359,7 +359,9 @@ local pushers = { } local opts = rspamd_config:get_all_opt(N) -if not opts then return end +if not opts then + return +end local process_settings = { select = function(val) selectors.custom = assert(load(val))() @@ -632,8 +634,10 @@ for k, v in pairs(settings.rules) do end local function gen_exporter(rule) - return function (task) - if task:has_flag('skip') then return end + return function(task) + if task:has_flag('skip') then + return + end local selector = rule.selector or 'default' local selected = selectors[selector](task) if selected then @@ -661,6 +665,6 @@ for k, r in pairs(settings.rules) do type = 'idempotent', callback = gen_exporter(r), flags = 'empty,explicit_disable,ignore_passthrough', - augmentations = {string.format("timeout=%f", r.timeout or 0.0)} + augmentations = { string.format("timeout=%f", r.timeout or 0.0) } }) end diff --git a/src/plugins/lua/metric_exporter.lua b/src/plugins/lua/metric_exporter.lua index 6e48ae533..75885516c 100644 --- a/src/plugins/lua/metric_exporter.lua +++ b/src/plugins/lua/metric_exporter.lua @@ -132,7 +132,7 @@ local function graphite_push(kwargs) data = { metrics_str, '\n', }, - callback = (function (err) + callback = (function(err) if err then logger.errx('Push failed: %1', err) return @@ -171,12 +171,14 @@ if not configure_metric_exporter() then return end -rspamd_config:add_on_load(function (_, ev_base, worker) +rspamd_config:add_on_load(function(_, ev_base, worker) -- Exit unless we're the first 'controller' worker - if not worker:is_primary_controller() then return end + if not worker:is_primary_controller() then + return + end -- Persist mempool variable to statefile on shutdown pool = mempool.create() - rspamd_config:register_finish_script(function () + rspamd_config:register_finish_script(function() local stamp = pool:get_variable(VAR_NAME, 'double') if not stamp then logger.warn('No last metric exporter push to persist to disk') @@ -207,14 +209,14 @@ rspamd_config:add_on_load(function (_, ev_base, worker) end -- Push metrics at regular intervals local function schedule_regular_push() - rspamd_config:add_periodic(ev_base, settings['interval'], function () + rspamd_config:add_periodic(ev_base, settings['interval'], function() push_metrics() return true end) end -- Push metrics to backend and reschedule check local function schedule_intermediate_push(when) - rspamd_config:add_periodic(ev_base, when, function () + rspamd_config:add_periodic(ev_base, when, function() push_metrics() schedule_regular_push() return false diff --git a/src/plugins/lua/mid.lua b/src/plugins/lua/mid.lua index d7bf02d1d..b8650c835 100644 --- a/src/plugins/lua/mid.lua +++ b/src/plugins/lua/mid.lua @@ -48,7 +48,7 @@ local function known_mid_cb(task) local header = task:get_header('Message-Id') local das = task:get_symbol(settings['symbol_dkim_allow']) if ((das or E)[1] or E).options then - for _,dkim_domain in ipairs(das[1]['options']) do + for _, dkim_domain in ipairs(das[1]['options']) do if dkim_domain then local v = map:get_key(dkim_domain:match "[^:]+") if v then @@ -70,9 +70,9 @@ local function known_mid_cb(task) end end -local opts = rspamd_config:get_all_opt('mid') +local opts = rspamd_config:get_all_opt('mid') if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end @@ -82,7 +82,7 @@ if opts then return end - map = rspamd_config:add_map{ + map = rspamd_config:add_map { url = opts.source, description = "Message-IDs map", type = 'map' diff --git a/src/plugins/lua/milter_headers.lua b/src/plugins/lua/milter_headers.lua index 02f50be27..b53a45457 100644 --- a/src/plugins/lua/milter_headers.lua +++ b/src/plugins/lua/milter_headers.lua @@ -137,11 +137,15 @@ local function milter_headers(task) local settings_override = false local function skip_wanted(hdr) - if settings_override then return true end + if settings_override then + return true + end -- Normal checks local function match_extended_headers_rcpt() local rcpts = task:get_recipients('smtp') - if not rcpts then return false end + if not rcpts then + return false + end local found for _, r in ipairs(rcpts) do found = false @@ -165,7 +169,9 @@ local function milter_headers(task) found = true end end - if found then break end + if found then + break + end end return found end @@ -176,11 +182,15 @@ local function milter_headers(task) if settings.skip_local and not settings.local_headers[hdr] then local ip = task:get_ip() - if (ip and ip:is_local()) then return true end + if (ip and ip:is_local()) then + return true + end end if settings.skip_authenticated and not settings.authenticated_headers[hdr] then - if task:get_user() ~= nil then return true end + if task:get_user() ~= nil then + return true + end end if settings.skip_all then @@ -211,7 +221,9 @@ local function milter_headers(task) routines['x-spamd-result'] = function() local local_mod = settings.routines['x-spamd-result'] - if skip_wanted('x-spamd-result') then return end + if skip_wanted('x-spamd-result') then + return + end if not common.symbols then common.symbols = task:get_symbols_all() end @@ -227,7 +239,7 @@ local function milter_headers(task) local buf = {} local verdict = string.format('default: %s [%.2f / %.2f]', - --TODO: (common.metric_action == 'no action') and 'False' or 'True', + --TODO: (common.metric_action == 'no action') and 'False' or 'True', (common.metric_action == 'reject') and 'True' or 'False', common.metric_score[1], common.metric_score[2]) table.insert(buf, verdict) @@ -247,12 +259,12 @@ local function milter_headers(task) for _, s in ipairs(common.symbols) do local sym_str = string.format('%s(%.2f)[%s]', - s.name, s.score, table.concat(s.options or {}, ',')) + s.name, s.score, table.concat(s.options or {}, ',')) table.insert(buf, sym_str) end add_header('x-spamd-result', table.concat(buf, '; '), ';') - local has_pr,action,message,module = task:has_pre_result() + local has_pr, action, message, module = task:has_pre_result() if has_pr then local pr_header = {} @@ -270,7 +282,9 @@ local function milter_headers(task) end routines['x-rspamd-queue-id'] = function() - if skip_wanted('x-rspamd-queue-id') then return end + if skip_wanted('x-rspamd-queue-id') then + return + end if common.queue_id ~= false then common.queue_id = task:get_queue_id() if not common.queue_id then @@ -286,21 +300,27 @@ local function milter_headers(task) end routines['remove-header'] = function() - if skip_wanted('remove-header') then return end + if skip_wanted('remove-header') then + return + end if settings.routines['remove-header'].header and settings.routines['remove-header'].remove then remove[settings.routines['remove-header'].header] = settings.routines['remove-header'].remove end end routines['remove-headers'] = function() - if skip_wanted('remove-headers') then return end + if skip_wanted('remove-headers') then + return + end for h, r in pairs(settings.routines['remove-headers'].headers) do remove[h] = r end end routines['add-headers'] = function() - if skip_wanted('add-headers') then return end + if skip_wanted('add-headers') then + return + end for h, r in pairs(settings.routines['add-headers'].headers) do add[h] = r remove[h] = settings.routines['add-headers'].remove @@ -309,7 +329,9 @@ local function milter_headers(task) routines['x-rspamd-server'] = function() local local_mod = settings.routines['x-rspamd-server'] - if skip_wanted('x-rspamd-server') then return end + if skip_wanted('x-rspamd-server') then + return + end if local_mod.remove then remove[local_mod.header] = local_mod.remove end @@ -319,7 +341,9 @@ local function milter_headers(task) routines['x-spamd-bar'] = function() local local_mod = settings.routines['x-spamd-bar'] - if skip_wanted('x-rspamd-bar') then return end + if skip_wanted('x-rspamd-bar') then + return + end if not common['metric_score'] then common['metric_score'] = task:get_metric_score() end @@ -342,7 +366,9 @@ local function milter_headers(task) routines['x-spam-level'] = function() local local_mod = settings.routines['x-spam-level'] - if skip_wanted('x-spam-level') then return end + if skip_wanted('x-spam-level') then + return + end if not common['metric_score'] then common['metric_score'] = task:get_metric_score() end @@ -358,7 +384,9 @@ local function milter_headers(task) routines['x-rspamd-action'] = function() local local_mod = settings.routines['x-rspamd-action'] - if skip_wanted('x-rspamd-action') then return end + if skip_wanted('x-rspamd-action') then + return + end if not common['metric_action'] then common['metric_action'] = task:get_metric_action() end @@ -370,7 +398,9 @@ local function milter_headers(task) end local function spam_header (class, name, value, remove_v) - if skip_wanted(class) then return end + if skip_wanted(class) then + return + end if not common['metric_action'] then common['metric_action'] = task:get_metric_action() end @@ -396,7 +426,9 @@ local function milter_headers(task) routines['x-virus'] = function() local local_mod = settings.routines['x-virus'] - if skip_wanted('x-virus') then return end + if skip_wanted('x-virus') then + return + end if not common.symbols_hash then if not common.symbols then common.symbols = task:get_symbols_all() @@ -453,17 +485,20 @@ local function milter_headers(task) end routines['x-os-fingerprint'] = function() - if skip_wanted('x-os-fingerprint') then return end + if skip_wanted('x-os-fingerprint') then + return + end local local_mod = settings.routines['x-os-fingerprint'] - local os_string, link_type, uptime_min, distance = - task:get_mempool():get_variable('os_fingerprint', + local os_string, link_type, uptime_min, distance = task:get_mempool():get_variable('os_fingerprint', 'string, string, double, double'); - if not os_string then return end + if not os_string then + return + end local value = string.format('%s, (up: %i min), (distance %i, link: %s)', - os_string, uptime_min, distance, link_type) + os_string, uptime_min, distance, link_type) if local_mod.remove then remove[local_mod.header] = local_mod.remove @@ -473,7 +508,9 @@ local function milter_headers(task) end routines['x-spam-status'] = function() - if skip_wanted('x-spam-status') then return end + if skip_wanted('x-spam-status') then + return + end if not common['metric_score'] then common['metric_score'] = task:get_metric_score() end @@ -498,12 +535,13 @@ local function milter_headers(task) end routines['authentication-results'] = function() - if skip_wanted('authentication-results') then return end + if skip_wanted('authentication-results') then + return + end local ar = require "lua_auth_results" if settings.routines['authentication-results'].remove then - remove[settings.routines['authentication-results'].header] = - settings.routines['authentication-results'].remove + remove[settings.routines['authentication-results'].header] = settings.routines['authentication-results'].remove end local res = ar.gen_auth_results(task, @@ -516,10 +554,11 @@ local function milter_headers(task) end routines['stat-signature'] = function() - if skip_wanted('stat-signature') then return end + if skip_wanted('stat-signature') then + return + end if settings.routines['stat-signature'].remove then - remove[settings.routines['stat-signature'].header] = - settings.routines['stat-signature'].remove + remove[settings.routines['stat-signature'].header] = settings.routines['stat-signature'].remove end local res = task:get_mempool():get_variable("stat_signature") if res then @@ -531,7 +570,7 @@ local function milter_headers(task) local res = task:get_mempool():get_variable("fuzzy_hashes", "fstrings") if res and #res > 0 then - for _,h in ipairs(res) do + for _, h in ipairs(res) do add_header('fuzzy-hashes', h) end end @@ -583,8 +622,12 @@ local function milter_headers(task) end end - if not next(add) then add = nil end - if not next(remove) then remove = nil end + if not next(add) then + add = nil + end + if not next(remove) then + remove = nil + end if add or remove then lua_mime.modify_headers(task, { @@ -595,7 +638,9 @@ local function milter_headers(task) end local config_schema = ts.shape({ - use = ts.array_of(ts.string) + ts.string / function(s) return {s} end, + use = ts.array_of(ts.string) + ts.string / function(s) + return { s } + end, remove_upstream_spam_flag = ts.boolean:is_optional(), extended_spam_headers = ts.boolean:is_optional(), skip_local = ts.boolean:is_optional(), @@ -609,13 +654,15 @@ local config_schema = ts.shape({ }) local opts = rspamd_config:get_all_opt(N) or - rspamd_config:get_all_opt('rmilter_headers') + rspamd_config:get_all_opt('rmilter_headers') -if not opts then return end +if not opts then + return +end -- Process config do - local res,err = config_schema:transform(opts) + local res, err = config_schema:transform(opts) if not res then logger.errx(rspamd_config, 'invalid config for %s: %s', N, err) return diff --git a/src/plugins/lua/mime_types.lua b/src/plugins/lua/mime_types.lua index 428637d70..939251630 100644 --- a/src/plugins/lua/mime_types.lua +++ b/src/plugins/lua/mime_types.lua @@ -224,10 +224,10 @@ local function check_mime_type(task) local ext = {} for n = 1, 2 do - ext[n] = #parts > n and string.lower(parts[#parts + 1 - n]) or nil + ext[n] = #parts > n and string.lower(parts[#parts + 1 - n]) or nil end - return ext[1],ext[2],parts + return ext[1], ext[2], parts end local function check_filename(fname, ct, is_archive, part, detected_ext, nfiles) @@ -243,7 +243,9 @@ local function check_mime_type(task) -- Decode hex encoded characters fname = string.gsub(fname, '%%(%x%x)', - function (hex) return string.char(tonumber(hex,16)) end ) + function(hex) + return string.char(tonumber(hex, 16)) + end) -- Replace potentially bad characters with '?' fname = fname:gsub('[^%s%g]', '?') @@ -256,7 +258,7 @@ local function check_mime_type(task) return end - local ext,ext2,parts = gen_extension(fname) + local ext, ext2, parts = gen_extension(fname) -- ext is the last extension, LOWERCASED -- ext2 is the one before last extension LOWERCASED @@ -272,10 +274,14 @@ local function check_mime_type(task) false, part, nil, 1) end - if not ext then return end + if not ext then + return + end local function check_extension(badness_mult, badness_mult2) - if not badness_mult and not badness_mult2 then return end + if not badness_mult and not badness_mult2 then + return + end if #parts > 2 then -- We need to ensure that next-to-last extension is an extension, -- so we check for its length and if it is not a number or date @@ -317,7 +323,9 @@ local function check_mime_type(task) if user_settings.bad_extensions[1] then -- Convert to a key-value map extra_table = fun.tomap( - fun.map(function(e) return e,1.0 end, + fun.map(function(e) + return e, 1.0 + end, user_settings.bad_extensions)) else extra_table = user_settings.bad_extensions @@ -327,7 +335,9 @@ local function check_mime_type(task) if user_settings.bad_archive_extensions[1] then -- Convert to a key-value map extra_archive_table = fun.tomap(fun.map( - function(e) return e,1.0 end, + function(e) + return e, 1.0 + end, user_settings.bad_archive_extensions)) else extra_archive_table = user_settings.bad_archive_extensions @@ -386,7 +396,7 @@ local function check_mime_type(task) if mt and ct and ct ~= 'application/octet-stream' then local found local mult - for _,v in ipairs(mt) do + for _, v in ipairs(mt) do mult = v.mult if ct == v.ct then found = true @@ -404,8 +414,8 @@ local function check_mime_type(task) local parts = task:get_parts() if parts then - for _,p in ipairs(parts) do - local mtype,subtype = p:get_type() + for _, p in ipairs(parts) do + local mtype, subtype = p:get_type() if not mtype then lua_util.debugm(N, task, "no content type for part: %s", p:get_id()) @@ -476,7 +486,7 @@ local function check_mime_type(task) local nfiles = #fl - for _,f in ipairs(fl) do + for _, f in ipairs(fl) do if f['encrypted'] then task:insert_result(settings['symbol_encrypted_archive'], 1.0, f['name']) @@ -497,7 +507,7 @@ local function check_mime_type(task) if nfiles == 1 and fl[1].name then -- We check that extension of the file inside archive is -- the same as double extension of the file - local _,ext2 = gen_extension(filename) + local _, ext2 = gen_extension(filename) if ext2 and #ext2 > 0 then local enc_ext = gen_extension(fl[1].name) @@ -525,7 +535,9 @@ local function check_mime_type(task) if detected_type and detected_type.ct ~= ct then local v_detected = map:get_key(detected_type.ct) - if not v or v_detected and v_detected > v then v = v_detected end + if not v or v_detected and v_detected > v then + v = v_detected + end detected_different = true end if v then @@ -567,19 +579,19 @@ local function check_mime_type(task) end end -local opts = rspamd_config:get_all_opt('mime_types') +local opts = rspamd_config:get_all_opt('mime_types') if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end settings.filename_whitelist = lua_maps.rspamd_map_add('mime_types', 'filename_whitelist', 'regexp', - 'filename whitelist') + 'filename whitelist') local function change_extension_map_entry(ext, ct, mult) if type(ct) == 'table' then local tbl = {} - for _,elt in ipairs(ct) do + for _, elt in ipairs(ct) do table.insert(tbl, { ct = elt, mult = mult, @@ -595,22 +607,24 @@ if opts then end -- Transform extension_map - for ext,ct in pairs(settings.extension_map) do + for ext, ct in pairs(settings.extension_map) do change_extension_map_entry(ext, ct, 1.0) end -- Add all extensions - for _,pair in ipairs(lua_mime_types.full_extensions_map) do + for _, pair in ipairs(lua_mime_types.full_extensions_map) do local ext, ct = pair[1], pair[2] if not settings.extension_map[ext] then - change_extension_map_entry(ext, ct, settings.other_extensions_mult) + change_extension_map_entry(ext, ct, settings.other_extensions_mult) end end local map_type = 'map' - if settings['regexp'] then map_type = 'regexp' end + if settings['regexp'] then + map_type = 'regexp' + end map = lua_maps.rspamd_map_add('mime_types', 'file', map_type, - 'mime types map') + 'mime types map') if map then local id = rspamd_config:register_symbol({ name = 'MIME_TYPES_CALLBACK', diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 0e78a6e2c..b48f7bdff 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -43,27 +43,24 @@ local function parse_multimap_value(parse_rule, p_ret) local number = {} local digit = lpeg.R("09") - number.integer = - (lpeg.S("+-") ^ -1) * - (digit ^ 1) + number.integer = (lpeg.S("+-") ^ -1) * + (digit ^ 1) -- Matches: .6, .899, .9999873 - number.fractional = - (lpeg.P(".") ) * + number.fractional = (lpeg.P(".")) * (digit ^ 1) -- Matches: 55.97, -90.8, .9 - number.decimal = - (number.integer * -- Integer - (number.fractional ^ -1)) + -- Fractional + number.decimal = (number.integer * -- Integer + (number.fractional ^ -1)) + -- Fractional (lpeg.S("+-") * number.fractional) -- Completely fractional number local sym_start = lpeg.R("az", "AZ") + lpeg.S("_") local sym_elt = sym_start + lpeg.R("09") - local symbol = sym_start * sym_elt ^0 + local symbol = sym_start * sym_elt ^ 0 local symbol_cap = lpeg.Cg(symbol, 'symbol') local score_cap = lpeg.Cg(number.decimal, 'score') - local opts_cap = lpeg.Cg(lpeg.Ct(lpeg.C(symbol) * (lpeg.P(",") * lpeg.C(symbol))^0), 'opts') + local opts_cap = lpeg.Cg(lpeg.Ct(lpeg.C(symbol) * (lpeg.P(",") * lpeg.C(symbol)) ^ 0), 'opts') local symscore_cap = (symbol_cap * lpeg.P(":") * score_cap) local symscoreopt_cap = symscore_cap * lpeg.P(":") * opts_cap local grammar = symscoreopt_cap + symscore_cap + symbol_cap + score_cap @@ -86,74 +83,107 @@ local function parse_multimap_value(parse_rule, p_ret) opts = tbl.opts end - return true,sym,score,opts + return true, sym, score, opts else if p_ret ~= '' then rspamd_logger.infox(rspamd_config, '%s: cannot parse string "%s"', parse_rule.symbol, p_ret) end - return true,nil,1.0,{} + return true, nil, 1.0, {} end elseif type(p_ret) == 'boolean' then - return p_ret,nil,1.0,{} + return p_ret, nil, 1.0, {} end - return false,nil,0.0,{} + return false, nil, 0.0, {} end - local value_types = { ip = { - get_value = function(ip) return ip:to_string() end, + get_value = function(ip) + return ip:to_string() + end, }, from = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, helo = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, header = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, rcpt = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, user = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, url = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, dnsbl = { - get_value = function(ip) return ip:to_string() end, + get_value = function(ip) + return ip:to_string() + end, }, filename = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, content = { - get_value = function() return nil end, + get_value = function() + return nil + end, }, hostname = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, asn = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, country = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, received = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, mempool = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, selector = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, symbol_options = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, } @@ -236,7 +266,7 @@ local function apply_url_filter(task, filter, url, r) return nil elseif string.find(filter, 'tld:regexp:') then if not r['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then r['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -254,7 +284,7 @@ local function apply_url_filter(task, filter, url, r) end elseif string.find(filter, 'full:regexp:') then if not r['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then r['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -272,7 +302,7 @@ local function apply_url_filter(task, filter, url, r) end elseif string.find(filter, 'regexp:') then if not r['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then r['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -305,27 +335,37 @@ local function apply_addr_filter(task, filter, input, rule) if filter == 'email:addr' or filter == 'email' then local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024) if addr and addr[1] then - return fun.totable(fun.map(function(a) return a.addr end, addr)) + return fun.totable(fun.map(function(a) + return a.addr + end, addr)) end elseif filter == 'email:user' then local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024) if addr and addr[1] then - return fun.totable(fun.map(function(a) return a.user end, addr)) + return fun.totable(fun.map(function(a) + return a.user + end, addr)) end elseif filter == 'email:domain' then local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024) if addr and addr[1] then - return fun.totable(fun.map(function(a) return a.domain end, addr)) + return fun.totable(fun.map(function(a) + return a.domain + end, addr)) end elseif filter == 'email:domain:tld' then local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024) if addr and addr[1] then - return fun.totable(fun.map(function(a) return rspamd_util.get_tld(a.domain) end, addr)) + return fun.totable(fun.map(function(a) + return rspamd_util.get_tld(a.domain) + end, addr)) end elseif filter == 'email:name' then local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024) if addr and addr[1] then - return fun.totable(fun.map(function(a) return a.name end, addr)) + return fun.totable(fun.map(function(a) + return a.name + end, addr)) end elseif filter == 'ip_addr' then local ip_addr = rspamd_ip.from_string(input) @@ -336,7 +376,7 @@ local function apply_addr_filter(task, filter, input, rule) else -- regexp case if not rule['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then rule['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -359,7 +399,7 @@ local function apply_filename_filter(task, filter, fn, r) return string.match(fn, '%.([^.]+)$') elseif string.find(filter, 'regexp:') then if not r['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then r['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -383,7 +423,7 @@ end local function apply_regexp_filter(task, filter, fn, r) if string.find(filter, 'regexp:') then if not r['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then r['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -406,26 +446,26 @@ end local function apply_content_filter(task, filter) if filter == 'body' then - return {task:get_rawbody()} + return { task:get_rawbody() } elseif filter == 'full' then - return {task:get_content()} + return { task:get_content() } elseif filter == 'headers' then - return {task:get_raw_headers()} + return { task:get_raw_headers() } elseif filter == 'text' then local ret = {} - for _,p in ipairs(task:get_text_parts()) do + for _, p in ipairs(task:get_text_parts()) do table.insert(ret, p:get_content()) end return ret elseif filter == 'rawtext' then local ret = {} - for _,p in ipairs(task:get_text_parts()) do + for _, p in ipairs(task:get_text_parts()) do table.insert(ret, p:get_raw_content()) end return ret elseif filter == 'oneline' then local ret = {} - for _,p in ipairs(task:get_text_parts()) do + for _, p in ipairs(task:get_text_parts()) do table.insert(ret, p:get_content_oneline()) end return ret @@ -456,7 +496,7 @@ local function multimap_query_redis(key, task, value, callback) cmd = 'HMGET' end - local srch = {key} + local srch = { key } -- Insert all ips for some mask :( if type(value) == 'userdata' and value.class == 'rspamd{ip}' then @@ -468,7 +508,7 @@ local function multimap_query_redis(key, task, value, callback) maxbits = 32 minbits = 8 end - for i=maxbits,minbits,-1 do + for i = maxbits, minbits, -1 do local nip = value:apply_mask(i):tostring() .. "/" .. i srch[#srch + 1] = nip end @@ -508,7 +548,7 @@ local function multimap_callback(task, rule) if ret then if type(err_or_data) == 'table' then - for _,elt in ipairs(err_or_data) do + for _, elt in ipairs(err_or_data) do callback(elt) end else @@ -516,7 +556,7 @@ local function multimap_callback(task, rule) end elseif err_code ~= 404 then rspamd_logger.infox(task, "map %s: get key returned error %s: %s", - rule.symbol, err_code, err_or_data) + rule.symbol, err_code, err_or_data) end end @@ -535,7 +575,7 @@ local function multimap_callback(task, rule) -- Here we need to spill this function into multiple queries if type(results) == 'table' then - for _,res in ipairs(results) do + for _, res in ipairs(results) do ret = multimap_query_redis(res, task, value, callback) if not ret then @@ -554,7 +594,7 @@ local function multimap_callback(task, rule) end local function insert_results(result, opt) - local _,symbol,score,opts = parse_multimap_value(rule, result) + local _, symbol, score, opts = parse_multimap_value(rule, result) local forced = false if symbol then if rule.symbols_set then @@ -569,9 +609,9 @@ local function multimap_callback(task, rule) if type(opt) == 'table' then table.insert(opt, result) elseif type(opt) ~= nil then - opt = {opt,result} + opt = { opt, result } else - opt = {result} + opt = { result } end else forced = not rule.dynamic_symbols @@ -614,7 +654,7 @@ local function multimap_callback(task, rule) local function rule_callback(result) if result then if type(result) == 'table' then - for _,rs in ipairs(result) do + for _, rs in ipairs(result) do if type(rs) ~= 'userdata' then rule_callback(rs) end @@ -639,7 +679,9 @@ local function multimap_callback(task, rule) end if type(value) == 'table' then - fun.each(function(elt) match_element(r, elt, rule_callback) end, value) + fun.each(function(elt) + match_element(r, elt, rule_callback) + end, value) else match_element(r, value, rule_callback) end @@ -659,17 +701,19 @@ local function multimap_callback(task, rule) end end, ls) else - fun.each(function(e) match_rule(r, e) end, ls) + fun.each(function(e) + match_rule(r, e) + end, ls) end end end local function match_addr(r, addr) - match_list(r, addr, {'addr'}) + match_list(r, addr, { 'addr' }) if not r.filter then - match_list(r, addr, {'domain'}) - match_list(r, addr, {'user'}) + match_list(r, addr, { 'domain' }) + match_list(r, addr, { 'user' }) end end @@ -703,7 +747,7 @@ local function multimap_callback(task, rule) return end else - if pos <= (total - (min_pos*-1)) then + if pos <= (total - (min_pos * -1)) then return end end @@ -713,7 +757,7 @@ local function multimap_callback(task, rule) end if max_pos then if max_pos < -1 then - if (total - (max_pos*-1)) >= pos then + if (total - (max_pos * -1)) >= pos then return end elseif max_pos > 0 then @@ -728,12 +772,16 @@ local function multimap_callback(task, rule) local got_flags = h['flags'] if match_flags then for _, flag in ipairs(match_flags) do - if not got_flags[flag] then return end + if not got_flags[flag] then + return + end end end if nmatch_flags then for _, flag in ipairs(nmatch_flags) do - if got_flags[flag] then return end + if got_flags[flag] then + return + end end end end @@ -759,16 +807,16 @@ local function multimap_callback(task, rule) if r['filter'] then data = apply_content_filter(task, r['filter'], r) else - data = {task:get_content()} + data = { task:get_content() } end - for _,v in ipairs(data) do + for _, v in ipairs(data) do match_rule(r, v) end end if rule.expression and not rule.combined then - local res,trace = rule['expression']:process_traced(task) + local res, trace = rule['expression']:process_traced(task) if not res or res == 0 then lua_util.debugm(N, task, 'condition is false for %s', @@ -811,7 +859,7 @@ local function multimap_callback(task, rule) end task:get_resolver():resolve_a({ - task= task, + task = task, name = to_resolve, callback = dns_cb, forced = true @@ -820,13 +868,13 @@ local function multimap_callback(task, rule) end, header = function() if type(rule['header']) == 'table' then - for _,rh in ipairs(rule['header']) do + for _, rh in ipairs(rule['header']) do local hv = task:get_header_full(rh) - match_list(rule, hv, {'decoded'}) + match_list(rule, hv, { 'decoded' }) end else local hv = task:get_header_full(rule['header']) - match_list(rule, hv, {'decoded'}) + match_list(rule, hv, { 'decoded' }) end end, rcpt = function() @@ -857,7 +905,7 @@ local function multimap_callback(task, rule) if task:has_urls() then local msg_urls = task:get_urls() - for _,url in ipairs(msg_urls) do + for _, url in ipairs(msg_urls) do match_url(rule, url) end end @@ -891,11 +939,11 @@ local function multimap_callback(task, rule) return p:is_archive() and det_type == 'archive' and not rule.skip_archives end - for _,p in fun.iter(fun.filter(filter_parts, parts)) do + for _, p in fun.iter(fun.filter(filter_parts, parts)) do if filter_archive(p) then local fnames = p:get_archive():get_files(1000) - for _,fn in ipairs(fnames) do + for _, fn in ipairs(fnames) do match_filename(rule, fn) end end @@ -971,7 +1019,7 @@ local function multimap_callback(task, rule) if elts then if type(elts) == 'table' then - for _,elt in ipairs(elts) do + for _, elt in ipairs(elts) do match_rule(rule, elt) end else @@ -980,9 +1028,9 @@ local function multimap_callback(task, rule) end end, combined = function() - local ret,trace = rule.combined:process(task) + local ret, trace = rule.combined:process(task) if ret and ret ~= 0 then - for n,t in pairs(trace) do + for n, t in pairs(trace) do insert_results(t.value, string.format("%s=%s", n, t.matched)) end @@ -999,7 +1047,6 @@ local function multimap_callback(task, rule) end end - local function gen_multimap_callback(rule) return function(task) multimap_callback(task, rule) @@ -1011,12 +1058,12 @@ local function multimap_on_load_gen(rule) lua_util.debugm(N, rspamd_config, "loaded map object for rule %s", rule['symbol']) local known_symbols = {} rule.map_obj:foreach(function(key, value) - local r,symbol,score,_ = parse_multimap_value(rule, value) + local r, symbol, score, _ = parse_multimap_value(rule, value) if r and symbol and not known_symbols[symbol] then lua_util.debugm(N, rspamd_config, "%s: adding new symbol %s (score = %s), triggered by %s", rule.symbol, symbol, score, key) - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = value, parent = rule.callback_id, type = 'virtual', @@ -1084,7 +1131,7 @@ local function add_multimap_rule(key, newrule) if newrule['url'] and not newrule['map'] then newrule['map'] = newrule['url'] end - if not (newrule.map or newrule.rules) then + if not (newrule.map or newrule.rules) then rspamd_logger.errx(rspamd_config, 'incomplete rule, missing map') return nil end @@ -1179,12 +1226,12 @@ local function add_multimap_rule(key, newrule) if type(newrule['flags']) == 'table' and newrule['flags'][1] then newrule['flags'] = newrule['flags'] elseif type(newrule['flags']) == 'string' then - newrule['flags'] = {newrule['flags']} + newrule['flags'] = { newrule['flags'] } end if type(newrule['nflags']) == 'table' and newrule['nflags'][1] then newrule['nflags'] = newrule['nflags'] elseif type(newrule['nflags']) == 'string' then - newrule['nflags'] = {newrule['nflags']} + newrule['nflags'] = { newrule['nflags'] } end local filter = newrule['filter'] or 'real_ip' if filter == 'real_ip' or filter == 'from_ip' then @@ -1259,7 +1306,7 @@ local function add_multimap_rule(key, newrule) end local expression = rspamd_expression.create(newrule['require_symbols'], - {parse_atom, process_atom}, rspamd_config:get_mempool()) + { parse_atom, process_atom }, rspamd_config:get_mempool()) if expression then newrule['expression'] = expression @@ -1280,11 +1327,11 @@ end local opts = rspamd_config:get_all_opt(N) if opts and type(opts) == 'table' then redis_params = rspamd_parse_redis_server(N) - for k,m in pairs(opts) do + for k, m in pairs(opts) do if type(m) == 'table' and m['type'] then local rule = add_multimap_rule(k, m) if not rule then - rspamd_logger.errx(rspamd_config, 'cannot add rule: "'..k..'"') + rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"') else rspamd_logger.infox(rspamd_config, 'added multimap rule: %s (%s)', k, rule.type) diff --git a/src/plugins/lua/mx_check.lua b/src/plugins/lua/mx_check.lua index 03d7f6087..71892b94b 100644 --- a/src/plugins/lua/mx_check.lua +++ b/src/plugins/lua/mx_check.lua @@ -80,7 +80,9 @@ local function mx_check(task) local valid = false local function check_results(mxes) - if fun.all(function(_, elt) return elt.checked end, mxes) then + if fun.all(function(_, elt) + return elt.checked + end, mxes) then -- Save cache local key = settings.key_prefix .. mx_domain local function redis_cache_cb(err) @@ -99,12 +101,12 @@ local function mx_check(task) task:insert_result(settings.symbol_bad_mx, 1.0) end local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_cache_cb, --callback - 'SETEX', -- command - {key, tostring(settings.expire_novalid), '0'} -- arguments + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_cache_cb, --callback + 'SETEX', -- command + { key, tostring(settings.expire_novalid), '0' } -- arguments ) lua_util.debugm(N, task, "set redis cache key: %s; invalid MX", key) if not ret then @@ -114,19 +116,21 @@ local function mx_check(task) local valid_mx = {} fun.each(function(k) table.insert(valid_mx, k) - end, fun.filter(function (_, elt) return elt.working end, mxes)) + end, fun.filter(function(_, elt) + return elt.working + end, mxes)) task:insert_result(settings.symbol_good_mx, 1.0, valid_mx) - local value = table.concat(valid_mx, ';') + local value = table.concat(valid_mx, ';') if mxes[mx_domain] and type(mxes[mx_domain]) == 'table' and mxes[mx_domain].mx_missing then value = mx_miss_cache_prefix .. value end local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_cache_cb, --callback - 'SETEX', -- command - {key, tostring(settings.expire), value} -- arguments + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_cache_cb, --callback + 'SETEX', -- command + { key, tostring(settings.expire), value } -- arguments ) lua_util.debugm(N, task, "set redis cache key: %s; %s", key, value) if not ret then @@ -230,11 +234,11 @@ local function mx_check(task) end) local max_mx_to_resolve = math.min(#results, settings.max_mx_a_records) - lua_util.debugm(N, task,'check %s MX records (%d actually returned)', + lua_util.debugm(N, task, 'check %s MX records (%d actually returned)', max_mx_to_resolve, #results) - for i=1,max_mx_to_resolve do + for i = 1, max_mx_to_resolve do local mx = results[i] - mxes[mx.name] = {checked = false, working = false, ips = {}} + mxes[mx.name] = { checked = false, working = false, ips = {} } local r = task:get_resolver() -- XXX: maybe add ipv6? r:resolve('a', { @@ -282,12 +286,12 @@ local function mx_check(task) local key = settings.key_prefix .. mx_domain local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_cache_get_cb, --callback - 'GET', -- command - {key} -- arguments + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_cache_get_cb, --callback + 'GET', -- command + { key } -- arguments ) if not ret then @@ -327,7 +331,7 @@ if opts then type = 'normal', callback = mx_check, flags = 'empty', - augmentations = {string.format("timeout=%f", settings.timeout + rspamd_config:get_dns_timeout() or 0.0)}, + augmentations = { string.format("timeout=%f", settings.timeout + rspamd_config:get_dns_timeout() or 0.0) }, }) rspamd_config:register_symbol({ name = settings.symbol_no_mx, @@ -379,7 +383,7 @@ if opts then }) if settings.exclude_domains then - exclude_domains = rspamd_config:add_map{ + exclude_domains = rspamd_config:add_map { type = 'set', description = 'Exclude specific domains from MX checks', url = settings.exclude_domains, diff --git a/src/plugins/lua/neural.lua b/src/plugins/lua/neural.lua index 33361b7df..f3b26f11a 100644 --- a/src/plugins/lua/neural.lua +++ b/src/plugins/lua/neural.lua @@ -35,7 +35,7 @@ local N = "neural" local settings = neural_common.settings -local redis_profile_schema = ts.shape{ +local redis_profile_schema = ts.shape { digest = ts.string, symbols = ts.array_of(ts.string), version = ts.number, @@ -77,7 +77,7 @@ local function new_ann_profile(task, rule, set, version) true, -- is write add_cb, --callback 'ZADD', -- command - {set.prefix, tostring(rspamd_util.get_time()), profile_serialized} + { set.prefix, tostring(rspamd_util.get_time()), profile_serialized } ) return profile @@ -87,7 +87,7 @@ end -- ANN filter function, used to insert scores based on the existing symbols local function ann_scores_filter(task) - for _,rule in pairs(settings.rules) do + for _, rule in pairs(settings.rules) do local sid = task:get_settings_id() or -1 local ann local profile @@ -228,14 +228,17 @@ local function ann_push_task_result(rule, task, verdict, score, set) end end - if learn_spam or learn_ham then local learn_type - if learn_spam then learn_type = 'spam' else learn_type = 'ham' end + if learn_spam then + learn_type = 'spam' + else + learn_type = 'ham' + end local function vectors_len_cb(err, data) if not err and type(data) == 'table' then - local nspam,nham = data[1],data[2] + local nspam, nham = data[1], data[2] if neural_common.can_push_train_vector(rule, task, learn_type, nspam, nham) then local vec = neural_common.result_to_vector(task, set) @@ -296,7 +299,7 @@ local function ann_push_task_result(rule, task, verdict, score, set) end lua_redis.exec_redis_script(neural_common.redis_script_id.vectors_len, - {task = task, is_write = false}, + { task = task, is_write = false }, vectors_len_cb, { set.ann.redis_key, @@ -318,7 +321,7 @@ end -- Utility to extract and split saved training vectors to a table of tables local function process_training_vectors(data) return fun.totable(fun.map(function(tok) - local _,str = rspamd_util.zstd_decompress(tok) + local _, str = rspamd_util.zstd_decompress(tok) return fun.totable(fun.map(tonumber, lua_util.str_split(tostring(str), ';'))) end, data)) end @@ -334,23 +337,23 @@ local function do_train_ann(worker, ev_base, rule, set, ann_key) local function redis_ham_cb(err, data) if err or type(data) ~= 'table' then rspamd_logger.errx(rspamd_config, 'cannot get ham tokens for ANN %s from redis: %s', - ann_key, err) + ann_key, err) -- Unlock on error lua_redis.redis_make_request_taskless(ev_base, - rspamd_config, - rule.redis, - nil, - true, -- is write + rspamd_config, + rule.redis, + nil, + true, -- is write neural_common.gen_unlock_cb(rule, set, ann_key), --callback - 'HDEL', -- command - {ann_key, 'lock'} + 'HDEL', -- command + { ann_key, 'lock' } ) else -- Decompress and convert to numbers each training vector ham_elts = process_training_vectors(data) - neural_common.spawn_train({worker = worker, ev_base = ev_base, - rule = rule, set = set, ann_key = ann_key, ham_vec = ham_elts, - spam_vec = spam_elts}) + neural_common.spawn_train({ worker = worker, ev_base = ev_base, + rule = rule, set = set, ann_key = ann_key, ham_vec = ham_elts, + spam_vec = spam_elts }) end end @@ -358,29 +361,29 @@ local function do_train_ann(worker, ev_base, rule, set, ann_key) local function redis_spam_cb(err, data) if err or type(data) ~= 'table' then rspamd_logger.errx(rspamd_config, 'cannot get spam tokens for ANN %s from redis: %s', - ann_key, err) + ann_key, err) -- Unlock ANN on error lua_redis.redis_make_request_taskless(ev_base, - rspamd_config, - rule.redis, - nil, - true, -- is write + rspamd_config, + rule.redis, + nil, + true, -- is write neural_common.gen_unlock_cb(rule, set, ann_key), --callback - 'HDEL', -- command - {ann_key, 'lock'} + 'HDEL', -- command + { ann_key, 'lock' } ) else -- Decompress and convert to numbers each training vector spam_elts = process_training_vectors(data) -- Now get ham vectors... lua_redis.redis_make_request_taskless(ev_base, - rspamd_config, - rule.redis, - nil, - false, -- is write - redis_ham_cb, --callback - 'SMEMBERS', -- command - {ann_key .. '_ham_set'} + rspamd_config, + rule.redis, + nil, + false, -- is write + redis_ham_cb, --callback + 'SMEMBERS', -- command + { ann_key .. '_ham_set' } ) end end @@ -388,21 +391,21 @@ local function do_train_ann(worker, ev_base, rule, set, ann_key) local function redis_lock_cb(err, data) if err then rspamd_logger.errx(rspamd_config, 'cannot call lock script for ANN %s from redis: %s', - ann_key, err) + ann_key, err) elseif type(data) == 'number' and data == 1 then -- ANN is locked, so we can extract SPAM and HAM vectors and spawn learning lua_redis.redis_make_request_taskless(ev_base, - rspamd_config, - rule.redis, - nil, - false, -- is write - redis_spam_cb, --callback - 'SMEMBERS', -- command - {ann_key .. '_spam_set'} + rspamd_config, + rule.redis, + nil, + false, -- is write + redis_spam_cb, --callback + 'SMEMBERS', -- command + { ann_key .. '_spam_set' } ) rspamd_logger.infox(rspamd_config, 'lock ANN %s:%s (key name %s) for learning', - rule.prefix, set.name, ann_key) + rule.prefix, set.name, ann_key) else local lock_tm = tonumber(data[1]) rspamd_logger.infox(rspamd_config, 'do not learn ANN %s:%s (key name %s), ' .. @@ -422,14 +425,14 @@ local function do_train_ann(worker, ev_base, rule, set, ann_key) -- This script returns either a boolean or a pair {'lock_time', 'hostname'} when -- ANN is locked by another host (or a process, meh) lua_redis.exec_redis_script(neural_common.redis_script_id.maybe_lock, - {ev_base = ev_base, is_write = true}, - redis_lock_cb, + { ev_base = ev_base, is_write = true }, + redis_lock_cb, { ann_key, tostring(os.time()), tostring(math.max(10.0, rule.watch_interval * 2)), rspamd_util.get_hostname() - }) + }) end -- This function loads new ann from Redis @@ -448,7 +451,7 @@ local function load_new_ann(rule, ev_base, set, profile, min_diff) else if type(data) == 'table' then if type(data[1]) == 'userdata' and data[1].cookie == text_cookie then - local _err,ann_data = rspamd_util.zstd_decompress(data[1]) + local _err, ann_data = rspamd_util.zstd_decompress(data[1]) local ann if _err or not ann_data then @@ -482,7 +485,7 @@ local function load_new_ann(rule, ev_base, set, profile, min_diff) true, -- is write rank_cb, --callback 'ZADD', -- command - {set.prefix, tostring(rspamd_util.get_time()), profile_serialized} + { set.prefix, tostring(rspamd_util.get_time()), profile_serialized } ) rspamd_logger.infox(rspamd_config, 'loaded ANN for %s:%s from %s; %s bytes compressed; version=%s', @@ -507,15 +510,15 @@ local function load_new_ann(rule, ev_base, set, profile, min_diff) local roc_thresholds = parser:get_object() set.ann.roc_thresholds = roc_thresholds rspamd_logger.infox(rspamd_config, - 'loaded ROC thresholds for %s:%s; version=%s', - rule.prefix, set.name, profile.version) + 'loaded ROC thresholds for %s:%s; version=%s', + rule.prefix, set.name, profile.version) rspamd_logger.debugx("ROC thresholds: %s", roc_thresholds) end end if set.ann and set.ann.ann and type(data[3]) == 'userdata' and data[3].cookie == text_cookie then -- PCA table - local _err,pca_data = rspamd_util.zstd_decompress(data[3]) + local _err, pca_data = rspamd_util.zstd_decompress(data[3]) if pca_data then if rule.max_inputs then -- We can use PCA @@ -555,8 +558,8 @@ local function load_new_ann(rule, ev_base, set, profile, min_diff) false, -- is write data_cb, --callback 'HMGET', -- command - {ann_key, 'ann', 'roc_thresholds', 'pca'}, -- arguments - {opaque_data = true} + { ann_key, 'ann', 'roc_thresholds', 'pca' }, -- arguments + { opaque_data = true } ) end @@ -570,7 +573,7 @@ local function process_existing_ann(_, ev_base, rule, set, profiles) local min_diff = math.huge local sel_elt - for _,elt in fun.iter(profiles) do + for _, elt in fun.iter(profiles) do if elt and elt.symbols then local dist = lua_util.distance_sorted(elt.symbols, my_symbols) -- Check distance @@ -641,7 +644,7 @@ local function maybe_train_existing_ann(worker, ev_base, rule, set, profiles) ham = 0, } - for _,elt in fun.iter(profiles) do + for _, elt in fun.iter(profiles) do if elt and elt.symbols then local dist = lua_util.distance_sorted(elt.symbols, my_symbols) -- Check distance @@ -734,7 +737,7 @@ local function maybe_train_existing_ann(worker, ev_base, rule, set, profiles) false, -- is write redis_len_cb_gen(initiate_train, 'ham', true), --callback 'SCARD', -- command - {ann_key .. '_ham_set'} + { ann_key .. '_ham_set' } ) end @@ -745,7 +748,7 @@ local function maybe_train_existing_ann(worker, ev_base, rule, set, profiles) false, -- is write redis_len_cb_gen(check_ham_len, 'spam', false), --callback 'SCARD', -- command - {ann_key .. '_spam_set'} + { ann_key .. '_spam_set' } ) end end @@ -755,14 +758,14 @@ local function load_ann_profile(element) local ucl = require "ucl" local parser = ucl.parser() - local res,ucl_err = parser:parse_string(element) + local res, ucl_err = parser:parse_string(element) if not res then rspamd_logger.warnx(rspamd_config, 'cannot parse ANN from redis: %s', ucl_err) return nil else local profile = parser:get_object() - local checked,schema_err = redis_profile_schema:transform(profile) + local checked, schema_err = redis_profile_schema:transform(profile) if not checked then rspamd_logger.errx(rspamd_config, "cannot parse profile schema: %s", schema_err) @@ -774,7 +777,7 @@ end -- Function to check or load ANNs from Redis local function check_anns(worker, cfg, ev_base, rule, process_callback, what) - for _,set in pairs(rule.settings) do + for _, set in pairs(rule.settings) do local function members_cb(err, data) if err then rspamd_logger.errx(cfg, 'cannot get ANNs list from redis: %s', @@ -800,7 +803,7 @@ local function check_anns(worker, cfg, ev_base, rule, process_callback, what) false, -- is write members_cb, --callback 'ZREVRANGE', -- command - {set.prefix, '0', tostring(settings.max_profiles)} -- arguments + { set.prefix, '0', tostring(settings.max_profiles) } -- arguments ) end end -- Cycle over all settings @@ -810,13 +813,13 @@ end -- Function to clean up old ANNs local function cleanup_anns(rule, cfg, ev_base) - for _,set in pairs(rule.settings) do + for _, set in pairs(rule.settings) do local function invalidate_cb(err, data) if err then rspamd_logger.errx(cfg, 'cannot exec invalidate script in redis: %s', err) elseif type(data) == 'table' then - for _,expired in ipairs(data) do + for _, expired in ipairs(data) do local profile = load_ann_profile(expired) rspamd_logger.infox(cfg, 'invalidated ANN for %s; redis key: %s; version=%s', rule.prefix .. ':' .. set.name, @@ -828,9 +831,9 @@ local function cleanup_anns(rule, cfg, ev_base) if type(set) == 'table' then lua_redis.exec_redis_script(neural_common.redis_script_id.maybe_invalidate, - {ev_base = ev_base, is_write = true}, + { ev_base = ev_base, is_write = true }, invalidate_cb, - {set.prefix, tostring(settings.max_profiles)}) + { set.prefix, tostring(settings.max_profiles) }) end end end @@ -845,7 +848,7 @@ local function ann_push_vector(task) return end - local verdict,score = lua_verdict.get_specific_verdict(N, task) + local verdict, score = lua_verdict.get_specific_verdict(N, task) if verdict == 'passthrough' then lua_util.debugm(N, task, 'ignore task as its verdict is %s(%s)', @@ -861,7 +864,7 @@ local function ann_push_vector(task) return end - for _,rule in pairs(settings.rules) do + for _, rule in pairs(settings.rules) do local set = neural_common.get_rule_settings(task, rule) if set then @@ -906,7 +909,7 @@ if settings.blacklisted_symbols and settings.blacklisted_symbols[1] then end -- Check all rules -for k,r in pairs(rules) do +for k, r in pairs(rules) do local rule_elt = lua_util.override_defaults(neural_common.default_options, r) rule_elt['redis'] = neural_common.redis_params rule_elt['anns'] = {} -- Store ANNs here @@ -921,7 +924,9 @@ for k,r in pairs(rules) do rule_elt.train.max_trains = rule_elt.train.max_train end - if not rule_elt.profile then rule_elt.profile = {} end + if not rule_elt.profile then + rule_elt.profile = {} + end if rule_elt.max_inputs and not has_blas then rspamd_logger.errx('cannot set max inputs to %s as BLAS is not compiled in', @@ -969,7 +974,7 @@ rspamd_config:register_symbol({ rspamd_config:add_post_init(neural_common.process_rules_settings) -- Add training scripts -for _,rule in pairs(settings.rules) do +for _, rule in pairs(settings.rules) do neural_common.load_scripts(rule.redis) -- This function will check ANNs in Redis when a worker is loaded rspamd_config:add_on_load(function(cfg, ev_base, worker) diff --git a/src/plugins/lua/once_received.lua b/src/plugins/lua/once_received.lua index 04e2168c1..2a5552ab9 100644 --- a/src/plugins/lua/once_received.lua +++ b/src/plugins/lua/once_received.lua @@ -71,11 +71,11 @@ local function check_quantity_received (task) else rspamd_logger.infox(task, 'source hostname has not been passed to Rspamd from MTA, ' .. 'but we could resolve source IP address PTR %s as "%s"', - to_resolve, results[1]) + to_resolve, results[1]) task:set_hostname(results[1]) if good_hosts then - for _,gh in ipairs(good_hosts) do + for _, gh in ipairs(good_hosts) do if string.find(results[1], gh) then return end @@ -84,7 +84,7 @@ local function check_quantity_received (task) if nreceived <= 1 then task:insert_result(symbol, 1) - for _,h in ipairs(bad_hosts) do + for _, h in ipairs(bad_hosts) do if string.find(results[1], h) then task:insert_result(symbol_strict, 1, h) @@ -104,17 +104,17 @@ local function check_quantity_received (task) end if whitelist and task_ip and whitelist:get_key(task_ip) then rspamd_logger.infox(task, 'whitelisted mail from %s', - task_ip:to_string()) + task_ip:to_string()) return end local hn = task:get_hostname() -- Here we don't care about received if (not hn) and task_ip and task_ip:is_valid() then - task:get_resolver():resolve_ptr({task = task, - name = task_ip:to_string(), - callback = recv_dns_cb, - forced = true + task:get_resolver():resolve_ptr({ task = task, + name = task_ip:to_string(), + callback = recv_dns_cb, + forced = true }) return end @@ -131,7 +131,7 @@ local function check_quantity_received (task) local rhn = string.lower(r['real_hostname']) -- Check for good hostname if rhn and good_hosts then - for _,gh in ipairs(good_hosts) do + for _, gh in ipairs(good_hosts) do if string.find(rhn, gh) then ret = false break @@ -146,8 +146,10 @@ local function check_quantity_received (task) -- Unresolved host task:insert_result(symbol, 1) - if not hn then return end - for _,h in ipairs(bad_hosts) do + if not hn then + return + end + for _, h in ipairs(bad_hosts) do if string.find(hn, h) then task:insert_result(symbol_strict, 1, h) return @@ -176,7 +178,7 @@ if opts then callback = check_quantity_received, }) - for n,v in pairs(opts) do + for n, v in pairs(opts) do if n == 'symbol_strict' then symbol_strict = v elseif n == 'symbol_rdns' then @@ -198,7 +200,7 @@ if opts then elseif n == 'whitelist' then local lua_maps = require "lua_maps" whitelist = lua_maps.map_add('once_received', 'whitelist', 'radix', - 'once received whitelist') + 'once received whitelist') elseif n == 'symbol_mx' then symbol_mx = v end @@ -214,7 +216,7 @@ if opts then type = 'virtual', parent = id }) - rspamd_config:register_symbol({ + rspamd_config:register_symbol({ name = symbol_strict, type = 'virtual', parent = id diff --git a/src/plugins/lua/p0f.lua b/src/plugins/lua/p0f.lua index d597ee25d..97757c23a 100644 --- a/src/plugins/lua/p0f.lua +++ b/src/plugins/lua/p0f.lua @@ -26,36 +26,36 @@ local N = 'p0f' if confighelp then rspamd_config:add_example(nil, N, - 'Detect remote OS via passive fingerprinting', - [[ -p0f { - # Enable module - enabled = true - - # Path to the unix socket that p0f listens on - socket = '/var/run/p0f.sock'; - - # Connection timeout - timeout = 5s; - - # If defined, insert symbol with lookup results - symbol = 'P0F'; - - # Patterns to match against results returned by p0f - # Symbol will be yielded on OS string, link type or distance matches - patterns = { - WINDOWS = '^Windows.*'; - #DSL = '^DSL$'; - #DISTANCE10 = '^distance:10$'; + 'Detect remote OS via passive fingerprinting', + [[ + p0f { + # Enable module + enabled = true + + # Path to the unix socket that p0f listens on + socket = '/var/run/p0f.sock'; + + # Connection timeout + timeout = 5s; + + # If defined, insert symbol with lookup results + symbol = 'P0F'; + + # Patterns to match against results returned by p0f + # Symbol will be yielded on OS string, link type or distance matches + patterns = { + WINDOWS = '^Windows.*'; + #DSL = '^DSL$'; + #DISTANCE10 = '^distance:10$'; + } + + # Cache lifetime in seconds (default - 2 hours) + expire = 7200; + + # Cache key prefix + prefix = 'p0f'; } - - # Cache lifetime in seconds (default - 2 hours) - expire = 7200; - - # Cache key prefix - prefix = 'p0f'; -} -]]) + ]]) return end @@ -90,7 +90,7 @@ if rule then priority = lua_util.symbols_priorities.medium, flags = 'empty,nostat', group = N, - augmentations = {string.format("timeout=%f", rule.timeout or 0.0)}, + augmentations = { string.format("timeout=%f", rule.timeout or 0.0) }, }) diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua index 628d6efa7..9dd03aa9d 100644 --- a/src/plugins/lua/phishing.lua +++ b/src/plugins/lua/phishing.lua @@ -48,7 +48,6 @@ local openphish_hash local generic_service_data = {} local openphish_data = {} - local opts = rspamd_config:get_all_opt(N) if not (opts and type(opts) == 'table') then rspamd_logger.infox(rspamd_config, 'Module is unconfigured') @@ -70,7 +69,7 @@ local function phishing_cb(task) local query = url:get_query() if path then - for _,d in ipairs(elt) do + for _, d in ipairs(elt) do if d['path'] == path then found_path = true data = d['data'] @@ -83,7 +82,7 @@ local function phishing_cb(task) end end else - for _,d in ipairs(elt) do + for _, d in ipairs(elt) do if not d['path'] then found_path = true end @@ -135,7 +134,9 @@ local function phishing_cb(task) local function compose_dns_query(elts) local cr = require "rspamd_cryptobox_hash" local h = cr.create() - for _,elt in ipairs(elts) do h:update(elt) end + for _, elt in ipairs(elts) do + h:update(elt) + end return string.format("%s.%s", h:base32():sub(1, 32), dns_suffix) end local r = task:get_resolver() @@ -154,14 +155,13 @@ local function phishing_cb(task) end end - local to_resolve_hp = compose_dns_query({host, path}) + local to_resolve_hp = compose_dns_query({ host, path }) rspamd_logger.debugm(N, task, 'try to resolve {%s, %s} -> %s', host, path, to_resolve_hp) r:resolve_txt({ task = task, name = to_resolve_hp, - callback = host_host_path_cb}) - + callback = host_host_path_cb }) if query then local function host_host_path_query_cb(_, _, results, err) @@ -170,13 +170,13 @@ local function phishing_cb(task) end end - local to_resolve_hpq = compose_dns_query({host, path, query}) + local to_resolve_hpq = compose_dns_query({ host, path, query }) rspamd_logger.debugm(N, task, 'try to resolve {%s, %s, %s} -> %s', host, path, query, to_resolve_hpq) r:resolve_txt({ task = task, name = to_resolve_hpq, - callback = host_host_path_query_cb}) + callback = host_host_path_query_cb }) end end @@ -193,8 +193,9 @@ local function phishing_cb(task) end local urls = task:get_urls() or {} - for _,url_iter in ipairs(urls) do - local function do_loop_iter() -- to emulate continue + for _, url_iter in ipairs(urls) do + local function do_loop_iter() + -- to emulate continue local url = url_iter if generic_service_hash then check_phishing_map(generic_service_data, url, generic_service_symbol) @@ -225,7 +226,6 @@ local function phishing_cb(task) purl = url:get_phished() end - if not purl then return end @@ -239,15 +239,15 @@ local function phishing_cb(task) if dmarc_dom and tld == dmarc_dom then lua_util.debugm(N, 'exclude phishing from %s -> %s by dmarc domain', tld, - ptld) + ptld) return end -- Now we can safely remove the last dot component if it is the same - local b,_ = string.find(tld, '%.[^%.]+$') - local b1,_ = string.find(ptld, '%.[^%.]+$') + local b, _ = string.find(tld, '%.[^%.]+$') + local b1, _ = string.find(ptld, '%.[^%.]+$') - local stripped_tld,stripped_ptld = tld, ptld + local stripped_tld, stripped_ptld = tld, ptld if b1 and b then if string.sub(tld, b) == string.sub(ptld, b1) then stripped_ptld = string.gsub(ptld, '%.[^%.]+$', '') @@ -260,7 +260,7 @@ local function phishing_cb(task) end local weight = 1.0 - local spoofed,why = util.is_utf_spoofed(tld, ptld) + local spoofed, why = util.is_utf_spoofed(tld, ptld) if spoofed then lua_util.debugm(N, task, "confusable: %1 -> %2: %3", tld, ptld, why) weight = 1.0 @@ -273,15 +273,19 @@ local function phishing_cb(task) weight = util.tanh(3 * (1 - dist + 0.1)) elseif dist > 1 then -- We also check if two labels are in the same ascii/non-ascii representation - local a1, a2 = false,false + local a1, a2 = false, false - if string.match(tld, '^[\001-\127]*$') then a1 = true end - if string.match(ptld, '^[\001-\127]*$') then a2 = true end + if string.match(tld, '^[\001-\127]*$') then + a1 = true + end + if string.match(ptld, '^[\001-\127]*$') then + a2 = true + end if a1 ~= a2 then weight = 1 lua_util.debugm(N, task, "confusable: %1 -> %2: different characters", - tld, ptld, why) + tld, ptld, why) else -- We have totally different strings in tld, so penalize it somehow weight = 0.5 @@ -292,20 +296,24 @@ local function phishing_cb(task) end local function is_url_in_map(map, furl) - for _,dn in ipairs({furl:get_tld(), furl:get_host()}) do + for _, dn in ipairs({ furl:get_tld(), furl:get_host() }) do if map:get_key(dn) then - return true,dn + return true, dn end end return false end local function found_in_map(map, furl, sweight) - if not furl then furl = url end - if not sweight then sweight = weight end + if not furl then + furl = url + end + if not sweight then + sweight = weight + end if #map > 0 then - for _,rule in ipairs(map) do - local found,dn = is_url_in_map(rule.map, furl) + for _, rule in ipairs(map) do + local found, dn = is_url_in_map(rule.map, furl) if found then task:insert_result(rule.symbol, sweight, string.format("%s->%s:%s", ptld, tld, dn)) return true @@ -342,13 +350,12 @@ local function phishing_map(mapname, phishmap, id) rspamd_logger.errx(rspamd_config, 'invalid exception table') end - - for sym,map_data in pairs(xd) do - local rmap = lua_maps.map_add_from_ucl (map_data, 'set', - 'Phishing ' .. mapname .. ' map') + for sym, map_data in pairs(xd) do + local rmap = lua_maps.map_add_from_ucl(map_data, 'set', + 'Phishing ' .. mapname .. ' map') if rmap then rspamd_config:register_virtual_symbol(sym, 1, id) - local rule = {symbol = sym, map = rmap} + local rule = { symbol = sym, map = rmap } table.insert(phishmap, rule) else rspamd_logger.infox(rspamd_config, 'cannot add map for symbol: %s', sym) @@ -360,8 +367,8 @@ end local function rspamd_str_split_fun(s, sep, func) local lpeg = require "lpeg" sep = lpeg.P(sep) - local elem = lpeg.P((1 - sep)^0 / func) - local p = lpeg.P(elem * (sep * elem)^0) + local elem = lpeg.P((1 - sep) ^ 0 / func) + local p = lpeg.P(elem * (sep * elem) ^ 0) return p:match(s) end @@ -382,7 +389,7 @@ local function insert_url_from_string(pool, tbl, str, data) if tbl[host] then table.insert(tbl[host], elt) else - tbl[host] = {elt} + tbl[host] = { elt } end return true @@ -408,7 +415,7 @@ local function generic_service_plain_cb(string) generic_service_data = new_data rspamd_logger.infox(generic_service_hash, "parsed %s elements from %s feed", - nelts, generic_service_name) + nelts, generic_service_name) pool:destroy() end @@ -424,7 +431,7 @@ local function openphish_json_cb(string) local function openphish_elt_parser(cap) if valid then local parser = ucl.parser() - local res,err = parser:parse_string(cap) + local res, err = parser:parse_string(cap) if not res then valid = false rspamd_logger.warnx(openphish_hash, 'cannot parse openphish map: ' .. err) @@ -445,7 +452,7 @@ local function openphish_json_cb(string) if valid then openphish_data = new_json_map rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed", - nelts) + nelts) end pool:destroy() @@ -467,7 +474,7 @@ local function openphish_plain_cb(s) openphish_data = new_data rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed", - nelts) + nelts) pool:destroy() end @@ -499,11 +506,11 @@ if opts then if opts['generic_service_enabled'] then generic_service_hash = rspamd_config:add_map({ - type = 'callback', - url = generic_service_map, - callback = generic_service_plain_cb, - description = 'Generic feed' - }) + type = 'callback', + url = generic_service_map, + callback = generic_service_plain_cb, + description = 'Generic feed' + }) end if opts['openphish_map'] then @@ -528,12 +535,12 @@ if opts then }) else openphish_hash = rspamd_config:add_map({ - type = 'callback', - url = openphish_map, - callback = openphish_json_cb, - opaque_data = true, - description = 'Open phishing premium feed map (see https://www.openphish.com for details)' - }) + type = 'callback', + url = openphish_map, + callback = openphish_json_cb, + opaque_data = true, + description = 'Open phishing premium feed map (see https://www.openphish.com for details)' + }) end end @@ -564,7 +571,7 @@ if opts then end if opts['domains'] and type(opts['domains']) == 'string' then domains = lua_maps.map_add_from_ucl(opts['domains'], 'set', - 'Phishing domains') + 'Phishing domains') end phishing_map('phishing_exceptions', phishing_exceptions_maps, id) phishing_map('exceptions', anchor_exceptions_maps, id) diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua index b225d0650..9f8292d6b 100644 --- a/src/plugins/lua/ratelimit.lua +++ b/src/plugins/lua/ratelimit.lua @@ -39,7 +39,7 @@ local redis_params -- Senders that are considered as bounce local settings = { bounce_senders = { 'postmaster', 'mailer-daemon', '', 'null', 'fetchmail-daemon', 'mdaemon' }, --- Do not check ratelimits for these recipients + -- Do not check ratelimits for these recipients whitelisted_rcpts = { 'postmaster', 'mailer-daemon' }, prefix = 'RL', ham_factor_rate = 1.01, @@ -54,11 +54,9 @@ local settings = { prefilter = true, } - local bucket_check_script = "ratelimit_check.lua" local bucket_check_id - local bucket_update_script = "ratelimit_update.lua" local bucket_update_id @@ -70,7 +68,6 @@ local message_func = function(_, limit_type, _, _, _) return string.format('Ratelimit "%s" exceeded', limit_type) end - local function load_scripts(_, _) bucket_check_id = lua_redis.load_redis_script_from_file(bucket_check_script, redis_params) bucket_update_id = lua_redis.load_redis_script_from_file(bucket_update_script, redis_params) @@ -106,27 +103,28 @@ local function parse_string_limit(lim, no_error) if not limit_parser then local digit = lpeg.R("09") limit_parser = {} - limit_parser.integer = - (lpeg.S("+-") ^ -1) * - (digit ^ 1) - limit_parser.fractional = - (lpeg.P(".") ) * - (digit ^ 1) - limit_parser.number = - (limit_parser.integer * - (limit_parser.fractional ^ -1)) + - (lpeg.S("+-") * limit_parser.fractional) + limit_parser.integer = (lpeg.S("+-") ^ -1) * + (digit ^ 1) + limit_parser.fractional = (lpeg.P(".")) * + (digit ^ 1) + limit_parser.number = (limit_parser.integer * + (limit_parser.fractional ^ -1)) + + (lpeg.S("+-") * limit_parser.fractional) limit_parser.time = lpeg.Cf(lpeg.Cc(1) * - (limit_parser.number / tonumber) * - ((lpeg.S("smhd") / parse_time_suffix) ^ -1), - function (acc, val) return acc * val end) + (limit_parser.number / tonumber) * + ((lpeg.S("smhd") / parse_time_suffix) ^ -1), + function(acc, val) + return acc * val + end) limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) * - (limit_parser.number / tonumber) * - ((lpeg.S("kmg") / parse_num_suffix) ^ -1), - function (acc, val) return acc * val end) + (limit_parser.number / tonumber) * + ((lpeg.S("kmg") / parse_num_suffix) ^ -1), + function(acc, val) + return acc * val + end) limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number * - (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) * - limit_parser.time) + (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) * + limit_parser.time) end local t = lpeg.match(limit_parser.limit, lim) @@ -142,7 +140,7 @@ local function parse_string_limit(lim, no_error) end local function str_to_rate(str) - local divider,divisor = parse_string_limit(str, false) + local divider, divisor = parse_string_limit(str, false) if not divisor then rspamd_logger.errx(rspamd_config, 'bad rate string: %s', str) @@ -153,7 +151,7 @@ local function str_to_rate(str) return divisor / divider end -local bucket_schema = ts.shape{ +local bucket_schema = ts.shape { burst = ts.number + ts.string / lua_util.dehumanize_number, rate = ts.number + ts.string / str_to_rate, skip_recipients = ts.boolean:is_optional(), @@ -184,7 +182,7 @@ local function parse_limit(name, data) return nil else - local parsed_bucket,err = bucket_schema:transform(data) + local parsed_bucket, err = bucket_schema:transform(data) if not parsed_bucket or err then rspamd_logger.errx(rspamd_config, 'cannot parse bucket for %s: %s; original value: %s', @@ -210,21 +208,27 @@ end --- Check whether this addr is bounce local function check_bounce(from) - return fun.any(function(b) return b == from end, settings.bounce_senders) + return fun.any(function(b) + return b == from + end, settings.bounce_senders) end local keywords = { ['ip'] = { ['get_value'] = function(task) local ip = task:get_ip() - if ip and ip:is_valid() then return tostring(ip) end + if ip and ip:is_valid() then + return tostring(ip) + end return nil end, }, ['rip'] = { ['get_value'] = function(task) local ip = task:get_ip() - if ip and ip:is_valid() and not ip:is_local() then return tostring(ip) end + if ip and ip:is_valid() and not ip:is_local() then + return tostring(ip) + end return nil end, }, @@ -243,7 +247,11 @@ local keywords = { if not ((from or E)[1] or E).user then return '_' end - if check_bounce(from[1]['user']) then return '_' else return nil end + if check_bounce(from[1]['user']) then + return '_' + else + return nil + end end, }, ['asn'] = { @@ -281,7 +289,7 @@ local keywords = { local parts = task:get_parts() or E local digests = {} - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do if p:get_filename() then table.insert(digests, p:get_digest()) end @@ -299,7 +307,7 @@ local keywords = { local parts = task:get_parts() or E local files = {} - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do local fname = p:get_filename() if fname then table.insert(files, fname) @@ -316,7 +324,7 @@ local keywords = { } local function gen_rate_key(task, rtype, bucket) - local key_t = {tostring(lua_util.round(100000.0 / bucket.burst))} + local key_t = { tostring(lua_util.round(100000.0 / bucket.burst)) } local key_keywords = lua_util.str_split(rtype, '_') local have_user = false @@ -326,9 +334,15 @@ local function gen_rate_key(task, rtype, bucket) if keywords[v] and type(keywords[v]['get_value']) == 'function' then ret = keywords[v]['get_value'](task) end - if not ret then return nil end - if v == 'user' then have_user = true end - if type(ret) ~= 'string' then ret = tostring(ret) end + if not ret then + return nil + end + if v == 'user' then + have_user = true + end + if type(ret) ~= 'string' then + ret = tostring(ret) + end table.insert(key_t, ret) end @@ -341,7 +355,9 @@ end local function make_prefix(redis_key, name, bucket) local hash_len = 24 - if hash_len > #redis_key then hash_len = #redis_key end + if hash_len > #redis_key then + hash_len = #redis_key + end local hash = settings.prefix .. string.sub(rspamd_hash.create(redis_key):base32(), 1, hash_len) -- Fill defaults @@ -367,7 +383,7 @@ end local function limit_to_prefixes(task, k, v, prefixes) local n = 0 - for _,bucket in ipairs(v.buckets) do + for _, bucket in ipairs(v.buckets) do if v.selector then local selectors = lua_selectors.process_selectors(task, v.selector) if selectors then @@ -403,7 +419,9 @@ end local function ratelimit_cb(task) if not settings.allow_local and - rspamd_lua_utils.is_rspamc_or_controller(task) then return end + rspamd_lua_utils.is_rspamc_or_controller(task) then + return + end -- Get initial task data local ip = task:get_from_ip() @@ -419,10 +437,14 @@ local function ratelimit_cb(task) local rcpts_user = {} if rcpts then fun.each(function(r) - fun.each(function(type) table.insert(rcpts_user, r[type]) end, {'user', 'addr'}) + fun.each(function(type) + table.insert(rcpts_user, r[type]) + end, { 'user', 'addr' }) end, rcpts) - if fun.any(function(r) return settings.whitelisted_rcpts:get_key(r) end, rcpts_user) then + if fun.any(function(r) + return settings.whitelisted_rcpts:get_key(r) + end, rcpts_user) then rspamd_logger.infox(task, 'skip ratelimit for whitelisted recipient') return end @@ -439,7 +461,7 @@ local function ratelimit_cb(task) local prefixes = {} local nprefixes = 0 - for k,v in pairs(settings.limits) do + for k, v in pairs(settings.limits) do nprefixes = nprefixes + limit_to_prefixes(task, k, v, prefixes) end @@ -504,9 +526,11 @@ local function ratelimit_cb(task) end -- Don't do anything if pre-result has been already set - if task:has_pre_result() then return end + if task:has_pre_result() then + return + end - local _,nrcpt = task:has_recipients('smtp') + local _, nrcpt = task:has_recipients('smtp') if not nrcpt or nrcpt <= 0 then nrcpt = 1 end @@ -518,19 +542,21 @@ local function ratelimit_cb(task) now = lua_util.round(now * 1000.0) -- Get milliseconds -- Now call check script for all defined prefixes - for pr,value in pairs(prefixes) do + for pr, value in pairs(prefixes) do local bucket = value.bucket local rate = (bucket.rate) / 1000.0 -- Leak rate in messages/ms local bincr = nrcpt - if bucket.skip_recipients then bincr = 1 end + if bucket.skip_recipients then + bincr = 1 + end lua_util.debugm(N, task, "check limit %s:%s -> %s (%s/%s)", value.name, pr, value.hash, bucket.burst, bucket.rate) lua_redis.exec_redis_script(bucket_check_id, - {key = value.hash, task = task, is_write = true}, + { key = value.hash, task = task, is_write = true }, gen_check_cb(pr, bucket, value.name, value.hash), - {value.hash, tostring(now), tostring(rate), tostring(bucket.burst), - tostring(settings.expire), tostring(bincr)}) + { value.hash, tostring(now), tostring(rate), tostring(bucket.burst), + tostring(settings.expire), tostring(bincr) }) end end end @@ -553,18 +579,20 @@ local function maybe_cleanup_pending(task) lua_util.debugm(N, task, 'cleaned pending bucked for %s: %s', k, data) end end - local _,nrcpt = task:has_recipients('smtp') + local _, nrcpt = task:has_recipients('smtp') if not nrcpt or nrcpt <= 0 then nrcpt = 1 end local bincr = nrcpt - if bucket.skip_recipients then bincr = 1 end + if bucket.skip_recipients then + bincr = 1 + end local now = task:get_timeval(true) now = lua_util.round(now * 1000.0) -- Get milliseconds lua_redis.exec_redis_script(bucket_cleanup_id, - {key = v.hash, task = task, is_write = true}, + { key = v.hash, task = task, is_write = true }, cleanup_cb, - {v.hash, tostring(now), tostring(settings.expire), tostring(bincr)}) + { v.hash, tostring(now), tostring(settings.expire), tostring(bincr) }) end end end @@ -590,7 +618,7 @@ local function ratelimit_update_cb(task) end local verdict = lua_verdict.get_specific_verdict(N, task) - local _,nrcpt = task:has_recipients('smtp') + local _, nrcpt = task:has_recipients('smtp') if not nrcpt or nrcpt <= 0 then nrcpt = 1 end @@ -624,14 +652,16 @@ local function ratelimit_update_cb(task) end local bincr = nrcpt - if bucket.skip_recipients then bincr = 1 end + if bucket.skip_recipients then + bincr = 1 + end lua_redis.exec_redis_script(bucket_update_id, - {key = v.hash, task = task, is_write = true}, + { key = v.hash, task = task, is_write = true }, update_bucket_cb, - {v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst), - tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult), - tostring(settings.expire), tostring(bincr)}) + { v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst), + tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult), + tostring(settings.expire), tostring(bincr) }) end end end @@ -653,7 +683,7 @@ if opts then if type(lim) == 'table' and lim.bucket then if lim.bucket[1] then - for _,bucket in ipairs(lim.bucket) do + for _, bucket in ipairs(lim.bucket) do local b = parse_limit(t, bucket) if not b then @@ -673,7 +703,7 @@ if opts then return end - buckets = {bucket} + buckets = { bucket } end settings.limits[t] = { @@ -696,7 +726,7 @@ if opts then buckets = parse_limit(t, lim) if buckets then settings.limits[t] = { - buckets = {buckets} + buckets = { buckets } } end end @@ -706,7 +736,7 @@ if opts then -- Display what's enabled fun.each(function(s) rspamd_logger.infox(rspamd_config, 'enabled ratelimit: %s', s) - end, fun.map(function(n,d) + end, fun.map(function(n, d) return string.format('%s [%s]', n, table.concat(fun.totable(fun.map(function(v) return string.format('symbol: %s, %s msgs burst, %s msgs/sec rate', @@ -723,14 +753,14 @@ if opts then if type(wrcpts) == 'string' then if string.find(wrcpts, ',') then settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl( - lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts') + lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts') else settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set', - 'Ratelimit whitelisted rcpts') + 'Ratelimit whitelisted rcpts') end elseif type(opts['whitelisted_rcpts']) == 'table' then settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set', - 'Ratelimit whitelisted rcpts') + 'Ratelimit whitelisted rcpts') else -- Stupid default... settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl( @@ -739,12 +769,12 @@ if opts then if opts['whitelisted_ip'] then settings.whitelisted_ip = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_ip', 'radix', - 'Ratelimit whitelist ip map') + 'Ratelimit whitelist ip map') end if opts['whitelisted_user'] then settings.whitelisted_user = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_user', 'set', - 'Ratelimit whitelist user map') + 'Ratelimit whitelist user map') end settings.custom_keywords = {} @@ -754,7 +784,7 @@ if opts then if ret then opts['custom_keywords'] = {} if type(res_or_err) == 'table' then - for k,hdl in pairs(res_or_err) do + for k, hdl in pairs(res_or_err) do settings['custom_keywords'][k] = hdl end elseif type(res_or_err) == 'function' then @@ -783,7 +813,7 @@ if opts then priority = lua_util.symbols_priorities.medium, callback = ratelimit_cb, flags = 'empty,nostat', - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}, + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }, } local id = rspamd_config:register_symbol(s) @@ -792,9 +822,9 @@ if opts then -- Display what's enabled fun.each(function(set, lim) if type(lim.buckets) == 'table' then - for _,b in ipairs(lim.buckets) do + for _, b in ipairs(lim.buckets) do if b.symbol then - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', name = b.symbol, score = 0.0, @@ -806,7 +836,7 @@ if opts then end, settings.limits) if settings.info_symbol then - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', name = settings.info_symbol, score = 0.0, @@ -814,7 +844,7 @@ if opts then } end if settings.symbol then - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', name = settings.symbol, score = 0.0, -- Might be overridden if needed @@ -827,7 +857,7 @@ if opts then name = 'RATELIMIT_UPDATE', flags = 'explicit_disable,ignore_passthrough', callback = ratelimit_update_cb, - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}, + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }, } end end diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua index 9d1097d94..b0ea8c63e 100644 --- a/src/plugins/lua/rbl.lua +++ b/src/plugins/lua/rbl.lua @@ -142,7 +142,9 @@ local function gen_check_rcvd_conditions(rbl, received_total) local nmatch_flags = rbl.received_nflags local function basic_received_check(rh) - if not (rh.real_ip and rh.real_ip:is_valid()) then return false end + if not (rh.real_ip and rh.real_ip:is_valid()) then + return false + end if ((rh.real_ip:get_version() == 6 and rbl.ipv6) or (rh.real_ip:get_version() == 4 and rbl.ipv4)) and ((rbl.exclude_private_ips and not rh.real_ip:is_local()) or @@ -155,7 +157,9 @@ local function gen_check_rcvd_conditions(rbl, received_total) end local function positioned_received_check(rh, pos) - if not rh or not basic_received_check(rh) then return false end + if not rh or not basic_received_check(rh) then + return false + end local got_flags = rh.flags or E if min_pos then if min_pos < 0 then @@ -200,7 +204,6 @@ local function gen_check_rcvd_conditions(rbl, received_total) return true end - if not (max_pos or min_pos or match_flags or nmatch_flags) then return basic_received_check else @@ -267,15 +270,15 @@ local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_tabl return end - for _,result in ipairs(results) do + for _, result in ipairs(results) do local ipstr = result:to_string() lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr) local foundrc = false -- Check return codes if rbl.returnbits then local ipnum = result:to_number() - for s,bits in pairs(rbl.returnbits) do - for _,check_bit in ipairs(bits) do + for s, bits in pairs(rbl.returnbits) do + for _, check_bit in ipairs(bits) do if bit.band(ipnum, check_bit) == check_bit then foundrc = true insert_results(s) @@ -285,7 +288,7 @@ local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_tabl end elseif rbl.returncodes then for s, codes in pairs(rbl.returncodes) do - for _,v in ipairs(codes) do + for _, v in ipairs(codes) do if string.find(ipstr, '^' .. v .. '$') then foundrc = true insert_results(s) @@ -370,7 +373,7 @@ local function gen_rbl_callback(rule) nreq.what[label] = true end - return true,nreq -- Duplicate + return true, nreq -- Duplicate else local nreq @@ -384,7 +387,7 @@ local function gen_rbl_callback(rule) n = processed, orig = req_str, resolve_ip = resolve_ip, - what = {[label] = true}, + what = { [label] = true }, } requests_table[req] = nreq end @@ -407,7 +410,7 @@ local function gen_rbl_callback(rule) n = to_resolve, orig = req_str, resolve_ip = resolve_ip, - what = {[label] = true}, + what = { [label] = true }, } requests_table[req] = nreq end @@ -431,7 +434,9 @@ local function gen_rbl_callback(rule) local function check_required_symbols(task, _) if rule.require_symbols then - return fun.all(function(sym) task:has_symbol(sym) end, rule.require_symbols) + return fun.all(function(sym) + task:has_symbol(sym) + end, rule.require_symbols) end return true @@ -484,7 +489,7 @@ local function gen_rbl_callback(rule) mime_from_domain = ((task:get_from('mime') or E)[1] or E).domain if mime_from_domain then local mime_from_domain_tld = rule.url_full_hostname and - mime_from_domain or rspamd_util.get_tld(mime_from_domain) + mime_from_domain or rspamd_util.get_tld(mime_from_domain) if rule.url_compose_map then mime_from_domain = rule.url_compose_map:process_url(task, mime_from_domain_tld, mime_from_domain) @@ -496,7 +501,7 @@ local function gen_rbl_callback(rule) for _, d in ipairs(das[1].options) do - local domain,result = d:match('^([^%:]*):([%+%-%~])$') + local domain, result = d:match('^([^%:]*):([%+%-%~])$') -- We must ignore bad signatures, omg if domain and result and result == '+' then @@ -516,7 +521,7 @@ local function gen_rbl_callback(rule) if mime_from_domain and mime_from_domain == domain_tld then add_dns_request(task, domain_tld, true, false, requests_table, - 'dkim', whitelist) + 'dkim', whitelist) end else if rule.dkim_domainonly then @@ -571,7 +576,7 @@ local function gen_rbl_callback(rule) local urls = lua_util.extract_specific_urls(ex_params) - for _,u in ipairs(urls) do + for _, u in ipairs(urls) do local flags = u:get_flags_num() if bit.band(flags, url_flag_bits.numeric) ~= 0 then @@ -616,14 +621,14 @@ local function gen_rbl_callback(rule) end local function check_received(task, requests_table, whitelist) - local received = fun.filter(function(h) + local received = fun .filter(function(h) return not h['flags']['artificial'] end, task:get_received_headers()):totable() local received_total = #received local check_conditions = gen_check_rcvd_conditions(rule, received_total) - for pos,rh in ipairs(received) do + for pos, rh in ipairs(received) do if check_conditions(rh, pos) then add_dns_request(task, rh.real_ip, false, true, requests_table, 'received', @@ -651,13 +656,13 @@ local function gen_rbl_callback(rule) local res = selector(task) if res and type(res) == 'table' then - for _,r in ipairs(res) do - add_dns_request(task, r, false, false, requests_table, - selector_label, whitelist) - end + for _, r in ipairs(res) do + add_dns_request(task, r, false, false, requests_table, + selector_label, whitelist) + end elseif res then add_dns_request(task, res, false, false, - requests_table, selector_label, whitelist) + requests_table, selector_label, whitelist) end end @@ -705,7 +710,9 @@ local function gen_rbl_callback(rule) local ex_params = { task = task, limit = rule.requests_limit, - filter = function(u) return u:get_protocol() == 'mailto' end, + filter = function(u) + return u:get_protocol() == 'mailto' + end, need_emails = true, prefix = 'rbl_email' } @@ -719,7 +726,7 @@ local function gen_rbl_callback(rule) local emails = lua_util.extract_specific_urls(ex_params) - for _,email in ipairs(emails) do + for _, email in ipairs(emails) do local domain if rule.emails_domainonly and not rule.url_full_hostname then if rule.url_compose_map then @@ -835,7 +842,7 @@ local function gen_rbl_callback(rule) end -- Execute functions pipeline - for i,f in ipairs(pipeline) do + for i, f in ipairs(pipeline) do if not f(task, dns_req, whitelist) then lua_util.debugm(N, task, "skip rbl check: %s; pipeline condition %s returned false", @@ -854,12 +861,12 @@ local function gen_rbl_callback(rule) local function gen_rbl_ip_dns_callback(orig_resolve_table_elt) return function(_, _, results, err) if not err then - for _,dns_res in ipairs(results) do + for _, dns_res in ipairs(results) do -- Check if we have rspamd{ip} userdata if type(dns_res) == 'userdata' then -- Add result as an actual RBL request local label = next(orig_resolve_table_elt.what) - local dup,nreq = add_dns_request(task, dns_res, false, true, + local dup, nreq = add_dns_request(task, dns_res, false, true, resolved_req, label) -- Add original name if not dup then @@ -932,7 +939,7 @@ local function gen_rbl_callback(rule) end end - return callback_f,string.format('checks: %s', table.concat(description, ',')) + return callback_f, string.format('checks: %s', table.concat(description, ',')) end local function add_rbl(key, rbl, global_opts) @@ -940,7 +947,7 @@ local function add_rbl(key, rbl, global_opts) rbl.symbol = key:upper() end - local flags_tbl = {'no_squeeze'} + local flags_tbl = { 'no_squeeze' } if rbl.is_whitelist then flags_tbl[#flags_tbl + 1] = 'nice' end @@ -955,7 +962,7 @@ local function add_rbl(key, rbl, global_opts) rbl.selectors = {} if type(rbl.selector) ~= 'table' then - rbl.selector = {['selector'] = rbl.selector} + rbl.selector = { ['selector'] = rbl.selector } end for selector_label, selector in pairs(rbl.selector) do @@ -970,7 +977,7 @@ local function add_rbl(key, rbl, global_opts) rbl.selector_flatten = true end local sel = selectors.create_selector_closure(rspamd_config, selector, '', - rbl.selector_flatten) + rbl.selector_flatten) if not sel then rspamd_logger.errx('invalid selector for rbl rule %s: %s', key, selector) @@ -1032,23 +1039,23 @@ local function add_rbl(key, rbl, global_opts) rbl.symbol) end - local callback,description = gen_rbl_callback(rbl) + local callback, description = gen_rbl_callback(rbl) if callback then local id if rbl.symbols_prefixes then - id = rspamd_config:register_symbol{ + id = rspamd_config:register_symbol { type = 'callback', callback = callback, - groups = {'rbl'}, + groups = { 'rbl' }, name = rbl.symbol .. '_CHECK', flags = table.concat(flags_tbl, ',') } - for _,prefix in pairs(rbl.symbols_prefixes) do + for _, prefix in pairs(rbl.symbols_prefixes) do -- For unknown results... - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', parent = id, group = 'rbl', @@ -1064,11 +1071,11 @@ local function add_rbl(key, rbl, global_opts) rbl.symbol, rbl.is_whitelist, rbl.ignore_whitelist) end else - id = rspamd_config:register_symbol{ + id = rspamd_config:register_symbol { type = 'callback', callback = callback, name = rbl.symbol, - groups = {'rbl'}, + groups = { 'rbl' }, group = 'rbl', score = 0, flags = table.concat(flags_tbl, ',') @@ -1082,7 +1089,6 @@ local function add_rbl(key, rbl, global_opts) end end - rspamd_logger.infox(rspamd_config, 'added rbl rule %s: %s', rbl.symbol, description) lua_util.debugm(N, rspamd_config, 'rule dump for %s: %s', @@ -1095,13 +1101,13 @@ local function add_rbl(key, rbl, global_opts) end if rbl.require_symbols then - for _,dep in ipairs(rbl.require_symbols) do + for _, dep in ipairs(rbl.require_symbols) do rspamd_config:register_dependency(check_sym, dep) end end -- Failure symbol - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', flags = 'nostat', name = rbl.symbol .. '_FAIL', @@ -1114,7 +1120,7 @@ local function add_rbl(key, rbl, global_opts) if s ~= rbl.symbol then -- hack - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', parent = id, name = s, @@ -1145,7 +1151,7 @@ local function add_rbl(key, rbl, global_opts) end if rbl.symbols_prefixes then - for _,prefix in pairs(rbl.symbols_prefixes) do + for _, prefix in pairs(rbl.symbols_prefixes) do process_specific_suffix(prefix .. '_' .. suffix) end else @@ -1155,13 +1161,13 @@ local function add_rbl(key, rbl, global_opts) end if rbl.returncodes then - for s,_ in pairs(rbl.returncodes) do + for s, _ in pairs(rbl.returncodes) do process_return_code(s) end end if rbl.returnbits then - for s,_ in pairs(rbl.returnbits) do + for s, _ in pairs(rbl.returnbits) do process_return_code(s) end end @@ -1200,16 +1206,16 @@ if opts.rules and opts.rbls then opts.rbls = lua_util.override_defaults(opts.rbls, opts.rules) end -if(opts['local_exclude_ip_map'] ~= nil) then +if (opts['local_exclude_ip_map'] ~= nil) then local_exclusions = lua_maps.map_add(N, 'local_exclude_ip_map', 'radix', - 'RBL exclusions map') + 'RBL exclusions map') end -- TODO: this code should be universal for all modules that use selectors to allow -- maps usage from selectors registered for a specific module if type(opts.attached_maps) == 'table' then opts.attached_maps_processed = {} - for i,map in ipairs(opts.attached_maps) do + for i, map in ipairs(opts.attached_maps) do -- Store maps in the configuration table to keep lifetime track opts.attached_maps_processed[i] = lua_maps.map_add_from_ucl(map) if opts.attached_maps_processed[i] == nil then @@ -1218,7 +1224,7 @@ if type(opts.attached_maps) == 'table' then end end -for key,rbl in pairs(opts.rbls) do +for key, rbl in pairs(opts.rbls) do if type(rbl) ~= 'table' or rbl.disabled == true or rbl.enabled == false then rspamd_logger.infox(rspamd_config, 'disable rbl "%s"', key) else @@ -1231,7 +1237,7 @@ for key,rbl in pairs(opts.rbls) do end -- Propagate default options from opts to rule if not rbl.ignore_defaults then - for default_opt_key,_ in pairs(rbl_common.default_options) do + for default_opt_key, _ in pairs(rbl_common.default_options) do local rbl_opt = default_opt_key:sub(#('default_') + 1) if rbl[rbl_opt] == nil then rbl[rbl_opt] = opts[default_opt_key] @@ -1243,15 +1249,19 @@ for key,rbl in pairs(opts.rbls) do rbl.requests_limit = rspamd_config:get_dns_max_requests() end - local res,err = rbl_common.rule_schema:transform(rbl) + local res, err = rbl_common.rule_schema:transform(rbl) if not res then rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED', key, err) else res = rbl_common.convert_checks(res) -- Aliases - if res.return_codes then res.returncodes = res.return_codes end - if res.return_bits then res.returnbits = res.return_bits end + if res.return_codes then + res.returncodes = res.return_codes + end + if res.return_bits then + res.returnbits = res.return_bits + end if not res then rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED', @@ -1272,10 +1282,12 @@ local function rbl_callback_white(task) local ws = task:get_symbol(w) if ws and ws[1] then ws = ws[1] - if not ws.options then ws.options = {} end - for _,opt in ipairs(ws.options) do - local elt,what = opt:match('^([^:]+):([^:]+)') - lua_util.debugm(N, task,'found whitelist from %s: %s(%s)', w, + if not ws.options then + ws.options = {} + end + for _, opt in ipairs(ws.options) do + local elt, what = opt:match('^([^:]+):([^:]+)') + lua_util.debugm(N, task, 'found whitelist from %s: %s(%s)', w, elt, what) if elt and what then whitelisted_elements[elt] = { @@ -1297,22 +1309,22 @@ local function rbl_callback_fin(task) lua_util.debugm(N, task, "finished rbl processing") end -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'callback', callback = rbl_callback_white, name = 'RBL_CALLBACK_WHITE', flags = 'nice,empty,no_squeeze', - groups = {'rbl'}, - augmentations = {string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0)}, + groups = { 'rbl' }, + augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0) }, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'callback', callback = rbl_callback_fin, name = 'RBL_CALLBACK', flags = 'empty,no_squeeze', - groups = {'rbl'}, - augmentations = {string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0)}, + groups = { 'rbl' }, + augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0) }, } for _, w in ipairs(white_symbols) do diff --git a/src/plugins/lua/replies.lua b/src/plugins/lua/replies.lua index 8debe3271..c4df9c97e 100644 --- a/src/plugins/lua/replies.lua +++ b/src/plugins/lua/replies.lua @@ -77,7 +77,9 @@ local function replies_check(task) return real_rcpt_h == stored_rcpt end - if fun.any(filter_predicate, fun.map(function(rcpt) return rcpt.addr or '' end, rcpts)) then + if fun.any(filter_predicate, fun.map(function(rcpt) + return rcpt.addr or '' + end, rcpts)) then lua_util.debugm(N, task, 'reply to %s validated', in_reply_to) return true end @@ -121,12 +123,12 @@ local function replies_check(task) local key = make_key(in_reply_to, settings.key_size, settings.key_prefix) local ret = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_get_cb, --callback - 'GET', -- command - {key} -- arguments + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_get_cb, --callback + 'GET', -- command + { key } -- arguments ) if not ret then @@ -136,7 +138,7 @@ end local function replies_set(task) local function redis_set_cb(err, _, addr) - if err ~=nil then + if err ~= nil then rspamd_logger.errx(task, 'redis_set_cb error when writing data to %s: %s', addr:get_addr(), err) end end @@ -162,14 +164,14 @@ local function replies_set(task) if sender then local sender_hash = make_key(sender:lower(), 8) lua_util.debugm(N, task, 'storing id: %s (%s), reply-to: %s (%s) for replies check', - msg_id, key, sender, sender_hash) + msg_id, key, sender, sender_hash) local ret = lua_redis.redis_make_request(task, redis_params, -- connect params key, -- hash key true, -- is write redis_set_cb, --callback 'PSETEX', -- command - {key, tostring(math.floor(settings['expire'] * 1000)), sender_hash} -- arguments + { key, tostring(math.floor(settings['expire'] * 1000)), sender_hash } -- arguments ) if not ret then rspamd_logger.errx(task, "redis request wasn't scheduled") @@ -181,7 +183,7 @@ end local function replies_check_cookie(task) local function cookie_matched(extra, ts) - local dt = task:get_date{format = 'connect', gmt = true} + local dt = task:get_date { format = 'connect', gmt = true } if dt < ts then rspamd_logger.infox(task, 'ignore cookie as its date is in future') @@ -232,7 +234,7 @@ local function replies_check_cookie(task) extracted_cookie = irt end - local dec_cookie,ts = cr.decrypt_cookie(settings.cookie_key, extracted_cookie) + local dec_cookie, ts = cr.decrypt_cookie(settings.cookie_key, extracted_cookie) if dec_cookie then -- We have something that looks like a cookie @@ -266,8 +268,10 @@ if opts then else -- Cookies mode -- Check key sanity: - local pattern = {'^'} - for i=1,32 do pattern[i + 1] = '[a-zA-Z0-9]' end + local pattern = { '^' } + for i = 1, 32 do + pattern[i + 1] = '[a-zA-Z0-9]' + end pattern[34] = '$' if not settings.cookie_key:match(table.concat(pattern, '')) then rspamd_logger.errx(rspamd_config, diff --git a/src/plugins/lua/rspamd_update.lua b/src/plugins/lua/rspamd_update.lua index 7800d419f..deda0381a 100644 --- a/src/plugins/lua/rspamd_update.lua +++ b/src/plugins/lua/rspamd_update.lua @@ -74,14 +74,14 @@ local function check_version(obj) if rspamd_version('cmp', obj['min_version']) > 0 then ret = false rspamd_logger.errx(rspamd_config, 'updates require at least %s version of rspamd', - obj['min_version']) + obj['min_version']) end end if obj['max_version'] then if rspamd_version('cmp', obj['max_version']) < 0 then ret = false rspamd_logger.errx(rspamd_config, 'updates require maximum %s version of rspamd', - obj['max_version']) + obj['max_version']) end end @@ -92,7 +92,7 @@ local function gen_callback() return function(data) local parser = ucl.parser() - local res,err = parser:parse_string(data) + local res, err = parser:parse_string(data) if not res then rspamd_logger.warnx(rspamd_config, 'cannot parse updates map: ' .. err) @@ -114,7 +114,7 @@ local function gen_callback() end rspamd_logger.infox(rspamd_config, 'loaded new rules with hash "%s"', - h:hex()) + h:hex()) end end @@ -131,7 +131,7 @@ if section and section.rules then end if type(section.rules) ~= 'table' then - section.rules = {section.rules} + section.rules = { section.rules } end fun.each(function(elt) diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua index db2f264e7..3027a541f 100644 --- a/src/plugins/lua/settings.lua +++ b/src/plugins/lua/settings.lua @@ -68,7 +68,7 @@ local function apply_settings(task, to_apply, id, name) end if to_apply.flags and type(to_apply.flags) == 'table' then - for _,fl in ipairs(to_apply.flags) do + for _, fl in ipairs(to_apply.flags) do task:set_flag(fl) end end @@ -77,12 +77,12 @@ local function apply_settings(task, to_apply, id, name) -- Add symbols, specified in the settings if #to_apply.symbols > 0 then -- Array like symbols - for _,val in ipairs(to_apply.symbols) do + for _, val in ipairs(to_apply.symbols) do task:insert_result(val, 1.0) end else -- Object like symbols - for k,v in pairs(to_apply.symbols) do + for k, v in pairs(to_apply.symbols) do if type(v) == 'table' then task:insert_result(k, v.score or 1.0, v.options or {}) elseif tonumber(v) then @@ -120,7 +120,7 @@ local function check_query_settings(task) if query_set then local parser = ucl.parser() - local res,err = parser:parse_text(query_set) + local res, err = parser:parse_text(query_set) if res then if settings_id then rspamd_logger.warnx(task, "both settings-id '%s' and settings headers are presented, ignore settings-id; ", @@ -129,7 +129,7 @@ local function check_query_settings(task) local settings_obj = parser:get_object() -- Treat as low priority - return settings_obj,nil,1 + return settings_obj, nil, 1 else rspamd_logger.errx(task, 'Parse error: %s', err) end @@ -141,8 +141,8 @@ local function check_query_settings(task) if query_maxscore then if settings_id then rspamd_logger.infox(task, "both settings id '%s' and maxscore '%s' headers are presented, merge them; " .. - "settings id has priority", - tostring(settings_id), tostring(query_maxscore)) + "settings id has priority", + tostring(settings_id), tostring(query_maxscore)) end -- We have score limits redefined by request local ms = tonumber(tostring(query_maxscore)) @@ -174,7 +174,7 @@ local function check_query_settings(task) if cached then local elt = cached.settings if elt['whitelist'] then - elt['apply'] = {whitelist = true} + elt['apply'] = { whitelist = true } end if elt.apply then @@ -290,7 +290,9 @@ local function check_settings(task) if elt then local input = elt.extract(task) - if not input then return false end + if not input then + return false + end if elt.check(input) then matched[#matched + 1] = atom @@ -307,7 +309,7 @@ local function check_settings(task) if res and res > 0 then if rule['whitelist'] then - rule['apply'] = {whitelist = true} + rule['apply'] = { whitelist = true } end return rule @@ -317,7 +319,7 @@ local function check_settings(task) end -- Check if we have override as query argument - local query_apply,id_elt,priority = check_query_settings(task) + local query_apply, id_elt, priority = check_query_settings(task) local function maybe_apply_query_settings() if query_apply then @@ -369,9 +371,9 @@ local function check_settings(task) -- Match rules according their order local applied = false - for pri = max_pri,min_pri,-1 do + for pri = max_pri, min_pri, -1 do if not applied and settings[pri] then - for _,s in ipairs(settings[pri]) do + for _, s in ipairs(settings[pri]) do local matched = {} local result = check_specific_setting(s.rule, matched) @@ -434,18 +436,18 @@ end local function convert_to_table(chk_elt, out) if type(chk_elt) == 'string' then - return {out} + return { out } end return out end local function gen_settings_external_cb(name) - return function (result, err_or_data, code, task) + return function(result, err_or_data, code, task) if result then local parser = ucl.parser() - local res,ucl_err = parser:parse_text(err_or_data) + local res, ucl_err = parser:parse_text(err_or_data) if not res then rspamd_logger.warnx(task, 'cannot parse settings from the external map %s: %s', name, ucl_err) @@ -467,7 +469,7 @@ local function process_ip_condition(ip) local out = {} if type(ip) == "table" then - for _,v in ipairs(ip) do + for _, v in ipairs(ip) do table.insert(out, process_ip_condition(v)) end elseif type(ip) == "string" then @@ -511,7 +513,7 @@ end local function process_email_condition(addr) local out = {} if type(addr) == "table" then - for _,v in ipairs(addr) do + for _, v in ipairs(addr) do table.insert(out, process_email_condition(v)) end elseif type(addr) == "string" then @@ -558,7 +560,7 @@ end local function process_string_condition(addr) local out = {} if type(addr) == "table" then - for _,v in ipairs(addr) do + for _, v in ipairs(addr) do table.insert(out, process_string_condition(v)) end elseif type(addr) == "string" then @@ -613,7 +615,9 @@ end -- Used to create a checking closure: if value matches expected somehow, return true local function gen_check_closure(expected, check_func) return function(value) - if not value then return false end + if not value then + return false + end if type(value) == 'function' then value = value() @@ -622,7 +626,9 @@ local function gen_check_closure(expected, check_func) if value then if not check_func then - check_func = function(a, b) return a == b end + check_func = function(a, b) + return a == b + end end local ret @@ -663,7 +669,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) check = gen_check_closure(convert_to_table(elt.ip, ips_table), check_ip_setting), extract = function(task) local ip = task:get_from_ip() - if ip and ip:is_valid() then return ip end + if ip and ip:is_valid() then + return ip + end return nil end, } @@ -680,7 +688,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) check = gen_check_closure(ips_map, check_map_setting), extract = function(task) local ip = task:get_from_ip() - if ip and ip:is_valid() then return ip end + if ip and ip:is_valid() then + return ip + end return nil end, } @@ -698,7 +708,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) check_ip_setting), extract = function(task) local ip = task:get_client_ip() - if ip:is_valid() then return ip end + if ip:is_valid() then + return ip + end return nil end, } @@ -715,7 +727,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) check = gen_check_closure(ips_map, check_map_setting), extract = function(task) local ip = task:get_client_ip() - if ip and ip:is_valid() then return ip end + if ip and ip:is_valid() then + return ip + end return nil end, } @@ -835,7 +849,12 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) lua_util.debugm(N, rspamd_config, 'added authenticated condition to "%s"', name) checks.authenticated = { - check = function(value) if value then return true end return false end, + check = function(value) + if value then + return true + end + return false + end, extract = function(task) return task:get_user() end @@ -846,7 +865,12 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) lua_util.debugm(N, rspamd_config, 'added local condition to "%s"', name) checks['local'] = { - check = function(value) if value then return true end return false end, + check = function(value) + if value then + return true + end + return false + end, extract = function(task) local ip = task:get_from_ip() if not ip or not ip:is_valid() then @@ -875,9 +899,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) checks[full_key] = cond -- Try numeric key - for i=1,1000 do + for i = 1, 1000 do local num_key = generic .. ':' .. tostring(i) - if not checks[num_key] then + if not checks[num_key] then checks[num_key] = cond aliases[num_key] = true break @@ -911,7 +935,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) if re then local cond = { check = function(values) - return fun.any(function(c) return re:match(c) end, values) + return fun.any(function(c) + return re:match(c) + end, values) end, extract = extractor_func(k), } @@ -923,7 +949,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) elseif type(v) == 'boolean' then local cond = { check = function(values) - if #values == 0 then return (not v) end + if #values == 0 then + return (not v) + end return v end, extract = extractor_func(k), @@ -943,7 +971,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) process_header_elt('request_header', function(hname) return function(task) local rh = task:get_request_header(hname) - if rh then return {rh} end + if rh then + return { rh } + end return {} end end) @@ -951,7 +981,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) return function(task) local rh = task:get_header_full(hname) if rh then - return fun.totable(fun.map(function(h) return h.decoded end, rh)) + return fun.totable(fun.map(function(h) + return h.decoded + end, rh)) end return {} end @@ -987,7 +1019,7 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) -- Count checks and create Rspamd expression from a set of rules local nchecks = 0 - for k,_ in pairs(checks) do + for k, _ in pairs(checks) do if not aliases[k] then nchecks = nchecks + 1 end @@ -1000,10 +1032,14 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) -- Here we get all keys and concatenate them with '&&' local s = ' && ' -- By De Morgan laws - if inverse then s = ' || ' end + if inverse then + s = ' || ' + end -- Exclude aliases and join all checks by key local expr_str = table.concat(lua_util.keys(fun.filter( - function(k, _) return not aliases[k] end, + function(k, _) + return not aliases[k] + end, checks)), s) if inverse then @@ -1030,7 +1066,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) rspamd_logger.errx(rspamd_config, 'use of undefined element "%s" when parsing settings expression, known checks: %s', - atom, table.concat(fun.totable(fun.map(function(k, _) return k end, checks)), ',')) + atom, table.concat(fun.totable(fun.map(function(k, _) + return k + end, checks)), ',')) return nil end @@ -1123,7 +1161,7 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) -- In fact, it is useless and evil but who cares... if elt.apply and elt.apply.symbols then -- Register virtual symbols - for k,v in pairs(elt.apply.symbols) do + for k, v in pairs(elt.apply.symbols) do local rtb = { type = 'virtual', parent = module_sym_id, @@ -1153,33 +1191,39 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) settings_initialized = false -- filter trash in the input local ft = fun.filter( - function(_, elt) - if type(elt) == "table" then - return true - end - return false - end, tbl) + function(_, elt) + if type(elt) == "table" then + return true + end + return false + end, tbl) -- clear all settings max_pri = 0 local nrules = 0 - for k in pairs(settings) do settings[k]={} end + for k in pairs(settings) do + settings[k] = {} + end -- fill new settings by priority fun.for_each(function(k, v) local pri = get_priority(v) - if pri > max_pri then max_pri = pri end + if pri > max_pri then + max_pri = pri + end if not settings[pri] then settings[pri] = {} end local s = process_setting_elt(k, v) if s then - table.insert(settings[pri], {name = k, rule = s}) + table.insert(settings[pri], { name = k, rule = s }) nrules = nrules + 1 end end, ft) -- sort settings with equal priorities in alphabetical order - for pri,_ in pairs(settings) do - table.sort(settings[pri], function(a,b) return a.name < b.name end) + for pri, _ in pairs(settings) do + table.sort(settings[pri], function(a, b) + return a.name < b.name + end) end settings_initialized = true @@ -1194,7 +1238,7 @@ local settings_map_pool local function process_settings_map(map_text) local parser = ucl.parser() - local res,err = parser:parse_text(map_text) + local res, err = parser:parse_text(map_text) if not res then rspamd_logger.warnx(rspamd_config, 'cannot parse settings map: ' .. err) @@ -1226,14 +1270,14 @@ local function gen_redis_callback(handler, id) for _, d in ipairs(data) do if type(d) == 'string' then local parser = ucl.parser() - local res,ucl_err = parser:parse_text(d) + local res, ucl_err = parser:parse_text(d) if not res then rspamd_logger.warnx(rspamd_config, 'cannot parse settings from redis: %s', - ucl_err) + ucl_err) else local obj = parser:get_object() rspamd_logger.infox(task, "<%1> apply settings according to redis rule %2", - task:get_message_id(), id) + task:get_message_id(), id) apply_settings(task, obj, nil, 'redis') break end @@ -1253,17 +1297,17 @@ local function gen_redis_callback(handler, id) if type(key) == 'table' then keys = key else - keys = {key} + keys = { key } end key = keys[1] - local ret,_,_ = rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_settings_cb, --callback - 'MGET', -- command - keys -- arguments + local ret, _, _ = rspamd_redis_make_request(task, + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_settings_cb, --callback + 'MGET', -- command + keys -- arguments ) if not ret then rspamd_logger.errx(task, 'Redis MGET failed: %s', ret) @@ -1279,17 +1323,17 @@ if redis_section then if redis_params then local handlers = redis_section.handlers - for id,h in pairs(handlers) do - local chunk,err = load(h) + for id, h in pairs(handlers) do + local chunk, err = load(h) if not chunk then rspamd_logger.errx(rspamd_config, 'Cannot load handler from string: %s', tostring(err)) else - local res,func = pcall(chunk) + local res, func = pcall(chunk) if not res then rspamd_logger.errx(rspamd_config, 'Cannot add handler from string: %s', - tostring(func)) + tostring(func)) else redis_key_handlers[id] = func end @@ -1304,7 +1348,7 @@ if redis_section then callback = gen_redis_callback(h, id), priority = lua_util.symbols_priorities.top, flags = 'empty,nostat', - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}, + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }, }) end, redis_key_handlers) end @@ -1337,7 +1381,7 @@ elseif set_section and type(set_section) == "table" then -- registered BEFORE settings plugin. Otherwise, we can have inconsistent settings expressions fun.each(function(_, elt) if elt.register_symbols then - for k,v in pairs(elt.register_symbols) do + for k, v in pairs(elt.register_symbols) do local rtb = { type = 'virtual', parent = module_sym_id, @@ -1347,7 +1391,7 @@ elseif set_section and type(set_section) == "table" then elseif type(k) == 'string' then rtb.name = k if type(v) == 'table' then - for kk,vv in pairs(v) do + for kk, vv in pairs(v) do -- Enrich table wih extra values rtb[kk] = vv end @@ -1358,7 +1402,7 @@ elseif set_section and type(set_section) == "table" then end if elt.apply and elt.apply.symbols then -- Register virtual symbols - for k,v in pairs(elt.apply.symbols) do + for k, v in pairs(elt.apply.symbols) do local rtb = { type = 'virtual', parent = module_sym_id, @@ -1372,7 +1416,7 @@ elseif set_section and type(set_section) == "table" then end end end, - -- Include only settings, exclude all maps + -- Include only settings, exclude all maps fun.filter( function(_, elt) if type(elt) == "table" then @@ -1382,7 +1426,7 @@ elseif set_section and type(set_section) == "table" then end, set_section) ) - rspamd_config:add_post_init(function () + rspamd_config:add_post_init(function() process_settings_table(set_section, true, settings_map_pool, true) end, 100) end diff --git a/src/plugins/lua/spamassassin.lua b/src/plugins/lua/spamassassin.lua index c98203fb6..3ea794495 100644 --- a/src/plugins/lua/spamassassin.lua +++ b/src/plugins/lua/spamassassin.lua @@ -159,7 +159,7 @@ end local ffi if type(jit) == 'table' then ffi = require("ffi") - ffi.cdef[[ + ffi.cdef [[ int rspamd_re_cache_type_from_string (const char *str); int rspamd_re_cache_process_ffi (void *ptask, void *pre, @@ -207,7 +207,7 @@ local function handle_header_def(hline, cur_rule) -- Check if an re is an ordinary re local ordinary = true - for _,h in ipairs(hdrs) do + for _, h in ipairs(hdrs) do if h == 'ALL' or h == 'ALL:raw' then ordinary = false cur_rule['type'] = 'function' @@ -239,71 +239,71 @@ local function handle_header_def(hline, cur_rule) end fun.each(function(func) - if func == 'addr' then - cur_param['function'] = function(str) - local addr_parsed = util.parse_mail_address(str) - local ret = {} - if addr_parsed then - for _,elt in ipairs(addr_parsed) do - if elt['addr'] then - table.insert(ret, elt['addr']) - end + if func == 'addr' then + cur_param['function'] = function(str) + local addr_parsed = util.parse_mail_address(str) + local ret = {} + if addr_parsed then + for _, elt in ipairs(addr_parsed) do + if elt['addr'] then + table.insert(ret, elt['addr']) end end - - return ret end - elseif func == 'name' then - cur_param['function'] = function(str) - local addr_parsed = util.parse_mail_address(str) - local ret = {} - if addr_parsed then - for _,elt in ipairs(addr_parsed) do - if elt['name'] then - table.insert(ret, elt['name']) - end - end - end - return ret - end - elseif func == 'raw' then - cur_param['raw'] = true - elseif func == 'case' then - cur_param['strong'] = true - else - rspamd_logger.warnx(rspamd_config, 'Function %1 is not supported in %2', - func, cur_rule['symbol']) + return ret end - end, fun.tail(args)) - - local function split_hdr_param(param, headers) - for _,hh in ipairs(headers) do - local nparam = {} - for k,v in pairs(param) do - if k ~= 'header' then - nparam[k] = v + elseif func == 'name' then + cur_param['function'] = function(str) + local addr_parsed = util.parse_mail_address(str) + local ret = {} + if addr_parsed then + for _, elt in ipairs(addr_parsed) do + if elt['name'] then + table.insert(ret, elt['name']) + end end end - nparam['header'] = hh - table.insert(hdr_params, nparam) + return ret end - end - -- Some header rules require splitting to check of multiple headers - if cur_param['header'] == 'MESSAGEID' then - -- Special case for spamassassin - ordinary = false - split_hdr_param(cur_param, { - 'Message-ID', - 'X-Message-ID', - 'Resent-Message-ID'}) - elseif cur_param['header'] == 'ToCc' then - ordinary = false - split_hdr_param(cur_param, { 'To', 'Cc', 'Bcc' }) + elseif func == 'raw' then + cur_param['raw'] = true + elseif func == 'case' then + cur_param['strong'] = true else - table.insert(hdr_params, cur_param) + rspamd_logger.warnx(rspamd_config, 'Function %1 is not supported in %2', + func, cur_rule['symbol']) end + end, fun.tail(args)) + + local function split_hdr_param(param, headers) + for _, hh in ipairs(headers) do + local nparam = {} + for k, v in pairs(param) do + if k ~= 'header' then + nparam[k] = v + end + end + + nparam['header'] = hh + table.insert(hdr_params, nparam) + end + end + -- Some header rules require splitting to check of multiple headers + if cur_param['header'] == 'MESSAGEID' then + -- Special case for spamassassin + ordinary = false + split_hdr_param(cur_param, { + 'Message-ID', + 'X-Message-ID', + 'Resent-Message-ID' }) + elseif cur_param['header'] == 'ToCc' then + ordinary = false + split_hdr_param(cur_param, { 'To', 'Cc', 'Bcc' }) + else + table.insert(hdr_params, cur_param) + end end cur_rule['ordinary'] = ordinary @@ -311,7 +311,6 @@ local function handle_header_def(hline, cur_rule) end end - local function freemail_search(input) local res = 0 local function trie_callback(number, pos) @@ -328,19 +327,19 @@ end local function gen_eval_rule(arg) local eval_funcs = { - {'check_freemail_from', function(task) - local from = task:get_from('mime') - if from and from[1] then - return freemail_search(string.lower(from[1]['addr'])) - end - return 0 - end}, - {'check_freemail_replyto', + { 'check_freemail_from', function(task) + local from = task:get_from('mime') + if from and from[1] then + return freemail_search(string.lower(from[1]['addr'])) + end + return 0 + end }, + { 'check_freemail_replyto', function(task) return freemail_search(task:get_header('Reply-To')) end }, - {'check_freemail_header', + { 'check_freemail_header', function(task, remain) -- Remain here contains one or two args: header and regexp to match local larg = string.match(remain, "^%(%s*['\"]([^%s]+)['\"]%s*%)$") @@ -353,7 +352,9 @@ local function gen_eval_rule(arg) local h if larg == 'EnvelopeFrom' then h = task:get_from('smtp') - if h then h = h[1]['addr'] end + if h then + h = h[1]['addr'] + end else h = task:get_header(larg) end @@ -382,7 +383,7 @@ local function gen_eval_rule(arg) }, { 'check_for_missing_to_header', - function (task) + function(task) local th = task:get_recipients('mime') if not th or #th == 0 then return 1 @@ -398,16 +399,20 @@ local function gen_eval_rule(arg) local rh_parsed = task:get_received_headers() local rh_cnt = 0 - if rh_mime then rh_cnt = #rh_mime end + if rh_mime then + rh_cnt = #rh_mime + end local parsed_cnt = 0 - if rh_parsed then parsed_cnt = #rh_parsed end + if rh_parsed then + parsed_cnt = #rh_parsed + end return rh_cnt - parsed_cnt end }, { 'check_for_shifted_date', - function (task, remain) + function(task, remain) -- Remain here contains two args: start and end hours shift local matches = internal_regexp['date_shift']:search(remain, true, true) if matches and matches[1] then @@ -426,8 +431,8 @@ local function gen_eval_rule(arg) end -- Now get the difference between Date and message received date - local dm = task:get_date { format = 'message', gmt = true} - local dt = task:get_date { format = 'connect', gmt = true} + local dm = task:get_date { format = 'message', gmt = true } + local dt = task:get_date { format = 'connect', gmt = true } local diff = dm - dt if (max_diff == 0 and diff >= min_diff) or @@ -449,7 +454,7 @@ local function gen_eval_rule(arg) if larg == 'mime_attachment' then local parts = task:get_parts() if parts then - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do if p:get_filename() then return 1 end @@ -507,7 +512,7 @@ local function gen_eval_rule(arg) function(task) local rcpt = task:get_recipients('mime') if rcpt then - for _,r in ipairs(rcpt) do + for _, r in ipairs(rcpt) do if sa_lists['to_blacklist'][string.lower(r['addr'])] then return 1 end @@ -522,7 +527,7 @@ local function gen_eval_rule(arg) function(task) local rcpt = task:get_recipients('mime') if rcpt then - for _,r in ipairs(rcpt) do + for _, r in ipairs(rcpt) do if sa_lists['to_whitelist'][string.lower(r['addr'])] then return 1 end @@ -537,7 +542,7 @@ local function gen_eval_rule(arg) function(task, remain) local tp = task:get_text_parts() - for _,p in ipairs(tp) do + for _, p in ipairs(tp) do if p:is_html() then local hc = p:get_html() @@ -552,9 +557,9 @@ local function gen_eval_rule(arg) } } - for _,f in ipairs(eval_funcs) do + for _, f in ipairs(eval_funcs) do local pat = string.format('^%s', f[1]) - local first,last = string.find(arg, pat) + local first, last = string.find(arg, pat) if first then local func_arg = string.sub(arg, last + 1) @@ -572,10 +577,11 @@ local function maybe_parse_sa_function(line) arg = elts[2] lua_util.debugm(N, rspamd_config, 'trying to parse SA function %1 with args %2', - elts[1], elts[2]) + elts[1], elts[2]) local substitutions = { - {'^exists:', - function(task) -- filter + { '^exists:', + function(task) + -- filter local hdrs_check if arg == 'MESSAGEID' then hdrs_check = { @@ -586,10 +592,10 @@ local function maybe_parse_sa_function(line) elseif arg == 'ToCc' then hdrs_check = { 'To', 'Cc', 'Bcc' } else - hdrs_check = {arg} + hdrs_check = { arg } end - for _,h in ipairs(hdrs_check) do + for _, h in ipairs(hdrs_check) do if task:has_header(h) then return 1 end @@ -597,7 +603,7 @@ local function maybe_parse_sa_function(line) return 0 end, }, - {'^eval:', + { '^eval:', function(task) local func = func_cache[arg] if not func then @@ -607,7 +613,7 @@ local function maybe_parse_sa_function(line) if not func then rspamd_logger.errx(task, 'cannot find appropriate eval rule for function %1', - arg) + arg) else return func(task) end @@ -617,7 +623,7 @@ local function maybe_parse_sa_function(line) }, } - for _,s in ipairs(substitutions) do + for _, s in ipairs(substitutions) do if string.find(line, s[1]) then return s[2] end @@ -664,26 +670,26 @@ local function process_sa_conf(f) local valid_rule = false local function insert_cur_rule() - if cur_rule['type'] ~= 'meta' and cur_rule['publish'] then - -- Create meta rule from this rule - local nsym = '__fake' .. cur_rule['symbol'] - local nrule = { - type = 'meta', - symbol = cur_rule['symbol'], - score = cur_rule['score'], - meta = nsym, - description = cur_rule['description'], - } - rules[nrule['symbol']] = nrule - cur_rule['symbol'] = nsym - end - -- We have previous rule valid - if not cur_rule['symbol'] then - rspamd_logger.errx(rspamd_config, 'bad rule definition: %1', cur_rule) - end - rules[cur_rule['symbol']] = cur_rule - cur_rule = {} - valid_rule = false + if cur_rule['type'] ~= 'meta' and cur_rule['publish'] then + -- Create meta rule from this rule + local nsym = '__fake' .. cur_rule['symbol'] + local nrule = { + type = 'meta', + symbol = cur_rule['symbol'], + score = cur_rule['score'], + meta = nsym, + description = cur_rule['description'], + } + rules[nrule['symbol']] = nrule + cur_rule['symbol'] = nsym + end + -- We have previous rule valid + if not cur_rule['symbol'] then + rspamd_logger.errx(rspamd_config, 'bad rule definition: %1', cur_rule) + end + rules[cur_rule['symbol']] = cur_rule + cur_rule = {} + valid_rule = false end local function parse_score(words) @@ -706,359 +712,370 @@ local function process_sa_conf(f) local skip_to_endif = false local if_nested = 0 for l in f:lines() do - (function () - l = lua_util.rspamd_str_trim(l) - -- Replace bla=~/re/ with bla =~ /re/ (#2372) - l = l:gsub('([^%s])%s*([=!]~)%s*([^%s])', '%1 %2 %3') + (function() + l = lua_util.rspamd_str_trim(l) + -- Replace bla=~/re/ with bla =~ /re/ (#2372) + l = l:gsub('([^%s])%s*([=!]~)%s*([^%s])', '%1 %2 %3') - if string.len(l) == 0 or string.sub(l, 1, 1) == '#' then - return - end + if string.len(l) == 0 or string.sub(l, 1, 1) == '#' then + return + end - -- Unbalanced if/endif - if if_nested < 0 then if_nested = 0 end - if skip_to_endif then - if string.match(l, '^endif') then - if_nested = if_nested - 1 + -- Unbalanced if/endif + if if_nested < 0 then + if_nested = 0 + end + if skip_to_endif then + if string.match(l, '^endif') then + if_nested = if_nested - 1 - if if_nested == 0 then + if if_nested == 0 then + skip_to_endif = false + end + elseif string.match(l, '^if') then + if_nested = if_nested + 1 + elseif string.match(l, '^else') then + -- Else counterpart for if skip_to_endif = false end - elseif string.match(l, '^if') then - if_nested = if_nested + 1 - elseif string.match(l, '^else') then - -- Else counterpart for if - skip_to_endif = false - end - return - else - if string.match(l, '^ifplugin') then - local ls = split(l) + return + else + if string.match(l, '^ifplugin') then + local ls = split(l) - if not fun.any(function(pl) - if pl == ls[2] then return true end + if not fun.any(function(pl) + if pl == ls[2] then + return true + end + return false + end, known_plugins) then + skip_to_endif = true + end + if_nested = if_nested + 1 + elseif string.match(l, '^if !plugin%(') then + local pname = string.match(l, '^if !plugin%(([A-Za-z:]+)%)') + if fun.any(function(pl) + if pl == pname then + return true + end return false - end, known_plugins) then + end, known_plugins) then + skip_to_endif = true + end + if_nested = if_nested + 1 + elseif string.match(l, '^if') then + -- Unknown if + skip_to_endif = true + if_nested = if_nested + 1 + elseif string.match(l, '^else') then + -- Else counterpart for if skip_to_endif = true + elseif string.match(l, '^endif') then + if_nested = if_nested - 1 end - if_nested = if_nested + 1 - elseif string.match(l, '^if !plugin%(') then - local pname = string.match(l, '^if !plugin%(([A-Za-z:]+)%)') - if fun.any(function(pl) - if pl == pname then return true end - return false - end, known_plugins) then - skip_to_endif = true - end - if_nested = if_nested + 1 - elseif string.match(l, '^if') then - -- Unknown if - skip_to_endif = true - if_nested = if_nested + 1 - elseif string.match(l, '^else') then - -- Else counterpart for if - skip_to_endif = true - elseif string.match(l, '^endif') then - if_nested = if_nested - 1 - end - end - - -- Skip comments - local words = fun.totable(fun.take_while( - function(w) return string.sub(w, 1, 1) ~= '#' end, - fun.filter(function(w) - return w ~= "" end, - fun.iter(split(l))))) - - if words[1] == "header" or words[1] == 'mimeheader' then - -- header SYMBOL Header ~= /regexp/ - if valid_rule then - insert_cur_rule() end - if words[4] and (words[4] == '=~' or words[4] == '!~') then - cur_rule['type'] = 'header' - cur_rule['symbol'] = words[2] - if words[4] == '!~' then - cur_rule['not'] = true + -- Skip comments + local words = fun.totable(fun.take_while( + function(w) + return string.sub(w, 1, 1) ~= '#' + end, + fun.filter(function(w) + return w ~= "" + end, + fun.iter(split(l))))) + + if words[1] == "header" or words[1] == 'mimeheader' then + -- header SYMBOL Header ~= /regexp/ + if valid_rule then + insert_cur_rule() end + if words[4] and (words[4] == '=~' or words[4] == '!~') then + cur_rule['type'] = 'header' + cur_rule['symbol'] = words[2] - cur_rule['re_expr'] = words_to_re(words, 4) - local unset_comp = string.find(cur_rule['re_expr'], '%s+%[if%-unset:') - if unset_comp then - -- We have optional part that needs to be processed - local unset = string.match(string.sub(cur_rule['re_expr'], unset_comp), - '%[if%-unset:%s*([^%]%s]+)]') - cur_rule['unset'] = unset - -- Cut it down - cur_rule['re_expr'] = string.sub(cur_rule['re_expr'], 1, unset_comp - 1) - end + if words[4] == '!~' then + cur_rule['not'] = true + end - cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) + cur_rule['re_expr'] = words_to_re(words, 4) + local unset_comp = string.find(cur_rule['re_expr'], '%s+%[if%-unset:') + if unset_comp then + -- We have optional part that needs to be processed + local unset = string.match(string.sub(cur_rule['re_expr'], unset_comp), + '%[if%-unset:%s*([^%]%s]+)]') + cur_rule['unset'] = unset + -- Cut it down + cur_rule['re_expr'] = string.sub(cur_rule['re_expr'], 1, unset_comp - 1) + end - if not cur_rule['re'] then - rspamd_logger.warnx(rspamd_config, "Cannot parse regexp '%1' for %2", - cur_rule['re_expr'], cur_rule['symbol']) - else - cur_rule['re']:set_max_hits(1) - handle_header_def(words[3], cur_rule) - end + cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) - if cur_rule['unset'] then - cur_rule['ordinary'] = false - end + if not cur_rule['re'] then + rspamd_logger.warnx(rspamd_config, "Cannot parse regexp '%1' for %2", + cur_rule['re_expr'], cur_rule['symbol']) + else + cur_rule['re']:set_max_hits(1) + handle_header_def(words[3], cur_rule) + end - if words[1] == 'mimeheader' then - cur_rule['mime'] = true - else - cur_rule['mime'] = false - end + if cur_rule['unset'] then + cur_rule['ordinary'] = false + end - if cur_rule['re'] and cur_rule['symbol'] and - (cur_rule['header'] or cur_rule['function']) then - valid_rule = true - cur_rule['re']:set_max_hits(1) - if cur_rule['header'] and cur_rule['ordinary'] then - for _,h in ipairs(cur_rule['header']) do - if type(h) == 'string' then - if cur_rule['mime'] then - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'mimeheader', - header = h, - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - else - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'header', - header = h, - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - end - else - h['mime'] = cur_rule['mime'] - if cur_rule['mime'] then - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'mimeheader', - header = h['header'], - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - else - if h['raw'] then + if words[1] == 'mimeheader' then + cur_rule['mime'] = true + else + cur_rule['mime'] = false + end + + if cur_rule['re'] and cur_rule['symbol'] and + (cur_rule['header'] or cur_rule['function']) then + valid_rule = true + cur_rule['re']:set_max_hits(1) + if cur_rule['header'] and cur_rule['ordinary'] then + for _, h in ipairs(cur_rule['header']) do + if type(h) == 'string' then + if cur_rule['mime'] then rspamd_config:register_regexp({ re = cur_rule['re'], - type = 'rawheader', - header = h['header'], + type = 'mimeheader', + header = h, pcre_only = is_pcre_only(cur_rule['symbol']), }) else rspamd_config:register_regexp({ re = cur_rule['re'], type = 'header', + header = h, + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + end + else + h['mime'] = cur_rule['mime'] + if cur_rule['mime'] then + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'mimeheader', header = h['header'], pcre_only = is_pcre_only(cur_rule['symbol']), }) + else + if h['raw'] then + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'rawheader', + header = h['header'], + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + else + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'header', + header = h['header'], + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + end end end end + cur_rule['re']:set_limit(match_limit) + cur_rule['re']:set_max_hits(1) end - cur_rule['re']:set_limit(match_limit) - cur_rule['re']:set_max_hits(1) end - end - else - -- Maybe we know the function and can convert it - local args = words_to_re(words, 2) - local func = maybe_parse_sa_function(args) - - if func then - cur_rule['type'] = 'function' - cur_rule['symbol'] = words[2] - cur_rule['function'] = func - valid_rule = true else - rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + -- Maybe we know the function and can convert it + local args = words_to_re(words, 2) + local func = maybe_parse_sa_function(args) + + if func then + cur_rule['type'] = 'function' + cur_rule['symbol'] = words[2] + cur_rule['function'] = func + valid_rule = true + else + rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + end end - end - elseif words[1] == "body" then - -- body SYMBOL /regexp/ - if valid_rule then - insert_cur_rule() - end - - cur_rule['symbol'] = words[2] - if words[3] and (string.sub(words[3], 1, 1) == '/' - or string.sub(words[3], 1, 1) == 'm') then - cur_rule['type'] = 'sabody' - cur_rule['re_expr'] = words_to_re(words, 2) - cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) - if cur_rule['re'] then - - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'sabody', - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - valid_rule = true - cur_rule['re']:set_limit(match_limit) - cur_rule['re']:set_max_hits(1) + elseif words[1] == "body" then + -- body SYMBOL /regexp/ + if valid_rule then + insert_cur_rule() end - else - -- might be function - local args = words_to_re(words, 2) - local func = maybe_parse_sa_function(args) - if func then - cur_rule['type'] = 'function' - cur_rule['symbol'] = words[2] - cur_rule['function'] = func - valid_rule = true + cur_rule['symbol'] = words[2] + if words[3] and (string.sub(words[3], 1, 1) == '/' + or string.sub(words[3], 1, 1) == 'm') then + cur_rule['type'] = 'sabody' + cur_rule['re_expr'] = words_to_re(words, 2) + cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) + if cur_rule['re'] then + + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'sabody', + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + valid_rule = true + cur_rule['re']:set_limit(match_limit) + cur_rule['re']:set_max_hits(1) + end else - rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + -- might be function + local args = words_to_re(words, 2) + local func = maybe_parse_sa_function(args) + + if func then + cur_rule['type'] = 'function' + cur_rule['symbol'] = words[2] + cur_rule['function'] = func + valid_rule = true + else + rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + end end - end - elseif words[1] == "rawbody" then - -- body SYMBOL /regexp/ - if valid_rule then - insert_cur_rule() - end - - cur_rule['symbol'] = words[2] - if words[3] and (string.sub(words[3], 1, 1) == '/' - or string.sub(words[3], 1, 1) == 'm') then - cur_rule['type'] = 'sarawbody' - cur_rule['re_expr'] = words_to_re(words, 2) - cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) - if cur_rule['re'] then - - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'sarawbody', - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - valid_rule = true - cur_rule['re']:set_limit(match_limit) - cur_rule['re']:set_max_hits(1) + elseif words[1] == "rawbody" then + -- body SYMBOL /regexp/ + if valid_rule then + insert_cur_rule() end - else - -- might be function - local args = words_to_re(words, 2) - local func = maybe_parse_sa_function(args) - if func then - cur_rule['type'] = 'function' - cur_rule['symbol'] = words[2] - cur_rule['function'] = func - valid_rule = true + cur_rule['symbol'] = words[2] + if words[3] and (string.sub(words[3], 1, 1) == '/' + or string.sub(words[3], 1, 1) == 'm') then + cur_rule['type'] = 'sarawbody' + cur_rule['re_expr'] = words_to_re(words, 2) + cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) + if cur_rule['re'] then + + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'sarawbody', + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + valid_rule = true + cur_rule['re']:set_limit(match_limit) + cur_rule['re']:set_max_hits(1) + end else - rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + -- might be function + local args = words_to_re(words, 2) + local func = maybe_parse_sa_function(args) + + if func then + cur_rule['type'] = 'function' + cur_rule['symbol'] = words[2] + cur_rule['function'] = func + valid_rule = true + else + rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + end + end + elseif words[1] == "full" then + -- body SYMBOL /regexp/ + if valid_rule then + insert_cur_rule() end - end - elseif words[1] == "full" then - -- body SYMBOL /regexp/ - if valid_rule then - insert_cur_rule() - end - cur_rule['symbol'] = words[2] + cur_rule['symbol'] = words[2] - if words[3] and (string.sub(words[3], 1, 1) == '/' - or string.sub(words[3], 1, 1) == 'm') then - cur_rule['type'] = 'message' + if words[3] and (string.sub(words[3], 1, 1) == '/' + or string.sub(words[3], 1, 1) == 'm') then + cur_rule['type'] = 'message' + cur_rule['re_expr'] = words_to_re(words, 2) + cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) + cur_rule['raw'] = true + if cur_rule['re'] then + valid_rule = true + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'body', + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + cur_rule['re']:set_limit(match_limit) + cur_rule['re']:set_max_hits(1) + end + else + -- might be function + local args = words_to_re(words, 2) + local func = maybe_parse_sa_function(args) + + if func then + cur_rule['type'] = 'function' + cur_rule['symbol'] = words[2] + cur_rule['function'] = func + valid_rule = true + else + rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + end + end + elseif words[1] == "uri" then + -- uri SYMBOL /regexp/ + if valid_rule then + insert_cur_rule() + end + cur_rule['type'] = 'uri' + cur_rule['symbol'] = words[2] cur_rule['re_expr'] = words_to_re(words, 2) cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) - cur_rule['raw'] = true - if cur_rule['re'] then + if cur_rule['re'] and cur_rule['symbol'] then valid_rule = true rspamd_config:register_regexp({ re = cur_rule['re'], - type = 'body', + type = 'url', pcre_only = is_pcre_only(cur_rule['symbol']), }) cur_rule['re']:set_limit(match_limit) cur_rule['re']:set_max_hits(1) end - else - -- might be function - local args = words_to_re(words, 2) - local func = maybe_parse_sa_function(args) - - if func then - cur_rule['type'] = 'function' - cur_rule['symbol'] = words[2] - cur_rule['function'] = func - valid_rule = true - else - rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + elseif words[1] == "meta" then + -- meta SYMBOL expression + if valid_rule then + insert_cur_rule() end - end - elseif words[1] == "uri" then - -- uri SYMBOL /regexp/ - if valid_rule then - insert_cur_rule() - end - cur_rule['type'] = 'uri' - cur_rule['symbol'] = words[2] - cur_rule['re_expr'] = words_to_re(words, 2) - cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) - if cur_rule['re'] and cur_rule['symbol'] then - valid_rule = true - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'url', - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - cur_rule['re']:set_limit(match_limit) - cur_rule['re']:set_max_hits(1) - end - elseif words[1] == "meta" then - -- meta SYMBOL expression - if valid_rule then - insert_cur_rule() - end - cur_rule['type'] = 'meta' - cur_rule['symbol'] = words[2] - cur_rule['meta'] = words_to_re(words, 2) - if cur_rule['meta'] and cur_rule['symbol'] - and cur_rule['meta'] ~= '0' then + cur_rule['type'] = 'meta' + cur_rule['symbol'] = words[2] + cur_rule['meta'] = words_to_re(words, 2) + if cur_rule['meta'] and cur_rule['symbol'] + and cur_rule['meta'] ~= '0' then valid_rule = true - end - elseif words[1] == "describe" and valid_rule then - cur_rule['description'] = words_to_re(words, 2) - elseif words[1] == "score" then - scores[words[2]] = parse_score(words) - elseif words[1] == 'freemail_domains' then - fun.each(function(dom) + end + elseif words[1] == "describe" and valid_rule then + cur_rule['description'] = words_to_re(words, 2) + elseif words[1] == "score" then + scores[words[2]] = parse_score(words) + elseif words[1] == 'freemail_domains' then + fun.each(function(dom) table.insert(freemail_domains, '@' .. dom) end, fun.drop_n(1, words)) - elseif words[1] == 'blacklist_from' then - sa_lists['from_blacklist'][words[2]] = 1 - sa_lists['elts'] = sa_lists['elts'] + 1 - elseif words[1] == 'whitelist_from' then - sa_lists['from_whitelist'][words[2]] = 1 - sa_lists['elts'] = sa_lists['elts'] + 1 - elseif words[1] == 'whitelist_to' then - sa_lists['to_whitelist'][words[2]] = 1 - sa_lists['elts'] = sa_lists['elts'] + 1 - elseif words[1] == 'blacklist_to' then - sa_lists['to_blacklist'][words[2]] = 1 - sa_lists['elts'] = sa_lists['elts'] + 1 - elseif words[1] == 'tflags' then - process_tflags(cur_rule, words) - elseif words[1] == 'replace_tag' then - process_replace(words, replace['tags']) - elseif words[1] == 'replace_pre' then - process_replace(words, replace['pre']) - elseif words[1] == 'replace_inter' then - process_replace(words, replace['inter']) - elseif words[1] == 'replace_post' then - process_replace(words, replace['post']) - elseif words[1] == 'replace_rules' then - fun.each(function(r) table.insert(replace['rules'], r) end, - fun.drop_n(1, words)) - end + elseif words[1] == 'blacklist_from' then + sa_lists['from_blacklist'][words[2]] = 1 + sa_lists['elts'] = sa_lists['elts'] + 1 + elseif words[1] == 'whitelist_from' then + sa_lists['from_whitelist'][words[2]] = 1 + sa_lists['elts'] = sa_lists['elts'] + 1 + elseif words[1] == 'whitelist_to' then + sa_lists['to_whitelist'][words[2]] = 1 + sa_lists['elts'] = sa_lists['elts'] + 1 + elseif words[1] == 'blacklist_to' then + sa_lists['to_blacklist'][words[2]] = 1 + sa_lists['elts'] = sa_lists['elts'] + 1 + elseif words[1] == 'tflags' then + process_tflags(cur_rule, words) + elseif words[1] == 'replace_tag' then + process_replace(words, replace['tags']) + elseif words[1] == 'replace_pre' then + process_replace(words, replace['pre']) + elseif words[1] == 'replace_inter' then + process_replace(words, replace['inter']) + elseif words[1] == 'replace_post' then + process_replace(words, replace['post']) + elseif words[1] == 'replace_rules' then + fun.each(function(r) + table.insert(replace['rules'], r) + end, + fun.drop_n(1, words)) + end end)() end if valid_rule then @@ -1069,7 +1086,9 @@ end -- Now check all valid rules and add the according rspamd rules local function calculate_score(sym, rule) - if fun.all(function(c) return c == '_' end, fun.take_n(2, fun.iter(sym))) then + if fun.all(function(c) + return c == '_' + end, fun.take_n(2, fun.iter(sym))) then return 0.0 end @@ -1102,7 +1121,9 @@ local function sa_regexp_match(data, re, raw, rule) end res = res + re:matchn(data, lim, raw) else - if re:match(data, raw) then res = 1 end + if re:match(data, raw) then + res = 1 + end end return res @@ -1117,26 +1138,26 @@ local function apply_replacements(str) local replacement = nil local ret = s fun.each(function(n, t) - local ns,matches = string.gsub(s, string.format("<%s%s>", prefix, n), "") + local ns, matches = string.gsub(s, string.format("<%s%s>", prefix, n), "") if matches > 0 then replacement = t ret = ns end end, tbl) - return ret,replacement + return ret, replacement end local repl - str,repl = check_specific_tag("pre ", str, replace['pre']) + str, repl = check_specific_tag("pre ", str, replace['pre']) if repl then pre = repl end - str,repl = check_specific_tag("inter ", str, replace['inter']) + str, repl = check_specific_tag("inter ", str, replace['inter']) if repl then inter = repl end - str,repl = check_specific_tag("post ", str, replace['post']) + str, repl = check_specific_tag("post ", str, replace['post']) if repl then post = repl end @@ -1160,12 +1181,11 @@ local function apply_replacements(str) local s = replace_all_tags(str) - if str ~= s then - return true,s + return true, s end - return false,str + return false, str end local function parse_atom(str) @@ -1180,7 +1200,7 @@ local function parse_atom(str) end local function gen_process_atom_cb(result_name, task) - return function (atom) + return function(atom) local atom_cb = atoms[atom] if atom_cb then @@ -1212,7 +1232,9 @@ local function post_process() -- Replace rule tags local ntags = {} local function rec_replace_tags(tag, tagv) - if ntags[tag] then return ntags[tag] end + if ntags[tag] then + return ntags[tag] + end fun.each(function(n, t) if n ~= tag then local s, matches = string.gsub(tagv, string.format("<%s>", n), t) @@ -1222,7 +1244,9 @@ local function post_process() end end, replace['tags']) - if not ntags[tag] then ntags[tag] = tagv end + if not ntags[tag] then + ntags[tag] = tagv + end return ntags[tag] end @@ -1283,7 +1307,7 @@ local function post_process() if not r['re'] then rspamd_logger.errx(task, 're is missing for rule %1 (%2 header)', k, - h['header']) + h['header']) return 0 end @@ -1333,7 +1357,9 @@ local function post_process() else str = rh['decoded'] end - if not str then return 0 end + if not str then + return 0 + end if h['function'] then str = h['function'](str) @@ -1353,7 +1379,9 @@ local function post_process() end, r['header']) if #check == 0 then - if r['not'] then return 1 end + if r['not'] then + return 1 + end return 0 end @@ -1375,10 +1403,10 @@ local function post_process() end atoms[k] = f end, - fun.filter(function(_, r) - return r['type'] == 'header' and r['header'] - end, - rules)) + fun.filter(function(_, r) + return r['type'] == 'header' and r['header'] + end, + rules)) -- Custom function rules fun.each(function(k, r) @@ -1397,10 +1425,10 @@ local function post_process() end atoms[k] = f end, - fun.filter(function(_, r) - return r['type'] == 'function' and r['function'] - end, - rules)) + fun.filter(function(_, r) + return r['type'] == 'function' and r['function'] + end, + rules)) -- Parts rules fun.each(function(k, r) @@ -1411,7 +1439,9 @@ local function post_process() end local t = 'mime' - if r['raw'] then t = 'rawmime' end + if r['raw'] then + t = 'rawmime' + end return process_regexp_opt(r.re, task, t) end @@ -1423,9 +1453,9 @@ local function post_process() end atoms[k] = f end, - fun.filter(function(_, r) - return r['type'] == 'part' - end, rules)) + fun.filter(function(_, r) + return r['type'] == 'part' + end, rules)) -- SA body rules fun.each(function(k, r) @@ -1448,9 +1478,9 @@ local function post_process() end atoms[k] = f end, - fun.filter(function(_, r) - return r['type'] == 'sabody' or r['type'] == 'message' or r['type'] == 'sarawbody' - end, rules)) + fun.filter(function(_, r) + return r['type'] == 'sabody' or r['type'] == 'message' or r['type'] == 'sarawbody' + end, rules)) -- URL rules fun.each(function(k, r) @@ -1470,156 +1500,156 @@ local function post_process() end atoms[k] = f end, - fun.filter(function(_, r) - return r['type'] == 'uri' - end, - rules)) + fun.filter(function(_, r) + return r['type'] == 'uri' + end, + rules)) -- Meta rules fun.each(function(k, r) - local expression = nil - -- Meta function callback - -- Here are dragons! - -- This function can be called from 2 DIFFERENT type of invocations: - -- 1) Invocation from Rspamd itself where `res_name` will be nil - -- 2) Invocation from other meta during expression:process_traced call - -- So we need to distinguish that and return different stuff to be able to deal with atoms - local meta_cb = function(task, res_name) - lua_util.debugm(N, task, 'meta callback for %s; result name: %s', k, res_name) - local cached = task:cache_get('sa_metas_processed') - - -- We avoid many task methods invocations here (likely) - if not cached then - cached = {} - task:cache_set('sa_metas_processed', cached) - end - - local already_processed = cached[k] - - -- Exclude elements that are named in the same way as the symbol itself - local function exclude_sym_filter(sopt) - return sopt ~= k - end + local expression = nil + -- Meta function callback + -- Here are dragons! + -- This function can be called from 2 DIFFERENT type of invocations: + -- 1) Invocation from Rspamd itself where `res_name` will be nil + -- 2) Invocation from other meta during expression:process_traced call + -- So we need to distinguish that and return different stuff to be able to deal with atoms + local meta_cb = function(task, res_name) + lua_util.debugm(N, task, 'meta callback for %s; result name: %s', k, res_name) + local cached = task:cache_get('sa_metas_processed') + + -- We avoid many task methods invocations here (likely) + if not cached then + cached = {} + task:cache_set('sa_metas_processed', cached) + end - if not (already_processed and already_processed[res_name or 'default']) then - -- Execute symbol - local function exec_symbol(cur_res) - local res,trace = expression:process_traced(gen_process_atom_cb(cur_res, task)) - lua_util.debugm(N, task, 'meta result for %s: %s; result name: %s', k, res, cur_res) - if res > 0 then - -- Symbol should be one shot to make it working properly - task:insert_result_named(cur_res, k, res, fun.totable(fun.filter(exclude_sym_filter, trace))) - end + local already_processed = cached[k] - if not cached[k] then - cached[k] = {} - end + -- Exclude elements that are named in the same way as the symbol itself + local function exclude_sym_filter(sopt) + return sopt ~= k + end - cached[k][cur_res] = res + if not (already_processed and already_processed[res_name or 'default']) then + -- Execute symbol + local function exec_symbol(cur_res) + local res, trace = expression:process_traced(gen_process_atom_cb(cur_res, task)) + lua_util.debugm(N, task, 'meta result for %s: %s; result name: %s', k, res, cur_res) + if res > 0 then + -- Symbol should be one shot to make it working properly + task:insert_result_named(cur_res, k, res, fun.totable(fun.filter(exclude_sym_filter, trace))) end - if not res_name then - -- Invoke for all named results - local named_results = task:get_all_named_results() - for _,cur_res in ipairs(named_results) do - exec_symbol(cur_res) - end - else - -- Invoked from another meta - exec_symbol(res_name) - return cached[k][res_name] or 0 + if not cached[k] then + cached[k] = {} end - else - -- We have cached the result - local res = already_processed[res_name or 'default'] or 0 - lua_util.debugm(N, task, 'cached meta result for %s: %s; result name: %s', - k, res, res_name) - if res_name then - return res + cached[k][cur_res] = res + end + + if not res_name then + -- Invoke for all named results + local named_results = task:get_all_named_results() + for _, cur_res in ipairs(named_results) do + exec_symbol(cur_res) end + else + -- Invoked from another meta + exec_symbol(res_name) + return cached[k][res_name] or 0 end + else + -- We have cached the result + local res = already_processed[res_name or 'default'] or 0 + lua_util.debugm(N, task, 'cached meta result for %s: %s; result name: %s', + k, res, res_name) - -- No return if invoked directly from Rspamd as we use task:insert_result_named directly + if res_name then + return res + end end - expression = rspamd_expression.create(r['meta'], parse_atom, rspamd_config:get_mempool()) - if not expression then - rspamd_logger.errx(rspamd_config, 'Cannot parse expression ' .. r['meta']) - else + -- No return if invoked directly from Rspamd as we use task:insert_result_named directly + end - if r['score'] then - rspamd_config:set_metric_symbol{ - name = k, score = r['score'], - description = r['description'], - priority = scores_priority, - one_shot = true - } - scores_added[k] = 1 - rspamd_config:register_symbol{ - name = k, - weight = calculate_score(k, r), - callback = meta_cb - } - else - -- Add 0 score to avoid issues - rspamd_config:register_symbol{ - name = k, - weight = calculate_score(k, r), - callback = meta_cb, - score = 0, - } - end + expression = rspamd_expression.create(r['meta'], parse_atom, rspamd_config:get_mempool()) + if not expression then + rspamd_logger.errx(rspamd_config, 'Cannot parse expression ' .. r['meta']) + else - r['expression'] = expression + if r['score'] then + rspamd_config:set_metric_symbol { + name = k, score = r['score'], + description = r['description'], + priority = scores_priority, + one_shot = true + } + scores_added[k] = 1 + rspamd_config:register_symbol { + name = k, + weight = calculate_score(k, r), + callback = meta_cb + } + else + -- Add 0 score to avoid issues + rspamd_config:register_symbol { + name = k, + weight = calculate_score(k, r), + callback = meta_cb, + score = 0, + } + end - if not atoms[k] then - atoms[k] = meta_cb - end + r['expression'] = expression + + if not atoms[k] then + atoms[k] = meta_cb end - end, - fun.filter(function(_, r) + end + end, + fun.filter(function(_, r) return r['type'] == 'meta' end, - rules)) + rules)) -- Check meta rules for foreign symbols and register dependencies -- First direct dependencies: fun.each(function(k, r) - if r['expression'] then - local expr_atoms = r['expression']:atoms() - - for _,a in ipairs(expr_atoms) do - if not atoms[a] then - local rspamd_symbol = replace_symbol(a) - if not external_deps[k] then - external_deps[k] = {} - end + if r['expression'] then + local expr_atoms = r['expression']:atoms() + + for _, a in ipairs(expr_atoms) do + if not atoms[a] then + local rspamd_symbol = replace_symbol(a) + if not external_deps[k] then + external_deps[k] = {} + end - if not external_deps[k][rspamd_symbol] then - rspamd_config:register_dependency(k, rspamd_symbol) - external_deps[k][rspamd_symbol] = true - lua_util.debugm(N, rspamd_config, + if not external_deps[k][rspamd_symbol] then + rspamd_config:register_dependency(k, rspamd_symbol) + external_deps[k][rspamd_symbol] = true + lua_util.debugm(N, rspamd_config, 'atom %1 is a direct foreign dependency, ' .. - 'register dependency for %2 on %3', + 'register dependency for %2 on %3', a, k, rspamd_symbol) - end end end end - end, - fun.filter(function(_, r) - return r['type'] == 'meta' - end, - rules)) + end + end, + fun.filter(function(_, r) + return r['type'] == 'meta' + end, + rules)) -- ... And then indirect ones ... local nchanges repeat - nchanges = 0 + nchanges = 0 fun.each(function(k, r) if r['expression'] then local expr_atoms = r['expression']:atoms() - for _,a in ipairs(expr_atoms) do + for _, a in ipairs(expr_atoms) do if type(external_deps[a]) == 'table' then for dep in pairs(external_deps[a]) do if not external_deps[k] then @@ -1629,16 +1659,16 @@ local function post_process() rspamd_config:register_dependency(k, dep) external_deps[k][dep] = true lua_util.debugm(N, rspamd_config, - 'atom %1 is an indirect foreign dependency, ' .. - 'register dependency for %2 on %3', - a, k, dep) - nchanges = nchanges + 1 + 'atom %1 is an indirect foreign dependency, ' .. + 'register dependency for %2 on %3', + a, k, dep) + nchanges = nchanges + 1 end end else local rspamd_symbol, replaced_symbol = replace_symbol(a) if replaced_symbol then - external_deps[a] = {[rspamd_symbol] = true} + external_deps[a] = { [rspamd_symbol] = true } else external_deps[a] = {} end @@ -1646,18 +1676,18 @@ local function post_process() end end end, - fun.filter(function(_, r) - return r['type'] == 'meta' - end, - rules)) + fun.filter(function(_, r) + return r['type'] == 'meta' + end, + rules)) until nchanges == 0 -- Set missing symbols fun.each(function(key, score) if not scores_added[key] then rspamd_config:set_metric_symbol({ - name = key, score = score, - priority = 2, flags = 'ignore'}) + name = key, score = score, + priority = 2, flags = 'ignore' }) end end, scores) @@ -1665,7 +1695,7 @@ local function post_process() if freemail_domains then freemail_trie = rspamd_trie.create(freemail_domains) rspamd_logger.infox(rspamd_config, 'loaded %1 freemail domains definitions', - #freemail_domains) + #freemail_domains) end rspamd_logger.infox(rspamd_config, 'loaded %1 blacklist/whitelist elements', sa_lists['elts']) @@ -1675,10 +1705,18 @@ local has_rules = false if type(section) == "table" then local keywords = { - pcre_only = {'table', function(v) pcre_only_regexps = lua_util.list_to_hash(v) end}, - alpha = {'number', function(v) meta_score_alpha = tonumber(v) end}, - match_limit = {'number', function(v) match_limit = tonumber(v) end}, - scores_priority = {'number', function(v) scores_priority = tonumber(v) end}, + pcre_only = { 'table', function(v) + pcre_only_regexps = lua_util.list_to_hash(v) + end }, + alpha = { 'number', function(v) + meta_score_alpha = tonumber(v) + end }, + match_limit = { 'number', function(v) + match_limit = tonumber(v) + end }, + scores_priority = { 'number', function(v) + scores_priority = tonumber(v) + end }, } for k, fn in pairs(section) do @@ -1694,7 +1732,7 @@ if type(section) == "table" then if not files or #files == 0 then rspamd_logger.errx(rspamd_config, "cannot find any files matching pattern %s", elt) else - for _,matched in ipairs(files) do + for _, matched in ipairs(files) do local f = io.open(matched, "r") if f then rspamd_logger.infox(rspamd_config, 'loading SA rules from %s', matched) @@ -1713,7 +1751,7 @@ if type(section) == "table" then if not files or #files == 0 then rspamd_logger.errx(rspamd_config, "cannot find any files matching pattern %s", fn) else - for _,matched in ipairs(files) do + for _, matched in ipairs(files) do local f = io.open(matched, "r") if f then rspamd_logger.infox(rspamd_config, 'loading SA rules from %s', matched) diff --git a/src/plugins/lua/spamtrap.lua b/src/plugins/lua/spamtrap.lua index 0f0bbe813..cd3b2968f 100644 --- a/src/plugins/lua/spamtrap.lua +++ b/src/plugins/lua/spamtrap.lua @@ -53,8 +53,8 @@ local function spamtrap_cb(task) local function do_action(rcpt) if settings['learn_fuzzy'] then rspamd_plugins.fuzzy_check.learn(task, - settings['fuzzy_flag'], - settings['fuzzy_weight']) + settings['fuzzy_flag'], + settings['fuzzy_weight']) end local act_flags = '' if settings['learn_spam'] then @@ -68,7 +68,7 @@ local function spamtrap_cb(task) rspamd_logger.infox(task, 'spamtrap found: <%s>', rcpt) local smtp_message if settings.smtp_message then - smtp_message = lua_util.template(settings.smtp_message, { rcpt = rcpt}) + smtp_message = lua_util.template(settings.smtp_message, { rcpt = rcpt }) else smtp_message = 'unknown error' if settings.action == 'no action' then @@ -77,10 +77,10 @@ local function spamtrap_cb(task) smtp_message = 'message rejected' end end - task:set_pre_result{action = settings.action, - message = smtp_message, - module = 'spamtrap', - flags = act_flags} + task:set_pre_result { action = settings.action, + message = smtp_message, + module = 'spamtrap', + flags = act_flags } end return true @@ -106,7 +106,7 @@ local function spamtrap_cb(task) false, -- is write gen_redis_spamtrap_cb(target), -- callback 'GET', -- command - {key} -- arguments + { key } -- arguments ) if not ret then rspamd_logger.errx(task, "redis request wasn't scheduled") @@ -120,8 +120,10 @@ local function spamtrap_cb(task) end -- Do not risk a FP by checking for more than one recipient - if rcpts and (#rcpts == 1 or (#rcpts > 0 and settings.allow_multiple_rcpts)) then - local targets = fun.map(function(r) return r['addr']:lower() end, rcpts) + if rcpts and (#rcpts == 1 or (#rcpts > 0 and settings.allow_multiple_rcpts)) then + local targets = fun.map(function(r) + return r['addr']:lower() + end, rcpts) if use_redis then fun.each(function(target) local key = settings['key_prefix'] .. target @@ -131,7 +133,7 @@ local function spamtrap_cb(task) false, -- is write gen_redis_spamtrap_cb(target), -- callback 'GET', -- command - {key} -- arguments + { key } -- arguments ) if not ret then rspamd_logger.errx(task, "redis request wasn't scheduled") @@ -159,18 +161,17 @@ if not (opts and type(opts) == 'table') then return end - local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, 'spamtrap', false, false) check_local = auth_and_local_conf[1] check_authed = auth_and_local_conf[2] if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end if settings['map'] then - settings['map'] = rspamd_config:add_map{ + settings['map'] = rspamd_config:add_map { url = settings['map'], description = string.format("Spamtrap map for %s", settings['symbol']), type = "regexp" @@ -179,7 +180,7 @@ if opts then redis_params = rspamd_parse_redis_server('spamtrap') if not redis_params then rspamd_logger.errx( - rspamd_config, 'no redis servers are specified, disabling module') + rspamd_config, 'no redis servers are specified, disabling module') return end use_redis = true; diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua index 997c07bca..5e151281a 100644 --- a/src/plugins/lua/spf.lua +++ b/src/plugins/lua/spf.lua @@ -88,7 +88,7 @@ local function spf_check_callback(task) local rh = task:get_received_headers() or {} local found = false - for i,hdr in ipairs(rh) do + for i, hdr in ipairs(rh) do if hdr.real_ip and local_config.external_relay:get_key(hdr.real_ip) then -- We can use the next header as a source of IP address if rh[i + 1] then @@ -129,16 +129,16 @@ local function spf_check_callback(task) local function policy_decode(res) if res == rspamd_spf.policy.fail then - return local_config.symbols.fail,'-' + return local_config.symbols.fail, '-' elseif res == rspamd_spf.policy.pass then - return local_config.symbols.allow,'+' + return local_config.symbols.allow, '+' elseif res == rspamd_spf.policy.soft_fail then - return local_config.symbols.softfail,'~' + return local_config.symbols.softfail, '~' elseif res == rspamd_spf.policy.neutral then - return local_config.symbols.neutral,'?' + return local_config.symbols.neutral, '?' end - return 'SPF_UNKNOWN','?' + return 'SPF_UNKNOWN', '?' end local function spf_resolved_cb(record, flags, err) @@ -153,7 +153,7 @@ local function spf_check_callback(task) ip, flags, err, error_or_addr) if result then - local sym,code = policy_decode(flag_or_policy) + local sym, code = policy_decode(flag_or_policy) local opt = string.format('%s%s', code, error_or_addr.str or '???') if bit.band(flags, rspamd_spf.flags.cached) ~= 0 then opt = opt .. ':c' @@ -203,17 +203,17 @@ end -- Register all symbols and init rspamd_spf library rspamd_spf.config(local_config) -local sym_id = rspamd_config:register_symbol{ +local sym_id = rspamd_config:register_symbol { name = 'SPF_CHECK', type = 'callback', flags = 'fine,empty', - groups = {'policies','spf'}, + groups = { 'policies', 'spf' }, score = 0.0, callback = spf_check_callback, -- We can merely estimate timeout here, as it is possible to construct an SPF record that would cause -- many DNS requests. But we won't like to set the maximum value for that all the time, as -- the majority of requests will typically have 1-4 subrequests - augmentations = {string.format("timeout=%f", rspamd_config:get_dns_timeout() * 4 or 0.0)}, + augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() * 4 or 0.0) }, } if local_config.whitelist then @@ -227,15 +227,15 @@ if local_config.external_relay then local lua_maps = require "lua_maps" local_config.external_relay = lua_maps.map_add_from_ucl(local_config.external_relay, - "radix", "External IP SPF map") + "radix", "External IP SPF map") end -for _,sym in pairs(local_config.symbols) do - rspamd_config:register_symbol{ +for _, sym in pairs(local_config.symbols) do + rspamd_config:register_symbol { name = sym, type = 'virtual', parent = sym_id, - groups = {'policies', 'spf'}, + groups = { 'policies', 'spf' }, } end diff --git a/src/plugins/lua/trie.lua b/src/plugins/lua/trie.lua index 0adf12dc1..7ba455289 100644 --- a/src/plugins/lua/trie.lua +++ b/src/plugins/lua/trie.lua @@ -56,14 +56,14 @@ local function tries_callback(task) params = body_params end - return function (idx, pos) + return function(idx, pos) local param = params[idx] local pattern = patterns[idx] local pattern_idx = pattern .. tostring(idx) .. type if param['multi'] or not matched[pattern_idx] then lua_util.debugm(N, task, "<%1> matched pattern %2 at pos %3", - task:get_message_id(), pattern, pos) + task:get_message_id(), pattern, pos) task:insert_result(param['symbol'], 1.0, type) if not param['multi'] then matched[pattern_idx] = true @@ -86,17 +86,19 @@ end local function process_single_pattern(pat, symbol, cf) if pat then local multi = false - if cf['multi'] then multi = true end + if cf['multi'] then + multi = true + end if cf['raw'] then table.insert(raw_patterns, pat) - table.insert(raw_params, {symbol=symbol, multi=multi}) + table.insert(raw_params, { symbol = symbol, multi = multi }) elseif cf['body'] then table.insert(body_patterns, pat) - table.insert(body_params, {symbol=symbol, multi=multi}) + table.insert(body_params, { symbol = symbol, multi = multi }) else table.insert(mime_patterns, pat) - table.insert(mime_params, {symbol=symbol, multi=multi}) + table.insert(mime_params, { symbol = symbol, multi = multi }) end end end @@ -109,7 +111,7 @@ local function process_trie_file(symbol, cf) else if cf['binary'] then rspamd_logger.errx(rspamd_config, 'binary trie patterns are not implemented yet: %1', - cf['file']) + cf['file']) else for line in file:lines() do local pat = string.match(line, '^([^#].*[^%s])%s*$') @@ -122,7 +124,7 @@ end local function process_trie_conf(symbol, cf) if type(cf) ~= 'table' then rspamd_logger.errx(rspamd_config, 'invalid value for symbol %1: "%2", expected table', - symbol, cf) + symbol, cf) return end @@ -135,16 +137,16 @@ local function process_trie_conf(symbol, cf) end end -local opts = rspamd_config:get_all_opt("trie") +local opts = rspamd_config:get_all_opt("trie") if opts then for sym, opt in pairs(opts) do - process_trie_conf(sym, opt) + process_trie_conf(sym, opt) end if #raw_patterns > 0 then raw_trie = rspamd_trie.create(raw_patterns) rspamd_logger.infox(rspamd_config, 'registered raw search trie from %1 patterns', #raw_patterns) - end + end if #mime_patterns > 0 then mime_trie = rspamd_trie.create(mime_patterns) diff --git a/src/plugins/lua/url_redirector.lua b/src/plugins/lua/url_redirector.lua index c5748647d..19189a5ee 100644 --- a/src/plugins/lua/url_redirector.lua +++ b/src/plugins/lua/url_redirector.lua @@ -60,7 +60,7 @@ local settings = { local function adjust_url(task, orig_url, redir_url) if type(redir_url) == 'string' then - redir_url = rspamd_url.create(task:get_mempool(), redir_url, {'redirect_target'}) + redir_url = rspamd_url.create(task:get_mempool(), redir_url, { 'redirect_target' }) end if redir_url then @@ -90,7 +90,7 @@ local function cache_url(task, orig_url, url, key, prefix) rspamd_logger.errx(task, 'got error while getting top urls count: %s', err) else rspamd_logger.infox(task, 'trimmed url set to %s elements', - settings.top_urls_count) + settings.top_urls_count) end end @@ -102,20 +102,20 @@ local function cache_url(task, orig_url, url, key, prefix) if data then if tonumber(data) > settings.top_urls_count * 2 then local ret = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_trim_cb, --callback - 'ZREMRANGEBYRANK', -- command - {settings.top_urls_key, '0', - tostring(-(settings.top_urls_count + 1))} -- arguments + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_trim_cb, --callback + 'ZREMRANGEBYRANK', -- command + { settings.top_urls_key, '0', + tostring(-(settings.top_urls_count + 1)) } -- arguments ) if not ret then rspamd_logger.errx(task, 'cannot trim top urls set') else rspamd_logger.infox(task, 'need to trim urls set from %s to %s elements', - data, - settings.top_urls_count) + data, + settings.top_urls_count) return end end @@ -128,12 +128,12 @@ local function cache_url(task, orig_url, url, key, prefix) rspamd_logger.errx(task, 'got error while setting redirect keys: %s', err) else local ret = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_card_cb, --callback - 'ZCARD', -- command - {settings.top_urls_key} -- arguments + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_card_cb, --callback + 'ZCARD', -- command + { settings.top_urls_key } -- arguments ) if not ret then rspamd_logger.errx(task, 'cannot make redis request to cache results') @@ -145,25 +145,27 @@ local function cache_url(task, orig_url, url, key, prefix) -- Save url with prefix str_url = string.format('^%s:%s', prefix, str_url) end - local ret,conn,_ = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_set_cb, --callback - 'SETEX', -- command - {key, tostring(settings.expire), str_url} -- arguments + local ret, conn, _ = lua_redis.redis_make_request(task, + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_set_cb, --callback + 'SETEX', -- command + { key, tostring(settings.expire), str_url } -- arguments ) if not ret then rspamd_logger.errx(task, 'cannot make redis request to cache results') else - conn:add_cmd('ZINCRBY', {settings.top_urls_key, '1', str_url}) + conn:add_cmd('ZINCRBY', { settings.top_urls_key, '1', str_url }) end end -- Reduce length of a string to a given length (16 by default) local function maybe_trim_url(url, limit) - if not limit then limit = 16 end + if not limit then + limit = 16 + end if #url > limit then return string.sub(url, 1, limit) .. '...' else @@ -179,7 +181,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) if ntries > settings.nested_limit then -- We cannot resolve more, stop rspamd_logger.debugm(N, task, 'cannot get more requests to resolve %s, stop on %s after %s attempts', - orig_url, url, ntries) + orig_url, url, ntries) cache_url(task, orig_url, url, key, 'nested') task:insert_result(settings.redirector_symbol_nested, 1.0, string.format('%s->%s:%d', maybe_trim_url(orig_url), maybe_trim_url(url), ntries)) @@ -198,16 +200,16 @@ local function resolve_cached(task, orig_url, url, key, ntries) local function http_callback(err, code, _, headers) if err then rspamd_logger.infox(task, 'found redirect error from %s to %s, err message: %s', - orig_url, url, err) + orig_url, url, err) cache_url(task, orig_url, url, key) else if code == 200 then if orig_url == url then rspamd_logger.infox(task, 'direct url %s, err code 200', - url) + url) else rspamd_logger.infox(task, 'found redirect from %s to %s, err code 200', - orig_url, url) + orig_url, url) end cache_url(task, orig_url, url, key) @@ -219,7 +221,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) redir_url = rspamd_url.create(task:get_mempool(), loc) end rspamd_logger.debugm(N, task, 'found redirect from %s to %s, err code %s', - orig_url, loc, code) + orig_url, loc, code) if redir_url then if settings.redirectors_only then @@ -227,7 +229,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) resolve_cached(task, orig_url, redir_url, key, ntries + 1) else lua_util.debugm(N, task, - "stop resolving redirects as %s is not a redirector", loc) + "stop resolving redirects as %s is not a redirector", loc) cache_url(task, orig_url, redir_url, key) end else @@ -239,7 +241,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) end else rspamd_logger.debugm(N, task, 'found redirect error from %s to %s, err code: %s', - orig_url, url, code) + orig_url, url, code) cache_url(task, orig_url, url, key) end end @@ -254,7 +256,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) lua_util.debugm(N, task, 'select user agent %s', ua) - rspamd_http.request{ + rspamd_http.request { headers = { ['User-Agent'] = ua, }, @@ -274,7 +276,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) if data ~= 'processing' then -- Got cached result rspamd_logger.debugm(N, task, 'found cached redirect from %s to %s', - url, data) + url, data) if data.sub(1, 1) == '^' then -- Prefixed url stored local prefix, new_url = data:match('^%^(%a+):(.+)$') @@ -302,12 +304,12 @@ local function resolve_cached(task, orig_url, url, key, ntries) if ntries == 1 then -- Reserve key in Redis that we are processing this redirection local ret = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_reserve_cb, --callback - 'SET', -- command - {key, 'processing', 'EX', tostring(settings.timeout * 2), 'NX'} -- arguments + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_reserve_cb, --callback + 'SET', -- command + { key, 'processing', 'EX', tostring(settings.timeout * 2), 'NX' } -- arguments ) if not ret then rspamd_logger.errx(task, 'Couldn\'t schedule SET') @@ -319,12 +321,12 @@ local function resolve_cached(task, orig_url, url, key, ntries) end local ret = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_get_cb, --callback - 'GET', -- command - {key} -- arguments + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_get_cb, --callback + 'GET', -- command + { key } -- arguments ) if not ret then rspamd_logger.errx(task, 'cannot make redis request to check results') @@ -354,13 +356,13 @@ local function url_redirector_handler(task) }) if sp_urls then - for _,u in ipairs(sp_urls) do + for _, u in ipairs(sp_urls) do url_redirector_process_url(task, u) end end end -local opts = rspamd_config:get_all_opt('url_redirector') +local opts = rspamd_config:get_all_opt('url_redirector') if opts then settings = lua_util.override_defaults(settings, opts) redis_params = lua_redis.parse_redis_server('url_redirector', settings) @@ -388,16 +390,16 @@ if opts then type = 'zlist', }) end - local id = rspamd_config:register_symbol{ + local id = rspamd_config:register_symbol { name = 'URL_REDIRECTOR_CHECK', type = 'callback,prefilter', priority = lua_util.symbols_priorities.medium, callback = url_redirector_handler, -- In fact, the real timeout is nested_limit * timeout... - augmentations = {string.format("timeout=%f", settings.timeout)} + augmentations = { string.format("timeout=%f", settings.timeout) } } - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = settings.redirector_symbol_nested, type = 'virtual', parent = id, @@ -405,7 +407,7 @@ if opts then } if settings.redirector_symbol then - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = settings.redirector_symbol, type = 'virtual', parent = id, diff --git a/src/plugins/lua/whitelist.lua b/src/plugins/lua/whitelist.lua index fc339c6ce..fa76da8d1 100644 --- a/src/plugins/lua/whitelist.lua +++ b/src/plugins/lua/whitelist.lua @@ -45,66 +45,70 @@ local function whitelist_cb(symbol, rule, task) local how = 'wl' -- Can be overridden - if rule.blacklist then how = 'bl' end + if rule.blacklist then + how = 'bl' + end local function parse_val(val) local how_override -- Strict is 'special' - if rule.strict then how_override = 'both' end + if rule.strict then + how_override = 'both' + end if val then lua_util.debugm(N, task, "found whitelist key: %s=%s", dom, val) if val == '' then - return (how_override or how),1.0 + return (how_override or how), 1.0 elseif val:match('^bl:') then - return (how_override or 'bl'),(tonumber(val:sub(4)) or 1.0) + return (how_override or 'bl'), (tonumber(val:sub(4)) or 1.0) elseif val:match('^wl:') then - return (how_override or 'wl'),(tonumber(val:sub(4)) or 1.0) + return (how_override or 'wl'), (tonumber(val:sub(4)) or 1.0) elseif val:match('^both:') then - return (how_override or 'both'),(tonumber(val:sub(6)) or 1.0) + return (how_override or 'both'), (tonumber(val:sub(6)) or 1.0) else - return (how_override or how),(tonumber(val) or 1.0) + return (how_override or how), (tonumber(val) or 1.0) end end - return (how_override or how),1.0 + return (how_override or how), 1.0 end if rule['map'] then local val = rule['map']:get_key(dom) if val then - how,mult = parse_val(val) + how, mult = parse_val(val) if not domains[check] then domains[check] = {} end domains[check] = { - [dom] = {how, mult} + [dom] = { how, mult } } lua_util.debugm(N, task, "final result: %s: %s->%s", dom, how, mult) - return true,mult,how + return true, mult, how end elseif rule['maps'] then - for _,v in pairs(rule['maps']) do + for _, v in pairs(rule['maps']) do local map = v.map if map then local val = map:get_key(dom) if val then - how,mult = parse_val(val) + how, mult = parse_val(val) if not domains[check] then domains[check] = {} end domains[check] = { - [dom] = {how, mult} + [dom] = { how, mult } } lua_util.debugm(N, task, "final result: %s: %s->%s", dom, how, mult) - return true,mult,how + return true, mult, how end end end @@ -116,14 +120,14 @@ local function whitelist_cb(symbol, rule, task) end domains[check] = { - [dom] = {how, mult} + [dom] = { how, mult } } - return true, mult,how + return true, mult, how end end - return false,0.0,how + return false, 0.0, how end local spf_violated = false @@ -164,14 +168,14 @@ local function whitelist_cb(symbol, rule, task) local dkim_opts = sym[1]['options'] if dkim_opts then fun.each(function(val) - if val[2] == '+' then - local tld = rspamd_util.get_tld(val[1]) - find_domain(tld, 'dkim_success') - elseif val[2] == '-' then - local tld = rspamd_util.get_tld(val[1]) - find_domain(tld, 'dkim_fail') - end - end, + if val[2] == '+' then + local tld = rspamd_util.get_tld(val[1]) + find_domain(tld, 'dkim_success') + elseif val[2] == '-' then + local tld = rspamd_util.get_tld(val[1]) + find_domain(tld, 'dkim_fail') + end + end, fun.map(function(s) return lua_util.rspamd_str_split(s, ':') end, dkim_opts)) @@ -198,7 +202,6 @@ local function whitelist_cb(symbol, rule, task) end end - local final_mult = 1.0 local found_wl, found_bl = false, false local opts = {} @@ -206,7 +209,7 @@ local function whitelist_cb(symbol, rule, task) if rule.valid_dkim then dkim_violated = true - for dom,val in pairs(domains.dkim_success or E) do + for dom, val in pairs(domains.dkim_success or E) do if val[1] == 'wl' or val[1] == 'both' then -- We have valid and whitelisted signature table.insert(opts, dom .. ':d:+') @@ -220,7 +223,7 @@ local function whitelist_cb(symbol, rule, task) end -- Blacklist counterpart - for dom,val in pairs(domains.dkim_fail or E) do + for dom, val in pairs(domains.dkim_fail or E) do if val[1] == 'bl' or val[1] == 'both' then -- We have valid and whitelisted signature table.insert(opts, dom .. ':d:-') @@ -255,7 +258,7 @@ local function whitelist_cb(symbol, rule, task) found_wl = false - for dom,val in pairs(domains.dmarc or E) do + for dom, val in pairs(domains.dmarc or E) do check_domain_violation('D', dom, val, (dmarc_violated or dkim_violated)) end @@ -264,7 +267,7 @@ local function whitelist_cb(symbol, rule, task) if rule.valid_spf then found_wl = false - for dom,val in pairs(domains.spf or E) do + for dom, val in pairs(domains.spf or E) do check_domain_violation('s', dom, val, (spf_violated or dkim_violated)) end @@ -318,7 +321,7 @@ end local configure_whitelist_module = function() local opts = rspamd_config:get_all_opt('whitelist') if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do options[k] = v end @@ -335,7 +338,7 @@ local configure_whitelist_module = function() fun.each(function(symbol, rule) if rule['domains'] then if type(rule['domains']) == 'string' then - rule['map'] = rspamd_config:add_map{ + rule['map'] = rspamd_config:add_map { url = rule['domains'], description = "Whitelist map for " .. symbol, type = 'map' @@ -347,7 +350,7 @@ local configure_whitelist_module = function() if type(v) == 'table' then return true elseif type(v) == 'string' and not (string.match(v, '^https?://') or - string.match(v, '^ftp://') or string.match(v, '^[./]')) then + string.match(v, '^ftp://') or string.match(v, '^[./]')) then return true end @@ -357,13 +360,13 @@ local configure_whitelist_module = function() if is_domains_list then rule['domains'] = fun.tomap(fun.map(function(d) if type(d) == 'table' then - return d[1],d[2] + return d[1], d[2] end - return d,1.0 + return d, 1.0 end, rule['domains'])) else - rule['map'] = rspamd_config:add_map{ + rule['map'] = rspamd_config:add_map { url = rule['domains'], description = "Whitelist map for " .. symbol, type = 'map' @@ -371,7 +374,7 @@ local configure_whitelist_module = function() end else rspamd_logger.errx(rspamd_config, 'whitelist %s has bad "domains" value', - symbol) + symbol) return end |