diff options
Diffstat (limited to 'lualib')
83 files changed, 2928 insertions, 2579 deletions
diff --git a/lualib/lua_auth_results.lua b/lualib/lua_auth_results.lua index d10e1081c..8c907d9b7 100644 --- a/lualib/lua_auth_results.lua +++ b/lualib/lua_auth_results.lua @@ -93,7 +93,7 @@ local function gen_auth_results(task, settings) else common.symbols[sym] = s if not auth_results[auth_type] then - auth_results[auth_type] = {key} + auth_results[auth_type] = { key } else table.insert(auth_results[auth_type], key) end @@ -111,7 +111,7 @@ local function gen_auth_results(task, settings) -- dkim=neutral (body hash did not verify) header.d=example.com header.s=sel header.b=fA8VVvJ8; -- dkim=neutral (body hash did not verify) header.d=example.com header.s=sel header.b=f8pM8o90; - for _,dres in ipairs(dkim_results) do + for _, dres in ipairs(dkim_results) do local ar_string = 'none' if dres.result == 'reject' then @@ -168,7 +168,7 @@ local function gen_auth_results(task, settings) hdr = hdr .. ' (policy=' .. lua_util.maybe_smtp_quote_value(opts[2]) .. ')' hdr = hdr .. ' header.from=' .. lua_util.maybe_smtp_quote_value(opts[1]) elseif key ~= 'none' then - local t = {opts[1]:match('^([^%s]+) : (.*)$')} + local t = { opts[1]:match('^([^%s]+) : (.*)$') } if #t > 0 then local dom = t[1] local rsn = t[2] @@ -197,7 +197,7 @@ local function gen_auth_results(task, settings) -- Main type local sender local sender_type - local smtp_from = task:get_from({'smtp','orig'}) + local smtp_from = task:get_from({ 'smtp', 'orig' }) if smtp_from and smtp_from[1] and @@ -241,26 +241,25 @@ local function gen_auth_results(task, settings) hdr = string.format('%s=%s', auth_type, key) end - table.insert(hdr_parts, hdr) end end end local u = task:get_user() - local smtp_from = task:get_from({'smtp','orig'}) + local smtp_from = task:get_from({ 'smtp', 'orig' }) if u and smtp_from then - local hdr = {[1] = 'auth=pass'} + local hdr = { [1] = 'auth=pass' } if settings['add_smtp_user'] then - table.insert(hdr,'smtp.auth=' .. lua_util.maybe_smtp_quote_value(u)) + table.insert(hdr, 'smtp.auth=' .. lua_util.maybe_smtp_quote_value(u)) end if smtp_from[1]['addr'] then - table.insert(hdr,'smtp.mailfrom=' .. lua_util.maybe_smtp_quote_value(smtp_from[1]['addr'])) + table.insert(hdr, 'smtp.mailfrom=' .. lua_util.maybe_smtp_quote_value(smtp_from[1]['addr'])) end - table.insert(hdr_parts, table.concat(hdr,' ')) + table.insert(hdr_parts, table.concat(hdr, ' ')) end if #hdr_parts > 0 then @@ -287,12 +286,12 @@ local function parse_ar_element(elt) local S = lpeg.S local V = lpeg.V local C = lpeg.C - local space = S(" ")^0 - local doublequoted = space * P'"' * ((1 - S'"\r\n\f\\') + (P'\\' * 1))^0 * '"' * space - local comment = space * P{ "(" * ((1 - S"()") + V(1))^0 * ")" } * space - local name = C((1 - S('=(" '))^1) * space + local space = S(" ") ^ 0 + local doublequoted = space * P '"' * ((1 - S '"\r\n\f\\') + (P '\\' * 1)) ^ 0 * '"' * space + local comment = space * P { "(" * ((1 - S "()") + V(1)) ^ 0 * ")" } * space + local name = C((1 - S('=(" ')) ^ 1) * space local pair = lpeg.Cg(name * "=" * space * name) * space - aar_elt_grammar = lpeg.Cf(lpeg.Ct("") * (pair + comment + doublequoted)^1, rawset) + aar_elt_grammar = lpeg.Cf(lpeg.Ct("") * (pair + comment + doublequoted) ^ 1, rawset) end return aar_elt_grammar:match(elt) diff --git a/lualib/lua_aws.lua b/lualib/lua_aws.lua index 5de61eb0c..e6c4b29a2 100644 --- a/lualib/lua_aws.lua +++ b/lualib/lua_aws.lua @@ -68,13 +68,12 @@ end local function save_cached_key(date_str, secret_key, region, service, req_type, key) local numdate = tonumber(date_str) -- expire old buckets - for k,_ in pairs(cached_keys) do + for k, _ in pairs(cached_keys) do if k < numdate then cached_keys[k] = nil end end - local bucket = cached_keys[tonumber(date_str)] local idx = string.format('%s.%s.%s.%s', secret_key, region, service, req_type) @@ -113,7 +112,7 @@ local function aws_signing_key(date_str, secret_key, region, service, req_type) end local hmac1 = rspamd_crypto_hash.create_specific_keyed("AWS4" .. secret_key, "sha256", date_str):bin() - local hmac2 = rspamd_crypto_hash.create_specific_keyed(hmac1, "sha256",region):bin() + local hmac2 = rspamd_crypto_hash.create_specific_keyed(hmac1, "sha256", region):bin() local hmac3 = rspamd_crypto_hash.create_specific_keyed(hmac2, "sha256", service):bin() local final_key = rspamd_crypto_hash.create_specific_keyed(hmac3, "sha256", req_type):bin() @@ -155,7 +154,7 @@ local function aws_canon_request_hash(method, uri, headers_to_sign, hex_hash) end, headers_to_sign)) local header_names = lua_util.keys(hdr_canon) table.sort(header_names) - for _,hn in ipairs(header_names) do + for _, hn in ipairs(header_names) do local v = hdr_canon[hn] lua_util.debugm(N, 'update signature with the header %s, %s', hn, v) @@ -165,19 +164,25 @@ local function aws_canon_request_hash(method, uri, headers_to_sign, hex_hash) lua_util.debugm(N, 'headers list to sign: %s', hdrs_list) sha_ctx:update(string.format('\n%s\n%s', hdrs_list, hex_hash)) - return sha_ctx:hex(),hdrs_list + return sha_ctx:hex(), hdrs_list end exports.aws_canon_request_hash = aws_canon_request_hash -local aws_authorization_hdr_args_schema = ts.shape{ +local aws_authorization_hdr_args_schema = ts.shape { date = ts.string + ts['nil'] / today_canonical, secret_key = ts.string, - method = ts.string + ts['nil'] / function() return 'GET' end, + method = ts.string + ts['nil'] / function() + return 'GET' + end, uri = ts.string, region = ts.string, - service = ts.string + ts['nil'] / function() return 's3' end, - req_type = ts.string + ts['nil'] / function() return 'aws4_request' end, + service = ts.string + ts['nil'] / function() + return 's3' + end, + req_type = ts.string + ts['nil'] / function() + return 'aws4_request' + end, headers = ts.map_of(ts.string, ts.string), key_id = ts.string, } @@ -199,9 +204,9 @@ ts.shape{ -- --]] local function aws_authorization_hdr(tbl, transformed) - local res,err + local res, err if not transformed then - res,err = aws_authorization_hdr_args_schema:transform(tbl) + res, err = aws_authorization_hdr_args_schema:transform(tbl) assert(res, err) else res = tbl @@ -210,7 +215,7 @@ local function aws_authorization_hdr(tbl, transformed) local signing_key = aws_signing_key(res.date, res.secret_key, res.region, res.service, res.req_type) assert(signing_key ~= nil) - local signed_sha,signed_hdrs = aws_canon_request_hash(res.method, res.uri, + local signed_sha, signed_hdrs = aws_canon_request_hash(res.method, res.uri, res.headers) if not signed_sha then @@ -224,7 +229,7 @@ local function aws_authorization_hdr(tbl, transformed) lua_util.debugm(N, "string to sign: %s", string_to_sign) local hmac = rspamd_crypto_hash.create_specific_keyed(signing_key, 'sha256', string_to_sign):hex() lua_util.debugm(N, "hmac: %s", hmac) - local auth_hdr = string.format('AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/%s,'.. + local auth_hdr = string.format('AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/%s,' .. 'SignedHeaders=%s,Signature=%s', res.key_id, res.date, res.region, res.service, res.req_type, signed_hdrs, hmac) @@ -255,7 +260,7 @@ This method returns new/modified in place table of the headers -- --]] local function aws_request_enrich(tbl, content) - local res,err = aws_authorization_hdr_args_schema:transform(tbl) + local res, err = aws_authorization_hdr_args_schema:transform(tbl) assert(res, err) local content_sha256 = rspamd_crypto_hash.create_specific('sha256', content):hex() local hdrs = res.headers @@ -281,7 +286,7 @@ local test_request_hdrs = { assert(aws_canon_request_hash('GET', '/test.txt', test_request_hdrs) == '7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972') -assert(aws_authorization_hdr{ +assert(aws_authorization_hdr { date = '20130524', region = 'us-east-1', headers = test_request_hdrs, diff --git a/lualib/lua_bayes_learn.lua b/lualib/lua_bayes_learn.lua index 7782d8196..ea97db6f8 100644 --- a/lualib/lua_bayes_learn.lua +++ b/lualib/lua_bayes_learn.lua @@ -40,7 +40,7 @@ exports.can_learn = function(task, is_spam, is_unlearn) end if in_class then - return false,string.format( + return false, string.format( 'already in class %s; probability %.2f%%', cl, math.abs((prob - 0.5) * 200.0)) end @@ -56,7 +56,7 @@ exports.autolearn = function(task, conf) local mime_rcpts = 'undef' local mr = task:get_recipients('mime') if mr then - for _,r in ipairs(mr) do + for _, r in ipairs(mr) do if mime_rcpts == 'undef' then mime_rcpts = r.addr else @@ -76,8 +76,8 @@ exports.autolearn = function(task, conf) end -- We have autolearn config so let's figure out what is requested - local verdict,score = lua_verdict.get_specific_verdict("bayes", task) - local learn_spam,learn_ham = false, false + local verdict, score = lua_verdict.get_specific_verdict("bayes", task) + local learn_spam, learn_ham = false, false if verdict == 'passthrough' then -- No need to autolearn @@ -117,12 +117,14 @@ exports.autolearn = function(task, conf) local ham_learns = task:get_mempool():get_variable('ham_learns', 'int64') or 0 local min_balance = 0.9 - if conf.min_balance then min_balance = conf.min_balance end + if conf.min_balance then + min_balance = conf.min_balance + end if spam_learns > 0 or ham_learns > 0 then local max_ratio = 1.0 / min_balance local spam_learns_ratio = spam_learns / (ham_learns + 1) - if spam_learns_ratio > max_ratio and learn_spam then + if spam_learns_ratio > max_ratio and learn_spam then lua_util.debugm(N, task, 'skip learning spam, balance is not satisfied: %s < %s; %s spam learns; %s ham learns', spam_learns_ratio, min_balance, spam_learns, ham_learns) @@ -130,7 +132,7 @@ exports.autolearn = function(task, conf) end local ham_learns_ratio = ham_learns / (spam_learns + 1) - if ham_learns_ratio > max_ratio and learn_ham then + if ham_learns_ratio > max_ratio and learn_ham then lua_util.debugm(N, task, 'skip learning ham, balance is not satisfied: %s < %s; %s spam learns; %s ham learns', ham_learns_ratio, min_balance, spam_learns, ham_learns) diff --git a/lualib/lua_cfg_transform.lua b/lualib/lua_cfg_transform.lua index 419ebf0f6..d6243ade1 100644 --- a/lualib/lua_cfg_transform.lua +++ b/lualib/lua_cfg_transform.lua @@ -32,24 +32,24 @@ local function metric_pairs(t) local function gen_keys(tbl) if implicit_array then - for _,v in ipairs(tbl) do + for _, v in ipairs(tbl) do if v.name then - table.insert(keys, {v.name, v}) + table.insert(keys, { v.name, v }) v.name = nil else -- Very tricky to distinguish: -- group {name = "foo" ... } + group "blah" { ... } - for gr_name,gr in pairs(v) do + for gr_name, gr in pairs(v) do if type(gr_name) ~= 'number' then -- We can also have implicit arrays here local gr_implicit = is_implicit(gr) if gr_implicit then - for _,gr_elt in ipairs(gr) do - table.insert(keys, {gr_name, gr_elt}) + for _, gr_elt in ipairs(gr) do + table.insert(keys, { gr_name, gr_elt }) end else - table.insert(keys, {gr_name, gr}) + table.insert(keys, { gr_name, gr }) end end end @@ -57,20 +57,20 @@ local function metric_pairs(t) end else if tbl.name then - table.insert(keys, {tbl.name, tbl}) + table.insert(keys, { tbl.name, tbl }) tbl.name = nil else - for k,v in pairs(tbl) do + for k, v in pairs(tbl) do if type(k) ~= 'number' then -- We can also have implicit arrays here local sym_implicit = is_implicit(v) if sym_implicit then - for _,elt in ipairs(v) do - table.insert(keys, {k, elt}) + for _, elt in ipairs(v) do + table.insert(keys, { k, elt }) end else - table.insert(keys, {k, v}) + table.insert(keys, { k, v }) end end end @@ -91,18 +91,26 @@ local function metric_pairs(t) end local function group_transform(cfg, k, v) - if v.name then k = v.name end + if v.name then + k = v.name + end local new_group = { symbols = {} } - if v.enabled then new_group.enabled = v.enabled end - if v.disabled then new_group.disabled = v.disabled end - if v.max_score then new_group.max_score = v.max_score end + if v.enabled then + new_group.enabled = v.enabled + end + if v.disabled then + new_group.disabled = v.disabled + end + if v.max_score then + new_group.max_score = v.max_score + end if v.symbol then - for sk,sv in metric_pairs(v.symbol) do + for sk, sv in metric_pairs(v.symbol) do if sv.name then sk = sv.name sv.name = nil -- Remove field @@ -112,7 +120,9 @@ local function group_transform(cfg, k, v) end end - if not cfg.group then cfg.group = {} end + if not cfg.group then + cfg.group = {} + end if cfg.group[k] then cfg.group[k] = lua_util.override_defaults(cfg.group[k], new_group) @@ -153,7 +163,9 @@ local function test_groups(groups) for gr_name, gr in pairs(groups) do if not gr.symbols then local cnt = 0 - for _,_ in pairs(gr) do cnt = cnt + 1 end + for _, _ in pairs(gr) do + cnt = cnt + 1 + end if cnt == 0 then logger.debugx('group %s is empty', gr_name) @@ -205,9 +217,9 @@ end -- merged group definition local function merge_groups(groups) local ret = {} - for k,gr in pairs(groups) do + for k, gr in pairs(groups) do if type(k) == 'number' then - for key,sec in pairs(gr) do + for key, sec in pairs(gr) do ret[key] = sec end else @@ -228,7 +240,7 @@ local function check_statistics_sanity() if rspamd_util.file_exists(local_stat) and rspamd_util.file_exists(local_bayes) then - logger.warnx(rspamd_config, 'conflicting files %s and %s are found: '.. + logger.warnx(rspamd_config, 'conflicting files %s and %s are found: ' .. 'Rspamd classifier configuration might be broken!', local_stat, local_bayes) end end @@ -237,7 +249,7 @@ end local function surbl_section_convert(cfg, section) local rbl_section = cfg.rbl.rbls local wl = section.whitelist - for name,value in pairs(section.rules or {}) do + for name, value in pairs(section.rules or {}) do if rbl_section[name] then logger.warnx(rspamd_config, 'conflicting names in surbl and rbl rules: %s, prefer surbl rule!', name) @@ -251,13 +263,21 @@ local function surbl_section_convert(cfg, section) converted.whitelist = wl end - for k,v in pairs(value) do + for k, v in pairs(value) do local skip = false -- Rename - if k == 'suffix' then k = 'rbl' end - if k == 'ips' then k = 'returncodes' end - if k == 'bits' then k = 'returnbits' end - if k == 'noip' then k = 'no_ip' end + if k == 'suffix' then + k = 'rbl' + end + if k == 'ips' then + k = 'returncodes' + end + if k == 'bits' then + k = 'returnbits' + end + if k == 'noip' then + k = 'no_ip' + end -- Crappy legacy if k == 'options' then if v == 'noip' or v == 'no_ip' then @@ -292,7 +312,7 @@ end local function emails_section_convert(cfg, section) local rbl_section = cfg.rbl.rbls local wl = section.whitelist - for name,value in pairs(section.rules or {}) do + for name, value in pairs(section.rules or {}) do if rbl_section[name] then logger.warnx(rspamd_config, 'conflicting names in emails and rbl rules: %s, prefer emails rule!', name) @@ -306,15 +326,27 @@ local function emails_section_convert(cfg, section) converted.whitelist = wl end - for k,v in pairs(value) do + for k, v in pairs(value) do local skip = false -- Rename - if k == 'dnsbl' then k = 'rbl' end - if k == 'check_replyto' then k = 'replyto' end - if k == 'hashlen' then k = 'hash_len' end - if k == 'encoding' then k = 'hash_format' end - if k == 'domain_only' then k = 'emails_domainonly' end - if k == 'delimiter' then k = 'emails_delimiter' end + if k == 'dnsbl' then + k = 'rbl' + end + if k == 'check_replyto' then + k = 'replyto' + end + if k == 'hashlen' then + k = 'hash_len' + end + if k == 'encoding' then + k = 'hash_format' + end + if k == 'domain_only' then + k = 'emails_domainonly' + end + if k == 'delimiter' then + k = 'emails_delimiter' + end if k == 'skip_body' then skip = true if v then @@ -365,14 +397,14 @@ return function(cfg) logger.errx('no actions defined') else -- Perform sanity check for actions - local actions_defs = {'no action', 'no_action', -- In case if that's added - 'greylist', 'add header', 'add_header', - 'rewrite subject', 'rewrite_subject', 'quarantine', - 'reject', 'discard'} + local actions_defs = { 'no action', 'no_action', -- In case if that's added + 'greylist', 'add header', 'add_header', + 'rewrite subject', 'rewrite_subject', 'quarantine', + 'reject', 'discard' } if not cfg.actions['no action'] and not cfg.actions['no_action'] and - not cfg.actions['accept'] then - for _,d in ipairs(actions_defs) do + not cfg.actions['accept'] then + for _, d in ipairs(actions_defs) do if cfg.actions[d] then local action_score = nil @@ -387,7 +419,7 @@ return function(cfg) elseif type(action_score) == 'number' and action_score < 0 then cfg.actions['no_action'] = cfg.actions[d] - 0.001 logger.infox(rspamd_config, 'set no_action score to: %s, as action %s has negative score', - cfg.actions['no_action'], d) + cfg.actions['no_action'], d) break end end @@ -401,7 +433,7 @@ return function(cfg) actions_set['grow_factor'] = true actions_set['subject'] = true - for k,_ in pairs(cfg.actions) do + for k, _ in pairs(cfg.actions) do if not actions_set[k] then logger.warnx(rspamd_config, 'unknown element in actions section: %s', k) end @@ -417,18 +449,18 @@ return function(cfg) 'reject', 'discard' } - for i=1,(#actions_order - 1) do + for i = 1, (#actions_order - 1) do local act = actions_order[i] if cfg.actions[act] and type(cfg.actions[act]) == 'number' then local score = cfg.actions[act] - for j=i+1,#actions_order do + for j = i + 1, #actions_order do local next_act = actions_order[j] if cfg.actions[next_act] and type(cfg.actions[next_act]) == 'number' then local next_score = cfg.actions[next_act] if next_score <= score then - logger.errx(rspamd_config, 'invalid actions thresholds order: action %s (%s) must have lower '.. + logger.errx(rspamd_config, 'invalid actions thresholds order: action %s (%s) must have lower ' .. 'score than action %s (%s)', act, score, next_act, next_score) ret = false end @@ -475,7 +507,9 @@ return function(cfg) -- Again: legacy stuff :( if not cfg.dkim.sign_headers then local sec = cfg.dkim_signing - if sec and sec[1] then sec = cfg.dkim_signing[1] end + if sec and sec[1] then + sec = cfg.dkim_signing[1] + end if sec and sec.sign_headers then cfg.dkim.sign_headers = sec.sign_headers @@ -483,7 +517,7 @@ return function(cfg) end -- DKIM signing/ARC legacy - for _, mod in ipairs({'dkim_signing', 'arc'}) do + for _, mod in ipairs({ 'dkim_signing', 'arc' }) do if cfg[mod] then if cfg[mod].auth_only ~= nil then if cfg[mod].sign_authenticated ~= nil then @@ -502,10 +536,10 @@ return function(cfg) end -- Try to find some obvious issues with configuration - for k,v in pairs(cfg) do - if type(v) == 'table' and v[k] and type (v[k]) == 'table' then + for k, v in pairs(cfg) do + if type(v) == 'table' and v[k] and type(v[k]) == 'table' then logger.errx('nested section: %s { %s { ... } }, it is likely a configuration error', - k, k) + k, k) end end @@ -531,9 +565,13 @@ return function(cfg) } end - if not cfg.reputation.rules then cfg.reputation.rules = {} end + if not cfg.reputation.rules then + cfg.reputation.rules = {} + end - if not fun.any(function(_, v) return v.selector and v.selector.ip end, + if not fun.any(function(_, v) + return v.selector and v.selector.ip + end, cfg.reputation.rules) then logger.infox(rspamd_config, 'attach ip reputation element to use it') diff --git a/lualib/lua_clickhouse.lua b/lualib/lua_clickhouse.lua index 7317cc392..28366d28a 100644 --- a/lualib/lua_clickhouse.lua +++ b/lualib/lua_clickhouse.lua @@ -35,7 +35,7 @@ local function escape_spaces(query) end local function ch_number(a) - if (a+2^52)-2^52 == a then + if (a + 2 ^ 52) - 2 ^ 52 == a then -- Integer return tostring(math.floor(a)) end @@ -59,7 +59,7 @@ end -- Converts an array to a string suitable for clickhouse local function array_to_string(ar) - for i,elt in ipairs(ar) do + for i, elt in ipairs(ar) do local t = type(elt) if t == 'string' then ar[i] = string.format('\'%s\'', clickhouse_quote(elt)) @@ -76,7 +76,7 @@ end -- Converts a row into TSV, taking extra care about arrays local function row_to_tsv(row) - for i,elt in ipairs(row) do + for i, elt in ipairs(row) do local t = type(elt) if t == 'table' then row[i] = '[' .. array_to_string(elt) .. ']' @@ -107,9 +107,9 @@ local function parse_clickhouse_response_json_eachrow(params, data, row_cb) local parser = ucl.parser() local res, err if type(s) == 'string' then - res,err = parser:parse_string(s) + res, err = parser:parse_string(s) else - res,err = parser:parse_text(s) + res, err = parser:parse_text(s) end if not res then @@ -151,9 +151,9 @@ local function parse_clickhouse_response_json(params, data) local res, err if type(s) == 'string' then - res,err = parser:parse_string(s) + res, err = parser:parse_string(s) else - res,err = parser:parse_text(s) + res, err = parser:parse_text(s) end if not res then @@ -169,14 +169,16 @@ local function parse_clickhouse_response_json(params, data) return 'bad json', {} end - return nil,json + return nil, json end -- Helper to generate HTTP closure local function mk_http_select_cb(upstream, params, ok_cb, fail_cb, row_cb) local function http_cb(err_message, code, data, _) if code ~= 200 or err_message then - if not err_message then err_message = data end + if not err_message then + err_message = data + end local ip_addr = upstream:get_addr():to_string(true) if fail_cb then @@ -205,8 +207,8 @@ local function mk_http_select_cb(upstream, params, ok_cb, fail_cb, row_cb) else local ip_addr = upstream:get_addr():to_string(true) rspamd_logger.errx(params.log_obj, - "request failed on clickhouse server %s: %s", - ip_addr, 'failed to parse reply') + "request failed on clickhouse server %s: %s", + ip_addr, 'failed to parse reply') end end end @@ -219,7 +221,9 @@ end local function mk_http_insert_cb(upstream, params, ok_cb, fail_cb) local function http_cb(err_message, code, data, _) if code ~= 200 or err_message then - if not err_message then err_message = data end + if not err_message then + err_message = data + end local ip_addr = upstream:get_addr():to_string(true) if fail_cb then @@ -234,7 +238,7 @@ local function mk_http_insert_cb(upstream, params, ok_cb, fail_cb) upstream:ok() if ok_cb then - local err,parsed = parse_clickhouse_response_json(data) + local err, parsed = parse_clickhouse_response_json(data) if err then fail_cb(params, err, data) @@ -273,10 +277,12 @@ end -- @example -- --]] -exports.select = function (upstream, settings, params, query, ok_cb, fail_cb, row_cb) +exports.select = function(upstream, settings, params, query, ok_cb, fail_cb, row_cb) local http_params = {} - for k,v in pairs(params) do http_params[k] = v end + for k, v in pairs(params) do + http_params[k] = v + end http_params.callback = mk_http_select_cb(upstream, http_params, ok_cb, fail_cb, row_cb) http_params.gzip = settings.use_gzip @@ -327,10 +333,12 @@ end -- @example -- --]] -exports.select_sync = function (upstream, settings, params, query, row_cb) +exports.select_sync = function(upstream, settings, params, query, row_cb) local http_params = {} - for k,v in pairs(params) do http_params[k] = v end + for k, v in pairs(params) do + http_params[k] = v + end http_params.gzip = settings.use_gzip http_params.mime_type = 'text/plain' @@ -388,11 +396,13 @@ end -- @example -- --]] -exports.insert = function (upstream, settings, params, query, rows, - ok_cb, fail_cb) +exports.insert = function(upstream, settings, params, query, rows, + ok_cb, fail_cb) local http_params = {} - for k,v in pairs(params) do http_params[k] = v end + for k, v in pairs(params) do + http_params[k] = v + end http_params.callback = mk_http_insert_cb(upstream, http_params, ok_cb, fail_cb) http_params.gzip = settings.use_gzip @@ -402,7 +412,7 @@ exports.insert = function (upstream, settings, params, query, rows, http_params.user = settings.user http_params.password = settings.password http_params.method = 'POST' - http_params.body = {rspamd_text.fromtable(rows, '\n'), '\n'} + http_params.body = { rspamd_text.fromtable(rows, '\n'), '\n' } http_params.log_obj = params.task or params.config if not http_params.url then @@ -441,11 +451,13 @@ end -- @example -- --]] -exports.generic = function (upstream, settings, params, query, +exports.generic = function(upstream, settings, params, query, ok_cb, fail_cb) local http_params = {} - for k,v in pairs(params) do http_params[k] = v end + for k, v in pairs(params) do + http_params[k] = v + end http_params.callback = mk_http_insert_cb(upstream, http_params, ok_cb, fail_cb) http_params.gzip = settings.use_gzip @@ -488,10 +500,12 @@ end -- @example -- --]] -exports.generic_sync = function (upstream, settings, params, query) +exports.generic_sync = function(upstream, settings, params, query) local http_params = {} - for k,v in pairs(params) do http_params[k] = v end + for k, v in pairs(params) do + http_params[k] = v + end http_params.gzip = settings.use_gzip http_params.mime_type = 'text/plain' @@ -521,10 +535,10 @@ exports.generic_sync = function (upstream, settings, params, query) return response.content, response else lua_util.debugm(N, http_params.log_obj, "clickhouse generic response: %1", response) - local e,obj = parse_clickhouse_response_json(params, response.content) + local e, obj = parse_clickhouse_response_json(params, response.content) if e then - return e,nil + return e, nil end return nil, obj end diff --git a/lualib/lua_content/ical.lua b/lualib/lua_content/ical.lua index 72db634ee..d018a8596 100644 --- a/lualib/lua_content/ical.lua +++ b/lualib/lua_content/ical.lua @@ -23,11 +23,15 @@ local ical_grammar local function gen_grammar() if not ical_grammar then local wsp = l.S(" \t\v\f") - local crlf = (l.P"\r"^-1 * l.P"\n") + l.P"\r" - local eol = (crlf * #crlf) + (crlf - (crlf^-1 * wsp)) - local name = l.C((l.P(1) - (l.P":"))^1) / function(v) return (v:gsub("[\n\r]+%s","")) end - local value = l.C((l.P(1) - eol)^0) / function(v) return (v:gsub("[\n\r]+%s","")) end - ical_grammar = name * ":" * wsp^0 * value * eol^-1 + local crlf = (l.P "\r" ^ -1 * l.P "\n") + l.P "\r" + local eol = (crlf * #crlf) + (crlf - (crlf ^ -1 * wsp)) + local name = l.C((l.P(1) - (l.P ":")) ^ 1) / function(v) + return (v:gsub("[\n\r]+%s", "")) + end + local value = l.C((l.P(1) - eol) ^ 0) / function(v) + return (v:gsub("[\n\r]+%s", "")) + end + ical_grammar = name * ":" * wsp ^ 0 * value * eol ^ -1 end return ical_grammar @@ -38,13 +42,15 @@ local exports = {} local function extract_text_data(specific) local fun = require "fun" - local tbl = fun.totable(fun.map(function(e) return e[2]:lower() end, specific.elts)) + local tbl = fun.totable(fun.map(function(e) + return e[2]:lower() + end, specific.elts)) return table.concat(tbl, '\n') end -- Keys that can have visible urls -local url_keys = lua_util.list_to_hash{ +local url_keys = lua_util.list_to_hash { 'description', 'location', 'summary', @@ -55,7 +61,7 @@ local url_keys = lua_util.list_to_hash{ } local function process_ical(input, mpart, task) - local control={n='\n', r=''} + local control = { n = '\n', r = '' } local rspamd_url = require "rspamd_url" local escaper = l.Ct((gen_grammar() / function(key, value) value = value:gsub("\\(.)", control) @@ -65,17 +71,17 @@ local function process_ical(input, mpart, task) local local_urls = rspamd_url.all(task:get_mempool(), value) if local_urls and #local_urls > 0 then - for _,u in ipairs(local_urls) do + for _, u in ipairs(local_urls) do lua_util.debugm(N, task, 'ical: found URL in ical key "%s": %s', - key, tostring(u)) + key, tostring(u)) task:inject_url(u, mpart) end end end lua_util.debugm(N, task, 'ical: ical key %s = "%s"', key, value) - return {key, value} - end)^1) + return { key, value } + end) ^ 1) local elts = escaper:match(input) diff --git a/lualib/lua_content/init.lua b/lualib/lua_content/init.lua index 1a13311b1..701d223a5 100644 --- a/lualib/lua_content/init.lua +++ b/lualib/lua_content/init.lua @@ -26,21 +26,21 @@ local lua_util = require "lua_util" local content_modules = { ical = { - mime_type = {"text/calendar", "application/calendar"}, + mime_type = { "text/calendar", "application/calendar" }, module = require "lua_content/ical", - extensions = {'ics'}, + extensions = { 'ics' }, output = "text" }, vcf = { - mime_type = {"text/vcard", "application/vcard"}, + mime_type = { "text/vcard", "application/vcard" }, module = require "lua_content/vcard", - extensions = {'vcf'}, + extensions = { 'vcf' }, output = "text" }, pdf = { mime_type = "application/pdf", module = require "lua_content/pdf", - extensions = {'pdf'}, + extensions = { 'pdf' }, output = "table" }, } @@ -51,20 +51,20 @@ local modules_by_extension local function init() modules_by_mime_type = {} modules_by_extension = {} - for k,v in pairs(content_modules) do + for k, v in pairs(content_modules) do if v.mime_type then if type(v.mime_type) == 'table' then - for _,mt in ipairs(v.mime_type) do - modules_by_mime_type[mt] = {k, v} + for _, mt in ipairs(v.mime_type) do + modules_by_mime_type[mt] = { k, v } end else - modules_by_mime_type[v.mime_type] = {k, v} + modules_by_mime_type[v.mime_type] = { k, v } end end if v.extensions then - for _,ext in ipairs(v.extensions) do - modules_by_extension[ext] = {k, v} + for _, ext in ipairs(v.extensions) do + modules_by_extension[ext] = { k, v } end end end @@ -106,5 +106,4 @@ exports.maybe_process_mime_part = function(part, task) end - return exports
\ No newline at end of file diff --git a/lualib/lua_content/pdf.lua b/lualib/lua_content/pdf.lua index 0677fab06..a8b6e984a 100644 --- a/lualib/lua_content/pdf.lua +++ b/lualib/lua_content/pdf.lua @@ -147,10 +147,10 @@ local function compile_tries() rspamd_trie.flags.no_start) local function compile_pats(patterns, indexes, compile_flags) local strs = {} - for what,data in pairs(patterns) do - for i,pat in ipairs(data.patterns) do + for what, data in pairs(patterns) do + for i, pat in ipairs(data.patterns) do strs[#strs + 1] = pat - indexes[#indexes + 1] = {what, data, pat, i} + indexes[#indexes + 1] = { what, data, pat, i } end end @@ -175,7 +175,7 @@ local function generic_grammar_elts() local S = lpeg.S local V = lpeg.V local C = lpeg.C - local D = R'09' -- Digits + local D = R '09' -- Digits local grammar_elts = {} @@ -214,37 +214,37 @@ local function generic_grammar_elts() end local function pdf_id_unescape(s) - return (s:gsub('#%d%d', function (cc) + return (s:gsub('#%d%d', function(cc) return string.char(tonumber(cc:sub(2), 16)) end)) end - local delim = S'()<>[]{}/%' - grammar_elts.ws = S'\0 \r\n\t\f' - local hex = R'af' + R'AF' + D + local delim = S '()<>[]{}/%' + grammar_elts.ws = S '\0 \r\n\t\f' + local hex = R 'af' + R 'AF' + D -- Comments. - local eol = P'\r\n' + '\n' - local line = (1 - S'\r\n\f')^0 * eol^-1 - grammar_elts.comment = P'%' * line + local eol = P '\r\n' + '\n' + local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1 + grammar_elts.comment = P '%' * line -- Numbers. - local sign = S'+-'^-1 - local decimal = D^1 - local float = D^1 * P'.' * D^0 + P'.' * D^1 + local sign = S '+-' ^ -1 + local decimal = D ^ 1 + local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1 grammar_elts.number = C(sign * (float + decimal)) / tonumber -- String - grammar_elts.str = P{ "(" * C(((1 - S"()\\") + (P '\\' * 1) + V(1))^0) / pdf_string_unescape * ")" } - grammar_elts.hexstr = P{"<" * C(hex^0) / pdf_hexstring_unescape * ">"} + grammar_elts.str = P { "(" * C(((1 - S "()\\") + (P '\\' * 1) + V(1)) ^ 0) / pdf_string_unescape * ")" } + grammar_elts.hexstr = P { "<" * C(hex ^ 0) / pdf_hexstring_unescape * ">" } -- Identifier - grammar_elts.id = P{'/' * C((1-(delim + grammar_elts.ws))^1) / pdf_id_unescape} + grammar_elts.id = P { '/' * C((1 - (delim + grammar_elts.ws)) ^ 1) / pdf_id_unescape } -- Booleans (who care about them?) grammar_elts.boolean = C(P("true") + P("false")) -- Stupid references - grammar_elts.ref = lpeg.Ct{lpeg.Cc("%REF%") * C(D^1) * " " * C(D^1) * " " * "R"} + grammar_elts.ref = lpeg.Ct { lpeg.Cc("%REF%") * C(D ^ 1) * " " * C(D ^ 1) * " " * "R" } return grammar_elts end @@ -255,16 +255,16 @@ local function gen_outer_grammar() local V = lpeg.V local gen = generic_grammar_elts() - return lpeg.P{ + return lpeg.P { "EXPR"; - EXPR = gen.ws^0 * V("ELT")^0 * gen.ws^0, + EXPR = gen.ws ^ 0 * V("ELT") ^ 0 * gen.ws ^ 0, ELT = V("ARRAY") + V("DICT") + V("ATOM"), - ATOM = gen.ws^0 * (gen.comment + gen.boolean + gen.ref + - gen.number + V("STRING") + gen.id) * gen.ws^0, - DICT = "<<" * gen.ws^0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR")^0, rawset) * gen.ws^0 * ">>", - KV_PAIR = lpeg.Cg(gen.id * gen.ws^0 * V("ELT") * gen.ws^0), - ARRAY = "[" * gen.ws^0 * lpeg.Ct(V("ELT")^0) * gen.ws^0 * "]", - STRING = lpeg.P{gen.str + gen.hexstr}, + ATOM = gen.ws ^ 0 * (gen.comment + gen.boolean + gen.ref + + gen.number + V("STRING") + gen.id) * gen.ws ^ 0, + DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>", + KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ELT") * gen.ws ^ 0), + ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ELT") ^ 0) * gen.ws ^ 0 * "]", + STRING = lpeg.P { gen.str + gen.hexstr }, } end @@ -274,7 +274,7 @@ local function gen_graphics_unary() local S = lpeg.S return P("q") + P("Q") + P("h") - + S("WSsFfBb") * P("*")^0 + P("n") + + S("WSsFfBb") * P("*") ^ 0 + P("n") end local function gen_graphics_binary() @@ -317,29 +317,29 @@ local function gen_text_grammar() local text_quote_op = P('"') local font_op = P("Tf") - return lpeg.P{ + return lpeg.P { "EXPR"; - EXPR = gen.ws^0 * lpeg.Ct(V("COMMAND")^0), + EXPR = gen.ws ^ 0 * lpeg.Ct(V("COMMAND") ^ 0), COMMAND = (V("UNARY") + V("BINARY") + V("TERNARY") + V("NARY") + V("TEXT") + - V("FONT") + gen.comment) * gen.ws^0, + V("FONT") + gen.comment) * gen.ws ^ 0, UNARY = unary_ops, - BINARY = V("ARG") / empty * gen.ws^1 * binary_ops, - TERNARY = V("ARG") / empty * gen.ws^1 * V("ARG") / empty * gen.ws^1 * ternary_ops, - NARY = (gen.number / 0 * gen.ws^1)^1 * (gen.id / empty * gen.ws^0)^-1 * nary_op, + BINARY = V("ARG") / empty * gen.ws ^ 1 * binary_ops, + TERNARY = V("ARG") / empty * gen.ws ^ 1 * V("ARG") / empty * gen.ws ^ 1 * ternary_ops, + NARY = (gen.number / 0 * gen.ws ^ 1) ^ 1 * (gen.id / empty * gen.ws ^ 0) ^ -1 * nary_op, ARG = V("ARRAY") + V("DICT") + V("ATOM"), ATOM = (gen.comment + gen.boolean + gen.ref + gen.number + V("STRING") + gen.id), - DICT = "<<" * gen.ws^0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR")^0, rawset) * gen.ws^0 * ">>", - KV_PAIR = lpeg.Cg(gen.id * gen.ws^0 * V("ARG") * gen.ws^0), - ARRAY = "[" * gen.ws^0 * lpeg.Ct(V("ARG")^0) * gen.ws^0 * "]", - STRING = lpeg.P{gen.str + gen.hexstr}, - TEXT = (V("TEXT_ARG") * gen.ws^1 * text_binary_op) + - (V("ARG") / 0 * gen.ws^1 * V("ARG") / 0 * gen.ws^1 * V("TEXT_ARG") * gen.ws^1 * text_quote_op), - FONT = (V("FONT_ARG") * gen.ws^1 * (gen.number / 0) * gen.ws^1 * font_op), + DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>", + KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ARG") * gen.ws ^ 0), + ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ARG") ^ 0) * gen.ws ^ 0 * "]", + STRING = lpeg.P { gen.str + gen.hexstr }, + TEXT = (V("TEXT_ARG") * gen.ws ^ 1 * text_binary_op) + + (V("ARG") / 0 * gen.ws ^ 1 * V("ARG") / 0 * gen.ws ^ 1 * V("TEXT_ARG") * gen.ws ^ 1 * text_quote_op), + FONT = (V("FONT_ARG") * gen.ws ^ 1 * (gen.number / 0) * gen.ws ^ 1 * font_op), FONT_ARG = lpeg.Ct(lpeg.Cc("%font%") * gen.id), TEXT_ARG = lpeg.Ct(V("STRING")) + V("TEXT_ARRAY"), TEXT_ARRAY = "[" * - lpeg.Ct(((gen.ws^0 * (gen.ws^0 * (gen.number / 0)^0 * gen.ws^0 * (gen.str + gen.hexstr)))^1)) * gen.ws^0 * "]", + lpeg.Ct(((gen.ws ^ 0 * (gen.ws ^ 0 * (gen.number / 0) ^ 0 * gen.ws ^ 0 * (gen.str + gen.hexstr))) ^ 1)) * gen.ws ^ 0 * "]", } end @@ -393,7 +393,7 @@ local function maybe_apply_filter(dict, data, pdf, task) if dict.Filter then local filt = dict.Filter if type(filt) == 'string' then - filt = {filt} + filt = { filt } end if dict.DecodeParms then @@ -401,19 +401,21 @@ local function maybe_apply_filter(dict, data, pdf, task) if type(decode_params) == 'table' then if decode_params.Predictor then - return nil,'predictor exists' + return nil, 'predictor exists' end end end - for _,f in ipairs(filt) do + for _, f in ipairs(filt) do uncompressed = apply_pdf_filter(uncompressed, f) - if not uncompressed then break end + if not uncompressed then + break + end end end - return uncompressed,nil + return uncompressed, nil end -- Conditionally extract stream data from object and attach it as obj.uncompressed @@ -428,7 +430,7 @@ local function maybe_extract_object_stream(obj, pdf, task) tonumber(maybe_dereference_object(dict.Length, pdf, task)) or 0) local real_stream = obj.stream.data:span(1, len) - local uncompressed,filter_err = maybe_apply_filter(dict, real_stream, pdf, task) + local uncompressed, filter_err = maybe_apply_filter(dict, real_stream, pdf, task) if uncompressed then obj.uncompressed = uncompressed @@ -442,7 +444,6 @@ local function maybe_extract_object_stream(obj, pdf, task) end end - local function parse_object_grammar(obj, task, pdf) -- Parse grammar local obj_dict_span @@ -453,7 +454,7 @@ local function parse_object_grammar(obj, task, pdf) end if obj_dict_span:len() < config.max_processing_size then - local ret,obj_or_err = pcall(pdf_outer_grammar.match, pdf_outer_grammar, obj_dict_span) + local ret, obj_or_err = pcall(pdf_outer_grammar.match, pdf_outer_grammar, obj_dict_span) if ret then if obj.stream then @@ -669,11 +670,11 @@ process_dict = function(task, pdf, obj, dict) if contents and type(contents) == 'table' then if contents[1] == '%REF%' then -- Single reference - contents = {contents} + contents = { contents } end obj.contents = {} - for _,c in ipairs(contents) do + for _, c in ipairs(contents) do local cobj = maybe_dereference_object(c, pdf, task) if cobj and type(cobj) == 'table' then obj.contents[#obj.contents + 1] = cobj @@ -719,25 +720,25 @@ process_dict = function(task, pdf, obj, dict) ---[[Disabled fonts extraction - local fonts = obj.resources.Font - if fonts and type(fonts) == 'table' then - obj.fonts = {} - for k,v in pairs(fonts) do - obj.fonts[k] = maybe_dereference_object(v, pdf, task) + --[[Disabled fonts extraction + local fonts = obj.resources.Font + if fonts and type(fonts) == 'table' then + obj.fonts = {} + for k,v in pairs(fonts) do + obj.fonts[k] = maybe_dereference_object(v, pdf, task) - if obj.fonts[k] then - local font = obj.fonts[k] + if obj.fonts[k] then + local font = obj.fonts[k] - if config.text_extraction then - process_font(task, pdf, font, k) - lua_util.debugm(N, task, 'found font "%s" for object %s:%s -> %s', - k, obj.major, obj.minor, font) + if config.text_extraction then + process_font(task, pdf, font, k) + lua_util.debugm(N, task, 'found font "%s" for object %s:%s -> %s', + k, obj.major, obj.minor, font) + end + end end end - end - end -]] + ]] lua_util.debugm(N, task, 'found resources for object %s:%s (%s): %s', obj.major, obj.minor, obj.type, obj.resources) @@ -783,8 +784,8 @@ local compound_obj_grammar local function compound_obj_grammar_gen() if not compound_obj_grammar then local gen = generic_grammar_elts() - compound_obj_grammar = gen.ws^0 * (gen.comment * gen.ws^1)^0 * - lpeg.Ct(lpeg.Ct(gen.number * gen.ws^1 * gen.number * gen.ws^0)^1) + compound_obj_grammar = gen.ws ^ 0 * (gen.comment * gen.ws ^ 1) ^ 0 * + lpeg.Ct(lpeg.Ct(gen.number * gen.ws ^ 1 * gen.number * gen.ws ^ 0) ^ 1) end return compound_obj_grammar @@ -798,8 +799,8 @@ local function pdf_compound_object_unpack(_, uncompressed, pdf, task, first) lua_util.debugm(N, task, 'compound elts (chunk length %s): %s', #uncompressed, elts) - for i,pair in ipairs(elts) do - local obj_number,offset = pair[1], pair[2] + for i, pair in ipairs(elts) do + local obj_number, offset = pair[1], pair[2] offset = offset + first if offset < #uncompressed then @@ -833,7 +834,7 @@ end -- PDF 1.5 ObjStmt local function extract_pdf_compound_objects(task, pdf) - for i,obj in ipairs(pdf.objects or {}) do + for i, obj in ipairs(pdf.objects or {}) do if i > 0 and i % 100 == 0 then local now = rspamd_util.get_ticks() @@ -894,7 +895,9 @@ local function extract_outer_objects(task, input, pdf) -- Also get the starting span and try to match it versus obj re to get numbers local obj_line_potential = first - 32 - if obj_line_potential < 1 then obj_line_potential = 1 end + if obj_line_potential < 1 then + obj_line_potential = 1 + end local prev_obj_end = pdf.end_objects[end_pos - 1] if end_pos > 1 and prev_obj_end >= obj_line_potential and prev_obj_end < first then obj_line_potential = prev_obj_end + 1 @@ -941,7 +944,7 @@ local function attach_pdf_streams(task, input, pdf) max_start_pos = math.min(config.max_pdf_objects, #pdf.start_streams) max_end_pos = math.min(config.max_pdf_objects, #pdf.end_streams) - for _,obj in ipairs(pdf.objects) do + for _, obj in ipairs(pdf.objects) do while start_pos <= max_start_pos and end_pos <= max_end_pos do local first = pdf.start_streams[start_pos] local last = pdf.end_streams[end_pos] @@ -957,7 +960,9 @@ local function attach_pdf_streams(task, input, pdf) -- Strip the first \n while first < last do local chr = input:byte(first) - if chr ~= 13 and chr ~= 10 then break end + if chr ~= 13 and chr ~= 10 then + break + end first = first + 1 end local len = last - first @@ -1000,7 +1005,7 @@ local function postprocess_pdf_objects(task, input, pdf) -- Now we have objects and we need to attach streams that are in bounds attach_pdf_streams(task, input, pdf) -- Parse grammar for outer objects - for i,obj in ipairs(pdf.objects) do + for i, obj in ipairs(pdf.objects) do if i > 0 and i % 100 == 0 then local now = rspamd_util.get_ticks() @@ -1031,7 +1036,7 @@ local function postprocess_pdf_objects(task, input, pdf) end -- Now we might probably have all objects being processed - for i,obj in ipairs(pdf.objects) do + for i, obj in ipairs(pdf.objects) do if obj.dict then -- Types processing if i > 0 and i % 100 == 0 then @@ -1076,10 +1081,10 @@ local function offsets_to_blocks(starts, ends, out) end local function search_text(task, pdf) - for _,obj in ipairs(pdf.objects) do + for _, obj in ipairs(pdf.objects) do if obj.type == 'Page' and obj.contents then local text = {} - for _,tobj in ipairs(obj.contents) do + for _, tobj in ipairs(obj.contents) do maybe_extract_object_stream(tobj, pdf, task) local matches = pdf_text_trie:match(tobj.uncompressed or '') if matches then @@ -1087,20 +1092,20 @@ local function search_text(task, pdf) local starts = {} local ends = {} - for npat,matched_positions in pairs(matches) do + for npat, matched_positions in pairs(matches) do if npat == 1 then - for _,pos in ipairs(matched_positions) do + for _, pos in ipairs(matched_positions) do starts[#starts + 1] = pos end else - for _,pos in ipairs(matched_positions) do + for _, pos in ipairs(matched_positions) do ends[#ends + 1] = pos end end end offsets_to_blocks(starts, ends, text_blocks) - for _,bl in ipairs(text_blocks) do + for _, bl in ipairs(text_blocks) do if bl.len > 2 then -- To remove \s+ET\b pattern (it can leave trailing space or not but it doesn't matter) bl.len = bl.len - 2 @@ -1111,7 +1116,7 @@ local function search_text(task, pdf) -- tobj.major, tobj.minor, bl.data) if bl.len < config.max_processing_size then - local ret,obj_or_err = pcall(pdf_text_grammar.match, pdf_text_grammar, + local ret, obj_or_err = pcall(pdf_text_grammar.match, pdf_text_grammar, bl.data) if ret then @@ -1147,13 +1152,13 @@ local function search_urls(task, pdf, mpart) return end - for k,v in pairs(dict) do + for k, v in pairs(dict) do if type(v) == 'table' then recursive_object_traverse(obj, v, rec + 1) elseif k == 'URI' then v = maybe_dereference_object(v, pdf, task) if type(v) == 'string' then - local url = rspamd_url.create(task:get_mempool(), v, {'content'}) + local url = rspamd_url.create(task:get_mempool(), v, { 'content' }) if url then lua_util.debugm(N, task, 'found url %s in object %s:%s', @@ -1165,7 +1170,7 @@ local function search_urls(task, pdf, mpart) end end - for _,obj in ipairs(pdf.objects) do + for _, obj in ipairs(pdf.objects) do if obj.dict and type(obj.dict) == 'table' then recursive_object_traverse(obj, obj.dict, 0) end @@ -1193,10 +1198,10 @@ local function process_pdf(input, mpart, task) -- Output object that excludes all internal stuff local pdf_output = lua_util.shallowcopy(pdf_object) local grouped_processors = {} - for npat,matched_positions in pairs(matches) do + for npat, matched_positions in pairs(matches) do local index = pdf_indexes[npat] - local proc_key,loc_npat = index[1], index[4] + local proc_key, loc_npat = index[1], index[4] if not grouped_processors[proc_key] then grouped_processors[proc_key] = { @@ -1206,16 +1211,18 @@ local function process_pdf(input, mpart, task) end local proc = grouped_processors[proc_key] -- Fill offsets - for _,pos in ipairs(matched_positions) do - proc.offsets[#proc.offsets + 1] = {pos, loc_npat} + for _, pos in ipairs(matched_positions) do + proc.offsets[#proc.offsets + 1] = { pos, loc_npat } end end - for name,processor in pairs(grouped_processors) do + for name, processor in pairs(grouped_processors) do -- Sort by offset lua_util.debugm(N, task, "pdf: process group %s with %s matches", name, #processor.offsets) - table.sort(processor.offsets, function(e1, e2) return e1[1] < e2[1] end) + table.sort(processor.offsets, function(e1, e2) + return e1[1] < e2[1] + end) processor.processor_func(input, task, processor.offsets, pdf_object, pdf_output) end @@ -1254,7 +1261,7 @@ local function process_pdf(input, mpart, task) end else -- All hashes - for h,sc in pairs(pdf_object.scripts) do + for h, sc in pairs(pdf_object.scripts) do if config.min_js_fuzzy and #sc.data >= config.min_js_fuzzy then lua_util.debugm(N, task, "pdf: add fuzzy hash from JavaScript: %s; size = %s; object: %s:%s", sc.hash, @@ -1323,7 +1330,7 @@ processors.suspicious = function(input, task, positions, pdf_object, pdf_output) local nencoded = 0 local close_encoded = 0 local last_encoded - for _,match in ipairs(positions) do + for _, match in ipairs(positions) do if match[2] == 1 then -- netsh suspicious_factor = suspicious_factor + 0.5 @@ -1386,7 +1393,7 @@ local function generic_table_inserter(positions, pdf_object, output_key) pdf_object[output_key] = {} end local shift = #pdf_object[output_key] - for i,pos in ipairs(positions) do + for i, pos in ipairs(positions) do pdf_object[output_key][i + shift] = pos[1] end end diff --git a/lualib/lua_content/vcard.lua b/lualib/lua_content/vcard.lua index 993a0828f..ed14412ea 100644 --- a/lualib/lua_content/vcard.lua +++ b/lualib/lua_content/vcard.lua @@ -24,11 +24,15 @@ local vcard_grammar local function gen_grammar() if not vcard_grammar then local wsp = l.S(" \t\v\f") - local crlf = (l.P"\r"^-1 * l.P"\n") + l.P"\r" - local eol = (crlf * #crlf) + (crlf - (crlf^-1 * wsp)) - local name = l.C((l.P(1) - (l.P":"))^1) / function(v) return (v:gsub("[\n\r]+%s","")) end - local value = l.C((l.P(1) - eol)^0) / function(v) return (v:gsub("[\n\r]+%s","")) end - vcard_grammar = name * ":" * wsp^0 * value * eol^-1 + local crlf = (l.P "\r" ^ -1 * l.P "\n") + l.P "\r" + local eol = (crlf * #crlf) + (crlf - (crlf ^ -1 * wsp)) + local name = l.C((l.P(1) - (l.P ":")) ^ 1) / function(v) + return (v:gsub("[\n\r]+%s", "")) + end + local value = l.C((l.P(1) - eol) ^ 0) / function(v) + return (v:gsub("[\n\r]+%s", "")) + end + vcard_grammar = name * ":" * wsp ^ 0 * value * eol ^ -1 end return vcard_grammar @@ -37,7 +41,7 @@ end local exports = {} local function process_vcard(input, mpart, task) - local control={n='\n', r=''} + local control = { n = '\n', r = '' } local rspamd_url = require "rspamd_url" local escaper = l.Ct((gen_grammar() / function(key, value) value = value:gsub("\\(.)", control) @@ -45,7 +49,7 @@ local function process_vcard(input, mpart, task) local local_urls = rspamd_url.all(task:get_mempool(), value) if local_urls and #local_urls > 0 then - for _,u in ipairs(local_urls) do + for _, u in ipairs(local_urls) do lua_util.debugm(N, task, 'vcard: found URL in vcard %s', tostring(u)) task:inject_url(u, mpart) @@ -53,8 +57,8 @@ local function process_vcard(input, mpart, task) end lua_util.debugm(N, task, 'vcard: vcard key %s = "%s"', key, value) - return {key, value} - end)^1) + return { key, value } + end) ^ 1) local elts = escaper:match(input) @@ -64,7 +68,9 @@ local function process_vcard(input, mpart, task) return { tag = 'vcard', - extract_text = function() return nil end, -- NYI + extract_text = function() + return nil + end, -- NYI elts = elts } end diff --git a/lualib/lua_dkim_tools.lua b/lualib/lua_dkim_tools.lua index 719b52067..1f396fbc4 100644 --- a/lualib/lua_dkim_tools.lua +++ b/lualib/lua_dkim_tools.lua @@ -27,7 +27,9 @@ local function check_violation(N, task, domain) -- Check for DKIM_REJECT local sym_check = 'R_DKIM_REJECT' - if N == 'arc' then sym_check = 'ARC_REJECT' end + if N == 'arc' then + sym_check = 'ARC_REJECT' + end if task:has_symbol(sym_check) then local sym = task:get_symbol(sym_check)[1] logger.infox(task, 'skip signing for %s: violation %s found: %s', @@ -92,7 +94,7 @@ local function parse_dkim_http_headers(N, task, settings) if not (domain and selector and key) then logger.errx(task, 'missing required headers to sign email') - return false,{} + return false, {} end -- Now check if we need to check the existing auth @@ -114,23 +116,23 @@ local function parse_dkim_http_headers(N, task, settings) end lua_util.debugm(N, task, 'no sign header %s', headers.sign_header) - return false,{} + return false, {} end local function prepare_dkim_signing(N, task, settings) local is_local, is_sign_networks, is_authed if settings.use_http_headers then - local res,tbl = parse_dkim_http_headers(N, task, settings) + local res, tbl = parse_dkim_http_headers(N, task, settings) if not res then if not settings.allow_headers_fallback then - return res,{} + return res, {} else lua_util.debugm(N, task, 'failed to read http headers, fallback to normal schema') end else - return res,tbl + return res, tbl end end @@ -139,13 +141,13 @@ local function prepare_dkim_signing(N, task, settings) local ret = settings.sign_condition(task) if not ret then - return false,{} + return false, {} end if ret[1] then - return true,ret + return true, ret else - return true,{ret} + return true, { ret } end end @@ -163,19 +165,19 @@ local function prepare_dkim_signing(N, task, settings) if metric_action == 'reject' or metric_action == 'drop' then -- No need to sign what we are already rejecting/dropping lua_util.debugm(N, task, 'task result is already %s, no need to sign', metric_action) - return false,{} + return false, {} end if metric_action == 'soft reject' then -- Same here, we are going to delay an email, signing is just a waste of time lua_util.debugm(N, task, 'task result is %s, skip signing', metric_action) - return false,{} + return false, {} end -- For spam actions, there is no clear distinction if metric_action ~= 'no action' and type(settings.skip_spam_sign) == 'boolean' and settings.skip_spam_sign then lua_util.debugm(N, task, 'task result is %s, no need to sign', metric_action) - return false,{} + return false, {} end end @@ -191,7 +193,7 @@ local function prepare_dkim_signing(N, task, settings) lua_util.debugm(N, task, 'mail was sent to us') else lua_util.debugm(N, task, 'mail is ineligible for signing') - return false,{} + return false, {} end local efrom = task:get_from('smtp') @@ -199,7 +201,7 @@ local function prepare_dkim_signing(N, task, settings) if #(((efrom or E)[1] or E).addr or '') == 0 then if not settings.allow_envfrom_empty then lua_util.debugm(N, task, 'empty envelope from not allowed') - return false,{} + return false, {} else empty_envelope = true end @@ -208,7 +210,7 @@ local function prepare_dkim_signing(N, task, settings) local hfrom = task:get_from('mime') if not settings.allow_hdrfrom_multiple and (hfrom or E)[2] then lua_util.debugm(N, task, 'multiple header from not allowed') - return false,{} + return false, {} end local eto = task:get_recipients(0) @@ -258,13 +260,13 @@ local function prepare_dkim_signing(N, task, settings) lua_util.debugm(N, task, 'skip signing: is_sign_network: %s, is_authed: %s, is_local: %s', is_sign_networks, is_authed, is_local) - return false,{} + return false, {} end if not hfrom or not hfrom[1] or not hfrom[1].addr then lua_util.debugm(N, task, 'signing_table: cannot get data when no header from is presented') - return false,{} + return false, {} end local sign_entry = settings.signing_table:get_key(hfrom[1].addr) @@ -277,8 +279,8 @@ local function prepare_dkim_signing(N, task, settings) end if settings.key_table then - -- Now search in key table - local key_entry = settings.key_table:get_key(sign_entry) + -- Now search in key table + local key_entry = settings.key_table:get_key(sign_entry) if key_entry then local parts = lua_util.str_split(key_entry, ':') @@ -290,7 +292,7 @@ local function prepare_dkim_signing(N, task, settings) if not selector then logger.errx(task, 'no selector defined for sign_entry %s, key_entry %s', sign_entry, key_entry) - return false,{} + return false, {} end local res = { @@ -310,7 +312,7 @@ local function prepare_dkim_signing(N, task, settings) hdom, selector, res.domain) end - return true,{res} + return true, { res } elseif #parts == 3 then -- domain, selector, key local selector = parts[2] @@ -332,11 +334,11 @@ local function prepare_dkim_signing(N, task, settings) hdom, selector, res.domain) end - return true,{res} + return true, { res } else logger.errx(task, 'invalid key entry for sign entry %s: %s; when signing %s domain', sign_entry, key_entry, hdom) - return false,{} + return false, {} end elseif settings.use_vault then -- Sign table is presented, the rest is covered by vault @@ -349,17 +351,17 @@ local function prepare_dkim_signing(N, task, settings) else logger.errx(task, 'missing key entry for sign entry %s; when signing %s domain', sign_entry, hdom) - return false,{} + return false, {} end else logger.errx(task, 'cannot get key entry for signing entry %s, when signing %s domain', sign_entry, hdom) - return false,{} + return false, {} end else lua_util.debugm(N, task, 'signing_table: no entry for %s', hfrom[1].addr) - return false,{} + return false, {} end else if settings.use_domain_sign_networks and is_sign_networks then @@ -409,7 +411,7 @@ local function prepare_dkim_signing(N, task, settings) if not dkim_domain then lua_util.debugm(N, task, 'could not extract dkim domain') - return false,{} + return false, {} end if settings.use_esld then @@ -435,7 +437,7 @@ local function prepare_dkim_signing(N, task, settings) lua_util.debugm(N, task, 'domain mismatch allowed for empty envelope: %1 != %2', hdom, edom) else lua_util.debugm(N, task, 'domain mismatch not allowed: %1 != %2', hdom, edom) - return false,{} + return false, {} end end end @@ -443,14 +445,14 @@ local function prepare_dkim_signing(N, task, settings) if auser and not settings.allow_username_mismatch then if not udom then lua_util.debugm(N, task, 'couldnt find domain in username') - return false,{} + return false, {} end if settings.use_esld then udom = rspamd_util.get_tld(udom) end if udom ~= dkim_domain then lua_util.debugm(N, task, 'user domain mismatch') - return false,{} + return false, {} end end @@ -465,8 +467,8 @@ local function prepare_dkim_signing(N, task, settings) } else lua_util.debugm(N, task, 'domain %s is not designated for vault', - dkim_domain) - return false,{} + dkim_domain) + return false, {} end else -- TODO: try every domain in the vault @@ -480,7 +482,7 @@ local function prepare_dkim_signing(N, task, settings) if settings.domain[dkim_domain] then -- support old style selector/paths if settings.domain[dkim_domain].selector or - settings.domain[dkim_domain].path then + settings.domain[dkim_domain].path then local k = {} k.selector = settings.domain[dkim_domain].selector k.key = settings.domain[dkim_domain].path @@ -500,7 +502,7 @@ local function prepare_dkim_signing(N, task, settings) if ret then table.insert(p, k) lua_util.debugm(N, task, 'using mempool selector %s with key %s', - k.selector, k.key) + k.selector, k.key) end end @@ -524,7 +526,7 @@ local function prepare_dkim_signing(N, task, settings) if #p == 0 and not settings.try_fallback then lua_util.debugm(N, task, 'dkim unconfigured and fallback disabled') - return false,{} + return false, {} end if not settings.use_redis then @@ -533,18 +535,18 @@ local function prepare_dkim_signing(N, task, settings) end insert_or_update_prop(N, task, p, 'selector', - 'default selector', settings.selector) + 'default selector', settings.selector) if settings.check_violation then if not check_violation(N, task, p.domain) then - return false,{} + return false, {} end end insert_or_update_prop(N, task, p, 'domain', 'dkim_domain', - dkim_domain) + dkim_domain) - return true,p + return true, p end exports.prepare_dkim_signing = prepare_dkim_signing @@ -575,11 +577,11 @@ exports.sign_using_redis = function(N, task, settings, selectors, sign_func, err false, -- is write redis_key_cb, --callback 'HGET', -- command - {settings.key_prefix, rk} -- arguments + { settings.key_prefix, rk } -- arguments ) if not rret then err_func(task, - string.format( "cannot make request to load DKIM key for %s", rk)) + string.format("cannot make request to load DKIM key for %s", rk)) end end @@ -601,7 +603,7 @@ exports.sign_using_redis = function(N, task, settings, selectors, sign_func, err false, -- is write redis_selector_cb, --callback 'HGET', -- command - {settings.selector_prefix, p.domain} -- arguments + { settings.selector_prefix, p.domain } -- arguments ) if not rret then err_func(task, string.format("cannot make Redis request to load DKIM selector for domain %s", @@ -627,7 +629,7 @@ exports.sign_using_vault = function(N, task, settings, selectors, sign_func, err full_url, err, body)) else local parser = ucl.parser() - local res,parser_err = parser:parse_string(body) + local res, parser_err = parser:parse_string(body) if not res then err_func(task, string.format('vault reply for %s (data=%s) cannot be parsed: %s', full_url, body, parser_err)) @@ -677,7 +679,7 @@ exports.sign_using_vault = function(N, task, settings, selectors, sign_func, err end end - local ret = http.request{ + local ret = http.request { task = task, url = full_url, callback = vault_callback, @@ -712,20 +714,20 @@ exports.process_signing_settings = function(N, settings, opts) local lua_maps = require "lua_maps" -- Used to convert plain options to the maps local maps_opts = { - sign_networks = {'radix', 'DKIM signing networks'}, - path_map = {'map', 'Paths to DKIM signing keys'}, - selector_map = {'map', 'DKIM selectors'}, - signing_table = {'glob', 'DKIM signing table'}, - key_table = {'glob', 'DKIM keys table'}, - vault_domains = {'glob', 'DKIM signing domains in vault'}, - whitelisted_signers_map = {'set', 'ARC trusted signers domains'} + sign_networks = { 'radix', 'DKIM signing networks' }, + path_map = { 'map', 'Paths to DKIM signing keys' }, + selector_map = { 'map', 'DKIM selectors' }, + signing_table = { 'glob', 'DKIM signing table' }, + key_table = { 'glob', 'DKIM keys table' }, + vault_domains = { 'glob', 'DKIM signing domains in vault' }, + whitelisted_signers_map = { 'set', 'ARC trusted signers domains' } } - for k,v in pairs(opts) do + for k, v in pairs(opts) do local maybe_map = maps_opts[k] if maybe_map then settings[k] = lua_maps.map_add_from_ucl(v, maybe_map[1], maybe_map[2]) elseif k == 'sign_condition' then - local ret,f = lua_util.callback_from_string(v) + local ret, f = lua_util.callback_from_string(v) if ret then settings[k] = f else diff --git a/lualib/lua_ffi/common.lua b/lualib/lua_ffi/common.lua index 4455c580e..4076cfad4 100644 --- a/lualib/lua_ffi/common.lua +++ b/lualib/lua_ffi/common.lua @@ -21,7 +21,7 @@ limitations under the License. local ffi = require 'ffi' -ffi.cdef[[ +ffi.cdef [[ struct GString { char *str; size_t len; diff --git a/lualib/lua_ffi/dkim.lua b/lualib/lua_ffi/dkim.lua index 16d7c11a1..e4592c2d2 100644 --- a/lualib/lua_ffi/dkim.lua +++ b/lualib/lua_ffi/dkim.lua @@ -21,7 +21,7 @@ limitations under the License. local ffi = require 'ffi' -ffi.cdef[[ +ffi.cdef [[ struct rspamd_dkim_sign_context_s; struct rspamd_dkim_key_s; struct rspamd_task; @@ -69,23 +69,22 @@ local function load_sign_key(what, format) elseif format == 'raw' then format = ffi.C.RSPAMD_DKIM_KEY_RAW else - return nil,'unknown key format' + return nil, 'unknown key format' end end return ffi.C.rspamd_dkim_sign_key_load(what, #what, format, nil) end -local default_dkim_headers = -"(o)from:(o)sender:(o)reply-to:(o)subject:(o)date:(o)message-id:" .. -"(o)to:(o)cc:(o)mime-version:(o)content-type:(o)content-transfer-encoding:" .. -"resent-to:resent-cc:resent-from:resent-sender:resent-message-id:" .. -"(o)in-reply-to:(o)references:list-id:list-owner:list-unsubscribe:" .. -"list-subscribe:list-post:(o)openpgp:(o)autocrypt" +local default_dkim_headers = "(o)from:(o)sender:(o)reply-to:(o)subject:(o)date:(o)message-id:" .. + "(o)to:(o)cc:(o)mime-version:(o)content-type:(o)content-transfer-encoding:" .. + "resent-to:resent-cc:resent-from:resent-sender:resent-message-id:" .. + "(o)in-reply-to:(o)references:list-id:list-owner:list-unsubscribe:" .. + "list-subscribe:list-post:(o)openpgp:(o)autocrypt" local function create_sign_context(task, privkey, dkim_headers, sign_type) if not task or not privkey then - return nil,'invalid arguments' + return nil, 'invalid arguments' end if not dkim_headers then @@ -103,10 +102,9 @@ local function create_sign_context(task, privkey, dkim_headers, sign_type) elseif sign_type == 'arc-seal' then sign_type = ffi.C.RSPAMD_DKIM_ARC_SEAL else - return nil,'invalid sign type' + return nil, 'invalid sign type' end - return ffi.C.rspamd_create_dkim_sign_context(task:topointer(), privkey, 1, 1, dkim_headers, sign_type, nil) end @@ -114,17 +112,23 @@ end local function do_sign(task, sign_context, selector, domain, expire, len, arc_idx) if not task or not sign_context or not selector or not domain then - return nil,'invalid arguments' + return nil, 'invalid arguments' end - if not expire then expire = 0 end - if not len then len = 0 end - if not arc_idx then arc_idx = 0 end + if not expire then + expire = 0 + end + if not len then + len = 0 + end + if not arc_idx then + arc_idx = 0 + end local gstring = ffi.C.rspamd_dkim_sign(task:topointer(), selector, domain, expire, len, arc_idx, nil, sign_context) if not gstring then - return nil,'cannot sign' + return nil, 'cannot sign' end local ret = ffi.string(gstring.str, gstring.len) diff --git a/lualib/lua_ffi/init.lua b/lualib/lua_ffi/init.lua index 6e61462f9..efbbc7ab6 100644 --- a/lualib/lua_ffi/init.lua +++ b/lualib/lua_ffi/init.lua @@ -31,7 +31,7 @@ if type(jit) == 'table' then return o ~= NULL end else - local ret,result_or_err = pcall(require, 'ffi') + local ret, result_or_err = pcall(require, 'ffi') if not ret then return {} @@ -51,7 +51,7 @@ exports.dkim = require "lua_ffi/dkim" exports.spf = require "lua_ffi/spf" exports.linalg = require "lua_ffi/linalg" -for k,v in pairs(ffi) do +for k, v in pairs(ffi) do -- Preserve all stuff to use lua_ffi as ffi itself exports[k] = v end diff --git a/lualib/lua_ffi/linalg.lua b/lualib/lua_ffi/linalg.lua index 6a5c53df0..2df488afe 100644 --- a/lualib/lua_ffi/linalg.lua +++ b/lualib/lua_ffi/linalg.lua @@ -23,15 +23,15 @@ local ffi = require 'ffi' local exports = {} -ffi.cdef[[ +ffi.cdef [[ void kad_sgemm_simple(int trans_A, int trans_B, int M, int N, int K, const float *A, const float *B, float *C); bool kad_ssyev_simple (int N, float *A, float *output); ]] local function table_to_ffi(a, m, n) local a_conv = ffi.new("float[?]", m * n) - for i=1,m or #a do - for j=1,n or #a[1] do + for i = 1, m or #a do + for j = 1, n or #a[1] do a_conv[(i - 1) * n + (j - 1)] = a[i][j] end end @@ -41,9 +41,9 @@ end local function ffi_to_table(a, m, n) local res = {} - for i=0,m-1 do + for i = 0, m - 1 do res[i + 1] = {} - for j=0,n-1 do + for j = 0, n - 1 do res[i + 1][j + 1] = a[i * n + j] end end @@ -75,7 +75,7 @@ exports.eigen = function(a, n) local res = ffi.new("float[?]", n) if ffi.C.kad_ssyev_simple(n, ffi.cast('float*', a), res) then - return res,a + return res, a end return nil diff --git a/lualib/lua_ffi/spf.lua b/lualib/lua_ffi/spf.lua index 783870eac..0f982f2f9 100644 --- a/lualib/lua_ffi/spf.lua +++ b/lualib/lua_ffi/spf.lua @@ -21,7 +21,7 @@ limitations under the License. local ffi = require 'ffi' -ffi.cdef[[ +ffi.cdef [[ enum spf_mech_e { SPF_FAIL, SPF_SOFT_FAIL, @@ -112,7 +112,7 @@ local function spf_resolve(task, cb) local digstr = ffi.new("char[64]") ffi.C.rspamd_snprintf(digstr, 64, "0x%xuL", rec.digest) res.digest = ffi.string(digstr) - for i = 1,nelts do + for i = 1, nelts do res.addrs[i] = spf_addr_tolua(elts[i - 1]) end diff --git a/lualib/lua_fuzzy.lua b/lualib/lua_fuzzy.lua index 332e828ac..f941436eb 100644 --- a/lualib/lua_fuzzy.lua +++ b/lualib/lua_fuzzy.lua @@ -38,7 +38,7 @@ local policies = { min_width = 500, min_length = 64, text_multiplier = 4.0, -- divide min_bytes by 4 for texts - mime_types = {"application/*"}, + mime_types = { "application/*" }, scan_archives = true, short_text_direct_hash = true, text_shingles = true, @@ -48,7 +48,7 @@ local policies = { local default_policy = policies.recommended -local policy_schema = ts.shape{ +local policy_schema = ts.shape { min_bytes = ts.number + ts.string / tonumber, min_height = ts.number + ts.string / tonumber, min_width = ts.number + ts.string / tonumber, @@ -61,7 +61,6 @@ local policy_schema = ts.shape{ skip_images = ts.boolean, } - local exports = {} @@ -74,7 +73,7 @@ exports.register_policy = function(name, policy) rspamd_logger.warnx(rspamd_config, "overriding policy %s", name) end - local parsed_policy,err = policy_schema:transform(policy) + local parsed_policy, err = policy_schema:transform(policy) if not parsed_policy then rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule policy %s: %s', @@ -160,7 +159,7 @@ local function check_length(task, part, rule) end local function check_text_part(task, part, rule, text) - local allow_direct,allow_shingles = false,false + local allow_direct, allow_shingles = false, false local id = part:get_id() lua_util.debugm(N, task, 'check text part %s', id) @@ -200,7 +199,7 @@ local function check_text_part(task, part, rule, text) allow_direct = check_length(task, part, rule) end - return allow_direct,allow_shingles + return allow_direct, allow_shingles end --local function has_sane_text_parts(task) @@ -211,7 +210,7 @@ end local function check_image_part(task, part, rule, image) if rule.skip_images then lua_util.debugm(N, task, 'skip image part as images are disabled') - return false,false + return false, false end local id = part:get_id() @@ -227,7 +226,7 @@ local function check_image_part(task, part, rule, image) if height and width then if height < min_height or width < min_width then lua_util.debugm(N, task, 'skip image part %s as it does not meet minimum sizes: %sx%s < %sx%s', - id, width, height, min_width, min_height) + id, width, height, min_width, min_height) return false, false else lua_util.debugm(N, task, 'allow image part %s: %sx%s', @@ -236,18 +235,20 @@ local function check_image_part(task, part, rule, image) end end - return check_length(task, part, rule),false + return check_length(task, part, rule), false end local function mime_types_check(task, part, rule) - local t,st = part:get_type() + local t, st = part:get_type() - if not t then return false, false end + if not t then + return false, false + end local ct = string.format('%s/%s', t, st) local detected_ct - t,st = part:get_detected_type() + t, st = part:get_detected_type() if t then detected_ct = string.format('%s/%s', t, st) else @@ -266,12 +267,12 @@ local function mime_types_check(task, part, rule) end opts = fun.tomap(fun.map(function(opt) local elts = lua_util.str_split(opt, ':') - return elts[1],elts[2] + return elts[1], elts[2] end, opts)) if opts[id] and opts[id] == '-' then lua_util.debugm(N, task, 'explicitly check binary part %s: bad mime type %s', id, ct) - return check_length(task, part, rule),false + return check_length(task, part, rule), false end if rule.mime_types then @@ -285,13 +286,13 @@ local function mime_types_check(task, part, rule) end, rule.mime_types) then lua_util.debugm(N, task, 'found mime type match for part %s: %s (%s detected)', id, ct, detected_ct) - return check_length(task, part, rule),false + return check_length(task, part, rule), false end return false, false end - return false,false + return false, false end exports.check_mime_part = function(task, part, rule_id) @@ -300,7 +301,7 @@ exports.check_mime_part = function(task, part, rule_id) if not rule then rspamd_logger.errx(task, 'cannot find rule with id %s', rule_id) - return false,false + return false, false end if part:is_text() then @@ -315,7 +316,7 @@ exports.check_mime_part = function(task, part, rule_id) -- Always send archives lua_util.debugm(N, task, 'check archive part %s', part:get_id()) - return true,false + return true, false end if part:is_specific() then @@ -323,7 +324,7 @@ exports.check_mime_part = function(task, part, rule_id) if type(sp) == 'table' and sp.fuzzy_hashes then lua_util.debugm(N, task, 'check specific part %s', part:get_id()) - return true,false + return true, false end end @@ -331,7 +332,7 @@ exports.check_mime_part = function(task, part, rule_id) return mime_types_check(task, part, rule) end - return false,false + return false, false end exports.cleanup_rules = function() diff --git a/lualib/lua_lexer.lua b/lualib/lua_lexer.lua index 324641013..54bbd7c04 100644 --- a/lualib/lua_lexer.lua +++ b/lualib/lua_lexer.lua @@ -22,36 +22,38 @@ local lpeg = require "lpeg" local P = lpeg.P local R = lpeg.R local S = lpeg.S -local D = R'09' -- Digits +local D = R '09' -- Digits local I = R('AZ', 'az', '\127\255') + '_' -- Identifiers local B = -(I + D) -- Word boundary local EOS = -lpeg.P(1) -- end of string -- Pattern for long strings and long comments. -local longstring = #(P'[[' + (P'[' * P'='^0 * '[')) * P(function(input, index) +local longstring = #(P '[[' + (P '[' * P '=' ^ 0 * '[')) * P(function(input, index) local level = input:match('^%[(=*)%[', index) if level then local _, last = input:find(']' .. level .. ']', index, true) - if last then return last + 1 end + if last then + return last + 1 + end end end) -- String literals. -local singlequoted = P"'" * ((1 - S"'\r\n\f\\") + (P'\\' * 1))^0 * "'" -local doublequoted = P'"' * ((1 - S'"\r\n\f\\') + (P'\\' * 1))^0 * '"' +local singlequoted = P "'" * ((1 - S "'\r\n\f\\") + (P '\\' * 1)) ^ 0 * "'" +local doublequoted = P '"' * ((1 - S '"\r\n\f\\') + (P '\\' * 1)) ^ 0 * '"' -- Comments. -local eol = P'\r\n' + '\n' -local line = (1 - S'\r\n\f')^0 * eol^-1 -local singleline = P'--' * line -local multiline = P'--' * longstring +local eol = P '\r\n' + '\n' +local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1 +local singleline = P '--' * line +local multiline = P '--' * longstring -- Numbers. -local sign = S'+-'^-1 -local decimal = D^1 -local hexadecimal = P'0' * S'xX' * R('09', 'AF', 'af') ^ 1 -local float = D^1 * P'.' * D^0 + P'.' * D^1 -local maybeexp = (float + decimal) * (S'eE' * sign * D^1)^-1 +local sign = S '+-' ^ -1 +local decimal = D ^ 1 +local hexadecimal = P '0' * S 'xX' * R('09', 'AF', 'af') ^ 1 +local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1 +local maybeexp = (float + decimal) * (S 'eE' * sign * D ^ 1) ^ -1 local function compile_keywords(keywords) local list = {} @@ -74,26 +76,26 @@ local function compile_keywords(keywords) end -- Identifiers -local ident = I * (I + D)^0 -local expr = ('.' * ident)^0 +local ident = I * (I + D) ^ 0 +local expr = ('.' * ident) ^ 0 local patterns = { - {'whitespace', S'\r\n\f\t\v '^1}, - {'constant', (P'true' + 'false' + 'nil') * B}, - {'string', singlequoted + doublequoted + longstring}, - {'comment', multiline + singleline}, - {'number', hexadecimal + maybeexp}, - {'operator', P'not' + '...' + 'and' + '..' + '~=' + '==' + '>=' + '<=' - + 'or' + S']{=>^[<;)*(%}+-:,/.#'}, - {'keyword', compile_keywords([[ + { 'whitespace', S '\r\n\f\t\v ' ^ 1 }, + { 'constant', (P 'true' + 'false' + 'nil') * B }, + { 'string', singlequoted + doublequoted + longstring }, + { 'comment', multiline + singleline }, + { 'number', hexadecimal + maybeexp }, + { 'operator', P 'not' + '...' + 'and' + '..' + '~=' + '==' + '>=' + '<=' + + 'or' + S ']{=>^[<;)*(%}+-:,/.#' }, + { 'keyword', compile_keywords([[ break do else elseif end for function if in local repeat return then until while - ]])}, - {'identifier', lpeg.Cmt(ident, + ]]) }, + { 'identifier', lpeg.Cmt(ident, function(input, index) return expr:match(input, index) end) }, - {'error', 1}, + { 'error', 1 }, } local compiled @@ -101,7 +103,7 @@ local compiled local function compile_patterns() if not compiled then local function process(elt) - local n,grammar = elt[1],elt[2] + local n, grammar = elt[1], elt[2] return lpeg.Cc(n) * lpeg.P(grammar) * lpeg.Cp() end local any = process(patterns[1]) @@ -151,7 +153,7 @@ exports.lex_to_table = function(input) local out = {} for kind, text, lnum, cnum in exports.gmatch(input) do - out[#out + 1] = {kind, text, lnum, cnum} + out[#out + 1] = { kind, text, lnum, cnum } end return out diff --git a/lualib/lua_magic/heuristics.lua b/lualib/lua_magic/heuristics.lua index 98cfb0eee..b8a1b4188 100644 --- a/lualib/lua_magic/heuristics.lua +++ b/lualib/lua_magic/heuristics.lua @@ -28,18 +28,18 @@ local fun = require "fun" local N = "lua_magic" local msoffice_trie local msoffice_patterns = { - doc = {[[WordDocument]]}, - xls = {[[Workbook]], [[Book]]}, - ppt = {[[PowerPoint Document]], [[Current User]]}, - vsd = {[[VisioDocument]]}, + doc = { [[WordDocument]] }, + xls = { [[Workbook]], [[Book]] }, + ppt = { [[PowerPoint Document]], [[Current User]] }, + vsd = { [[VisioDocument]] }, } local msoffice_trie_clsid local msoffice_clsids = { - doc = {[[0609020000000000c000000000000046]]}, - xls = {[[1008020000000000c000000000000046]], [[2008020000000000c000000000000046]]}, - ppt = {[[108d81649b4fcf1186ea00aa00b929e8]]}, - msg = {[[46f0060000000000c000000000000046]], [[0b0d020000000000c000000000000046]]}, - msi = {[[84100c0000000000c000000000000046]]}, + doc = { [[0609020000000000c000000000000046]] }, + xls = { [[1008020000000000c000000000000046]], [[2008020000000000c000000000000046]] }, + ppt = { [[108d81649b4fcf1186ea00aa00b929e8]] }, + msg = { [[46f0060000000000c000000000000046]], [[0b0d020000000000c000000000000046]] }, + msi = { [[84100c0000000000c000000000000046]] }, } local zip_trie local zip_patterns = { @@ -54,37 +54,37 @@ local zip_patterns = { [[mimetypeapplication/vnd\.oasis\.opendocument\.formula]], [[mimetypeapplication/vnd\.oasis\.opendocument\.chart]] }, - odp = {[[mimetypeapplication/vnd\.oasis\.opendocument\.presentation]]}, - epub = {[[epub\+zip]]}, - asice = {[[mimetypeapplication/vnd\.etsi\.asic-e\+zipPK]]}, - asics = {[[mimetypeapplication/vnd\.etsi\.asic-s\+zipPK]]}, + odp = { [[mimetypeapplication/vnd\.oasis\.opendocument\.presentation]] }, + epub = { [[epub\+zip]] }, + asice = { [[mimetypeapplication/vnd\.etsi\.asic-e\+zipPK]] }, + asics = { [[mimetypeapplication/vnd\.etsi\.asic-s\+zipPK]] }, } local txt_trie local txt_patterns = { html = { - {[=[(?i)<html[\s>]]=], 32}, - {[[(?i)<script\b]], 20}, -- Commonly used by spammers - {[[<script\s+type="text\/javascript">]], 31}, -- Another spammy pattern - {[[(?i)<\!DOCTYPE HTML\b]], 33}, - {[[(?i)<body\b]], 20}, - {[[(?i)<table\b]], 20}, - {[[(?i)<a\s]], 10}, - {[[(?i)<p\b]], 10}, - {[[(?i)<div\b]], 10}, - {[[(?i)<span\b]], 10}, + { [=[(?i)<html[\s>]]=], 32 }, + { [[(?i)<script\b]], 20 }, -- Commonly used by spammers + { [[<script\s+type="text\/javascript">]], 31 }, -- Another spammy pattern + { [[(?i)<\!DOCTYPE HTML\b]], 33 }, + { [[(?i)<body\b]], 20 }, + { [[(?i)<table\b]], 20 }, + { [[(?i)<a\s]], 10 }, + { [[(?i)<p\b]], 10 }, + { [[(?i)<div\b]], 10 }, + { [[(?i)<span\b]], 10 }, }, csv = { - {[[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]], 20} + { [[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]], 20 } }, ics = { - {[[^BEGIN:VCALENDAR\r?\n]], 40}, + { [[^BEGIN:VCALENDAR\r?\n]], 40 }, }, vcf = { - {[[^BEGIN:VCARD\r?\n]], 40}, + { [[^BEGIN:VCARD\r?\n]], 40 }, }, xml = { - {[[<\?xml\b.+\?>]], 31}, + { [[<\?xml\b.+\?>]], 31 }, } } @@ -103,11 +103,11 @@ local function compile_tries() rspamd_trie.flags.no_start) local function compile_pats(patterns, indexes, transform_func, compile_flags) local strs = {} - for ext,pats in pairs(patterns) do - for _,pat in ipairs(pats) do + for ext, pats in pairs(patterns) do + for _, pat in ipairs(pats) do -- These are utf16 strings in fact... strs[#strs + 1] = transform_func(pat) - indexes[#indexes + 1] = {ext, pat} + indexes[#indexes + 1] = { ext, pat } end end @@ -120,12 +120,14 @@ local function compile_tries() return '^' .. table.concat( fun.totable( - fun.map(function(c) return c .. [[\x{00}]] end, + fun.map(function(c) + return c .. [[\x{00}]] + end, fun.iter(pat)))) end local function msoffice_clsid_transform(pat) local hex_table = {} - for i=1,#pat,2 do + for i = 1, #pat, 2 do local subc = pat:sub(i, i + 1) hex_table[#hex_table + 1] = string.format('\\x{%s}', subc) end @@ -140,10 +142,14 @@ local function compile_tries() msoffice_clsid_transform) -- Misc zip patterns at the initial fragment zip_trie = compile_pats(zip_patterns, zip_patterns_indexes, - function(pat) return pat end) + function(pat) + return pat + end) -- Text patterns at the initial fragment txt_trie = compile_pats(txt_patterns, txt_patterns_indexes, - function(pat_tbl) return pat_tbl[1] end, + function(pat_tbl) + return pat_tbl[1] + end, bit.bor(rspamd_trie.flags.re, rspamd_trie.flags.dot_all, rspamd_trie.flags.no_start)) @@ -160,12 +166,13 @@ local function detect_ole_format(input, log_obj, _, part) return nil end - local bom,sec_size = rspamd_util.unpack('<I2<I2', input:span(29, 4)) + local bom, sec_size = rspamd_util.unpack('<I2<I2', input:span(29, 4)) if bom == 0xFFFE then bom = '<' else lua_util.debugm(N, log_obj, "bom file!: %s", bom) - bom = '>'; sec_size = bit.bswap(sec_size) + bom = '>'; + sec_size = bit.bswap(sec_size) end if sec_size < 7 or sec_size > 31 then @@ -194,39 +201,39 @@ local function detect_ole_format(input, log_obj, _, part) -- Extract clsid local matches = msoffice_trie_clsid:match(input:span(offset + 80, 16)) if matches then - for n,_ in pairs(matches) do + for n, _ in pairs(matches) do if msoffice_clsid_indexes[n] then lua_util.debugm(N, log_obj, "found valid clsid for %s", msoffice_clsid_indexes[n][1]) - return true,msoffice_clsid_indexes[n][1] + return true, msoffice_clsid_indexes[n][1] end end end - return true,nil + return true, nil elseif dtype == 2 then local matches = msoffice_trie:match(input:span(offset, 64)) if matches then - for n,_ in pairs(matches) do + for n, _ in pairs(matches) do if msoffice_patterns_indexes[n] then - return true,msoffice_patterns_indexes[n][1] + return true, msoffice_patterns_indexes[n][1] end end end - return true,nil + return true, nil elseif dtype >= 0 and dtype < 5 then -- Bad type - return true,nil + return true, nil end end - return false,nil + return false, nil end repeat - local res,ext = process_dir_entry(directory_offset) + local res, ext = process_dir_entry(directory_offset) if res and ext then - return ext,60 + return ext, 60 end if not res then @@ -247,7 +254,7 @@ local function process_top_detected(res) return res[ex1] > res[ex2] end) - return extensions[1],res[extensions[1]] + return extensions[1], res[extensions[1]] end return nil @@ -276,7 +283,7 @@ local function detect_archive_flaw(part, arch, log_obj, _) if arch_type == 'zip' then -- Find specific files/folders in zip file local files = arch:get_files(100) or {} - for _,file in ipairs(files) do + for _, file in ipairs(files) do if file == '[Content_Types].xml' then add_msoffice_confidence(10) elseif file:sub(1, 3) == 'xl/' then @@ -292,10 +299,10 @@ local function detect_archive_flaw(part, arch, log_obj, _) end end - local ext,weight = process_top_detected(res) + local ext, weight = process_top_detected(res) if weight >= 40 then - return ext,weight + return ext, weight end -- Apply misc Zip detection logic @@ -306,32 +313,34 @@ local function detect_archive_flaw(part, arch, log_obj, _) local matches = zip_trie:match(start_span) if matches then - for n,_ in pairs(matches) do + for n, _ in pairs(matches) do if zip_patterns_indexes[n] then lua_util.debugm(N, log_obj, "found zip pattern for %s", zip_patterns_indexes[n][1]) - return zip_patterns_indexes[n][1],40 + return zip_patterns_indexes[n][1], 40 end end end end end - return arch_type:lower(),40 + return arch_type:lower(), 40 end local csv_grammar -- Returns a grammar that will count commas local function get_csv_grammar() if not csv_grammar then - local lpeg = require'lpeg' + local lpeg = require 'lpeg' - local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P'""' / '"')^0) * '"' + - lpeg.C((1 - lpeg.S',\n"')^0) + local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P '""' / '"') ^ 0) * '"' + + lpeg.C((1 - lpeg.S ',\n"') ^ 0) - csv_grammar = lpeg.Cf(lpeg.Cc(0) * field * lpeg.P( (lpeg.P(',') + - lpeg.P('\t')) * field)^1 * (lpeg.S'\r\n' + -1), - function(acc) return acc + 1 end) + csv_grammar = lpeg.Cf(lpeg.Cc(0) * field * lpeg.P((lpeg.P(',') + + lpeg.P('\t')) * field) ^ 1 * (lpeg.S '\r\n' + -1), + function(acc) + return acc + 1 + end) end return csv_grammar @@ -402,17 +411,17 @@ exports.text_part_heuristic = function(part, log_obj, _) while b >= 127 and idx < len do -- utf8 part if bit.band(b, 0xe0) == 0xc0 and remain > 1 and - bit.band(bytes[idx + 1], 0xc0) == 0x80 then - return true,1 + bit.band(bytes[idx + 1], 0xc0) == 0x80 then + return true, 1 elseif bit.band(b, 0xf0) == 0xe0 and remain > 2 and - bit.band(bytes[idx + 1], 0xc0) == 0x80 and - bit.band(bytes[idx + 2], 0xc0) == 0x80 then - return true,2 + bit.band(bytes[idx + 1], 0xc0) == 0x80 and + bit.band(bytes[idx + 2], 0xc0) == 0x80 then + return true, 2 elseif bit.band(b, 0xf8) == 0xf0 and remain > 3 and - bit.band(bytes[idx + 1], 0xc0) == 0x80 and - bit.band(bytes[idx + 2], 0xc0) == 0x80 and - bit.band(bytes[idx + 3], 0xc0) == 0x80 then - return true,3 + bit.band(bytes[idx + 1], 0xc0) == 0x80 and + bit.band(bytes[idx + 2], 0xc0) == 0x80 and + bit.band(bytes[idx + 3], 0xc0) == 0x80 then + return true, 3 end n8bit = n8bit + 1 @@ -422,10 +431,10 @@ exports.text_part_heuristic = function(part, log_obj, _) end if n8bit >= 3 then - return true,n8bit + return true, n8bit end - return false,0 + return false, 0 end -- Convert to string as LuaJIT can optimise string.sub (and fun.iter) but not C calls @@ -439,7 +448,7 @@ exports.text_part_heuristic = function(part, log_obj, _) if (b < 0x20) and not (b == 0x0d or b == 0x0a or b == 0x09) then non_printable = non_printable + 1 elseif b >= 127 then - local c,nskip = rough_8bit_check(bytes, i, tlen - i, tlen) + local c, nskip = rough_8bit_check(bytes, i, tlen - i, tlen) if not c then non_printable = non_printable + 1 @@ -462,7 +471,7 @@ exports.text_part_heuristic = function(part, log_obj, _) local parent = part:get_parent() if parent then - local parent_type,parent_subtype = parent:get_type() + local parent_type, parent_subtype = parent:get_type() if parent_type == 'multipart' and parent_subtype == 'encrypted' then -- Skip text heuristics for encrypted parts @@ -473,7 +482,7 @@ exports.text_part_heuristic = function(part, log_obj, _) end local content = part:get_content() - local mtype,msubtype = part:get_type() + local mtype, msubtype = part:get_type() local clen = #content local is_text @@ -495,8 +504,8 @@ exports.text_part_heuristic = function(part, log_obj, _) if matches then -- Require at least 2 occurrences of those patterns - for n,positions in pairs(matches) do - local ext,weight = txt_patterns_indexes[n][1], txt_patterns_indexes[n][2][2] + for n, positions in pairs(matches) do + local ext, weight = txt_patterns_indexes[n][1], txt_patterns_indexes[n][2][2] if ext then res[ext] = (res[ext] or 0) + weight * #positions lua_util.debugm(N, log_obj, "found txt pattern for %s: %s, total: %s; %s/%s announced", @@ -504,7 +513,7 @@ exports.text_part_heuristic = function(part, log_obj, _) end end - if res.html and res.html >= 40 then + if res.html and res.html >= 40 then -- HTML has priority over something like js... return 'html', res.html end @@ -525,7 +534,7 @@ exports.text_part_heuristic = function(part, log_obj, _) -- Content type stuff if (mtype == 'text' or mtype == 'application') and - (msubtype == 'html' or msubtype == 'xhtml+xml') then + (msubtype == 'html' or msubtype == 'xhtml+xml') then return 'html', 21 end @@ -539,12 +548,12 @@ exports.text_part_heuristic = function(part, log_obj, _) local function has_extension(file, ext) local ext_len = ext:len() return file:len() > ext_len + 1 - and file:sub(-ext_len):lower() == ext - and file:sub(-ext_len - 1, -ext_len - 1) == '.' + and file:sub(-ext_len):lower() == ext + and file:sub(-ext_len - 1, -ext_len - 1) == '.' end if fname and (has_extension(fname, 'htm') or has_extension(fname, 'html')) then - return 'html',21 + return 'html', 21 end if mtype ~= 'text' then @@ -552,7 +561,7 @@ exports.text_part_heuristic = function(part, log_obj, _) return nil end - return 'txt',40 + return 'txt', 40 end end end @@ -569,7 +578,7 @@ exports.pdf_format_heuristic = function(input, log_obj, pos, part) weight = weight + 30 end - return 'pdf',weight + return 'pdf', weight end exports.pe_part_heuristic = function(input, log_obj, pos, part) @@ -590,7 +599,7 @@ exports.pe_part_heuristic = function(input, log_obj, pos, part) return end - return 'exe',30 + return 'exe', 30 end return exports diff --git a/lualib/lua_magic/init.lua b/lualib/lua_magic/init.lua index 4ad027168..38bfddbf2 100644 --- a/lualib/lua_magic/init.lua +++ b/lualib/lua_magic/init.lua @@ -70,7 +70,6 @@ local function process_patterns(log_obj) str, pattern.ext) end - if max_short_offset < match.position then max_short_offset = match.position end @@ -93,10 +92,10 @@ local function process_patterns(log_obj) end if not compiled_patterns then - for ext,pattern in pairs(patterns) do + for ext, pattern in pairs(patterns) do assert(types[ext], 'not found type: ' .. ext) pattern.ext = ext - for _,match in ipairs(pattern.matches) do + for _, match in ipairs(pattern.matches) do if match.string then if match.relative_position and not match.position then match.position = match.relative_position + #match.string @@ -111,7 +110,7 @@ local function process_patterns(log_obj) elseif match.hex then local hex_table = {} - for i=1,#match.hex,2 do + for i = 1, #match.hex, 2 do local subc = match.hex:sub(i, i + 1) hex_table[#hex_table + 1] = string.format('\\x{%s}', subc) end @@ -131,15 +130,21 @@ local function process_patterns(log_obj) compile_flags = bit.bor(compile_flags, rspamd_trie.flags.single_match) compile_flags = bit.bor(compile_flags, rspamd_trie.flags.no_start) compiled_patterns = rspamd_trie.create(fun.totable( - fun.map(function(t) return t[1] end, processed_patterns)), + fun.map(function(t) + return t[1] + end, processed_patterns)), compile_flags ) compiled_short_patterns = rspamd_trie.create(fun.totable( - fun.map(function(t) return t[1] end, short_patterns)), + fun.map(function(t) + return t[1] + end, short_patterns)), compile_flags ) compiled_tail_patterns = rspamd_trie.create(fun.totable( - fun.map(function(t) return t[1] end, tail_patterns)), + fun.map(function(t) + return t[1] + end, tail_patterns)), compile_flags ) @@ -167,24 +172,36 @@ local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_ res[ext] = res[ext] + 1 end - lua_util.debugm(N, log_obj,'add pattern for %s, weight %s, total weight %s', + lua_util.debugm(N, log_obj, 'add pattern for %s, weight %s, total weight %s', ext, weight, res[ext]) end local function match_position(pos, expected) - local cmp = function(a, b) return a == b end + local cmp = function(a, b) + return a == b + end if type(expected) == 'table' then -- Something like {'>', 0} if expected[1] == '>' then - cmp = function(a, b) return a > b end + cmp = function(a, b) + return a > b + end elseif expected[1] == '>=' then - cmp = function(a, b) return a >= b end + cmp = function(a, b) + return a >= b + end elseif expected[1] == '<' then - cmp = function(a, b) return a < b end + cmp = function(a, b) + return a < b + end elseif expected[1] == '<=' then - cmp = function(a, b) return a <= b end + cmp = function(a, b) + return a <= b + end elseif expected[1] == '!=' then - cmp = function(a, b) return a ~= b end + cmp = function(a, b) + return a ~= b + end end expected = expected[2] end @@ -196,7 +213,7 @@ local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_ return cmp(pos, expected) end - for npat,matched_positions in pairs(matches) do + for npat, matched_positions in pairs(matches) do local pat_data = processed_tbl[npat] local pattern = pat_data[3] local match = pat_data[2] @@ -205,12 +222,12 @@ local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_ if match.position then local position = match.position - for _,pos in ipairs(matched_positions) do + for _, pos in ipairs(matched_positions) do lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)', pattern.ext, pos, offset) if match_position(pos + offset, position) then if match.heuristic then - local ext,weight = match.heuristic(input, log_obj, pos + offset, part) + local ext, weight = match.heuristic(input, log_obj, pos + offset, part) if ext then add_result(weight, ext) @@ -226,9 +243,9 @@ local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_ -- Match all positions local all_right = true local matched_pos = 0 - for _,position in ipairs(match.positions) do + for _, position in ipairs(match.positions) do local matched = false - for _,pos in ipairs(matched_positions) do + for _, pos in ipairs(matched_positions) do lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)', pattern.ext, pos, offset) if not match_position(pos + offset, position) then @@ -245,7 +262,7 @@ local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_ if all_right then if match.heuristic then - local ext,weight = match.heuristic(input, log_obj, matched_pos + offset, part) + local ext, weight = match.heuristic(input, log_obj, matched_pos + offset, part) if ext then add_result(weight, ext) @@ -269,14 +286,16 @@ local function process_detected(res) return res[ex1] > res[ex2] end) - return extensions,res[extensions[1]] + return extensions, res[extensions[1]] end return nil end exports.detect = function(part, log_obj) - if not log_obj then log_obj = rspamd_config end + if not log_obj then + log_obj = rspamd_config + end local input = part:get_content() local res = {} @@ -286,7 +305,6 @@ exports.detect = function(part, log_obj) input = rspamd_text.fromstring(input) end - if type(input) == 'userdata' then local inplen = #input @@ -303,18 +321,17 @@ exports.detect = function(part, log_obj) compiled_short_patterns, short_patterns, log_obj, res, part) -- Check if we have enough data or go to long patterns - local extensions,confidence = process_detected(res) + local extensions, confidence = process_detected(res) if extensions and #extensions > 0 and confidence > 30 then -- We are done on short patterns - return extensions[1],types[extensions[1]] + return extensions[1], types[extensions[1]] end -- No way, let's check data in chunks or just the whole input if it is small enough if #input > exports.chunk_size * 3 then -- Chunked version as input is too long - local chunk1, chunk2 = - input:span(1, exports.chunk_size * 2), + local chunk1, chunk2 = input:span(1, exports.chunk_size * 2), input:span(inplen - exports.chunk_size, exports.chunk_size) local offset1, offset2 = 0, inplen - exports.chunk_size @@ -335,7 +352,7 @@ exports.detect = function(part, log_obj) local extensions = process_detected(res) if extensions and #extensions > 0 then - return extensions[1],types[extensions[1]] + return extensions[1], types[extensions[1]] end -- Nothing found @@ -343,22 +360,22 @@ exports.detect = function(part, log_obj) end exports.detect_mime_part = function(part, log_obj) - local ext,weight = heuristics.mime_part_heuristic(part, log_obj) + local ext, weight = heuristics.mime_part_heuristic(part, log_obj) if ext and weight and weight > 20 then - return ext,types[ext] + return ext, types[ext] end ext = exports.detect(part, log_obj) if ext then - return ext,types[ext] + return ext, types[ext] end -- Text/html and other parts - ext,weight = heuristics.text_part_heuristic(part, log_obj) + ext, weight = heuristics.text_part_heuristic(part, log_obj) if ext and weight and weight > 20 then - return ext,types[ext] + return ext, types[ext] end end diff --git a/lualib/lua_magic/patterns.lua b/lualib/lua_magic/patterns.lua index b1eb5ad4d..971ddd95f 100644 --- a/lualib/lua_magic/patterns.lua +++ b/lualib/lua_magic/patterns.lua @@ -27,13 +27,13 @@ local patterns = { matches = { { string = [[%PDF-[12]\.\d]], - position = {'<=', 1024}, + position = { '<=', 1024 }, weight = 60, heuristic = heuristics.pdf_format_heuristic }, { string = [[%FDF-[12]\.\d]], - position = {'<=', 1024}, + position = { '<=', 1024 }, weight = 60, heuristic = heuristics.pdf_format_heuristic }, @@ -103,7 +103,7 @@ local patterns = { -- PE part { string = [[PE\x{00}\x{00}]], - position = {'>=', 0x3c + 4}, + position = { '>=', 0x3c + 4 }, weight = 15, heuristic = heuristics.pe_part_heuristic, } @@ -131,7 +131,7 @@ local patterns = { matches = { { string = [[(?i)@\s*ECHO\s+OFF]], - position = {'>=', 0}, + position = { '>=', 0 }, weight = 60, }, } @@ -189,7 +189,7 @@ local patterns = { matches = { { hex = [[4d53434600000000]], -- Can be anywhere for SFX :( - position = {'>=', 8}, + position = { '>=', 8 }, weight = 60, }, } @@ -268,7 +268,7 @@ local patterns = { matches = { { string = [[\x{01}CD001\x{01}]], - position = {'>=', 0x8000 + 7}, -- first 32k is unused + position = { '>=', 0x8000 + 7 }, -- first 32k is unused weight = 60, }, } diff --git a/lualib/lua_maps.lua b/lualib/lua_maps.lua index d65481177..baaed5b50 100644 --- a/lualib/lua_maps.lua +++ b/lualib/lua_maps.lua @@ -36,15 +36,15 @@ local function map_hash_key(data, mtype) return st:hex() end -local function starts(where,st) - return string.sub(where,1,string.len(st))==st +local function starts(where, st) + return string.sub(where, 1, string.len(st)) == st end -local function cut_prefix(where,st) - return string.sub(where,#st + 1) +local function cut_prefix(where, st) + return string.sub(where, #st + 1) end -local function maybe_adjust_type(data,mtype) +local function maybe_adjust_type(data, mtype) local function check_prefix(prefix, t) if starts(data, prefix) then data = cut_prefix(data, prefix) @@ -57,41 +57,40 @@ local function maybe_adjust_type(data,mtype) end local known_types = { - {'regexp;', 'regexp'}, - {'re;', 'regexp'}, - {'regexp_multi;', 'regexp_multi'}, - {'re_multi;', 'regexp_multi'}, - {'glob;', 'glob'}, - {'glob_multi;', 'glob_multi'}, - {'radix;', 'radix'}, - {'ipnet;', 'radix'}, - {'set;', 'set'}, - {'hash;', 'hash'}, - {'plain;', 'hash'}, - {'cdb;', 'cdb'}, - {'cdb:/', 'cdb'}, + { 'regexp;', 'regexp' }, + { 're;', 'regexp' }, + { 'regexp_multi;', 'regexp_multi' }, + { 're_multi;', 'regexp_multi' }, + { 'glob;', 'glob' }, + { 'glob_multi;', 'glob_multi' }, + { 'radix;', 'radix' }, + { 'ipnet;', 'radix' }, + { 'set;', 'set' }, + { 'hash;', 'hash' }, + { 'plain;', 'hash' }, + { 'cdb;', 'cdb' }, + { 'cdb:/', 'cdb' }, } if mtype == 'callback' then return mtype end - for _,t in ipairs(known_types) do + for _, t in ipairs(known_types) do if check_prefix(t[1], t[2]) then - return data,mtype + return data, mtype end end -- No change - return data,mtype + return data, mtype end - -local external_map_schema = ts.shape{ +local external_map_schema = ts.shape { external = ts.equivalent(true), -- must be true backend = ts.string, -- where to get data, required - method = ts.one_of{"body", "header", "query"}, -- how to pass input - encode = ts.one_of{"json", "messagepack"}:is_optional(), -- how to encode input (if relevant) + method = ts.one_of { "body", "header", "query" }, -- how to pass input + encode = ts.one_of { "json", "messagepack" }:is_optional(), -- how to encode input (if relevant) timeout = (ts.number + ts.string / lua_util.parse_time_interval):is_optional(), } @@ -100,7 +99,9 @@ local ucl = require "ucl" local function url_encode_string(str) str = string.gsub(str, "([^%w _%%%-%.~])", - function(c) return string.format("%%%02X", string.byte(c)) end) + function(c) + return string.format("%%%02X", string.byte(c)) + end) str = string.gsub(str, " ", "+") return str end @@ -147,7 +148,7 @@ local function query_external_map(map_config, upstreams, key, callback, task) -- query/header and no encode if map_config.method == 'query' then local params_table = {} - for k,v in pairs(key) do + for k, v in pairs(key) do if type(v) == 'string' then table.insert(params_table, string.format('%s=%s', url_encode_string(k), url_encode_string(v))) end @@ -176,7 +177,7 @@ local function query_external_map(map_config, upstreams, key, callback, task) end end - local ret = rspamd_http.request{ + local ret = rspamd_http.request { task = task, url = url, callback = map_callback, @@ -261,7 +262,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) end if type(opt) == 'string' then - opt,mtype = maybe_adjust_type(opt, mtype) + opt, mtype = maybe_adjust_type(opt, mtype) local cache_key = map_hash_key(opt, mtype) if not callback and maps_cache[cache_key] then rspamd_logger.infox(rspamd_config, 'reuse url for %s(%s)', @@ -270,7 +271,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) return maps_cache[cache_key] end -- We have a single string, so we treat it as a map - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = opt, @@ -287,7 +288,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) local cache_key = lua_util.table_digest(opt) if not callback and maps_cache[cache_key] then rspamd_logger.infox(rspamd_config, 'reuse url for complex map definition %s: %s', - cache_key:sub(1,8), description) + cache_key:sub(1, 8), description) return maps_cache[cache_key] end @@ -295,8 +296,8 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) if opt[1] then -- Adjust each element if needed local adjusted - for i,source in ipairs(opt) do - local nsrc,ntype = maybe_adjust_type(source, mtype) + for i, source in ipairs(opt) do + local nsrc, ntype = maybe_adjust_type(source, mtype) if mtype ~= ntype then if not adjusted then @@ -322,7 +323,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) end else -- Plain table - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = opt, @@ -339,7 +340,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) elseif mtype == 'regexp' or mtype == 'glob' then if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then -- Plain table - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = opt, @@ -353,7 +354,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) return ret end else - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = { @@ -373,7 +374,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) else if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then -- Plain table - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = opt, @@ -390,7 +391,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) local data = {} local nelts = 0 -- Plain array of keys, count merely numeric elts - for _,elt in ipairs(opt) do + for _, elt in ipairs(opt) do if type(elt) == 'string' then -- Numeric table if mtype == 'hash' then @@ -421,7 +422,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) return nil end ret.foreach = function(_, func) - for k,v in pairs(ret.__data) do + for k, v in pairs(ret.__data) do if not func(k, v) then return false end @@ -449,7 +450,7 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) else if opt.external then -- External map definition, missing fields are handled by schema - local parse_res,parse_err = external_map_schema(opt) + local parse_res, parse_err = external_map_schema(opt) if parse_res then ret.__upstreams = lua_util.http_upstreams_by_url(rspamd_config:get_mempool(), opt.backend) @@ -471,14 +472,14 @@ local function rspamd_map_add_from_ucl(opt, mtype, description, callback) else -- Adjust lua specific augmentations in a trivial case if type(opt.url) == 'string' then - local nsrc,ntype = maybe_adjust_type(opt.url, mtype) + local nsrc, ntype = maybe_adjust_type(opt.url, mtype) if nsrc and ntype then opt.url = nsrc mtype = ntype end end -- We have some non-trivial object so let C code to deal with it somehow... - local map = rspamd_config:add_map{ + local map = rspamd_config:add_map { type = mtype, description = description, url = opt, @@ -526,7 +527,9 @@ local function rspamd_maybe_check_map(key, what) local fun = require "fun" if type(what) == "table" then - return fun.any(function(elt) return rspamd_maybe_check_map(key, elt) end, what) + return fun.any(function(elt) + return rspamd_maybe_check_map(key, elt) + end, what) end if type(rspamd_maps) == "table" then local mn @@ -572,7 +575,7 @@ exports.fill_config_maps = function(mname, opts, map_defs) rspamd_logger.errx(rspamd_config, 'map add error %s for module %s', k, mname) return false end - opts[k..'_orig'] = opts[k] + opts[k .. '_orig'] = opts[k] opts[k] = map elseif not v.optional then rspamd_logger.errx(rspamd_config, 'cannot find non optional map %s for module %s', k, mname) @@ -583,27 +586,27 @@ exports.fill_config_maps = function(mname, opts, map_defs) return true end -local direct_map_schema = ts.shape{ -- complex object +local direct_map_schema = ts.shape { -- complex object name = ts.string:is_optional(), description = ts.string:is_optional(), selector_alias = ts.string:is_optional(), -- an optional alias for the selectos framework timeout = ts.number, data = ts.array_of(ts.string):is_optional(), -- Tableshape has no options support for something like key1 or key2? - upstreams = ts.one_of{ + upstreams = ts.one_of { ts.string, ts.array_of(ts.string), - }:is_optional(), - url = ts.one_of{ + } :is_optional(), + url = ts.one_of { ts.string, ts.array_of(ts.string), - }:is_optional(), + } :is_optional(), } -exports.map_schema = ts.one_of{ +exports.map_schema = ts.one_of { ts.string, -- 'http://some_map' ts.array_of(ts.string), -- ['foo', 'bar'] - ts.one_of{direct_map_schema, external_map_schema} + ts.one_of { direct_map_schema, external_map_schema } } return exports diff --git a/lualib/lua_maps_expressions.lua b/lualib/lua_maps_expressions.lua index 0d030b4c4..996de99c0 100644 --- a/lualib/lua_maps_expressions.lua +++ b/lualib/lua_maps_expressions.lua @@ -64,7 +64,7 @@ local function process_func(elt, task) if values then if type(values) == 'table' then - for _,val in ipairs(values) do + for _, val in ipairs(values) do if res == 0 then match_rule(val) end @@ -80,17 +80,16 @@ local function process_func(elt, task) local res = elt.expr:process(process_atom) if res > 0 then - return res,matched + return res, matched end return nil end - -exports.schema = ts.shape{ +exports.schema = ts.shape { expression = ts.string, rules = ts.array_of( - ts.shape{ + ts.shape { selector = ts.string, map = lua_maps.map_schema, } @@ -119,7 +118,9 @@ exports.schema = ts.shape{ -- --]] local function create(cfg, obj, module_name) - if not module_name then module_name = 'lua_maps_expressions' end + if not module_name then + module_name = 'lua_maps_expressions' + end if not obj or not obj.rules or not obj.expression then rspamd_logger.errx(cfg, 'cannot add maps combination for module %s: required elements are missing', @@ -133,7 +134,7 @@ local function create(cfg, obj, module_name) module_name = module_name } - for name,rule in pairs(obj.rules) do + for name, rule in pairs(obj.rules) do local sel = lua_selectors.create_selector_closure(cfg, rule.selector) if not sel then @@ -201,7 +202,7 @@ local function create(cfg, obj, module_name) ret.expr = expr if obj.symbol then - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual,ghost', name = obj.symbol, score = 0.0, diff --git a/lualib/lua_meta.lua b/lualib/lua_meta.lua index 2de26cf44..340d89ee8 100644 --- a/lualib/lua_meta.lua +++ b/lualib/lua_meta.lua @@ -42,13 +42,13 @@ local function meta_size_function(task) } local size = task:get_size() - for i = 1,#sizes do + for i = 1, #sizes do if sizes[i] >= size then - return {(1.0 * i) / #sizes} + return { (1.0 * i) / #sizes } end end - return {0} + return { 0 } end local function meta_images_function(task) @@ -60,7 +60,7 @@ local function meta_images_function(task) local nsmall = 0 if images then - for _,img in ipairs(images) do + for _, img in ipairs(images) do if img:get_type() == 'png' then npng = npng + 1 elseif img:get_type() == 'jpeg' then @@ -87,7 +87,7 @@ local function meta_images_function(task) nlarge = 1.0 * nlarge / ntotal nsmall = 1.0 * nsmall / ntotal end - return {ntotal,njpg,npng,nlarge,nsmall} + return { ntotal, njpg, npng, nlarge, nsmall } end local function meta_nparts_function(task) @@ -103,7 +103,7 @@ local function meta_nparts_function(task) local parts = task:get_parts() if parts then - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do if p:is_attachment() then nattachments = nattachments + 1 end @@ -111,7 +111,7 @@ local function meta_nparts_function(task) end end - return {(1.0 * ntextparts)/totalparts, (1.0 * nattachments)/totalparts} + return { (1.0 * ntextparts) / totalparts, (1.0 * nattachments) / totalparts } end local function meta_encoding_function(task) @@ -120,7 +120,7 @@ local function meta_encoding_function(task) local tp = task:get_text_parts() if tp and #tp > 0 then - for _,p in ipairs(tp) do + for _, p in ipairs(tp) do if p:is_utf() then nutf = nutf + 1 else @@ -128,10 +128,10 @@ local function meta_encoding_function(task) end end - return {nutf / #tp, nother / #tp} + return { nutf / #tp, nother / #tp } end - return {0, 0} + return { 0, 0 } end local function meta_recipients_function(task) @@ -145,10 +145,14 @@ local function meta_recipients_function(task) nsmtp = #(task:get_recipients('smtp')) end - if nmime > 0 then nmime = 1.0 / nmime end - if nsmtp > 0 then nsmtp = 1.0 / nsmtp end + if nmime > 0 then + nmime = 1.0 / nmime + end + if nsmtp > 0 then + nsmtp = 1.0 / nsmtp + end - return {nmime,nsmtp} + return { nmime, nsmtp } end local function meta_received_function(task) @@ -180,7 +184,9 @@ local function meta_received_function(task) secure_factor = secure_factor + 1.0 end end, - fun.filter(function(rc) return not rc.flags or not rc.flags['artificial'] end, rh)) + fun.filter(function(rc) + return not rc.flags or not rc.flags['artificial'] + end, rh)) if ntotal > 0 then invalid_factor = invalid_factor / ntotal @@ -193,16 +199,16 @@ local function meta_received_function(task) end end - return {count_factor, invalid_factor, time_factor, secure_factor} + return { count_factor, invalid_factor, time_factor, secure_factor } end local function meta_urls_function(task) - local has_urls,nurls = task:has_urls() + local has_urls, nurls = task:has_urls() if has_urls and nurls > 0 then - return {1.0 / nurls} + return { 1.0 / nurls } end - return {0} + return { 0 } end local function meta_words_function(task) @@ -224,7 +230,7 @@ local function meta_words_function(task) 20, } - for i = 1,#lens do + for i = 1, #lens do if lens[i] >= avg_len then ret_len = (1.0 * i) / #lens break @@ -241,7 +247,7 @@ local function meta_words_function(task) 0, -- capital characters rate 0, -- numeric characters } - for _,p in ipairs(tp) do + for _, p in ipairs(tp) do local stats = p:get_stats() local len = p:get_length() @@ -266,7 +272,7 @@ local function meta_words_function(task) divisor = #tp end - for _,wr in ipairs(wres) do + for _, wr in ipairs(wres) do table.insert(ret, wr / divisor) end @@ -401,7 +407,7 @@ local metafunctions = { }, } -local meta_schema = ts.shape{ +local meta_schema = ts.shape { cb = ts.func, ninputs = ts.number, names = ts.array_of(ts.string), @@ -413,8 +419,8 @@ local metatokens_by_name = {} local function fill_metatokens_by_name() metatokens_by_name = {} - for _,mt in ipairs(metafunctions) do - for i=1,mt.ninputs do + for _, mt in ipairs(metafunctions) do + for i = 1, mt.ninputs do local name = mt.names[i] metatokens_by_name[name] = function(task) @@ -429,8 +435,8 @@ local function calculate_digest() local cr = require "rspamd_cryptobox_hash" local h = cr.create() - for _,mt in ipairs(metafunctions) do - for i=1,mt.ninputs do + for _, mt in ipairs(metafunctions) do + for i = 1, mt.ninputs do local name = mt.names[i] h:update(name) end @@ -450,9 +456,9 @@ local function rspamd_gen_metatokens(task, names) if cached then return cached else - for _,mt in ipairs(metafunctions) do + for _, mt in ipairs(metafunctions) do local ct = mt.cb(task) - for i,tok in ipairs(ct) do + for i, tok in ipairs(ct) do lua_util.debugm(N, task, "metatoken: %s = %s", mt.names[i], tok) if tok ~= tok or tok == math.huge then @@ -468,7 +474,7 @@ local function rspamd_gen_metatokens(task, names) end else - for _,n in ipairs(names) do + for _, n in ipairs(names) do if metatokens_by_name[n] then local tok = metatokens_by_name[n](task) if tok ~= tok or tok == math.huge then @@ -484,7 +490,7 @@ local function rspamd_gen_metatokens(task, names) end return metatokens - end +end exports.rspamd_gen_metatokens = rspamd_gen_metatokens exports.gen_metatokens = rspamd_gen_metatokens @@ -492,9 +498,9 @@ exports.gen_metatokens = rspamd_gen_metatokens local function rspamd_gen_metatokens_table(task) local metatokens = {} - for _,mt in ipairs(metafunctions) do + for _, mt in ipairs(metafunctions) do local ct = mt.cb(task) - for i,tok in ipairs(ct) do + for i, tok in ipairs(ct) do if tok ~= tok or tok == math.huge then logger.errx(task, 'metatoken %s returned %s; replace it with 0 for sanity', mt.names[i], tok) @@ -514,7 +520,7 @@ exports.gen_metatokens_table = rspamd_gen_metatokens_table local function rspamd_count_metatokens() local ipairs = ipairs local total = 0 - for _,mt in ipairs(metafunctions) do + for _, mt in ipairs(metafunctions) do total = total + mt.ninputs end diff --git a/lualib/lua_mime.lua b/lualib/lua_mime.lua index 038dccceb..0f5aa75c0 100644 --- a/lualib/lua_mime.lua +++ b/lualib/lua_mime.lua @@ -64,7 +64,7 @@ local function do_append_footer(task, part, footer, is_multipart, out, state) end if is_multipart then - out[#out + 1] = string.format('Content-Type: %s; charset=utf-8%s'.. + out[#out + 1] = string.format('Content-Type: %s; charset=utf-8%s' .. 'Content-Transfer-Encoding: %s', ct, newline_s, cte) out[#out + 1] = '' @@ -79,11 +79,11 @@ local function do_append_footer(task, part, footer, is_multipart, out, state) if content:sub(-(nlen), nlen + 1) == double_nline then -- content without last newline content = content:sub(-(#newline_s), #newline_s + 1) .. footer - out[#out + 1] = {encode_func(content), true} + out[#out + 1] = { encode_func(content), true } out[#out + 1] = '' else content = content .. footer - out[#out + 1] = {encode_func(content), true} + out[#out + 1] = { encode_func(content), true } out[#out + 1] = '' end @@ -156,7 +156,7 @@ exports.add_text_footer = function(task, html_footer, text_footer) local boundaries = {} local cur_boundary - for _,part in ipairs(task:get_parts()) do + for _, part in ipairs(task:get_parts()) do local boundary = part:get_boundary() if part:is_multipart() then if cur_boundary then @@ -169,7 +169,7 @@ exports.add_text_footer = function(task, html_footer, text_footer) local rh = part:get_raw_headers() if #rh > 0 then - out[#out + 1] = {rh, true} + out[#out + 1] = { rh, true } end elseif part:is_message() then if boundary then @@ -184,14 +184,14 @@ exports.add_text_footer = function(task, html_footer, text_footer) boundary) end - out[#out + 1] = {part:get_raw_headers(), true} + out[#out + 1] = { part:get_raw_headers(), true } else local append_footer = false local skip_footer = part:is_attachment() local parent = part:get_parent() if parent then - local t,st = parent:get_type() + local t, st = parent:get_type() if t == 'multipart' and st == 'signed' then -- Do not modify signed parts @@ -230,8 +230,8 @@ exports.add_text_footer = function(task, html_footer, text_footer) do_append_footer(task, part, append_footer, parent and parent:is_multipart(), out, state) else - out[#out + 1] = {part:get_raw_headers(), true} - out[#out + 1] = {part:get_raw_content(), false} + out[#out + 1] = { part:get_raw_headers(), true } + out[#out + 1] = { part:get_raw_content(), false } end end end @@ -285,38 +285,40 @@ local function do_replacement (task, part, mp, replacements, -- sort matches and form the table: -- start .. end for inclusion position local matches_flattened = {} - for npat,matches in pairs(match_pos) do - for _,m in ipairs(matches) do - table.insert(matches_flattened, {m, npat}) + for npat, matches in pairs(match_pos) do + for _, m in ipairs(matches) do + table.insert(matches_flattened, { m, npat }) end end -- Handle the case of empty match if #matches_flattened == 0 then - out[#out + 1] = {part:get_raw_headers(), true} - out[#out + 1] = {part:get_raw_content(), false} + out[#out + 1] = { part:get_raw_headers(), true } + out[#out + 1] = { part:get_raw_content(), false } return end if is_multipart then - out[#out + 1] = {string.format('Content-Type: %s; charset="utf-8"%s'.. + out[#out + 1] = { string.format('Content-Type: %s; charset="utf-8"%s' .. 'Content-Transfer-Encoding: %s', - ct, newline_s, cte), true} - out[#out + 1] = {'', true} + ct, newline_s, cte), true } + out[#out + 1] = { '', true } else state.new_cte = cte end state.has_matches = true -- now sort flattened by start of match and eliminate all overlaps - table.sort(matches_flattened, function(m1, m2) return m1[1][1] < m2[1][1] end) + table.sort(matches_flattened, function(m1, m2) + return m1[1][1] < m2[1][1] + end) - for i=1,#matches_flattened - 1 do + for i = 1, #matches_flattened - 1 do local st = matches_flattened[i][1][1] -- current start of match local e = matches_flattened[i][1][2] -- current end of match local max_npat = matches_flattened[i][2] - for j=i+1,#matches_flattened do + for j = i + 1, #matches_flattened do if matches_flattened[j][1][1] == st then -- overlap if matches_flattened[j][1][2] > e then @@ -329,7 +331,7 @@ local function do_replacement (task, part, mp, replacements, end end -- Maximum overlap for all matches - for j=i,#matches_flattened do + for j = i, #matches_flattened do if matches_flattened[j][1][1] == st then if e > matches_flattened[j][1][2] then matches_flattened[j][1][2] = e @@ -341,7 +343,7 @@ local function do_replacement (task, part, mp, replacements, end end -- Off-by one: match returns 0 based positions while we use 1 based in Lua - for _,m in ipairs(matches_flattened) do + for _, m in ipairs(matches_flattened) do m[1][1] = m[1][1] + 1 m[1][2] = m[1][2] + 1 end @@ -354,7 +356,7 @@ local function do_replacement (task, part, mp, replacements, local cur_start = 1 local fragments = {} - for _,m in ipairs(matches_flattened) do + for _, m in ipairs(matches_flattened) do if m[1][1] >= cur_start then fragments[#fragments + 1] = content:sub(cur_start, m[1][1] - 1) fragments[#fragments + 1] = replacements[m[2]] @@ -368,11 +370,11 @@ local function do_replacement (task, part, mp, replacements, end -- Final stuff - out[#out + 1] = {encode_func(rspamd_text.fromtable(fragments)), false} + out[#out + 1] = { encode_func(rspamd_text.fromtable(fragments)), false } else -- No matches - out[#out + 1] = {part:get_raw_headers(), true} - out[#out + 1] = {part:get_raw_content(), false} + out[#out + 1] = { part:get_raw_headers(), true } + out[#out + 1] = { part:get_raw_content(), false } end end @@ -429,12 +431,12 @@ exports.multipattern_text_replace = function(task, mp, replacements) local boundaries = {} local cur_boundary - for _,part in ipairs(task:get_parts()) do + for _, part in ipairs(task:get_parts()) do local boundary = part:get_boundary() if part:is_multipart() then if cur_boundary then - out[#out + 1] = {string.format('--%s', - boundaries[#boundaries]), true} + out[#out + 1] = { string.format('--%s', + boundaries[#boundaries]), true } end boundaries[#boundaries + 1] = boundary or '--XXX' @@ -442,28 +444,28 @@ exports.multipattern_text_replace = function(task, mp, replacements) local rh = part:get_raw_headers() if #rh > 0 then - out[#out + 1] = {rh, true} + out[#out + 1] = { rh, true } end elseif part:is_message() then if boundary then if cur_boundary and boundary ~= cur_boundary then -- Need to close boundary - out[#out + 1] = {string.format('--%s--', - boundaries[#boundaries]), true} + out[#out + 1] = { string.format('--%s--', + boundaries[#boundaries]), true } table.remove(boundaries) cur_boundary = nil end - out[#out + 1] = {string.format('--%s', - boundary), true} + out[#out + 1] = { string.format('--%s', + boundary), true } end - out[#out + 1] = {part:get_raw_headers(), true} + out[#out + 1] = { part:get_raw_headers(), true } else local skip_replacement = part:is_attachment() local parent = part:get_parent() if parent then - local t,st = parent:get_type() + local t, st = parent:get_type() if t == 'multipart' and st == 'signed' then -- Do not modify signed parts @@ -477,13 +479,13 @@ exports.multipattern_text_replace = function(task, mp, replacements) if boundary then if cur_boundary and boundary ~= cur_boundary then -- Need to close boundary - out[#out + 1] = {string.format('--%s--', - boundaries[#boundaries]), true} + out[#out + 1] = { string.format('--%s--', + boundaries[#boundaries]), true } table.remove(boundaries) cur_boundary = boundary end - out[#out + 1] = {string.format('--%s', - boundary), true} + out[#out + 1] = { string.format('--%s', + boundary), true } end if not skip_replacement then @@ -491,8 +493,8 @@ exports.multipattern_text_replace = function(task, mp, replacements) parent and parent:is_multipart(), out, state) else -- Append as is - out[#out + 1] = {part:get_raw_headers(), true} - out[#out + 1] = {part:get_raw_content(), false} + out[#out + 1] = { part:get_raw_headers(), true } + out[#out + 1] = { part:get_raw_content(), false } end end end @@ -500,9 +502,9 @@ exports.multipattern_text_replace = function(task, mp, replacements) -- Close remaining local b = table.remove(boundaries) while b do - out[#out + 1] = {string.format('--%s--', b), true} + out[#out + 1] = { string.format('--%s--', b), true } if #boundaries > 0 then - out[#out + 1] = {'', true} + out[#out + 1] = { '', true } end b = table.remove(boundaries) end @@ -519,7 +521,9 @@ end --]] exports.modify_headers = function(task, hdr_alterations, mode) -- Assume default mode compatibility - if not mode then mode = 'compat' end + if not mode then + mode = 'compat' + end local add = hdr_alterations.add or {} local remove = hdr_alterations.remove or {} @@ -531,7 +535,7 @@ exports.modify_headers = function(task, hdr_alterations, mode) add_headers[hname] = {} end if not hdr_flattened[hname] then - hdr_flattened[hname] = {add = {}} + hdr_flattened[hname] = { add = {} } end local add_tbl = hdr_flattened[hname].add if hdr.value then @@ -539,9 +543,9 @@ exports.modify_headers = function(task, hdr_alterations, mode) order = (tonumber(hdr.order) or -1), value = hdr.value, }) - table.insert(add_tbl, {tonumber(hdr.order) or -1, hdr.value}) + table.insert(add_tbl, { tonumber(hdr.order) or -1, hdr.value }) elseif type(hdr) == 'table' then - for _,v in ipairs(hdr) do + for _, v in ipairs(hdr) do flatten_add_header(hname, v) end elseif type(hdr) == 'string' then @@ -549,7 +553,7 @@ exports.modify_headers = function(task, hdr_alterations, mode) order = -1, value = hdr, }) - table.insert(add_tbl, {-1, hdr}) + table.insert(add_tbl, { -1, hdr }) else logger.errx(task, 'invalid modification of header: %s', hdr) end @@ -561,19 +565,18 @@ exports.modify_headers = function(task, hdr_alterations, mode) end if hdr_alterations.order then -- Get headers alterations ordered - for _,hname in ipairs(hdr_alterations.order) do + for _, hname in ipairs(hdr_alterations.order) do flatten_add_header(hname, add[hname]) end else - for hname,hdr in pairs(add) do + for hname, hdr in pairs(add) do flatten_add_header(hname, hdr) end end - - for hname,hdr in pairs(remove) do + for hname, hdr in pairs(remove) do if not hdr_flattened[hname] then - hdr_flattened[hname] = {remove = {}} + hdr_flattened[hname] = { remove = {} } end if not hdr_flattened[hname].remove then hdr_flattened[hname].remove = {} @@ -582,7 +585,7 @@ exports.modify_headers = function(task, hdr_alterations, mode) if type(hdr) == 'number' then table.insert(remove_tbl, hdr) else - for _,num in ipairs(hdr) do + for _, num in ipairs(hdr) do table.insert(remove_tbl, num) end end @@ -590,15 +593,19 @@ exports.modify_headers = function(task, hdr_alterations, mode) if mode == 'compat' then -- Clear empty alterations in the compat mode - if add_headers and not next(add_headers) then add_headers = nil end - if hdr_alterations.remove and not next(hdr_alterations.remove) then hdr_alterations.remove = nil end + if add_headers and not next(add_headers) then + add_headers = nil + end + if hdr_alterations.remove and not next(hdr_alterations.remove) then + hdr_alterations.remove = nil + end end task:set_milter_reply({ add_headers = add_headers, remove_headers = hdr_alterations.remove }) - for hname,flat_rules in pairs(hdr_flattened) do + for hname, flat_rules in pairs(hdr_flattened) do task:modify_header(hname, flat_rules) end end @@ -611,7 +618,9 @@ exports.message_to_ucl = function(task, stringify_content) local E = {} local maybe_stringify_f = stringify_content and - tostring or function(t) return t end + tostring or function(t) + return t + end local result = { size = task:get_size(), digest = task:get_digest(), @@ -643,7 +652,7 @@ exports.message_to_ucl = function(task, stringify_content) local parts = task:get_parts() or E result.parts = {} - for _,part in ipairs(parts) do + for _, part in ipairs(parts) do if not part:is_multipart() and not part:is_message() then local p = { size = part:get_length(), @@ -651,7 +660,7 @@ exports.message_to_ucl = function(task, stringify_content) detected_type = string.format('%s/%s', part:get_detected_type()), filename = part:get_filename(), content = maybe_stringify_f(part:get_content()), - headers = part:get_headers(true) or E, + headers = part:get_headers(true) or E, boundary = part:get_enclosing_boundary(), } table.insert(result.parts, p) @@ -659,7 +668,7 @@ exports.message_to_ucl = function(task, stringify_content) -- Service part: multipart container or message/rfc822 local p = { type = string.format('%s/%s', part:get_type()), - headers = part:get_headers(true) or E, + headers = part:get_headers(true) or E, boundary = part:get_enclosing_boundary(), size = 0, } @@ -683,8 +692,8 @@ exports.message_to_ucl_schema = function() local ts = require("tableshape").types local function headers_schema() - return ts.shape{ - order = ts.integer:describe('Header order in a message'), + return ts.shape { + order = ts.integer:describe('Header order in a message'), raw = ts.string:describe('Raw header value'):is_optional(), empty_separator = ts.boolean:describe('Whether header has an empty separator'), separator = ts.string:describe('Separator between a header and a value'), @@ -696,8 +705,8 @@ exports.message_to_ucl_schema = function() end local function part_schema() - return ts.shape{ - content = ts.string:describe('Decoded content'):is_optional(), + return ts.shape { + content = ts.string:describe('Decoded content'):is_optional(), multipart_boundary = ts.string:describe('Multipart service boundary'):is_optional(), size = ts.integer:describe('Size of the part'), type = ts.string:describe('Announced type'):is_optional(), @@ -709,10 +718,10 @@ exports.message_to_ucl_schema = function() end local function email_addr_schema() - return ts.shape{ - addr = ts.string:describe('Parsed address'):is_optional(), + return ts.shape { + addr = ts.string:describe('Parsed address'):is_optional(), raw = ts.string:describe('Raw address'), - flags = ts.shape{ + flags = ts.shape { valid = ts.boolean:describe('Valid address'):is_optional(), ip = ts.boolean:describe('IP like address'):is_optional(), braced = ts.boolean:describe('Have braces around address'):is_optional(), @@ -721,13 +730,13 @@ exports.message_to_ucl_schema = function() backslash = ts.boolean:describe('Backslash in address'):is_optional(), ['8bit'] = ts.boolean:describe('8 bit characters in address'):is_optional(), }, - user = ts.string:describe('Parsed user part'):is_optional(), - name = ts.string:describe('Displayed name'):is_optional(), - domain = ts.string:describe('Parsed domain part'):is_optional(), + user = ts.string:describe('Parsed user part'):is_optional(), + name = ts.string:describe('Displayed name'):is_optional(), + domain = ts.string:describe('Parsed domain part'):is_optional(), } end local function envelope_schema() - return ts.shape{ + return ts.shape { from_smtp = email_addr_schema():describe('SMTP from'):is_optional(), recipients_smtp = ts.array_of(email_addr_schema()):describe('SMTP recipients'):is_optional(), helo = ts.string:describe('SMTP Helo'):is_optional(), @@ -737,12 +746,12 @@ exports.message_to_ucl_schema = function() } end - return ts.shape{ + return ts.shape { headers = ts.array_of(headers_schema()), parts = ts.array_of(part_schema()), digest = ts.pattern(string.format('^%s$', string.rep('%x', 32))) - :describe('Message digest'), - newlines = ts.one_of({"cr", "lf", "crlf"}):describe('Newlines type'), + :describe('Message digest'), + newlines = ts.one_of({ "cr", "lf", "crlf" }):describe('Newlines type'), size = ts.integer:describe('Size of the message in bytes'), envelope = envelope_schema() } diff --git a/lualib/lua_mime_types.lua b/lualib/lua_mime_types.lua index 6bb20f1d9..ba55f9740 100644 --- a/lualib/lua_mime_types.lua +++ b/lualib/lua_mime_types.lua @@ -23,723 +23,723 @@ local exports = {} -- All mime extensions with corresponding content types exports.full_extensions_map = { - {"323", "text/h323"}, - {"3g2", "video/3gpp2"}, - {"3gp", "video/3gpp"}, - {"3gp2", "video/3gpp2"}, - {"3gpp", "video/3gpp"}, - {"7z", {"application/x-7z-compressed", "application/7z"}}, - {"aa", "audio/audible"}, - {"AAC", "audio/aac"}, - {"aaf", "application/octet-stream"}, - {"aax", "audio/vnd.audible.aax"}, - {"ac3", "audio/ac3"}, - {"aca", "application/octet-stream"}, - {"accda", "application/msaccess.addin"}, - {"accdb", "application/msaccess"}, - {"accdc", "application/msaccess.cab"}, - {"accde", "application/msaccess"}, - {"accdr", "application/msaccess.runtime"}, - {"accdt", "application/msaccess"}, - {"accdw", "application/msaccess.webapplication"}, - {"accft", "application/msaccess.ftemplate"}, - {"acx", "application/internet-property-stream"}, - {"AddIn", "text/xml"}, - {"ade", "application/msaccess"}, - {"adobebridge", "application/x-bridge-url"}, - {"adp", "application/msaccess"}, - {"ADT", "audio/vnd.dlna.adts"}, - {"ADTS", "audio/aac"}, - {"afm", "application/octet-stream"}, - {"ai", "application/postscript"}, - {"aif", "audio/aiff"}, - {"aifc", "audio/aiff"}, - {"aiff", "audio/aiff"}, - {"air", "application/vnd.adobe.air-application-installer-package+zip"}, - {"amc", "application/mpeg"}, - {"anx", "application/annodex"}, - {"apk", "application/vnd.android.package-archive" }, - {"application", "application/x-ms-application"}, - {"art", "image/x-jg"}, - {"asa", "application/xml"}, - {"asax", "application/xml"}, - {"ascx", "application/xml"}, - {"asd", "application/octet-stream"}, - {"asf", "video/x-ms-asf"}, - {"ashx", "application/xml"}, - {"asi", "application/octet-stream"}, - {"asm", "text/plain"}, - {"asmx", "application/xml"}, - {"aspx", "application/xml"}, - {"asr", "video/x-ms-asf"}, - {"asx", "video/x-ms-asf"}, - {"atom", "application/atom+xml"}, - {"au", "audio/basic"}, - {"avi", "video/x-msvideo"}, - {"axa", "audio/annodex"}, - {"axs", "application/olescript"}, - {"axv", "video/annodex"}, - {"bas", "text/plain"}, - {"bcpio", "application/x-bcpio"}, - {"bin", "application/octet-stream"}, - {"bmp", {"image/bmp", "image/x-ms-bmp"}}, - {"c", "text/plain"}, - {"cab", "application/octet-stream"}, - {"caf", "audio/x-caf"}, - {"calx", "application/vnd.ms-office.calx"}, - {"cat", "application/vnd.ms-pki.seccat"}, - {"cc", "text/plain"}, - {"cd", "text/plain"}, - {"cdda", "audio/aiff"}, - {"cdf", "application/x-cdf"}, - {"cer", "application/x-x509-ca-cert"}, - {"cfg", "text/plain"}, - {"chm", "application/octet-stream"}, - {"class", "application/x-java-applet"}, - {"clp", "application/x-msclip"}, - {"cmd", "text/plain"}, - {"cmx", "image/x-cmx"}, - {"cnf", "text/plain"}, - {"cod", "image/cis-cod"}, - {"config", "application/xml"}, - {"contact", "text/x-ms-contact"}, - {"coverage", "application/xml"}, - {"cpio", "application/x-cpio"}, - {"cpp", "text/plain"}, - {"crd", "application/x-mscardfile"}, - {"crl", "application/pkix-crl"}, - {"crt", "application/x-x509-ca-cert"}, - {"cs", "text/plain"}, - {"csdproj", "text/plain"}, - {"csh", "application/x-csh"}, - {"csproj", "text/plain"}, - {"css", "text/css"}, - {"csv", {"application/vnd.ms-excel", "text/csv", "text/plain"}}, - {"cur", "application/octet-stream"}, - {"cxx", "text/plain"}, - {"dat", {"application/octet-stream", "application/ms-tnef"}}, - {"datasource", "application/xml"}, - {"dbproj", "text/plain"}, - {"dcr", "application/x-director"}, - {"def", "text/plain"}, - {"deploy", "application/octet-stream"}, - {"der", "application/x-x509-ca-cert"}, - {"dgml", "application/xml"}, - {"dib", "image/bmp"}, - {"dif", "video/x-dv"}, - {"dir", "application/x-director"}, - {"disco", "text/xml"}, - {"divx", "video/divx"}, - {"dll", "application/x-msdownload"}, - {"dll.config", "text/xml"}, - {"dlm", "text/dlm"}, - {"doc", "application/msword"}, - {"docm", "application/vnd.ms-word.document.macroEnabled.12"}, - {"docx", { - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/msword", - "application/vnd.ms-word.document.12", - "application/octet-stream", - }}, - {"dot", "application/msword"}, - {"dotm", "application/vnd.ms-word.template.macroEnabled.12"}, - {"dotx", "application/vnd.openxmlformats-officedocument.wordprocessingml.template"}, - {"dsp", "application/octet-stream"}, - {"dsw", "text/plain"}, - {"dtd", "text/xml"}, - {"dtsConfig", "text/xml"}, - {"dv", "video/x-dv"}, - {"dvi", "application/x-dvi"}, - {"dwf", "drawing/x-dwf"}, - {"dwg", {"application/acad", "image/vnd.dwg"}}, - {"dwp", "application/octet-stream"}, - {"dxf", "application/x-dxf" }, - {"dxr", "application/x-director"}, - {"eml", "message/rfc822"}, - {"emz", "application/octet-stream"}, - {"eot", "application/vnd.ms-fontobject"}, - {"eps", "application/postscript"}, - {"etl", "application/etl"}, - {"etx", "text/x-setext"}, - {"evy", "application/envoy"}, - {"exe", { - "application/x-dosexec", - "application/x-msdownload", - "application/x-executable", - }}, - {"exe.config", "text/xml"}, - {"fdf", "application/vnd.fdf"}, - {"fif", "application/fractals"}, - {"filters", "application/xml"}, - {"fla", "application/octet-stream"}, - {"flac", "audio/flac"}, - {"flr", "x-world/x-vrml"}, - {"flv", "video/x-flv"}, - {"fsscript", "application/fsharp-script"}, - {"fsx", "application/fsharp-script"}, - {"generictest", "application/xml"}, - {"gif", "image/gif"}, - {"gpx", "application/gpx+xml"}, - {"group", "text/x-ms-group"}, - {"gsm", "audio/x-gsm"}, - {"gtar", "application/x-gtar"}, - {"gz", {"application/gzip", "application/x-gzip", "application/tlsrpt+gzip"}}, - {"h", "text/plain"}, - {"hdf", "application/x-hdf"}, - {"hdml", "text/x-hdml"}, - {"hhc", "application/x-oleobject"}, - {"hhk", "application/octet-stream"}, - {"hhp", "application/octet-stream"}, - {"hlp", "application/winhlp"}, - {"hpp", "text/plain"}, - {"hqx", "application/mac-binhex40"}, - {"hta", "application/hta"}, - {"htc", "text/x-component"}, - {"htm", "text/html"}, - {"html", "text/html"}, - {"htt", "text/webviewhtml"}, - {"hxa", "application/xml"}, - {"hxc", "application/xml"}, - {"hxd", "application/octet-stream"}, - {"hxe", "application/xml"}, - {"hxf", "application/xml"}, - {"hxh", "application/octet-stream"}, - {"hxi", "application/octet-stream"}, - {"hxk", "application/xml"}, - {"hxq", "application/octet-stream"}, - {"hxr", "application/octet-stream"}, - {"hxs", "application/octet-stream"}, - {"hxt", "text/html"}, - {"hxv", "application/xml"}, - {"hxw", "application/octet-stream"}, - {"hxx", "text/plain"}, - {"i", "text/plain"}, - {"ico", "image/x-icon"}, - {"ics", {"text/calendar", "application/ics", "application/octet-stream"}}, - {"idl", "text/plain"}, - {"ief", "image/ief"}, - {"iii", "application/x-iphone"}, - {"inc", "text/plain"}, - {"inf", "application/octet-stream"}, - {"ini", "text/plain"}, - {"inl", "text/plain"}, - {"ins", "application/x-internet-signup"}, - {"ipa", "application/x-itunes-ipa"}, - {"ipg", "application/x-itunes-ipg"}, - {"ipproj", "text/plain"}, - {"ipsw", "application/x-itunes-ipsw"}, - {"iqy", "text/x-ms-iqy"}, - {"isp", "application/x-internet-signup"}, - {"ite", "application/x-itunes-ite"}, - {"itlp", "application/x-itunes-itlp"}, - {"itms", "application/x-itunes-itms"}, - {"itpc", "application/x-itunes-itpc"}, - {"IVF", "video/x-ivf"}, - {"jar", "application/java-archive"}, - {"java", "application/octet-stream"}, - {"jck", "application/liquidmotion"}, - {"jcz", "application/liquidmotion"}, - {"jfif", {"image/jpeg", "image/pjpeg"}}, - {"jnlp", "application/x-java-jnlp-file"}, - {"jpb", "application/octet-stream"}, - {"jpe", {"image/jpeg", "image/pjpeg"}}, - {"jpeg", {"image/jpeg", "image/pjpeg"}}, - {"jpg", {"image/jpeg", "image/pjpeg"}}, - {"js", "application/javascript"}, - {"json", "application/json"}, - {"jsx", "text/jscript"}, - {"jsxbin", "text/plain"}, - {"latex", "application/x-latex"}, - {"library-ms", "application/windows-library+xml"}, - {"lit", "application/x-ms-reader"}, - {"loadtest", "application/xml"}, - {"lpk", "application/octet-stream"}, - {"lsf", "video/x-la-asf"}, - {"lst", "text/plain"}, - {"lsx", "video/x-la-asf"}, - {"lzh", "application/octet-stream"}, - {"m13", "application/x-msmediaview"}, - {"m14", "application/x-msmediaview"}, - {"m1v", "video/mpeg"}, - {"m2t", "video/vnd.dlna.mpeg-tts"}, - {"m2ts", "video/vnd.dlna.mpeg-tts"}, - {"m2v", "video/mpeg"}, - {"m3u", "audio/x-mpegurl"}, - {"m3u8", "audio/x-mpegurl"}, - {"m4a", {"audio/m4a", "audio/x-m4a"}}, - {"m4b", "audio/m4b"}, - {"m4p", "audio/m4p"}, - {"m4r", "audio/x-m4r"}, - {"m4v", "video/x-m4v"}, - {"mac", "image/x-macpaint"}, - {"mak", "text/plain"}, - {"man", "application/x-troff-man"}, - {"manifest", "application/x-ms-manifest"}, - {"map", "text/plain"}, - {"master", "application/xml"}, - {"mbox", "application/mbox"}, - {"mda", "application/msaccess"}, - {"mdb", "application/x-msaccess"}, - {"mde", "application/msaccess"}, - {"mdp", "application/octet-stream"}, - {"me", "application/x-troff-me"}, - {"mfp", "application/x-shockwave-flash"}, - {"mht", "message/rfc822"}, - {"mhtml", "message/rfc822"}, - {"mid", "audio/mid"}, - {"midi", "audio/mid"}, - {"mix", "application/octet-stream"}, - {"mk", "text/plain"}, - {"mmf", "application/x-smaf"}, - {"mno", "text/xml"}, - {"mny", "application/x-msmoney"}, - {"mod", "video/mpeg"}, - {"mov", "video/quicktime"}, - {"movie", "video/x-sgi-movie"}, - {"mp2", "video/mpeg"}, - {"mp2v", "video/mpeg"}, - {"mp3", {"audio/mpeg", "audio/mpeg3", "audio/mp3", "audio/x-mpeg-3"}}, - {"mp4", "video/mp4"}, - {"mp4v", "video/mp4"}, - {"mpa", "video/mpeg"}, - {"mpe", "video/mpeg"}, - {"mpeg", "video/mpeg"}, - {"mpf", "application/vnd.ms-mediapackage"}, - {"mpg", "video/mpeg"}, - {"mpp", "application/vnd.ms-project"}, - {"mpv2", "video/mpeg"}, - {"mqv", "video/quicktime"}, - {"ms", "application/x-troff-ms"}, - {"msg", "application/vnd.ms-outlook"}, - {"msi", {"application/x-msi", "application/octet-stream"}}, - {"mso", "application/octet-stream"}, - {"mts", "video/vnd.dlna.mpeg-tts"}, - {"mtx", "application/xml"}, - {"mvb", "application/x-msmediaview"}, - {"mvc", "application/x-miva-compiled"}, - {"mxp", "application/x-mmxp"}, - {"nc", "application/x-netcdf"}, - {"nsc", "video/x-ms-asf"}, - {"nws", "message/rfc822"}, - {"ocx", "application/octet-stream"}, - {"oda", "application/oda"}, - {"odb", "application/vnd.oasis.opendocument.database"}, - {"odc", "application/vnd.oasis.opendocument.chart"}, - {"odf", "application/vnd.oasis.opendocument.formula"}, - {"odg", "application/vnd.oasis.opendocument.graphics"}, - {"odh", "text/plain"}, - {"odi", "application/vnd.oasis.opendocument.image"}, - {"odl", "text/plain"}, - {"odm", "application/vnd.oasis.opendocument.text-master"}, - {"odp", "application/vnd.oasis.opendocument.presentation"}, - {"ods", "application/vnd.oasis.opendocument.spreadsheet"}, - {"odt", "application/vnd.oasis.opendocument.text"}, - {"oga", "audio/ogg"}, - {"ogg", "audio/ogg"}, - {"ogv", "video/ogg"}, - {"ogx", "application/ogg"}, - {"one", "application/onenote"}, - {"onea", "application/onenote"}, - {"onepkg", "application/onenote"}, - {"onetmp", "application/onenote"}, - {"onetoc", "application/onenote"}, - {"onetoc2", "application/onenote"}, - {"opus", "audio/ogg"}, - {"orderedtest", "application/xml"}, - {"osdx", "application/opensearchdescription+xml"}, - {"otf", "application/font-sfnt"}, - {"otg", "application/vnd.oasis.opendocument.graphics-template"}, - {"oth", "application/vnd.oasis.opendocument.text-web"}, - {"otp", "application/vnd.oasis.opendocument.presentation-template"}, - {"ots", "application/vnd.oasis.opendocument.spreadsheet-template"}, - {"ott", "application/vnd.oasis.opendocument.text-template"}, - {"oxt", "application/vnd.openofficeorg.extension"}, - {"p10", "application/pkcs10"}, - {"p12", "application/x-pkcs12"}, - {"p7b", "application/x-pkcs7-certificates"}, - {"p7c", "application/pkcs7-mime"}, - {"p7m", "application/pkcs7-mime", "application/x-pkcs7-mime"}, - {"p7r", "application/x-pkcs7-certreqresp"}, - {"p7s", {"application/pkcs7-signature", "application/x-pkcs7-signature", "text/plain"}}, - {"pbm", "image/x-portable-bitmap"}, - {"pcast", "application/x-podcast"}, - {"pct", "image/pict"}, - {"pcx", "application/octet-stream"}, - {"pcz", "application/octet-stream"}, - {"pdf", "application/pdf"}, - {"pfb", "application/octet-stream"}, - {"pfm", "application/octet-stream"}, - {"pfx", "application/x-pkcs12"}, - {"pgm", "image/x-portable-graymap"}, - {"pic", "image/pict"}, - {"pict", "image/pict"}, - {"pkgdef", "text/plain"}, - {"pkgundef", "text/plain"}, - {"pko", "application/vnd.ms-pki.pko"}, - {"pls", "audio/scpls"}, - {"pma", "application/x-perfmon"}, - {"pmc", "application/x-perfmon"}, - {"pml", "application/x-perfmon"}, - {"pmr", "application/x-perfmon"}, - {"pmw", "application/x-perfmon"}, - {"png", "image/png"}, - {"pnm", "image/x-portable-anymap"}, - {"pnt", "image/x-macpaint"}, - {"pntg", "image/x-macpaint"}, - {"pnz", "image/png"}, - {"pot", "application/vnd.ms-powerpoint"}, - {"potm", "application/vnd.ms-powerpoint.template.macroEnabled.12"}, - {"potx", "application/vnd.openxmlformats-officedocument.presentationml.template"}, - {"ppa", "application/vnd.ms-powerpoint"}, - {"ppam", "application/vnd.ms-powerpoint.addin.macroEnabled.12"}, - {"ppm", "image/x-portable-pixmap"}, - {"pps", "application/vnd.ms-powerpoint"}, - {"ppsm", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12"}, - {"ppsx", "application/vnd.openxmlformats-officedocument.presentationml.slideshow"}, - {"ppt", "application/vnd.ms-powerpoint"}, - {"pptm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12"}, - {"pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, - {"prf", "application/pics-rules"}, - {"prm", "application/octet-stream"}, - {"prx", "application/octet-stream"}, - {"ps", "application/postscript"}, - {"psc1", "application/PowerShell"}, - {"psd", "application/octet-stream"}, - {"psess", "application/xml"}, - {"psm", "application/octet-stream"}, - {"psp", "application/octet-stream"}, - {"pst", "application/vnd.ms-outlook"}, - {"pub", "application/x-mspublisher"}, - {"pwz", "application/vnd.ms-powerpoint"}, - {"qht", "text/x-html-insertion"}, - {"qhtm", "text/x-html-insertion"}, - {"qt", "video/quicktime"}, - {"qti", "image/x-quicktime"}, - {"qtif", "image/x-quicktime"}, - {"qtl", "application/x-quicktimeplayer"}, - {"qxd", "application/octet-stream"}, - {"ra", "audio/x-pn-realaudio"}, - {"ram", "audio/x-pn-realaudio"}, - {"rar", {"application/x-rar-compressed", "application/x-rar", "application/rar", "application/octet-stream"}}, - {"ras", "image/x-cmu-raster"}, - {"rat", "application/rat-file"}, - {"rc", "text/plain"}, - {"rc2", "text/plain"}, - {"rct", "text/plain"}, - {"rdlc", "application/xml"}, - {"reg", "text/plain"}, - {"resx", "application/xml"}, - {"rf", "image/vnd.rn-realflash"}, - {"rgb", "image/x-rgb"}, - {"rgs", "text/plain"}, - {"rm", "application/vnd.rn-realmedia"}, - {"rmi", "audio/mid"}, - {"rmp", "application/vnd.rn-rn_music_package"}, - {"roff", "application/x-troff"}, - {"rpm", "audio/x-pn-realaudio-plugin"}, - {"rqy", "text/x-ms-rqy"}, - {"rtf", {"application/rtf","application/msword", "text/richtext", "text/rtf"}}, - {"rtx", "text/richtext"}, - {"rvt", "application/octet-stream" }, - {"ruleset", "application/xml"}, - {"s", "text/plain"}, - {"safariextz", "application/x-safari-safariextz"}, - {"scd", "application/x-msschedule"}, - {"scr", "text/plain"}, - {"sct", "text/scriptlet"}, - {"sd2", "audio/x-sd2"}, - {"sdp", "application/sdp"}, - {"sea", "application/octet-stream"}, - {"searchConnector-ms", "application/windows-search-connector+xml"}, - {"setpay", "application/set-payment-initiation"}, - {"setreg", "application/set-registration-initiation"}, - {"settings", "application/xml"}, - {"sgimb", "application/x-sgimb"}, - {"sgml", "text/sgml"}, - {"sh", "application/x-sh"}, - {"shar", "application/x-shar"}, - {"shtml", "text/html"}, - {"sit", "application/x-stuffit"}, - {"sitemap", "application/xml"}, - {"skin", "application/xml"}, - {"skp", "application/x-koan" }, - {"sldm", "application/vnd.ms-powerpoint.slide.macroEnabled.12"}, - {"sldx", "application/vnd.openxmlformats-officedocument.presentationml.slide"}, - {"slk", "application/vnd.ms-excel"}, - {"sln", "text/plain"}, - {"slupkg-ms", "application/x-ms-license"}, - {"smd", "audio/x-smd"}, - {"smi", "application/octet-stream"}, - {"smx", "audio/x-smd"}, - {"smz", "audio/x-smd"}, - {"snd", "audio/basic"}, - {"snippet", "application/xml"}, - {"snp", "application/octet-stream"}, - {"sol", "text/plain"}, - {"sor", "text/plain"}, - {"spc", "application/x-pkcs7-certificates"}, - {"spl", "application/futuresplash"}, - {"spx", "audio/ogg"}, - {"src", "application/x-wais-source"}, - {"srf", "text/plain"}, - {"SSISDeploymentManifest", "text/xml"}, - {"ssm", "application/streamingmedia"}, - {"sst", "application/vnd.ms-pki.certstore"}, - {"stl", "application/vnd.ms-pki.stl"}, - {"sv4cpio", "application/x-sv4cpio"}, - {"sv4crc", "application/x-sv4crc"}, - {"svc", "application/xml"}, - {"svg", "image/svg+xml"}, - {"swf", "application/x-shockwave-flash"}, - {"step", "application/step"}, - {"stp", "application/step"}, - {"t", "application/x-troff"}, - {"tar", "application/x-tar"}, - {"tcl", "application/x-tcl"}, - {"testrunconfig", "application/xml"}, - {"testsettings", "application/xml"}, - {"tex", "application/x-tex"}, - {"texi", "application/x-texinfo"}, - {"texinfo", "application/x-texinfo"}, - {"tgz", "application/x-compressed"}, - {"thmx", "application/vnd.ms-officetheme"}, - {"thn", "application/octet-stream"}, - {"tif", {"image/tiff", "application/octet-stream"}}, - {"tiff", "image/tiff"}, - {"tlh", "text/plain"}, - {"tli", "text/plain"}, - {"toc", "application/octet-stream"}, - {"tr", "application/x-troff"}, - {"trm", "application/x-msterminal"}, - {"trx", "application/xml"}, - {"ts", "video/vnd.dlna.mpeg-tts"}, - {"tsv", "text/tab-separated-values"}, - {"ttf", "application/font-sfnt"}, - {"tts", "video/vnd.dlna.mpeg-tts"}, - {"txt", "text/plain"}, - {"u32", "application/octet-stream"}, - {"uls", "text/iuls"}, - {"user", "text/plain"}, - {"ustar", "application/x-ustar"}, - {"vb", "text/plain"}, - {"vbdproj", "text/plain"}, - {"vbk", "video/mpeg"}, - {"vbproj", "text/plain"}, - {"vbs", "text/vbscript"}, - {"vcf", {"text/x-vcard", "text/vcard"}}, - {"vcproj", "application/xml"}, - {"vcs", "text/plain"}, - {"vcxproj", "application/xml"}, - {"vddproj", "text/plain"}, - {"vdp", "text/plain"}, - {"vdproj", "text/plain"}, - {"vdx", "application/vnd.ms-visio.viewer"}, - {"vml", "text/xml"}, - {"vscontent", "application/xml"}, - {"vsct", "text/xml"}, - {"vsd", "application/vnd.visio"}, - {"vsi", "application/ms-vsi"}, - {"vsix", "application/vsix"}, - {"vsixlangpack", "text/xml"}, - {"vsixmanifest", "text/xml"}, - {"vsmdi", "application/xml"}, - {"vspscc", "text/plain"}, - {"vss", "application/vnd.visio"}, - {"vsscc", "text/plain"}, - {"vssettings", "text/xml"}, - {"vssscc", "text/plain"}, - {"vst", "application/vnd.visio"}, - {"vstemplate", "text/xml"}, - {"vsto", "application/x-ms-vsto"}, - {"vsw", "application/vnd.visio"}, - {"vsx", "application/vnd.visio"}, - {"vtx", "application/vnd.visio"}, - {"wav", {"audio/wav", "audio/vnd.wave", "audio/x-wav"}}, - {"wave", "audio/wav"}, - {"wax", "audio/x-ms-wax"}, - {"wbk", "application/msword"}, - {"wbmp", "image/vnd.wap.wbmp"}, - {"wcm", "application/vnd.ms-works"}, - {"wdb", "application/vnd.ms-works"}, - {"wdp", "image/vnd.ms-photo"}, - {"webarchive", "application/x-safari-webarchive"}, - {"webm", "video/webm"}, - {"webp", "image/webp"}, - {"webtest", "application/xml"}, - {"wiq", "application/xml"}, - {"wiz", "application/msword"}, - {"wks", "application/vnd.ms-works"}, - {"WLMP", "application/wlmoviemaker"}, - {"wlpginstall", "application/x-wlpg-detect"}, - {"wlpginstall3", "application/x-wlpg3-detect"}, - {"wm", "video/x-ms-wm"}, - {"wma", "audio/x-ms-wma"}, - {"wmd", "application/x-ms-wmd"}, - {"wmf", {"application/x-msmetafile", "image/wmf", "image/x-wmf"}}, - {"wml", "text/vnd.wap.wml"}, - {"wmlc", "application/vnd.wap.wmlc"}, - {"wmls", "text/vnd.wap.wmlscript"}, - {"wmlsc", "application/vnd.wap.wmlscriptc"}, - {"wmp", "video/x-ms-wmp"}, - {"wmv", "video/x-ms-wmv"}, - {"wmx", "video/x-ms-wmx"}, - {"wmz", "application/x-ms-wmz"}, - {"woff", "application/font-woff"}, - {"wpl", "application/vnd.ms-wpl"}, - {"wps", "application/vnd.ms-works"}, - {"wri", "application/x-mswrite"}, - {"wrl", "x-world/x-vrml"}, - {"wrz", "x-world/x-vrml"}, - {"wsc", "text/scriptlet"}, - {"wsdl", "text/xml"}, - {"wvx", "video/x-ms-wvx"}, - {"x", "application/directx"}, - {"xaf", "x-world/x-vrml"}, - {"xaml", "application/xaml+xml"}, - {"xap", "application/x-silverlight-app"}, - {"xbap", "application/x-ms-xbap"}, - {"xbm", "image/x-xbitmap"}, - {"xdr", "text/plain"}, - {"xht", "application/xhtml+xml"}, - {"xhtml", "application/xhtml+xml"}, - {"xla", "application/vnd.ms-excel"}, - {"xlam", "application/vnd.ms-excel.addin.macroEnabled.12"}, - {"xlc", "application/vnd.ms-excel"}, - {"xld", "application/vnd.ms-excel"}, - {"xlk", "application/vnd.ms-excel"}, - {"xll", "application/vnd.ms-excel"}, - {"xlm", "application/vnd.ms-excel"}, - {"xls", { - "application/excel", - "application/vnd.ms-excel", - "application/vnd.ms-office", - "application/x-excel", - "application/octet-stream" - }}, - {"xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12"}, - {"xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12"}, - {"xlsx", { - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.ms-excel.12", - "application/octet-stream" - }}, - {"xlt", "application/vnd.ms-excel"}, - {"xltm", "application/vnd.ms-excel.template.macroEnabled.12"}, - {"xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template"}, - {"xlw", "application/vnd.ms-excel"}, - {"xml", {"application/xml", "text/xml", "application/octet-stream"}}, - {"xmp", "application/octet-stream" }, - {"xmta", "application/xml"}, - {"xof", "x-world/x-vrml"}, - {"XOML", "text/plain"}, - {"xpm", "image/x-xpixmap"}, - {"xps", "application/vnd.ms-xpsdocument"}, - {"xrm-ms", "text/xml"}, - {"xsc", "application/xml"}, - {"xsd", "text/xml"}, - {"xsf", "text/xml"}, - {"xsl", "text/xml"}, - {"xslt", "text/xml"}, - {"xsn", "application/octet-stream"}, - {"xss", "application/xml"}, - {"xspf", "application/xspf+xml"}, - {"xtp", "application/octet-stream"}, - {"xwd", "image/x-xwindowdump"}, - {"z", "application/x-compress"}, - {"zip", { - "application/zip", - "application/x-zip-compressed", - "application/octet-stream" - }}, - {"zlib", "application/zlib"}, + { "323", "text/h323" }, + { "3g2", "video/3gpp2" }, + { "3gp", "video/3gpp" }, + { "3gp2", "video/3gpp2" }, + { "3gpp", "video/3gpp" }, + { "7z", { "application/x-7z-compressed", "application/7z" } }, + { "aa", "audio/audible" }, + { "AAC", "audio/aac" }, + { "aaf", "application/octet-stream" }, + { "aax", "audio/vnd.audible.aax" }, + { "ac3", "audio/ac3" }, + { "aca", "application/octet-stream" }, + { "accda", "application/msaccess.addin" }, + { "accdb", "application/msaccess" }, + { "accdc", "application/msaccess.cab" }, + { "accde", "application/msaccess" }, + { "accdr", "application/msaccess.runtime" }, + { "accdt", "application/msaccess" }, + { "accdw", "application/msaccess.webapplication" }, + { "accft", "application/msaccess.ftemplate" }, + { "acx", "application/internet-property-stream" }, + { "AddIn", "text/xml" }, + { "ade", "application/msaccess" }, + { "adobebridge", "application/x-bridge-url" }, + { "adp", "application/msaccess" }, + { "ADT", "audio/vnd.dlna.adts" }, + { "ADTS", "audio/aac" }, + { "afm", "application/octet-stream" }, + { "ai", "application/postscript" }, + { "aif", "audio/aiff" }, + { "aifc", "audio/aiff" }, + { "aiff", "audio/aiff" }, + { "air", "application/vnd.adobe.air-application-installer-package+zip" }, + { "amc", "application/mpeg" }, + { "anx", "application/annodex" }, + { "apk", "application/vnd.android.package-archive" }, + { "application", "application/x-ms-application" }, + { "art", "image/x-jg" }, + { "asa", "application/xml" }, + { "asax", "application/xml" }, + { "ascx", "application/xml" }, + { "asd", "application/octet-stream" }, + { "asf", "video/x-ms-asf" }, + { "ashx", "application/xml" }, + { "asi", "application/octet-stream" }, + { "asm", "text/plain" }, + { "asmx", "application/xml" }, + { "aspx", "application/xml" }, + { "asr", "video/x-ms-asf" }, + { "asx", "video/x-ms-asf" }, + { "atom", "application/atom+xml" }, + { "au", "audio/basic" }, + { "avi", "video/x-msvideo" }, + { "axa", "audio/annodex" }, + { "axs", "application/olescript" }, + { "axv", "video/annodex" }, + { "bas", "text/plain" }, + { "bcpio", "application/x-bcpio" }, + { "bin", "application/octet-stream" }, + { "bmp", { "image/bmp", "image/x-ms-bmp" } }, + { "c", "text/plain" }, + { "cab", "application/octet-stream" }, + { "caf", "audio/x-caf" }, + { "calx", "application/vnd.ms-office.calx" }, + { "cat", "application/vnd.ms-pki.seccat" }, + { "cc", "text/plain" }, + { "cd", "text/plain" }, + { "cdda", "audio/aiff" }, + { "cdf", "application/x-cdf" }, + { "cer", "application/x-x509-ca-cert" }, + { "cfg", "text/plain" }, + { "chm", "application/octet-stream" }, + { "class", "application/x-java-applet" }, + { "clp", "application/x-msclip" }, + { "cmd", "text/plain" }, + { "cmx", "image/x-cmx" }, + { "cnf", "text/plain" }, + { "cod", "image/cis-cod" }, + { "config", "application/xml" }, + { "contact", "text/x-ms-contact" }, + { "coverage", "application/xml" }, + { "cpio", "application/x-cpio" }, + { "cpp", "text/plain" }, + { "crd", "application/x-mscardfile" }, + { "crl", "application/pkix-crl" }, + { "crt", "application/x-x509-ca-cert" }, + { "cs", "text/plain" }, + { "csdproj", "text/plain" }, + { "csh", "application/x-csh" }, + { "csproj", "text/plain" }, + { "css", "text/css" }, + { "csv", { "application/vnd.ms-excel", "text/csv", "text/plain" } }, + { "cur", "application/octet-stream" }, + { "cxx", "text/plain" }, + { "dat", { "application/octet-stream", "application/ms-tnef" } }, + { "datasource", "application/xml" }, + { "dbproj", "text/plain" }, + { "dcr", "application/x-director" }, + { "def", "text/plain" }, + { "deploy", "application/octet-stream" }, + { "der", "application/x-x509-ca-cert" }, + { "dgml", "application/xml" }, + { "dib", "image/bmp" }, + { "dif", "video/x-dv" }, + { "dir", "application/x-director" }, + { "disco", "text/xml" }, + { "divx", "video/divx" }, + { "dll", "application/x-msdownload" }, + { "dll.config", "text/xml" }, + { "dlm", "text/dlm" }, + { "doc", "application/msword" }, + { "docm", "application/vnd.ms-word.document.macroEnabled.12" }, + { "docx", { + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/msword", + "application/vnd.ms-word.document.12", + "application/octet-stream", + } }, + { "dot", "application/msword" }, + { "dotm", "application/vnd.ms-word.template.macroEnabled.12" }, + { "dotx", "application/vnd.openxmlformats-officedocument.wordprocessingml.template" }, + { "dsp", "application/octet-stream" }, + { "dsw", "text/plain" }, + { "dtd", "text/xml" }, + { "dtsConfig", "text/xml" }, + { "dv", "video/x-dv" }, + { "dvi", "application/x-dvi" }, + { "dwf", "drawing/x-dwf" }, + { "dwg", { "application/acad", "image/vnd.dwg" } }, + { "dwp", "application/octet-stream" }, + { "dxf", "application/x-dxf" }, + { "dxr", "application/x-director" }, + { "eml", "message/rfc822" }, + { "emz", "application/octet-stream" }, + { "eot", "application/vnd.ms-fontobject" }, + { "eps", "application/postscript" }, + { "etl", "application/etl" }, + { "etx", "text/x-setext" }, + { "evy", "application/envoy" }, + { "exe", { + "application/x-dosexec", + "application/x-msdownload", + "application/x-executable", + } }, + { "exe.config", "text/xml" }, + { "fdf", "application/vnd.fdf" }, + { "fif", "application/fractals" }, + { "filters", "application/xml" }, + { "fla", "application/octet-stream" }, + { "flac", "audio/flac" }, + { "flr", "x-world/x-vrml" }, + { "flv", "video/x-flv" }, + { "fsscript", "application/fsharp-script" }, + { "fsx", "application/fsharp-script" }, + { "generictest", "application/xml" }, + { "gif", "image/gif" }, + { "gpx", "application/gpx+xml" }, + { "group", "text/x-ms-group" }, + { "gsm", "audio/x-gsm" }, + { "gtar", "application/x-gtar" }, + { "gz", { "application/gzip", "application/x-gzip", "application/tlsrpt+gzip" } }, + { "h", "text/plain" }, + { "hdf", "application/x-hdf" }, + { "hdml", "text/x-hdml" }, + { "hhc", "application/x-oleobject" }, + { "hhk", "application/octet-stream" }, + { "hhp", "application/octet-stream" }, + { "hlp", "application/winhlp" }, + { "hpp", "text/plain" }, + { "hqx", "application/mac-binhex40" }, + { "hta", "application/hta" }, + { "htc", "text/x-component" }, + { "htm", "text/html" }, + { "html", "text/html" }, + { "htt", "text/webviewhtml" }, + { "hxa", "application/xml" }, + { "hxc", "application/xml" }, + { "hxd", "application/octet-stream" }, + { "hxe", "application/xml" }, + { "hxf", "application/xml" }, + { "hxh", "application/octet-stream" }, + { "hxi", "application/octet-stream" }, + { "hxk", "application/xml" }, + { "hxq", "application/octet-stream" }, + { "hxr", "application/octet-stream" }, + { "hxs", "application/octet-stream" }, + { "hxt", "text/html" }, + { "hxv", "application/xml" }, + { "hxw", "application/octet-stream" }, + { "hxx", "text/plain" }, + { "i", "text/plain" }, + { "ico", "image/x-icon" }, + { "ics", { "text/calendar", "application/ics", "application/octet-stream" } }, + { "idl", "text/plain" }, + { "ief", "image/ief" }, + { "iii", "application/x-iphone" }, + { "inc", "text/plain" }, + { "inf", "application/octet-stream" }, + { "ini", "text/plain" }, + { "inl", "text/plain" }, + { "ins", "application/x-internet-signup" }, + { "ipa", "application/x-itunes-ipa" }, + { "ipg", "application/x-itunes-ipg" }, + { "ipproj", "text/plain" }, + { "ipsw", "application/x-itunes-ipsw" }, + { "iqy", "text/x-ms-iqy" }, + { "isp", "application/x-internet-signup" }, + { "ite", "application/x-itunes-ite" }, + { "itlp", "application/x-itunes-itlp" }, + { "itms", "application/x-itunes-itms" }, + { "itpc", "application/x-itunes-itpc" }, + { "IVF", "video/x-ivf" }, + { "jar", "application/java-archive" }, + { "java", "application/octet-stream" }, + { "jck", "application/liquidmotion" }, + { "jcz", "application/liquidmotion" }, + { "jfif", { "image/jpeg", "image/pjpeg" } }, + { "jnlp", "application/x-java-jnlp-file" }, + { "jpb", "application/octet-stream" }, + { "jpe", { "image/jpeg", "image/pjpeg" } }, + { "jpeg", { "image/jpeg", "image/pjpeg" } }, + { "jpg", { "image/jpeg", "image/pjpeg" } }, + { "js", "application/javascript" }, + { "json", "application/json" }, + { "jsx", "text/jscript" }, + { "jsxbin", "text/plain" }, + { "latex", "application/x-latex" }, + { "library-ms", "application/windows-library+xml" }, + { "lit", "application/x-ms-reader" }, + { "loadtest", "application/xml" }, + { "lpk", "application/octet-stream" }, + { "lsf", "video/x-la-asf" }, + { "lst", "text/plain" }, + { "lsx", "video/x-la-asf" }, + { "lzh", "application/octet-stream" }, + { "m13", "application/x-msmediaview" }, + { "m14", "application/x-msmediaview" }, + { "m1v", "video/mpeg" }, + { "m2t", "video/vnd.dlna.mpeg-tts" }, + { "m2ts", "video/vnd.dlna.mpeg-tts" }, + { "m2v", "video/mpeg" }, + { "m3u", "audio/x-mpegurl" }, + { "m3u8", "audio/x-mpegurl" }, + { "m4a", { "audio/m4a", "audio/x-m4a" } }, + { "m4b", "audio/m4b" }, + { "m4p", "audio/m4p" }, + { "m4r", "audio/x-m4r" }, + { "m4v", "video/x-m4v" }, + { "mac", "image/x-macpaint" }, + { "mak", "text/plain" }, + { "man", "application/x-troff-man" }, + { "manifest", "application/x-ms-manifest" }, + { "map", "text/plain" }, + { "master", "application/xml" }, + { "mbox", "application/mbox" }, + { "mda", "application/msaccess" }, + { "mdb", "application/x-msaccess" }, + { "mde", "application/msaccess" }, + { "mdp", "application/octet-stream" }, + { "me", "application/x-troff-me" }, + { "mfp", "application/x-shockwave-flash" }, + { "mht", "message/rfc822" }, + { "mhtml", "message/rfc822" }, + { "mid", "audio/mid" }, + { "midi", "audio/mid" }, + { "mix", "application/octet-stream" }, + { "mk", "text/plain" }, + { "mmf", "application/x-smaf" }, + { "mno", "text/xml" }, + { "mny", "application/x-msmoney" }, + { "mod", "video/mpeg" }, + { "mov", "video/quicktime" }, + { "movie", "video/x-sgi-movie" }, + { "mp2", "video/mpeg" }, + { "mp2v", "video/mpeg" }, + { "mp3", { "audio/mpeg", "audio/mpeg3", "audio/mp3", "audio/x-mpeg-3" } }, + { "mp4", "video/mp4" }, + { "mp4v", "video/mp4" }, + { "mpa", "video/mpeg" }, + { "mpe", "video/mpeg" }, + { "mpeg", "video/mpeg" }, + { "mpf", "application/vnd.ms-mediapackage" }, + { "mpg", "video/mpeg" }, + { "mpp", "application/vnd.ms-project" }, + { "mpv2", "video/mpeg" }, + { "mqv", "video/quicktime" }, + { "ms", "application/x-troff-ms" }, + { "msg", "application/vnd.ms-outlook" }, + { "msi", { "application/x-msi", "application/octet-stream" } }, + { "mso", "application/octet-stream" }, + { "mts", "video/vnd.dlna.mpeg-tts" }, + { "mtx", "application/xml" }, + { "mvb", "application/x-msmediaview" }, + { "mvc", "application/x-miva-compiled" }, + { "mxp", "application/x-mmxp" }, + { "nc", "application/x-netcdf" }, + { "nsc", "video/x-ms-asf" }, + { "nws", "message/rfc822" }, + { "ocx", "application/octet-stream" }, + { "oda", "application/oda" }, + { "odb", "application/vnd.oasis.opendocument.database" }, + { "odc", "application/vnd.oasis.opendocument.chart" }, + { "odf", "application/vnd.oasis.opendocument.formula" }, + { "odg", "application/vnd.oasis.opendocument.graphics" }, + { "odh", "text/plain" }, + { "odi", "application/vnd.oasis.opendocument.image" }, + { "odl", "text/plain" }, + { "odm", "application/vnd.oasis.opendocument.text-master" }, + { "odp", "application/vnd.oasis.opendocument.presentation" }, + { "ods", "application/vnd.oasis.opendocument.spreadsheet" }, + { "odt", "application/vnd.oasis.opendocument.text" }, + { "oga", "audio/ogg" }, + { "ogg", "audio/ogg" }, + { "ogv", "video/ogg" }, + { "ogx", "application/ogg" }, + { "one", "application/onenote" }, + { "onea", "application/onenote" }, + { "onepkg", "application/onenote" }, + { "onetmp", "application/onenote" }, + { "onetoc", "application/onenote" }, + { "onetoc2", "application/onenote" }, + { "opus", "audio/ogg" }, + { "orderedtest", "application/xml" }, + { "osdx", "application/opensearchdescription+xml" }, + { "otf", "application/font-sfnt" }, + { "otg", "application/vnd.oasis.opendocument.graphics-template" }, + { "oth", "application/vnd.oasis.opendocument.text-web" }, + { "otp", "application/vnd.oasis.opendocument.presentation-template" }, + { "ots", "application/vnd.oasis.opendocument.spreadsheet-template" }, + { "ott", "application/vnd.oasis.opendocument.text-template" }, + { "oxt", "application/vnd.openofficeorg.extension" }, + { "p10", "application/pkcs10" }, + { "p12", "application/x-pkcs12" }, + { "p7b", "application/x-pkcs7-certificates" }, + { "p7c", "application/pkcs7-mime" }, + { "p7m", "application/pkcs7-mime", "application/x-pkcs7-mime" }, + { "p7r", "application/x-pkcs7-certreqresp" }, + { "p7s", { "application/pkcs7-signature", "application/x-pkcs7-signature", "text/plain" } }, + { "pbm", "image/x-portable-bitmap" }, + { "pcast", "application/x-podcast" }, + { "pct", "image/pict" }, + { "pcx", "application/octet-stream" }, + { "pcz", "application/octet-stream" }, + { "pdf", "application/pdf" }, + { "pfb", "application/octet-stream" }, + { "pfm", "application/octet-stream" }, + { "pfx", "application/x-pkcs12" }, + { "pgm", "image/x-portable-graymap" }, + { "pic", "image/pict" }, + { "pict", "image/pict" }, + { "pkgdef", "text/plain" }, + { "pkgundef", "text/plain" }, + { "pko", "application/vnd.ms-pki.pko" }, + { "pls", "audio/scpls" }, + { "pma", "application/x-perfmon" }, + { "pmc", "application/x-perfmon" }, + { "pml", "application/x-perfmon" }, + { "pmr", "application/x-perfmon" }, + { "pmw", "application/x-perfmon" }, + { "png", "image/png" }, + { "pnm", "image/x-portable-anymap" }, + { "pnt", "image/x-macpaint" }, + { "pntg", "image/x-macpaint" }, + { "pnz", "image/png" }, + { "pot", "application/vnd.ms-powerpoint" }, + { "potm", "application/vnd.ms-powerpoint.template.macroEnabled.12" }, + { "potx", "application/vnd.openxmlformats-officedocument.presentationml.template" }, + { "ppa", "application/vnd.ms-powerpoint" }, + { "ppam", "application/vnd.ms-powerpoint.addin.macroEnabled.12" }, + { "ppm", "image/x-portable-pixmap" }, + { "pps", "application/vnd.ms-powerpoint" }, + { "ppsm", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12" }, + { "ppsx", "application/vnd.openxmlformats-officedocument.presentationml.slideshow" }, + { "ppt", "application/vnd.ms-powerpoint" }, + { "pptm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12" }, + { "pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation" }, + { "prf", "application/pics-rules" }, + { "prm", "application/octet-stream" }, + { "prx", "application/octet-stream" }, + { "ps", "application/postscript" }, + { "psc1", "application/PowerShell" }, + { "psd", "application/octet-stream" }, + { "psess", "application/xml" }, + { "psm", "application/octet-stream" }, + { "psp", "application/octet-stream" }, + { "pst", "application/vnd.ms-outlook" }, + { "pub", "application/x-mspublisher" }, + { "pwz", "application/vnd.ms-powerpoint" }, + { "qht", "text/x-html-insertion" }, + { "qhtm", "text/x-html-insertion" }, + { "qt", "video/quicktime" }, + { "qti", "image/x-quicktime" }, + { "qtif", "image/x-quicktime" }, + { "qtl", "application/x-quicktimeplayer" }, + { "qxd", "application/octet-stream" }, + { "ra", "audio/x-pn-realaudio" }, + { "ram", "audio/x-pn-realaudio" }, + { "rar", { "application/x-rar-compressed", "application/x-rar", "application/rar", "application/octet-stream" } }, + { "ras", "image/x-cmu-raster" }, + { "rat", "application/rat-file" }, + { "rc", "text/plain" }, + { "rc2", "text/plain" }, + { "rct", "text/plain" }, + { "rdlc", "application/xml" }, + { "reg", "text/plain" }, + { "resx", "application/xml" }, + { "rf", "image/vnd.rn-realflash" }, + { "rgb", "image/x-rgb" }, + { "rgs", "text/plain" }, + { "rm", "application/vnd.rn-realmedia" }, + { "rmi", "audio/mid" }, + { "rmp", "application/vnd.rn-rn_music_package" }, + { "roff", "application/x-troff" }, + { "rpm", "audio/x-pn-realaudio-plugin" }, + { "rqy", "text/x-ms-rqy" }, + { "rtf", { "application/rtf", "application/msword", "text/richtext", "text/rtf" } }, + { "rtx", "text/richtext" }, + { "rvt", "application/octet-stream" }, + { "ruleset", "application/xml" }, + { "s", "text/plain" }, + { "safariextz", "application/x-safari-safariextz" }, + { "scd", "application/x-msschedule" }, + { "scr", "text/plain" }, + { "sct", "text/scriptlet" }, + { "sd2", "audio/x-sd2" }, + { "sdp", "application/sdp" }, + { "sea", "application/octet-stream" }, + { "searchConnector-ms", "application/windows-search-connector+xml" }, + { "setpay", "application/set-payment-initiation" }, + { "setreg", "application/set-registration-initiation" }, + { "settings", "application/xml" }, + { "sgimb", "application/x-sgimb" }, + { "sgml", "text/sgml" }, + { "sh", "application/x-sh" }, + { "shar", "application/x-shar" }, + { "shtml", "text/html" }, + { "sit", "application/x-stuffit" }, + { "sitemap", "application/xml" }, + { "skin", "application/xml" }, + { "skp", "application/x-koan" }, + { "sldm", "application/vnd.ms-powerpoint.slide.macroEnabled.12" }, + { "sldx", "application/vnd.openxmlformats-officedocument.presentationml.slide" }, + { "slk", "application/vnd.ms-excel" }, + { "sln", "text/plain" }, + { "slupkg-ms", "application/x-ms-license" }, + { "smd", "audio/x-smd" }, + { "smi", "application/octet-stream" }, + { "smx", "audio/x-smd" }, + { "smz", "audio/x-smd" }, + { "snd", "audio/basic" }, + { "snippet", "application/xml" }, + { "snp", "application/octet-stream" }, + { "sol", "text/plain" }, + { "sor", "text/plain" }, + { "spc", "application/x-pkcs7-certificates" }, + { "spl", "application/futuresplash" }, + { "spx", "audio/ogg" }, + { "src", "application/x-wais-source" }, + { "srf", "text/plain" }, + { "SSISDeploymentManifest", "text/xml" }, + { "ssm", "application/streamingmedia" }, + { "sst", "application/vnd.ms-pki.certstore" }, + { "stl", "application/vnd.ms-pki.stl" }, + { "sv4cpio", "application/x-sv4cpio" }, + { "sv4crc", "application/x-sv4crc" }, + { "svc", "application/xml" }, + { "svg", "image/svg+xml" }, + { "swf", "application/x-shockwave-flash" }, + { "step", "application/step" }, + { "stp", "application/step" }, + { "t", "application/x-troff" }, + { "tar", "application/x-tar" }, + { "tcl", "application/x-tcl" }, + { "testrunconfig", "application/xml" }, + { "testsettings", "application/xml" }, + { "tex", "application/x-tex" }, + { "texi", "application/x-texinfo" }, + { "texinfo", "application/x-texinfo" }, + { "tgz", "application/x-compressed" }, + { "thmx", "application/vnd.ms-officetheme" }, + { "thn", "application/octet-stream" }, + { "tif", { "image/tiff", "application/octet-stream" } }, + { "tiff", "image/tiff" }, + { "tlh", "text/plain" }, + { "tli", "text/plain" }, + { "toc", "application/octet-stream" }, + { "tr", "application/x-troff" }, + { "trm", "application/x-msterminal" }, + { "trx", "application/xml" }, + { "ts", "video/vnd.dlna.mpeg-tts" }, + { "tsv", "text/tab-separated-values" }, + { "ttf", "application/font-sfnt" }, + { "tts", "video/vnd.dlna.mpeg-tts" }, + { "txt", "text/plain" }, + { "u32", "application/octet-stream" }, + { "uls", "text/iuls" }, + { "user", "text/plain" }, + { "ustar", "application/x-ustar" }, + { "vb", "text/plain" }, + { "vbdproj", "text/plain" }, + { "vbk", "video/mpeg" }, + { "vbproj", "text/plain" }, + { "vbs", "text/vbscript" }, + { "vcf", { "text/x-vcard", "text/vcard" } }, + { "vcproj", "application/xml" }, + { "vcs", "text/plain" }, + { "vcxproj", "application/xml" }, + { "vddproj", "text/plain" }, + { "vdp", "text/plain" }, + { "vdproj", "text/plain" }, + { "vdx", "application/vnd.ms-visio.viewer" }, + { "vml", "text/xml" }, + { "vscontent", "application/xml" }, + { "vsct", "text/xml" }, + { "vsd", "application/vnd.visio" }, + { "vsi", "application/ms-vsi" }, + { "vsix", "application/vsix" }, + { "vsixlangpack", "text/xml" }, + { "vsixmanifest", "text/xml" }, + { "vsmdi", "application/xml" }, + { "vspscc", "text/plain" }, + { "vss", "application/vnd.visio" }, + { "vsscc", "text/plain" }, + { "vssettings", "text/xml" }, + { "vssscc", "text/plain" }, + { "vst", "application/vnd.visio" }, + { "vstemplate", "text/xml" }, + { "vsto", "application/x-ms-vsto" }, + { "vsw", "application/vnd.visio" }, + { "vsx", "application/vnd.visio" }, + { "vtx", "application/vnd.visio" }, + { "wav", { "audio/wav", "audio/vnd.wave", "audio/x-wav" } }, + { "wave", "audio/wav" }, + { "wax", "audio/x-ms-wax" }, + { "wbk", "application/msword" }, + { "wbmp", "image/vnd.wap.wbmp" }, + { "wcm", "application/vnd.ms-works" }, + { "wdb", "application/vnd.ms-works" }, + { "wdp", "image/vnd.ms-photo" }, + { "webarchive", "application/x-safari-webarchive" }, + { "webm", "video/webm" }, + { "webp", "image/webp" }, + { "webtest", "application/xml" }, + { "wiq", "application/xml" }, + { "wiz", "application/msword" }, + { "wks", "application/vnd.ms-works" }, + { "WLMP", "application/wlmoviemaker" }, + { "wlpginstall", "application/x-wlpg-detect" }, + { "wlpginstall3", "application/x-wlpg3-detect" }, + { "wm", "video/x-ms-wm" }, + { "wma", "audio/x-ms-wma" }, + { "wmd", "application/x-ms-wmd" }, + { "wmf", { "application/x-msmetafile", "image/wmf", "image/x-wmf" } }, + { "wml", "text/vnd.wap.wml" }, + { "wmlc", "application/vnd.wap.wmlc" }, + { "wmls", "text/vnd.wap.wmlscript" }, + { "wmlsc", "application/vnd.wap.wmlscriptc" }, + { "wmp", "video/x-ms-wmp" }, + { "wmv", "video/x-ms-wmv" }, + { "wmx", "video/x-ms-wmx" }, + { "wmz", "application/x-ms-wmz" }, + { "woff", "application/font-woff" }, + { "wpl", "application/vnd.ms-wpl" }, + { "wps", "application/vnd.ms-works" }, + { "wri", "application/x-mswrite" }, + { "wrl", "x-world/x-vrml" }, + { "wrz", "x-world/x-vrml" }, + { "wsc", "text/scriptlet" }, + { "wsdl", "text/xml" }, + { "wvx", "video/x-ms-wvx" }, + { "x", "application/directx" }, + { "xaf", "x-world/x-vrml" }, + { "xaml", "application/xaml+xml" }, + { "xap", "application/x-silverlight-app" }, + { "xbap", "application/x-ms-xbap" }, + { "xbm", "image/x-xbitmap" }, + { "xdr", "text/plain" }, + { "xht", "application/xhtml+xml" }, + { "xhtml", "application/xhtml+xml" }, + { "xla", "application/vnd.ms-excel" }, + { "xlam", "application/vnd.ms-excel.addin.macroEnabled.12" }, + { "xlc", "application/vnd.ms-excel" }, + { "xld", "application/vnd.ms-excel" }, + { "xlk", "application/vnd.ms-excel" }, + { "xll", "application/vnd.ms-excel" }, + { "xlm", "application/vnd.ms-excel" }, + { "xls", { + "application/excel", + "application/vnd.ms-excel", + "application/vnd.ms-office", + "application/x-excel", + "application/octet-stream" + } }, + { "xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12" }, + { "xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12" }, + { "xlsx", { + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel.12", + "application/octet-stream" + } }, + { "xlt", "application/vnd.ms-excel" }, + { "xltm", "application/vnd.ms-excel.template.macroEnabled.12" }, + { "xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template" }, + { "xlw", "application/vnd.ms-excel" }, + { "xml", { "application/xml", "text/xml", "application/octet-stream" } }, + { "xmp", "application/octet-stream" }, + { "xmta", "application/xml" }, + { "xof", "x-world/x-vrml" }, + { "XOML", "text/plain" }, + { "xpm", "image/x-xpixmap" }, + { "xps", "application/vnd.ms-xpsdocument" }, + { "xrm-ms", "text/xml" }, + { "xsc", "application/xml" }, + { "xsd", "text/xml" }, + { "xsf", "text/xml" }, + { "xsl", "text/xml" }, + { "xslt", "text/xml" }, + { "xsn", "application/octet-stream" }, + { "xss", "application/xml" }, + { "xspf", "application/xspf+xml" }, + { "xtp", "application/octet-stream" }, + { "xwd", "image/x-xwindowdump" }, + { "z", "application/x-compress" }, + { "zip", { + "application/zip", + "application/x-zip-compressed", + "application/octet-stream" + } }, + { "zlib", "application/zlib" }, } -- Used to match extension by content type exports.reversed_extensions_map = { - ["text/html"] = "html", - ["text/css"] = "css", - ["text/xml"] = "xml", - ["image/gif"] = "gif", - ["image/jpeg"] = "jpeg", - ["application/javascript"] = "js", - ["application/atom+xml"] = "atom", - ["application/rss+xml"] = "rss", - ["application/csv"] = "csv", - ["text/mathml"] = "mml", - ["text/plain"] = "txt", - ["text/vnd.sun.j2me.app-descriptor"] = "jad", - ["text/vnd.wap.wml"] = "wml", - ["text/x-component"] = "htc", - ["image/png"] = "png", - ["image/svg+xml"] = "svg", - ["image/tiff"] = "tiff", - ["image/vnd.wap.wbmp"] = "wbmp", - ["image/webp"] = "webp", - ["image/x-icon"] = "ico", - ["image/x-jng"] = "jng", - ["image/x-ms-bmp"] = "bmp", - ["font/woff"] = "woff", - ["font/woff2"] = "woff2", - ["application/java-archive"] = "jar", - ["application/json"] = "json", - ["application/mac-binhex40"] = "hqx", - ["application/msword"] = "doc", - ["application/pdf"] = "pdf", - ["application/postscript"] = "ps", - ["application/rtf"] = "rtf", - ["application/vnd.apple.mpegurl"] = "m3u8", - ["application/vnd.google-earth.kml+xml"] = "kml", - ["application/vnd.google-earth.kmz"] = "kmz", - ["application/vnd.ms-excel"] = "xls", - ["application/vnd.ms-fontobject"] = "eot", - ["application/vnd.ms-powerpoint"] = "ppt", - ["application/vnd.oasis.opendocument.graphics"] = "odg", - ["application/vnd.oasis.opendocument.presentation"] = "odp", - ["application/vnd.oasis.opendocument.spreadsheet"] = "ods", - ["application/vnd.oasis.opendocument.text"] = "odt", - ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "pptx", - ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "xlsx", - ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "docx", - ["application/x-7z-compressed"] = "7z", - ["application/x-cocoa"] = "cco", - ["application/x-java-archive-diff"] = "jardiff", - ["application/x-java-jnlp-file"] = "jnlp", - ["application/x-makeself"] = "run", - ["application/x-perl"] = "pl", - ["application/x-pilot"] = "pdb", - ["application/x-rar-compressed"] = "rar", - ["application/x-redhat-package-manager"] = "rpm", - ["application/x-sea"] = "sea", - ["application/x-shockwave-flash"] = "swf", - ["application/x-stuffit"] = "sit", - ["application/x-tcl"] = "tcl", - ["application/x-x509-ca-cert"] = "crt", - ["application/x-xpinstall"] = "xpi", - ["application/xhtml+xml"] = "xhtml", - ["application/xspf+xml"] = "xspf", - ["application/zip"] = "zip", - ["application/x-dosexec"] = "exe", - ["application/x-msdownload"] = "exe", - ["application/x-executable"] = "exe", - ["text/x-msdos-batch"] = "bat", + ["text/html"] = "html", + ["text/css"] = "css", + ["text/xml"] = "xml", + ["image/gif"] = "gif", + ["image/jpeg"] = "jpeg", + ["application/javascript"] = "js", + ["application/atom+xml"] = "atom", + ["application/rss+xml"] = "rss", + ["application/csv"] = "csv", + ["text/mathml"] = "mml", + ["text/plain"] = "txt", + ["text/vnd.sun.j2me.app-descriptor"] = "jad", + ["text/vnd.wap.wml"] = "wml", + ["text/x-component"] = "htc", + ["image/png"] = "png", + ["image/svg+xml"] = "svg", + ["image/tiff"] = "tiff", + ["image/vnd.wap.wbmp"] = "wbmp", + ["image/webp"] = "webp", + ["image/x-icon"] = "ico", + ["image/x-jng"] = "jng", + ["image/x-ms-bmp"] = "bmp", + ["font/woff"] = "woff", + ["font/woff2"] = "woff2", + ["application/java-archive"] = "jar", + ["application/json"] = "json", + ["application/mac-binhex40"] = "hqx", + ["application/msword"] = "doc", + ["application/pdf"] = "pdf", + ["application/postscript"] = "ps", + ["application/rtf"] = "rtf", + ["application/vnd.apple.mpegurl"] = "m3u8", + ["application/vnd.google-earth.kml+xml"] = "kml", + ["application/vnd.google-earth.kmz"] = "kmz", + ["application/vnd.ms-excel"] = "xls", + ["application/vnd.ms-fontobject"] = "eot", + ["application/vnd.ms-powerpoint"] = "ppt", + ["application/vnd.oasis.opendocument.graphics"] = "odg", + ["application/vnd.oasis.opendocument.presentation"] = "odp", + ["application/vnd.oasis.opendocument.spreadsheet"] = "ods", + ["application/vnd.oasis.opendocument.text"] = "odt", + ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "pptx", + ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "xlsx", + ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "docx", + ["application/x-7z-compressed"] = "7z", + ["application/x-cocoa"] = "cco", + ["application/x-java-archive-diff"] = "jardiff", + ["application/x-java-jnlp-file"] = "jnlp", + ["application/x-makeself"] = "run", + ["application/x-perl"] = "pl", + ["application/x-pilot"] = "pdb", + ["application/x-rar-compressed"] = "rar", + ["application/x-redhat-package-manager"] = "rpm", + ["application/x-sea"] = "sea", + ["application/x-shockwave-flash"] = "swf", + ["application/x-stuffit"] = "sit", + ["application/x-tcl"] = "tcl", + ["application/x-x509-ca-cert"] = "crt", + ["application/x-xpinstall"] = "xpi", + ["application/xhtml+xml"] = "xhtml", + ["application/xspf+xml"] = "xspf", + ["application/zip"] = "zip", + ["application/x-dosexec"] = "exe", + ["application/x-msdownload"] = "exe", + ["application/x-executable"] = "exe", + ["text/x-msdos-batch"] = "bat", - ["audio/midi"] = "mid", - ["audio/mpeg"] = "mp3", - ["audio/ogg"] = "ogg", - ["audio/x-m4a"] = "m4a", - ["audio/x-realaudio"] = "ra", - ["video/3gpp"] = "3gpp", - ["video/mp2t"] = "ts", - ["video/mp4"] = "mp4", - ["video/mpeg"] = "mpeg", - ["video/quicktime"] = "mov", - ["video/webm"] = "webm", - ["video/x-flv"] = "flv", - ["video/x-m4v"] = "m4v", - ["video/x-mng"] = "mng", - ["video/x-ms-asf"] = "asx", - ["video/x-ms-wmv"] = "wmv", - ["video/x-msvideo"] = "avi", + ["audio/midi"] = "mid", + ["audio/mpeg"] = "mp3", + ["audio/ogg"] = "ogg", + ["audio/x-m4a"] = "m4a", + ["audio/x-realaudio"] = "ra", + ["video/3gpp"] = "3gpp", + ["video/mp2t"] = "ts", + ["video/mp4"] = "mp4", + ["video/mpeg"] = "mpeg", + ["video/quicktime"] = "mov", + ["video/webm"] = "webm", + ["video/x-flv"] = "flv", + ["video/x-m4v"] = "m4v", + ["video/x-mng"] = "mng", + ["video/x-ms-asf"] = "asx", + ["video/x-ms-wmv"] = "wmv", + ["video/x-msvideo"] = "avi", } return exports diff --git a/lualib/lua_scanners/avast.lua b/lualib/lua_scanners/avast.lua index 5c54f8c40..7e778979a 100644 --- a/lualib/lua_scanners/avast.lua +++ b/lualib/lua_scanners/avast.lua @@ -189,7 +189,7 @@ local function avast_check(task, content, digest, rule, maybe_part) -- Define callbacks - avast_helo_cb = function (merr, mdata, conn) + avast_helo_cb = function(merr, mdata, conn) -- Called when we have established a connection but not read anything tcp_conn = conn @@ -204,7 +204,6 @@ local function avast_check(task, content, digest, rule, maybe_part) end end - avast_scan_cb = function(merr) -- Called when we have send request to avast and are waiting for reply if no_connection_error(merr) then diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua index f984864e7..fc99ab0b9 100644 --- a/lualib/lua_scanners/clamav.lua +++ b/lualib/lua_scanners/clamav.lua @@ -137,7 +137,7 @@ local function clamav_check(task, content, digest, rule, maybe_part) local vname = string.match(data, 'stream: (.+) FOUND') if string.find(vname, '^Heuristics%.Encrypted') then rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix) - common.yield_result(task, rule, 'File is encrypted: '.. vname, + common.yield_result(task, rule, 'File is encrypted: ' .. vname, 0.0, 'encrypted', maybe_part) cached = 'ENCRYPTED' elseif string.find(vname, '^Heuristics%.OLE2%.ContainsMacros') then @@ -146,7 +146,7 @@ local function clamav_check(task, content, digest, rule, maybe_part) cached = 'MACRO' elseif string.find(vname, '^Heuristics%.Limits%.Exceeded') then rspamd_logger.errx(task, '%s: ClamAV Limits Exceeded', rule.log_prefix) - common.yield_result(task, rule, 'Limits Exceeded: '.. vname, 0.0, + common.yield_result(task, rule, 'Limits Exceeded: ' .. vname, 0.0, 'fail', maybe_part) elseif vname then common.yield_result(task, rule, vname, 1.0, nil, maybe_part) diff --git a/lualib/lua_scanners/cloudmark.lua b/lualib/lua_scanners/cloudmark.lua index 5993c4fe2..b07f238ea 100644 --- a/lualib/lua_scanners/cloudmark.lua +++ b/lualib/lua_scanners/cloudmark.lua @@ -84,7 +84,7 @@ local function cloudmark_preload(rule, cfg, ev_base, _) http.request({ ev_base = ev_base, config = cfg, - url = cloudmark_url(rule, addr, '/score/v2/max-message-size'), + url = cloudmark_url(rule, addr, '/score/v2/max-message-size'), callback = max_message_size_cb, }) end @@ -142,8 +142,8 @@ local function cloudmark_config(opts) if cloudmark_conf.upstreams then - cloudmark_conf.symbols = {{ symbol = cloudmark_conf.symbol_spam, score = 5.0 }} - cloudmark_conf.preloads = {cloudmark_preload} + cloudmark_conf.symbols = { { symbol = cloudmark_conf.symbol_spam, score = 5.0 } } + cloudmark_conf.preloads = { cloudmark_preload } lua_util.add_debug_alias('external_services', cloudmark_conf.name) return cloudmark_conf end @@ -162,7 +162,7 @@ local function table_to_multipart_body(tbl, boundary) local seen_data = false local out = {} - for k,v in pairs(tbl) do + for k, v in pairs(tbl) do if v.data then seen_data = true table.insert(out, string.format('--%s\r\n', boundary)) @@ -229,7 +229,7 @@ local function parse_cloudmark_reply(task, rule, body) if rule.add_headers and type(obj.appendHeaders) == 'table' then local headers_add = fun.tomap(fun.map(function(h) - return h.headerField,{ + return h.headerField, { order = 1, value = h.body } end, obj.appendHeaders)) @@ -275,7 +275,9 @@ local function cloudmark_check(task, content, digest, rule, maybe_part) local rcpt_to = task:get_recipients('smtp') if rcpt_to then request['rcptTo'] = { - data = table.concat(fun.totable(fun.map(function(r) return r.addr end, rcpt_to)), ',') + data = table.concat(fun.totable(fun.map(function(r) + return r.addr + end, rcpt_to)), ',') } end @@ -325,9 +327,9 @@ local function cloudmark_check(task, content, digest, rule, maybe_part) http.request(request_data) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. 'exceed', rule.log_prefix) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '.. + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' .. 'retransmits exceed') upstream:fail() end @@ -337,7 +339,9 @@ local function cloudmark_check(task, content, digest, rule, maybe_part) cloudmark_requery() else -- Parse the response - if upstream then upstream:ok() end + if upstream then + upstream:ok() + end if code ~= 200 then rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) @@ -360,7 +364,7 @@ local function cloudmark_check(task, content, digest, rule, maybe_part) end return { - type = {'cloudmark', 'scanner'}, + type = { 'cloudmark', 'scanner' }, description = 'Cloudmark cartridge interface', configure = cloudmark_config, check = cloudmark_check, diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua index c2e314d39..11f5e1f87 100644 --- a/lualib/lua_scanners/common.lua +++ b/lualib/lua_scanners/common.lua @@ -42,7 +42,9 @@ local function log_clean(task, rule, msg) end local function match_patterns(default_sym, found, patterns, dyn_weight) - if type(patterns) ~= 'table' then return default_sym, dyn_weight end + if type(patterns) ~= 'table' then + return default_sym, dyn_weight + end if not patterns[1] then for sym, pat in pairs(patterns) do if pat:match(found) then @@ -71,7 +73,7 @@ local function yield_result(task, rule, vname, dyn_weight, is_fail, maybe_part) local flags if type(vname) == 'string' then - threat_table = {vname} + threat_table = { vname } elseif type(vname) == 'table' then threat_table = vname end @@ -82,7 +84,9 @@ local function yield_result(task, rule, vname, dyn_weight, is_fail, maybe_part) patterns = rule.patterns symbol = rule.symbol threat_info = rule.detection_category .. 'found' - if not dyn_weight then dyn_weight = 1.0 end + if not dyn_weight then + dyn_weight = 1.0 + end elseif is_fail == 'fail' then patterns = rule.patterns_fail symbol = rule.symbol_fail @@ -100,7 +104,6 @@ local function yield_result(task, rule, vname, dyn_weight, is_fail, maybe_part) dyn_weight = 1.0 end - for _, tm in ipairs(threat_table) do local symname, symscore = match_patterns(symbol, tm, patterns, dyn_weight) if rule.whitelist and rule.whitelist:get_key(tm) then @@ -136,7 +139,9 @@ end local function message_not_too_large(task, content, rule) local max_size = tonumber(rule.max_size) - if not max_size then return true end + if not max_size then + return true + end if #content > max_size then rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)", rule.log_prefix, #content, max_size) @@ -147,7 +152,9 @@ end local function message_not_too_small(task, content, rule) local min_size = tonumber(rule.min_size) - if not min_size then return true end + if not min_size then + return true + end if #content < min_size then rspamd_logger.infox(task, "skip %s check as it is too small: %s (%s is allowed)", rule.log_prefix, #content, min_size) @@ -171,7 +178,7 @@ local function message_min_words(task, rule) if not text_part_above_limit then rspamd_logger.infox(task, '%s: #words in all text parts is below text_part_min_words limit: %s', - rule.log_prefix, rule.text_part_min_words) + rule.log_prefix, rule.text_part_min_words) end return text_part_above_limit @@ -192,7 +199,7 @@ local function dynamic_scan(task, rule) -- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, "result is already reject") -- return false --elseif metric_result[1] > metric_result[2]*2 then - if metric_result[1] > metric_result[2]*2 then + if metric_result[1] > metric_result[2] * 2 then rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'score > 2 * reject_level: ' .. metric_result[1]) return false elseif has_pre_result and metric_action == 'reject' then @@ -236,7 +243,7 @@ local function need_check(task, content, rule, digest, fn, maybe_part) else lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s', - rule.log_prefix, key, threat_string[1]) + rule.log_prefix, key, threat_string[1]) end uncached = false else @@ -251,10 +258,10 @@ local function need_check(task, content, rule, digest, fn, maybe_part) local f_dynamic_scan = dynamic_scan(task, rule) if uncached and - f_message_not_too_large and - f_message_not_too_small and - f_message_min_words and - f_dynamic_scan then + f_message_not_too_large and + f_message_not_too_small and + f_message_min_words and + f_dynamic_scan then fn() @@ -272,7 +279,7 @@ local function need_check(task, content, rule, digest, fn, maybe_part) false, -- is write redis_av_cb, --callback 'GET', -- command - {key} -- arguments) + { key } -- arguments) ) then return true end @@ -284,7 +291,9 @@ end local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part) local key = digest - if not dyn_weight then dyn_weight = 1.0 end + if not dyn_weight then + dyn_weight = 1.0 + end local function redis_set_cb(err) -- Do nothing @@ -293,7 +302,7 @@ local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part) rule.detection_category, to_save, key, err) else lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s - ttl %s', - rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire) + rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire) end end @@ -301,7 +310,7 @@ local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part) to_save = table.concat(to_save, '\v') end - local value_tbl = {to_save, dyn_weight} + local value_tbl = { to_save, dyn_weight } if maybe_part and rule.show_attachments and maybe_part:get_filename() then local fname = maybe_part:get_filename() table.insert(value_tbl, fname) @@ -381,24 +390,24 @@ local function gen_extension(fname) local ext = {} for n = 1, 2 do - ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil + ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil end - return ext[1],ext[2],filename_parts + return ext[1], ext[2], filename_parts end local function check_parts_match(task, rule) local filter_func = function(p) - local mtype,msubtype = p:get_type() + local mtype, msubtype = p:get_type() local detected_ext = p:get_detected_ext() local fname = p:get_filename() local ext, ext2 if rule.scan_all_mime_parts == false then - -- check file extension and filename regex matching + -- check file extension and filename regex matching --lua_util.debugm(rule.name, task, '%s: filename: |%s|%s|', rule.log_prefix, fname) if fname ~= nil then - ext,ext2 = gen_extension(fname) + ext, ext2 = gen_extension(fname) --lua_util.debugm(rule.name, task, '%s: extension, fname: |%s|%s|%s|', rule.log_prefix, ext, ext2, fname) if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext') or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then @@ -433,8 +442,8 @@ local function check_parts_match(task, rule) if p:is_archive() then local arch = p:get_archive() local filelist = arch:get_files_full(1000) - for _,f in ipairs(filelist) do - ext,ext2 = gen_extension(f.name) + for _, f in ipairs(filelist) do + ext, ext2 = gen_extension(f.name) if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext') or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then lua_util.debugm(rule.name, task, '%s: extension matched in archive: |%s|%s|', rule.log_prefix, ext, ext2) @@ -488,7 +497,7 @@ local function check_metric_results(task, rule) if rule.symbol_type == 'postfilter' and metric_action == 'reject' then return true, 'result is already reject' - elseif metric_result[1] > metric_result[2]*2 then + elseif metric_result[1] > metric_result[2] * 2 then return true, 'score > 2 * reject_level: ' .. metric_result[1] elseif has_pre_result and metric_action == 'reject' then return true, 'pre_result reject is set' diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua index 5cf0ef4a6..8d5e9e10a 100644 --- a/lualib/lua_scanners/dcc.lua +++ b/lualib/lua_scanners/dcc.lua @@ -90,7 +90,7 @@ local function dcc_check(task, content, digest, rule) local upstream = rule.upstreams:get_upstream_round_robin() local addr = upstream:get_addr() local retransmits = rule.retransmits - local client = rule.client + local client = rule.client local client_ip = task:get_from_ip() if client_ip and client_ip:is_valid() then @@ -116,7 +116,8 @@ local function dcc_check(task, content, digest, rule) local rcpts = task:get_recipients(); if rcpts then local dcc_recipients = table.concat(fun.totable(fun.map(function(rcpt) - return rcpt['addr'] end, + return rcpt['addr'] + end, rcpts)), '\n') if dcc_recipients then envrcpt = dcc_recipients @@ -164,8 +165,8 @@ local function dcc_check(task, content, digest, rule) fuz2_max = 999999, }) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. - 'exceed', rule.log_prefix) + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed', rule.log_prefix) common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail') end end @@ -176,14 +177,14 @@ local function dcc_check(task, content, digest, rule) else -- Parse the response - local _,_,result,disposition,header = tostring(data):find("(.-)\n(.-)\n(.-)$") + local _, _, result, disposition, header = tostring(data):find("(.-)\n(.-)\n(.-)$") lua_util.debugm(rule.name, task, 'DCC result=%1 disposition=%2 header="%3"', result, disposition, header) if header then -- Unfold header header = header:gsub('\r?\n%s*', ' ') - local _,_,info = header:find("; (.-)$") + local _, _, info = header:find("; (.-)$") if (result == 'R') then -- Reject common.yield_result(task, rule, info, rule.default_score) @@ -194,63 +195,65 @@ local function dcc_check(task, content, digest, rule) dcc_requery() elseif result == 'A' then - local opts = {} - local score = 0.0 - info = info:lower() - local rep = info:match('rep=([^=%s]+)') + local opts = {} + local score = 0.0 + info = info:lower() + local rep = info:match('rep=([^=%s]+)') - -- Adjust reputation if available - if rep then rep = tonumber(rep) end - if not rep then - rep = 1.0 + -- Adjust reputation if available + if rep then + rep = tonumber(rep) + end + if not rep then + rep = 1.0 + end + + local function check_threshold(what, num, lim) + local rnum + if num == 'many' then + rnum = lim + else + rnum = tonumber(num) end - local function check_threshold(what, num, lim) - local rnum - if num == 'many' then - rnum = lim - else - rnum = tonumber(num) - end - - if rnum and rnum >= lim then - opts[#opts + 1] = string.format('%s=%s', what, num) - score = score + rep / 3.0 - end + if rnum and rnum >= lim then + opts[#opts + 1] = string.format('%s=%s', what, num) + score = score + rep / 3.0 end + end - info = info:lower() - local body = info:match('body=([^=%s]+)') + info = info:lower() + local body = info:match('body=([^=%s]+)') - if body then - check_threshold('body', body, rule.body_max) - end + if body then + check_threshold('body', body, rule.body_max) + end - local fuz1 = info:match('fuz1=([^=%s]+)') + local fuz1 = info:match('fuz1=([^=%s]+)') - if fuz1 then - check_threshold('fuz1', fuz1, rule.fuz1_max) - end + if fuz1 then + check_threshold('fuz1', fuz1, rule.fuz1_max) + end - local fuz2 = info:match('fuz2=([^=%s]+)') + local fuz2 = info:match('fuz2=([^=%s]+)') - if fuz2 then - check_threshold('fuz2', fuz2, rule.fuz2_max) - end + if fuz2 then + check_threshold('fuz2', fuz2, rule.fuz2_max) + end - if #opts > 0 and score > 0 then - task:insert_result(rule.symbol_bulk, - score, - opts) - common.save_cache(task, digest, rule, opts, score) + if #opts > 0 and score > 0 then + task:insert_result(rule.symbol_bulk, + score, + opts) + common.save_cache(task, digest, rule, opts, score) + else + common.save_cache(task, digest, rule, 'OK') + if rule.log_clean then + rspamd_logger.infox(task, '%s: clean, returned result A - info: %s', + rule.log_prefix, info) else - common.save_cache(task, digest, rule, 'OK') - if rule.log_clean then - rspamd_logger.infox(task, '%s: clean, returned result A - info: %s', - rule.log_prefix, info) - else - lua_util.debugm(rule.name, task, '%s: returned result A - info: %s', - rule.log_prefix, info) + lua_util.debugm(rule.name, task, '%s: returned result A - info: %s', + rule.log_prefix, info) end end elseif result == 'G' then @@ -302,7 +305,7 @@ local function dcc_check(task, content, digest, rule) end return { - type = {'dcc','bulk', 'hash', 'scanner'}, + type = { 'dcc', 'bulk', 'hash', 'scanner' }, description = 'dcc bulk scanner', configure = dcc_config, check = dcc_check, diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua index d2153f7ac..5a469c327 100644 --- a/lualib/lua_scanners/fprot.lua +++ b/lualib/lua_scanners/fprot.lua @@ -84,7 +84,9 @@ local function fprot_check(task, content, digest, rule, maybe_part) local addr = upstream:get_addr() local retransmits = rule.retransmits local scan_id = task:get_queue_id() - if not scan_id then scan_id = task:get_uid() end + if not scan_id then + scan_id = task:get_uid() + end local header = string.format('SCAN STREAM %s SIZE %d\n', scan_id, #content) local footer = '\n' diff --git a/lualib/lua_scanners/icap.lua b/lualib/lua_scanners/icap.lua index ba571b13d..16459b06e 100644 --- a/lualib/lua_scanners/icap.lua +++ b/lualib/lua_scanners/icap.lua @@ -91,7 +91,6 @@ local common = require "lua_scanners/common" local rspamd_util = require "rspamd_util" local rspamd_version = rspamd_version - local N = 'icap' local function icap_config(opts) @@ -151,8 +150,8 @@ local function icap_config(opts) end icap_conf.upstreams = upstream_list.create(rspamd_config, - icap_conf.servers, - icap_conf.default_port) + icap_conf.servers, + icap_conf.default_port) if icap_conf.upstreams then lua_util.add_debug_alias('external_services', icap_conf.name) @@ -160,7 +159,7 @@ local function icap_config(opts) end rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', - icap_conf.servers) + icap_conf.servers) return nil end @@ -180,7 +179,7 @@ local function icap_check(task, content, digest, rule, maybe_part) rspamd_version('main'), rspamd_version('id'), rspamd_util.get_hostname(), - string.sub(task:get_uid(), 1,6)) + string.sub(task:get_uid(), 1, 6)) end -- Build the icap queries @@ -196,9 +195,9 @@ local function icap_check(task, content, digest, rule, maybe_part) end local respond_headers = { - -- Add main RESPMOD header before any other - string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme), - string.format('Host: %s\r\n', addr:to_string()), + -- Add main RESPMOD header before any other + string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme), + string.format('Host: %s\r\n', addr:to_string()), } local size = tonumber(#content) @@ -221,7 +220,7 @@ local function icap_check(task, content, digest, rule, maybe_part) addr = upstream:get_addr() lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', - rule.log_prefix, addr, addr:get_port()) + rule.log_prefix, addr, addr:get_port()) tcp_options.host = addr:to_string() tcp_options.port = addr:get_port() @@ -232,8 +231,8 @@ local function icap_check(task, content, digest, rule, maybe_part) tcp.request(tcp_options) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. - 'exceed - error: %s', rule.log_prefix, err_m or '') + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed - error: %s', rule.log_prefix, err_m or '') common.yield_result(task, rule, string.format('failed - error: %s', err_m), 0.0, 'fail', maybe_part) end @@ -309,9 +308,13 @@ local function icap_check(task, content, digest, rule, maybe_part) end table.insert(respond_headers, '\r\n') - for _,h in ipairs(resp_req_headers) do table.insert(respond_headers, h) end + for _, h in ipairs(resp_req_headers) do + table.insert(respond_headers, h) + end table.insert(respond_headers, '\r\n') - for _,h in ipairs(resp_http_headers) do table.insert(respond_headers, h) end + for _, h in ipairs(resp_http_headers) do + table.insert(respond_headers, h) + end table.insert(respond_headers, '\r\n') table.insert(respond_headers, chunked_size .. '\r\n') table.insert(respond_headers, content) @@ -333,7 +336,7 @@ local function icap_check(task, content, digest, rule, maybe_part) elseif string.find(s, '^HTTP') then icap_headers['http'] = tostring(s) elseif string.find(s, '[%a%d-+]-:') then - local _,_,key,value = tostring(s):find("([%a%d-+]-):%s?(.+)") + local _, _, key, value = tostring(s):find("([%a%d-+]-):%s?(.+)") if key ~= nil then icap_headers[key:lower()] = tostring(value) end @@ -349,7 +352,7 @@ local function icap_check(task, content, digest, rule, maybe_part) if maybe_split and string.find(icap_threat, ',') then local threats = lua_util.str_split(string.gsub(icap_threat, "%s", ""), ',') or {} - for _,v in ipairs(threats) do + for _, v in ipairs(threats) do table.insert(threat_table, v) end else @@ -421,8 +424,7 @@ local function icap_check(task, content, digest, rule, maybe_part) -- Generic ICAP Headers if headers['x-infection-found'] then - local _,_,icap_type,_,icap_threat = - headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$") + local _, _, icap_type, _, icap_threat = headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$") -- Type=2 is typical for scan error returns if icap_type and icap_type == '2' then @@ -435,8 +437,8 @@ local function icap_check(task, content, digest, rule, maybe_part) lua_util.debugm(rule.name, task, '%s: icap X-Infection-Found: %s', rule.log_prefix, icap_threat) threat_table_add(icap_threat, false) - -- stupid workaround for unuseable x-infection-found header - -- but also x-virus-name set (McAfee Web Gateway 9) + -- stupid workaround for unuseable x-infection-found header + -- but also x-virus-name set (McAfee Web Gateway 9) elseif not icap_threat and headers['x-virus-name'] then threat_table_add(headers['x-virus-name'], true) else @@ -450,7 +452,7 @@ local function icap_check(task, content, digest, rule, maybe_part) lua_util.debugm(rule.name, task, '%s: icap X-Virus-ID: %s', rule.log_prefix, headers['x-virus-id']) threat_table_add(headers['x-virus-id'], true) - -- FSecure X-Headers + -- FSecure X-Headers elseif headers['x-fsecure-scan-result'] and headers['x-fsecure-scan-result'] ~= "clean" then local infected_filename = "" @@ -468,23 +470,23 @@ local function icap_check(task, content, digest, rule, maybe_part) rule.log_prefix, infection_name, infected_filename) threat_table_add(infection_name, true) - -- McAfee Web Gateway manual extra headers + -- McAfee Web Gateway manual extra headers elseif headers['x-mwg-block-reason'] and headers['x-mwg-block-reason'] ~= "" then threat_table_add(headers['x-mwg-block-reason'], false) - -- Sophos SAVDI special http headers + -- Sophos SAVDI special http headers elseif headers['x-blocked'] and headers['x-blocked'] ~= "" then threat_table_add(headers['x-blocked'], false) elseif headers['x-block-reason'] and headers['x-block-reason'] ~= "" then threat_table_add(headers['x-block-reason'], false) - -- last try HTTP [4]xx return + -- last try HTTP [4]xx return elseif headers.http and string.find(headers.http, '^HTTP%/[12]%.. [4]%d%d') then threat_table_add( - string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) + string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) elseif rule.use_http_3xx_as_threat and headers.http and string.find(headers.http, '^HTTP%/[12]%.. [3]%d%d') - then + then threat_table_add( - string.format("pseudo-virus (redirect): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) + string.format("pseudo-virus (redirect): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false) end if #threat_table > 0 then @@ -515,7 +517,7 @@ local function icap_check(task, content, digest, rule, maybe_part) end else rspamd_logger.errx(task, '%s: unhandled response |%s|', - rule.log_prefix, string.gsub(result, "\r\n", ", ")) + rule.log_prefix, string.gsub(result, "\r\n", ", ")) common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_http_headers.icap), 0.0, 'fail', maybe_part) end @@ -536,13 +538,13 @@ local function icap_check(task, content, digest, rule, maybe_part) -- Threat found - close connection connection:close() elseif not icap_header_result - and rule.use_http_result_header - and icap_headers.encapsulated - and not string.find(icap_headers.encapsulated, 'null%-body=0') - then + and rule.use_http_result_header + and icap_headers.encapsulated + and not string.find(icap_headers.encapsulated, 'null%-body=0') + then -- Try to read encapsulated HTTP Headers lua_util.debugm(rule.name, task, '%s: no ICAP virus header found - try HTTP headers', - rule.log_prefix) + rule.log_prefix) connection:add_read(icap_r_respond_http_cb, '\r\n\r\n') else connection:close() @@ -569,7 +571,7 @@ local function icap_check(task, content, digest, rule, maybe_part) return false else rspamd_logger.errx(task, '%s: unhandled response |%s|', - rule.log_prefix, string.gsub(result, "\r\n", ", ")) + rule.log_prefix, string.gsub(result, "\r\n", ", ")) common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_headers.icap), 0.0, 'fail', maybe_part) end @@ -600,7 +602,9 @@ local function icap_check(task, content, digest, rule, maybe_part) if rule.x_client_header then local client = task:get_from_ip() - if client then add_respond_header('X-Client-IP', client:to_string()) end + if client then + add_respond_header('X-Client-IP', client:to_string()) + end end -- F-Secure extra headers @@ -608,19 +612,23 @@ local function icap_check(task, content, digest, rule, maybe_part) if rule.x_rcpt_header then local rcpt_to = task:get_principal_recipient() - if rcpt_to then add_respond_header('X-Rcpt-To', rcpt_to) end + if rcpt_to then + add_respond_header('X-Rcpt-To', rcpt_to) + end end if rule.x_from_header then local mail_from = task:get_principal_recipient() - if mail_from and mail_from[1] then add_respond_header('X-Rcpt-To', mail_from[1].addr) end + if mail_from and mail_from[1] then + add_respond_header('X-Rcpt-To', mail_from[1].addr) + end end end if icap_headers.connection and icap_headers.connection:lower() == 'close' then lua_util.debugm(rule.name, task, '%s: OPTIONS request Connection: %s - using new connection', - rule.log_prefix, icap_headers.connection) + rule.log_prefix, icap_headers.connection) connection:close() tcp_options.callback = icap_w_respond_cb tcp_options.data = get_respond_query() @@ -631,13 +639,13 @@ local function icap_check(task, content, digest, rule, maybe_part) else rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s', - rule.log_prefix, icap_headers['methods']) + rule.log_prefix, icap_headers['methods']) common.yield_result(task, rule, 'NO RESPMOD', 0.0, 'fail', maybe_part) end else rspamd_logger.errx(task, '%s: OPTIONS query failed: %s', - rule.log_prefix, icap_headers.icap or "-") + rule.log_prefix, icap_headers.icap or "-") common.yield_result(task, rule, 'OPTIONS query failed', 0.0, 'fail', maybe_part) end @@ -682,7 +690,7 @@ local function icap_check(task, content, digest, rule, maybe_part) end return { - type = {N, 'virus', 'virus', 'scanner'}, + type = { N, 'virus', 'virus', 'scanner' }, description = 'generic icap antivirus', configure = icap_config, check = icap_check, diff --git a/lualib/lua_scanners/init.lua b/lualib/lua_scanners/init.lua index 20bea7a47..e47cebeec 100644 --- a/lualib/lua_scanners/init.lua +++ b/lualib/lua_scanners/init.lua @@ -25,7 +25,7 @@ local exports = { } local function require_scanner(name) - local sc = require ("lua_scanners/" .. name) + local sc = require("lua_scanners/" .. name) exports[sc.name or name] = sc end @@ -65,7 +65,9 @@ exports.filter = function(t) return fun.tomap(fun.filter(function(_, elt) return type(elt) == 'table' and elt.type and ( (type(elt.type) == 'string' and elt.type == t) or - (type(elt.type) == 'table' and fun.any(function(tt) return tt == t end, elt.type)) + (type(elt.type) == 'table' and fun.any(function(tt) + return tt == t + end, elt.type)) ) end, exports)) end diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua index 69aaad7e5..d52cef041 100644 --- a/lualib/lua_scanners/kaspersky_av.lua +++ b/lualib/lua_scanners/kaspersky_av.lua @@ -107,7 +107,6 @@ local function kaspersky_check(task, content, digest, rule, maybe_part) rspamd_util.close_file(message_fd) end) - local function kaspersky_callback(err, data) if err then diff --git a/lualib/lua_scanners/kaspersky_se.lua b/lualib/lua_scanners/kaspersky_se.lua index b0031ad69..5e0f2eaad 100644 --- a/lualib/lua_scanners/kaspersky_se.lua +++ b/lualib/lua_scanners/kaspersky_se.lua @@ -125,7 +125,7 @@ local function kaspersky_se_check(task, content, digest, rule, maybe_part) local req_body if rule.use_files then - local fname = string.format('%s/%s.tmp', + local fname = string.format('%s/%s.tmp', rule.tmpdir, rspamd_util.random_hex(32)) local message_fd = rspamd_util.create_file(fname) @@ -187,9 +187,9 @@ local function kaspersky_se_check(task, content, digest, rule, maybe_part) http.request(request_data) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. 'exceed', rule.log_prefix) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '.. + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' .. 'retransmits exceed') end end @@ -198,7 +198,9 @@ local function kaspersky_se_check(task, content, digest, rule, maybe_part) requery() else -- Parse the response - if upstream then upstream:ok() end + if upstream then + upstream:ok() + end if code ~= 200 then rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) @@ -242,7 +244,7 @@ local function kaspersky_se_check(task, content, digest, rule, maybe_part) if why == 'PASSWORD PROTECTED' then rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix) - common.yield_result(task, rule, 'File is encrypted: '.. why, + common.yield_result(task, rule, 'File is encrypted: ' .. why, 0.0, 'encrypted', maybe_part) cached = 'ENCRYPTED' else diff --git a/lualib/lua_scanners/oletools.lua b/lualib/lua_scanners/oletools.lua index 8513699a0..378e09467 100644 --- a/lualib/lua_scanners/oletools.lua +++ b/lualib/lua_scanners/oletools.lua @@ -122,7 +122,7 @@ local function oletools_check(task, content, digest, rule, maybe_part) callback = oletools_callback, }) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. 'exceed - err: %s', rule.log_prefix, error) common.yield_result(task, rule, 'failed to scan, maximum retransmits exceed - err: ' .. error, @@ -137,9 +137,9 @@ local function oletools_check(task, content, digest, rule, maybe_part) else json_response = json_response .. tostring(data) - if not string.find(json_response, '\t\n\n\t') and #data == 8192 then + if not string.find(json_response, '\t\n\n\t') and #data == 8192 then lua_util.debugm(rule.name, task, '%s: no stop word: add_read - #json: %s / current packet: %s', - rule.log_prefix, #json_response, #data) + rule.log_prefix, #json_response, #data) conn:add_read(oletools_callback) else @@ -243,7 +243,7 @@ local function oletools_check(task, content, digest, rule, maybe_part) rule.log_prefix, v.message) --common.yield_result(task, rule, 'failed - err: ' .. v.error, 0.0, 'fail') - elseif type(v.analysis) == 'table' and type(v.macros) == 'table' then + elseif type(v.analysis) == 'table' and type(v.macros) == 'table' then -- analysis + macro found - evaluate response if type(v.analysis) == 'table' and #v.analysis == 0 and #v.macros == 0 then @@ -259,20 +259,20 @@ local function oletools_check(task, content, digest, rule, maybe_part) lua_util.debugm(rule.name, task, '%s: type: %s', rule.log_prefix, result[2]['type']) - for _,m in ipairs(v.macros) do - lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, '.. + for _, m in ipairs(v.macros) do + lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, ' .. 'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename) end - for _,a in ipairs(v.analysis) do - lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, '.. + for _, a in ipairs(v.analysis) do + lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, ' .. 'description: %s', rule.log_prefix, a.type, a.keyword, a.description) if a.type == 'AutoExec' then analysis_cat_table.autoexec = 'A' table.insert(analysis_keyword_table, a.keyword) elseif a.type == 'Suspicious' then if rule.extended == true or - (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings') + (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings') then analysis_cat_table.suspicious = 'S' table.insert(analysis_keyword_table, a.keyword) @@ -312,7 +312,7 @@ local function oletools_check(task, content, digest, rule, maybe_part) local analysis_cat_table_keys_sorted = lua_util.keys(analysis_cat_table) table.sort(analysis_cat_table_keys_sorted) - for _,v in ipairs(analysis_cat_table_keys_sorted) do + for _, v in ipairs(analysis_cat_table_keys_sorted) do table.insert(analysis_cat_table_values_sorted, analysis_cat_table[v]) end @@ -361,7 +361,7 @@ local function oletools_check(task, content, digest, rule, maybe_part) end return { - type = {N, 'attachment scanner', 'hash', 'scanner'}, + type = { N, 'attachment scanner', 'hash', 'scanner' }, description = 'oletools office macro scanner', configure = oletools_config, check = oletools_check, diff --git a/lualib/lua_scanners/p0f.lua b/lualib/lua_scanners/p0f.lua index c5ccea1fe..7785f83ce 100644 --- a/lualib/lua_scanners/p0f.lua +++ b/lualib/lua_scanners/p0f.lua @@ -30,8 +30,8 @@ local common = require "lua_scanners/common" -- SEE: https://github.com/p0f/p0f/blob/v3.06b/docs/README#L317 local S = { BAD_QUERY = 0x0, - OK = 0x10, - NO_MATCH = 0x20 + OK = 0x10, + NO_MATCH = 0x20 } local N = 'p0f' @@ -49,7 +49,7 @@ local function p0f_check(task, ip, rule) end local function trim(...) - local vars = {...} + local vars = { ... } for k, v in ipairs(vars) do -- skip numbers, trim only strings @@ -73,22 +73,22 @@ local function p0f_check(task, ip, rule) -- API response must be 232 bytes long if #data ~= 232 then rspamd_logger.errx(task, 'malformed response from p0f on %s, %s bytes', - rule.socket, #data) + rule.socket, #data) common.yield_result(task, rule, 'Malformed Response: ' .. rule.socket, - 0.0, 'fail') + 0.0, 'fail') return end local _, status, _, _, _, uptime_min, _, _, _, distance, _, _, os_name, - os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack( + os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack( 'I4I4I4I4I4I4I4I4I4hbbc32c32c32c32c32c32', data)) if status ~= S.OK then if status == S.BAD_QUERY then rspamd_logger.errx(task, 'malformed p0f query on %s', rule.socket) common.yield_result(task, rule, 'Malformed Query: ' .. rule.socket, - 0.0, 'fail') + 0.0, 'fail') end return @@ -97,19 +97,19 @@ local function p0f_check(task, ip, rule) local os_string = #os_name == 0 and 'unknown' or os_name .. ' ' .. os_flavor task:get_mempool():set_variable('os_fingerprint', os_string, link_type, - uptime_min, distance) + uptime_min, distance) if link_type and #link_type > 0 then common.yield_result(task, rule, { os_string, 'link=' .. link_type, - 'distance=' .. distance}, + 'distance=' .. distance }, 0.0) else common.yield_result(task, rule, { os_string, 'link=unknown', - 'distance=' .. distance}, + 'distance=' .. distance }, 0.0) end @@ -153,7 +153,7 @@ local function p0f_check(task, ip, rule) end local query = rspamd_util.pack('I4 I1 c16', 0x50304601, - ip:get_version(), ip2bin(ip)) + ip:get_version(), ip2bin(ip)) tcp.request({ host = rule.socket, @@ -176,12 +176,12 @@ local function p0f_check(task, ip, rule) if rule.redis_params then local key = rule.prefix .. ip:to_string() ret = lua_redis.redis_make_request(task, - rule.redis_params, - key, - false, - redis_get_cb, - 'GET', - { key } + rule.redis_params, + key, + false, + redis_get_cb, + 'GET', + { key } ) end @@ -219,7 +219,7 @@ local function p0f_config(opts) end return { - type = {N, 'fingerprint', 'scanner'}, + type = { N, 'fingerprint', 'scanner' }, description = 'passive OS fingerprinter', configure = p0f_config, check = p0f_check, diff --git a/lualib/lua_scanners/pyzor.lua b/lualib/lua_scanners/pyzor.lua index 78250a3ad..75c1b4a15 100644 --- a/lualib/lua_scanners/pyzor.lua +++ b/lualib/lua_scanners/pyzor.lua @@ -28,7 +28,7 @@ local rspamd_logger = require "rspamd_logger" local common = require "lua_scanners/common" local N = 'pyzor' -local categories = {'pyzor','bulk', 'hash', 'scanner'} +local categories = { 'pyzor', 'bulk', 'hash', 'scanner' } local function pyzor_config(opts) @@ -174,10 +174,10 @@ local function pyzor_check(task, content, digest, rule) end end - if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then - rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix) - return - end + if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then + rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix) + return + end tcp.request({ task = task, @@ -197,7 +197,6 @@ local function pyzor_check(task, content, digest, rule) end end - return { type = categories, description = 'pyzor bulk scanner', diff --git a/lualib/lua_scanners/razor.lua b/lualib/lua_scanners/razor.lua index 7de4c84eb..fcc0a8e3a 100644 --- a/lualib/lua_scanners/razor.lua +++ b/lualib/lua_scanners/razor.lua @@ -80,7 +80,6 @@ local function razor_config(opts) return nil end - local function razor_check(task, content, digest, rule) local function razor_check_uncached () local upstream = rule.upstreams:get_upstream_round_robin() @@ -96,14 +95,14 @@ local function razor_check(task, content, digest, rule) retransmits = retransmits - 1 lua_util.debugm(rule.name, task, '%s: Request Error: %s - retries left: %s', - rule.log_prefix, err, retransmits) + rule.log_prefix, err, retransmits) -- Select a different upstream! upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', - rule.log_prefix, addr, addr:get_port()) + rule.log_prefix, addr, addr:get_port()) tcp.request({ task = task, @@ -116,8 +115,8 @@ local function razor_check(task, content, digest, rule) callback = razor_callback, }) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. - 'exceed', rule.log_prefix) + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed', rule.log_prefix) common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail') end end @@ -148,7 +147,7 @@ local function razor_check(task, content, digest, rule) end common.save_cache(task, digest, rule, 'OK', rule.default_score) else - rspamd_logger.errx(task,"%s - unknown response from razorfy: %s", addr:to_string(), threat_string) + rspamd_logger.errx(task, "%s - unknown response from razorfy: %s", addr:to_string(), threat_string) end end @@ -174,7 +173,7 @@ local function razor_check(task, content, digest, rule) end return { - type = {'razor','spam', 'hash', 'scanner'}, + type = { 'razor', 'spam', 'hash', 'scanner' }, description = 'razor bulk scanner', configure = razor_config, check = razor_check, diff --git a/lualib/lua_scanners/savapi.lua b/lualib/lua_scanners/savapi.lua index 430009df3..08f7b6682 100644 --- a/lualib/lua_scanners/savapi.lua +++ b/lualib/lua_scanners/savapi.lua @@ -116,13 +116,13 @@ local function savapi_check(task, content, digest, rule) local function savapi_fin_cb(err, conn) local vnames_reordered = {} -- Swap table - for virus,_ in pairs(vnames) do + for virus, _ in pairs(vnames) do table.insert(vnames_reordered, virus) end lua_util.debugm(rule.name, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered) if #vnames_reordered > 0 then local vname = {} - for _,virus in ipairs(vnames_reordered) do + for _, virus in ipairs(vnames_reordered) do table.insert(vname, virus) end @@ -181,8 +181,8 @@ local function savapi_check(task, content, digest, rule) if string.find(result, '100 PRODUCT') then lua_util.debugm(rule.name, task, "%s: scanning file: %s", rule['type'], fname) - conn:add_write(savapi_scan1_cb, {string.format('SCAN %s\n', - fname)}) + conn:add_write(savapi_scan1_cb, { string.format('SCAN %s\n', + fname) }) else rspamd_logger.errx(task, '%s: invalid product id %s', rule['type'], rule['product_id']) @@ -217,7 +217,7 @@ local function savapi_check(task, content, digest, rule) upstream = upstream, timeout = rule['timeout'], callback = savapi_callback_init, - stop_pattern = {'\n'}, + stop_pattern = { '\n' }, }) else rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) @@ -228,7 +228,7 @@ local function savapi_check(task, content, digest, rule) -- 100 SAVAPI:4.0 greeting if string.find(result, '100') then - conn:add_write(savapi_greet1_cb, {string.format('SET PRODUCT %s\n', rule['product_id'])}) + conn:add_write(savapi_greet1_cb, { string.format('SET PRODUCT %s\n', rule['product_id']) }) end end end @@ -240,7 +240,7 @@ local function savapi_check(task, content, digest, rule) upstream = upstream, timeout = rule['timeout'], callback = savapi_callback_init, - stop_pattern = {'\n'}, + stop_pattern = { '\n' }, }) end diff --git a/lualib/lua_scanners/spamassassin.lua b/lualib/lua_scanners/spamassassin.lua index c4fd0d2e5..f425924d5 100644 --- a/lualib/lua_scanners/spamassassin.lua +++ b/lualib/lua_scanners/spamassassin.lua @@ -71,8 +71,8 @@ local function spamassassin_config(opts) end spamassassin_conf.upstreams = upstream_list.create(rspamd_config, - spamassassin_conf.servers, - spamassassin_conf.default_port) + spamassassin_conf.servers, + spamassassin_conf.default_port) if spamassassin_conf.upstreams then lua_util.add_debug_alias('external_services', spamassassin_conf.N) @@ -80,7 +80,7 @@ local function spamassassin_config(opts) end rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', - spamassassin_conf.servers) + spamassassin_conf.servers) return nil end @@ -95,7 +95,7 @@ local function spamassassin_check(task, content, digest, rule) local request_data = { "HEADERS SPAMC/1.5\r\n", "User: root\r\n", - "Content-length: ".. #content .. "\r\n", + "Content-length: " .. #content .. "\r\n", "\r\n", content, } @@ -110,14 +110,14 @@ local function spamassassin_check(task, content, digest, rule) retransmits = retransmits - 1 lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s', - rule.log_prefix, error, retransmits) + rule.log_prefix, error, retransmits) -- Select a different upstream! upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s', - rule.log_prefix, addr, addr:get_port()) + rule.log_prefix, addr, addr:get_port()) tcp.request({ task = task, @@ -129,8 +129,8 @@ local function spamassassin_check(task, content, digest, rule) callback = spamassassin_callback, }) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. - 'exceed - err: %s', rule.log_prefix, error) + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. + 'exceed - err: %s', rule.log_prefix, error) common.yield_result(task, rule, 'failed to scan and retransmits exceed: ' .. error, 0.0, 'fail') end end @@ -164,7 +164,7 @@ local function spamassassin_check(task, content, digest, rule) end lua_util.debugm(rule.N, task, '%s: spam_score: %s, symbols: %s, int spam_score: |%s|, type spam_score: |%s|', - rule.log_prefix, spam_score, symbols, tonumber(spam_score), type(spam_score)) + rule.log_prefix, spam_score, symbols, tonumber(spam_score), type(spam_score)) if tonumber(spam_score) > 0 and #symbols > 0 and symbols ~= "none" then @@ -205,7 +205,7 @@ local function spamassassin_check(task, content, digest, rule) end return { - type = {N,'spam', 'scanner'}, + type = { N, 'spam', 'scanner' }, description = 'spamassassin spam scanner', configure = spamassassin_config, check = spamassassin_check, diff --git a/lualib/lua_scanners/vadesecure.lua b/lualib/lua_scanners/vadesecure.lua index b912eb46f..826573a89 100644 --- a/lualib/lua_scanners/vadesecure.lua +++ b/lualib/lua_scanners/vadesecure.lua @@ -85,17 +85,17 @@ local function vade_config(opts) score = 8.0, description = 'VadeSecure decided message to be phishing' }, - commercial = { + commercial = { symbol = 'VADE_COMMERCIAL', score = 0.0, description = 'VadeSecure decided message to be commercial message' }, - community = { + community = { symbol = 'VADE_COMMUNITY', score = 0.0, description = 'VadeSecure decided message to be community message' }, - transactional = { + transactional = { symbol = 'VADE_TRANSACTIONAL', score = 0.0, description = 'VadeSecure decided message to be transactional message' @@ -250,9 +250,9 @@ local function vade_check(task, content, digest, rule, maybe_part) http.request(request_data) else - rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. + rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' .. 'exceed', rule.log_prefix) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '.. + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' .. 'retransmits exceed') end end @@ -261,7 +261,9 @@ local function vade_check(task, content, digest, rule, maybe_part) vade_requery() else -- Parse the response - if upstream then upstream:ok() end + if upstream then + upstream:ok() + end if code ~= 200 then rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) @@ -341,7 +343,7 @@ local function vade_check(task, content, digest, rule, maybe_part) end return { - type = {'vadesecure', 'scanner'}, + type = { 'vadesecure', 'scanner' }, description = 'VadeSecure Filterd interface', configure = vade_config, check = vade_check, diff --git a/lualib/lua_scanners/virustotal.lua b/lualib/lua_scanners/virustotal.lua index e53795b84..d937c4128 100644 --- a/lualib/lua_scanners/virustotal.lua +++ b/lualib/lua_scanners/virustotal.lua @@ -123,7 +123,7 @@ local function virustotal_check(task, content, digest, rule, maybe_part) else local ucl = require "ucl" local parser = ucl.parser() - local res,json_err = parser:parse_string(body) + local res, json_err = parser:parse_string(body) lua_util.debugm(rule.name, task, '%s: got reply data: "%s"', rule.log_prefix, body) diff --git a/lualib/lua_selectors/common.lua b/lualib/lua_selectors/common.lua index c2fe9b58f..7b2372da8 100644 --- a/lualib/lua_selectors/common.lua +++ b/lualib/lua_selectors/common.lua @@ -21,8 +21,8 @@ local cr_hash = require 'rspamd_cryptobox_hash' local blake2b_key = cr_hash.create_specific('blake2'):update('rspamd'):bin() local function digest_schema() - return {ts.one_of{'hex', 'base32', 'bleach32', 'rbase32', 'base64'}:is_optional(), - ts.one_of{'blake2', 'sha256', 'sha1', 'sha512', 'md5'}:is_optional()} + return { ts.one_of { 'hex', 'base32', 'bleach32', 'rbase32', 'base64' }:is_optional(), + ts.one_of { 'blake2', 'sha256', 'sha1', 'sha512', 'md5' }:is_optional() } end exports.digest_schema = digest_schema @@ -66,7 +66,6 @@ local function create_digest(data, args) return encode_digest(h, args) end - local function get_cached_or_raw_digest(task, idx, mime_part, args) if #args == 0 then -- Optimise as we already have this hash in the API diff --git a/lualib/lua_selectors/extractors.lua b/lualib/lua_selectors/extractors.lua index 484adec56..81dfa9d4d 100644 --- a/lualib/lua_selectors/extractors.lua +++ b/lualib/lua_selectors/extractors.lua @@ -30,7 +30,9 @@ local function gen_exclude_flags_filter(exclude_flags) return function(u) local got_flags = u:get_flags() for _, flag in ipairs(exclude_flags) do - if got_flags[flag] then return false end + if got_flags[flag] then + return false + end end return true end @@ -44,11 +46,11 @@ local extractors = { return args[1], 'string' end - return '','string' + return '', 'string' end, ['description'] = [[Return value from function's argument or an empty string, For example, `id('Something')` returns a string 'Something']], - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, -- Similar but for making lists ['list'] = { @@ -57,7 +59,7 @@ For example, `id('Something')` returns a string 'Something']], return fun.map(tostring, args), 'string_list' end - return {},'string_list' + return {}, 'string_list' end, ['description'] = [[Return a list from function's arguments or an empty list, For example, `list('foo', 'bar')` returns a list {'foo', 'bar'}]], @@ -66,7 +68,9 @@ For example, `list('foo', 'bar')` returns a list {'foo', 'bar'}]], ['ip'] = { ['get_value'] = function(task) local ip = task:get_ip() - if ip and ip:is_valid() then return ip,'userdata' end + if ip and ip:is_valid() then + return ip, 'userdata' + end return nil end, ['description'] = [[Get source IP address]], @@ -81,7 +85,7 @@ For example, `list('foo', 'bar')` returns a list {'foo', 'bar'}]], from = task:get_from(0) end if ((from or E)[1] or E).addr then - return from[1],'table' + return from[1], 'table' end return nil end, @@ -97,7 +101,7 @@ uses any type by default)]], rcpts = task:get_recipients(0) end if ((rcpts or E)[1] or E).addr then - return rcpts,'table_list' + return rcpts, 'table_list' end return nil end, @@ -111,7 +115,7 @@ uses any type by default)]], if not country then return nil else - return country,'string' + return country, 'string' end end, ['description'] = [[Get country (ASN module must be executed first)]], @@ -124,7 +128,7 @@ uses any type by default)]], if not asn then return nil else - return asn,'string' + return asn, 'string' end end, ['description'] = [[Get AS number (ASN module must be executed first)]], @@ -136,7 +140,7 @@ uses any type by default)]], if not auser then return nil else - return auser,'string' + return auser, 'string' end end, ['description'] = 'Get authenticated user name', @@ -144,14 +148,14 @@ uses any type by default)]], -- Get principal recipient ['to'] = { ['get_value'] = function(task) - return task:get_principal_recipient(),'string' + return task:get_principal_recipient(), 'string' end, ['description'] = 'Get principal recipient', }, -- Get content digest ['digest'] = { ['get_value'] = function(task) - return task:get_digest(),'string' + return task:get_digest(), 'string' end, ['description'] = 'Get content digest', }, @@ -160,14 +164,14 @@ uses any type by default)]], ['get_value'] = function(task, args) local parts = task:get_parts() or E local digests = {} - for i,p in ipairs(parts) do + for i, p in ipairs(parts) do if p:is_attachment() then table.insert(digests, common.get_cached_or_raw_digest(task, i, p, args)) end end if #digests > 0 then - return digests,'string_list' + return digests, 'string_list' end return nil @@ -184,7 +188,7 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, local parts = task:get_parts() or E local files = {} - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do local fname = p:get_filename() if fname then table.insert(files, fname) @@ -192,7 +196,7 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, end if #files > 0 then - return files,'string_list' + return files, 'string_list' end return nil @@ -205,7 +209,7 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, local text_parts = task:get_text_parts() or E local languages = {} - for _,p in ipairs(text_parts) do + for _, p in ipairs(text_parts) do local lang = p:get_language() if lang then table.insert(languages, lang) @@ -213,7 +217,7 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, end if #languages > 0 then - return languages,'string_list' + return languages, 'string_list' end return nil @@ -223,7 +227,7 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, -- Get helo value ['helo'] = { ['get_value'] = function(task) - return task:get_helo(),'string' + return task:get_helo(), 'string' end, ['description'] = 'Get helo value', }, @@ -238,20 +242,20 @@ the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, end if args[2]:match('full') then - return task:get_header_full(args[1], strong),'table_list' + return task:get_header_full(args[1], strong), 'table_list' end - return task:get_header(args[1], strong),'string' + return task:get_header(args[1], strong), 'string' else - return task:get_header(args[1]),'string' + return task:get_header(args[1]), 'string' end end, ['description'] = [[Get header with the name that is expected as an argument. The optional second argument accepts list of flags: - `full`: returns all headers with this name with all data (like task:get_header_full()) - `strong`: use case sensitive match when matching header's name]], - ['args_schema'] = {ts.string, - (ts.pattern("strong") + ts.pattern("full")):is_optional()} + ['args_schema'] = { ts.string, + (ts.pattern("strong") + ts.pattern("full")):is_optional() } }, -- Get list of received headers (returns list of tables) ['received'] = { @@ -261,10 +265,12 @@ The optional second argument accepts list of flags: return nil end if args[1] then - return fun.map(function(r) return r[args[1]] end, rh), 'string_list' + return fun.map(function(r) + return r[args[1]] + end, rh), 'string_list' end - return rh,'table_list' + return rh, 'table_list' end, ['description'] = [[Get list of received headers. If no arguments specified, returns list of tables. Otherwise, selects a specific element, @@ -278,9 +284,11 @@ e.g. `by_hostname`]], return nil end if args[1] then - return fun.map(function(r) return r[args[1]](r) end, urls), 'string_list' + return fun.map(function(r) + return r[args[1]](r) + end, urls), 'string_list' end - return urls,'userdata_list' + return urls, 'userdata_list' end, ['description'] = [[Get list of all urls. If no arguments specified, returns list of url objects. Otherwise, calls a specific method, @@ -299,21 +307,21 @@ e.g. `get_tld`]], if not urls[1] then return nil end - return urls,'userdata_list' + return urls, 'userdata_list' end, ['description'] = [[Get most specific urls. Arguments are equal to the Lua API function]], - ['args_schema'] = {ts.shape{ + ['args_schema'] = { ts.shape { limit = ts.number + ts.string / tonumber, esld_limit = (ts.number + ts.string / tonumber):is_optional(), exclude_flags = url_flags_ts, flags = url_flags_ts, - flags_mode = ts.one_of{'explicit'}:is_optional(), + flags_mode = ts.one_of { 'explicit' }:is_optional(), prefix = ts.string:is_optional(), need_content = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), need_emails = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), need_images = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), ignore_redirected = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), - }} + } } }, ['specific_urls_filter_map'] = { ['get_value'] = function(task, args) @@ -331,21 +339,23 @@ e.g. `get_tld`]], if not urls[1] then return nil end - return fun.filter(function(u) return map:get_key(tostring(u)) end, urls),'userdata_list' + return fun.filter(function(u) + return map:get_key(tostring(u)) + end, urls), 'userdata_list' end, ['description'] = [[Get most specific urls, filtered by some map. Arguments are equal to the Lua API function]], - ['args_schema'] = {ts.string, ts.shape{ + ['args_schema'] = { ts.string, ts.shape { limit = ts.number + ts.string / tonumber, esld_limit = (ts.number + ts.string / tonumber):is_optional(), exclude_flags = url_flags_ts, flags = url_flags_ts, - flags_mode = ts.one_of{'explicit'}:is_optional(), + flags_mode = ts.one_of { 'explicit' }:is_optional(), prefix = ts.string:is_optional(), need_content = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), need_emails = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), need_images = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), ignore_redirected = (ts.boolean + ts.string / lua_util.toboolean):is_optional(), - }} + } } }, -- URLs filtered by flags ['urls_filtered'] = { @@ -354,13 +364,13 @@ e.g. `get_tld`]], if not urls[1] then return nil end - return urls,'userdata_list' + return urls, 'userdata_list' end, ['description'] = [[Get list of all urls filtered by flags_include/exclude (see rspamd_task:get_urls_filtered for description)]], - ['args_schema'] = {ts.array_of{ + ['args_schema'] = { ts.array_of { url_flags_ts:is_optional(), url_flags_ts:is_optional() - }} + } } }, -- Get all emails ['emails'] = { @@ -370,9 +380,11 @@ e.g. `get_tld`]], return nil end if args[1] then - return fun.map(function(r) return r[args[1]](r) end, urls), 'string_list' + return fun.map(function(r) + return r[args[1]](r) + end, urls), 'string_list' end - return urls,'userdata_list' + return urls, 'userdata_list' end, ['description'] = [[Get list of all emails. If no arguments specified, returns list of url objects. Otherwise, calls a specific method, @@ -383,11 +395,11 @@ e.g. `get_user`]], ['pool_var'] = { ['get_value'] = function(task, args) local type = args[2] or 'string' - return task:get_mempool():get_variable(args[1], type),(type) + return task:get_mempool():get_variable(args[1], type), (type) end, ['description'] = [[Get specific pool var. The first argument must be variable name, the second argument is optional and defines the type (string by default)]], - ['args_schema'] = {ts.string, ts.string:is_optional()} + ['args_schema'] = { ts.string, ts.string:is_optional() } }, -- Get value of specific key from task cache ['task_cache'] = { @@ -406,35 +418,35 @@ the second argument is optional and defines the type (string by default)]], end, ['description'] = [[Get value of specific key from task cache. The first argument must be the key name]], - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Get specific HTTP request header. The first argument must be header name. ['request_header'] = { ['get_value'] = function(task, args) local hdr = task:get_request_header(args[1]) if hdr then - return hdr,'string' + return hdr, 'string' end return nil end, ['description'] = [[Get specific HTTP request header. The first argument must be header name.]], - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Get task date, optionally formatted ['time'] = { ['get_value'] = function(task, args) local what = args[1] or 'message' - local dt = task:get_date{format = what, gmt = true} + local dt = task:get_date { format = what, gmt = true } if dt then if args[2] then -- Should be in format !xxx, as dt is in GMT - return os.date(args[2], dt),'string' + return os.date(args[2], dt), 'string' end - return tostring(dt),'string' + return tostring(dt), 'string' end return nil @@ -444,8 +456,8 @@ The first argument must be header name.]], - `message`: timestamp as defined by `Date` header The second argument is optional time format, see [os.date](http://pgl.yoyo.org/luai/i/os.date) description]], - ['args_schema'] = {ts.one_of{'connect', 'message'}:is_optional(), - ts.string:is_optional()} + ['args_schema'] = { ts.one_of { 'connect', 'message' }:is_optional(), + ts.string:is_optional() } }, -- Get text words from a message ['words'] = { @@ -473,13 +485,15 @@ The first argument must be header name.]], - `norm`: normalised words (lowercased) - `full`: list of tables ]], - ['args_schema'] = { ts.one_of { 'stem', 'raw', 'norm', 'full' }:is_optional()}, + ['args_schema'] = { ts.one_of { 'stem', 'raw', 'norm', 'full' }:is_optional() }, }, -- Get queue ID ['queueid'] = { ['get_value'] = function(task) local queueid = task:get_queue_id() - if queueid then return queueid,'string' end + if queueid then + return queueid, 'string' + end return nil end, ['description'] = [[Get queue ID]], @@ -488,7 +502,9 @@ The first argument must be header name.]], ['uid'] = { ['get_value'] = function(task) local uid = task:get_uid() - if uid then return uid,'string' end + if uid then + return uid, 'string' + end return nil end, ['description'] = [[Get ID of the task being processed]], @@ -497,7 +513,9 @@ The first argument must be header name.]], ['messageid'] = { ['get_value'] = function(task) local mid = task:get_message_id() - if mid then return mid,'string' end + if mid then + return mid, 'string' + end return nil end, ['description'] = [[Get message ID]], @@ -507,25 +525,25 @@ The first argument must be header name.]], ['get_value'] = function(task, args) local symbol = task:get_symbol(args[1], args[2]) if symbol then - return symbol[1],'table' + return symbol[1], 'table' end end, ['description'] = 'Get specific symbol. The first argument must be the symbol name. ' .. - 'The second argument is an optional shadow result name. ' .. - 'Returns the symbol table. See task:get_symbol()', - ['args_schema'] = {ts.string, ts.string:is_optional()} + 'The second argument is an optional shadow result name. ' .. + 'Returns the symbol table. See task:get_symbol()', + ['args_schema'] = { ts.string, ts.string:is_optional() } }, -- Get full scan result ['scan_result'] = { ['get_value'] = function(task, args) local res = task:get_metric_result(args[1]) if res then - return res,'table' + return res, 'table' end end, ['description'] = 'Get full scan result (either default or shadow if shadow result name is specified)' .. 'Returns the result table. See task:get_metric_result()', - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, -- Get list of metatokens as strings ['metatokens'] = { diff --git a/lualib/lua_selectors/init.lua b/lualib/lua_selectors/init.lua index 5823fef96..6cdd11584 100644 --- a/lualib/lua_selectors/init.lua +++ b/lualib/lua_selectors/init.lua @@ -51,21 +51,21 @@ local function implicit_tostring(t, ud_or_table) if t == 'table' then -- Table (very special) if ud_or_table.value then - return ud_or_table.value,'string' + return ud_or_table.value, 'string' elseif ud_or_table.addr then - return ud_or_table.addr,'string' + return ud_or_table.addr, 'string' end - return logger.slog("%s", ud_or_table),'string' + return logger.slog("%s", ud_or_table), 'string' elseif (t == 'string' or t == 'text') and type(ud_or_table) == 'userdata' then if ud_or_table.cookie and ud_or_table.cookie == text_cookie then -- Preserve opaque - return ud_or_table,'string' + return ud_or_table, 'string' else - return tostring(ud_or_table),'string' + return tostring(ud_or_table), 'string' end elseif t ~= 'nil' then - return tostring(ud_or_table),'string' + return tostring(ud_or_table), 'string' end return nil @@ -84,7 +84,7 @@ local function process_selector(task, sel) return pure_type(t) end - local input,etype = sel.selector.get_value(task, sel.selector.args) + local input, etype = sel.selector.get_value(task, sel.selector.args) if not input then lua_util.debugm(M, task, 'no value extracted for %s', sel.selector.name) @@ -98,14 +98,16 @@ local function process_selector(task, sel) local first_elt = pipe[1] if first_elt and (first_elt.method or - fun.any(function(t) return t == 'userdata' or t == 'table' end, first_elt.types)) then + fun.any(function(t) + return t == 'userdata' or t == 'table' + end, first_elt.types)) then -- Explicit conversion local meth = first_elt if meth.types[etype] then lua_util.debugm(M, task, 'apply method `%s` to %s', meth.name, etype) - input,etype = meth.process(input, etype, meth.args) + input, etype = meth.process(input, etype, meth.args) else local pt = pure_type(etype) @@ -114,7 +116,9 @@ local function process_selector(task, sel) meth.name, pt) -- Map method to a list of inputs, excluding empty elements -- We need to fold it down here to get a proper type resolution - input = fun.totable(fun.filter(function(map_elt, _) return map_elt end, + input = fun.totable(fun.filter(function(map_elt, _) + return map_elt + end, fun.map(function(list_elt) local ret, ty = meth.process(list_elt, pt, meth.args) etype = ty @@ -139,7 +143,9 @@ local function process_selector(task, sel) etype = 'string' else lua_util.debugm(M, task, 'apply implicit map %s->string', pt) - input = fun.filter(function(map_elt) return map_elt end, + input = fun.filter(function(map_elt) + return map_elt + end, fun.map(function(list_elt) local ret = implicit_tostring(pt, list_elt) return ret @@ -166,16 +172,20 @@ local function process_selector(task, sel) if pt and x.types['list'] then -- Generic list processor lua_util.debugm(M, task, 'apply list function `%s` to %s', x.name, t) - return {x.process(value, t, x.args)} + return { x.process(value, t, x.args) } elseif pt and x.map_type and x.types[pt] then local map_type = x.map_type .. '_list' lua_util.debugm(M, task, 'map `%s` to list of %s resulting %s', x.name, pt, map_type) -- Apply map, filtering empty values return { - fun.filter(function(map_elt) return map_elt end, + fun.filter(function(map_elt) + return map_elt + end, fun.map(function(list_elt) - if not list_elt then return nil end + if not list_elt then + return nil + end local ret, _ = x.process(list_elt, pt, x.args) return ret end, value)), @@ -187,14 +197,16 @@ local function process_selector(task, sel) end lua_util.debugm(M, task, 'apply %s to %s', x.name, t) - return {x.process(value, t, x.args)} + return { x.process(value, t, x.args) } end local res = fun.foldl(fold_function, - {input, etype}, + { input, etype }, pipe) - if not res or not res[1] then return nil end -- Pipeline failed + if not res or not res[1] then + return nil + end -- Pipeline failed if not allowed_type(res[2]) then -- Search for implicit conversion @@ -202,7 +214,9 @@ local function process_selector(task, sel) if pt then lua_util.debugm(M, task, 'apply implicit map %s->string_list', pt) - res[1] = fun.map(function(e) return implicit_tostring(pt, e) end, res[1]) + res[1] = fun.map(function(e) + return implicit_tostring(pt, e) + end, res[1]) res[2] = 'string_list' else res[1] = implicit_tostring(res[2], res[1]) @@ -222,14 +236,14 @@ end local function make_grammar() local l = require "lpeg" - local spc = l.S(" \t\n")^0 + local spc = l.S(" \t\n") ^ 0 local cont = l.R("\128\191") -- continuation byte local utf8_high = l.R("\194\223") * cont + l.R("\224\239") * cont * cont + l.R("\240\244") * cont * cont * cont - local atom = l.C((l.R("az") + l.R("AZ") + l.R("09") + l.S("_-") + utf8_high)^1) - local singlequoted_string = l.P "'" * l.C(((1 - l.S "'\r\n\f\\") + (l.P'\\' * 1))^0) * "'" - local doublequoted_string = l.P '"' * l.C(((1 - l.S'"\r\n\f\\') + (l.P'\\' * 1))^0) * '"' + local atom = l.C((l.R("az") + l.R("AZ") + l.R("09") + l.S("_-") + utf8_high) ^ 1) + local singlequoted_string = l.P "'" * l.C(((1 - l.S "'\r\n\f\\") + (l.P '\\' * 1)) ^ 0) * "'" + local doublequoted_string = l.P '"' * l.C(((1 - l.S '"\r\n\f\\') + (l.P '\\' * 1)) ^ 0) * '"' local argument = atom + singlequoted_string + doublequoted_string local dot = l.P(".") local semicolon = l.P(":") @@ -239,20 +253,22 @@ local function make_grammar() local tbl_ebrace = spc * "}" local ebrace = spc * ")" local comma = spc * "," * spc - local sel_separator = spc * l.S";*" * spc + local sel_separator = spc * l.S ";*" * spc - return l.P{ + return l.P { "LIST"; - LIST = l.Ct(l.V("EXPR")) * (sel_separator * l.Ct(l.V("EXPR")))^0, - EXPR = l.V("FUNCTION") * (semicolon * l.V("METHOD"))^-1 * (dot * l.V("PROCESSOR"))^0, - PROCESSOR = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace)^0), - FUNCTION = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace)^0), - METHOD = l.Ct(atom / function(e) return '__' .. e end * spc * (obrace * l.V("ARG_LIST") * ebrace)^0), - ARG_LIST = l.Ct((l.V("ARG") * comma^0)^0), + LIST = l.Ct(l.V("EXPR")) * (sel_separator * l.Ct(l.V("EXPR"))) ^ 0, + EXPR = l.V("FUNCTION") * (semicolon * l.V("METHOD")) ^ -1 * (dot * l.V("PROCESSOR")) ^ 0, + PROCESSOR = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0), + FUNCTION = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0), + METHOD = l.Ct(atom / function(e) + return '__' .. e + end * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0), + ARG_LIST = l.Ct((l.V("ARG") * comma ^ 0) ^ 0), ARG = l.Cf(tbl_obrace * l.V("NAMED_ARG") * tbl_ebrace, rawset) + argument + l.V("LIST_ARGS"), - NAMED_ARG = (l.Ct("") * l.Cg(argument * eqsign * (argument + l.V("LIST_ARGS")) * comma^0)^0), + NAMED_ARG = (l.Ct("") * l.Cg(argument * eqsign * (argument + l.V("LIST_ARGS")) * comma ^ 0) ^ 0), LIST_ARGS = l.Ct(tbl_obrace * l.V("LIST_ARG") * tbl_ebrace), - LIST_ARG = l.Cg(argument * comma^0)^0, + LIST_ARG = l.Cg(argument * comma ^ 0) ^ 0, } end @@ -262,27 +278,29 @@ local parser = make_grammar() -- @function lua_selectors.parse_selector(cfg, str) --]] exports.parse_selector = function(cfg, str) - local parsed = {parser:match(str)} + local parsed = { parser:match(str) } local output = {} - if not parsed or not parsed[1] then return nil end + if not parsed or not parsed[1] then + return nil + end local function check_args(name, schema, args) if schema then if getmetatable(schema) then -- Schema covers all arguments - local res,err = schema:transform(args) + local res, err = schema:transform(args) if not res then logger.errx(rspamd_config, 'invalid arguments for %s: %s', name, err) return false else - for i,elt in ipairs(res) do + for i, elt in ipairs(res) do args[i] = elt end end else - for i,selt in ipairs(schema) do - local res,err = selt:transform(args[i]) + for i, selt in ipairs(schema) do + local res, err = selt:transform(args[i]) if err then logger.errx(rspamd_config, 'invalid arguments for %s: argument number: %s, error: %s', name, i, err) @@ -301,7 +319,7 @@ exports.parse_selector = function(cfg, str) -- table of individual selectors -- each selector: list of functions -- each function: function name + optional list of arguments - for _,sel in ipairs(parsed) do + for _, sel in ipairs(parsed) do local res = { selector = {}, processor_pipe = {}, @@ -364,13 +382,15 @@ exports.parse_selector = function(cfg, str) local ret_type = type(ret) - if ret_type == 'nil' then return nil end + if ret_type == 'nil' then + return nil + end -- Now apply types heuristic if ret_type == 'string' then - return ret,'string' + return ret, 'string' elseif ret_type == 'table' then -- TODO: we need to ensure that 1) table is numeric 2) table has merely strings - return ret,'string_list' + return ret, 'string_list' else return implicit_tostring(ret_type, ret) end @@ -452,11 +472,13 @@ end exports.process_selectors = function(task, selectors_pipe) local ret = {} - for _,sel in ipairs(selectors_pipe) do + for _, sel in ipairs(selectors_pipe) do local r = process_selector(task, sel) -- If any element is nil, then the whole selector is nil - if not r then return nil end + if not r then + return nil + end table.insert(ret, r) end @@ -467,13 +489,17 @@ end -- @function lua_selectors.combine_selectors(task, selectors, delimiter) --]] exports.combine_selectors = function(_, selectors, delimiter) - if not delimiter then delimiter = '' end + if not delimiter then + delimiter = '' + end - if not selectors then return nil end + if not selectors then + return nil + end local have_tables, have_userdata - for _,s in ipairs(selectors) do + for _, s in ipairs(selectors) do if type(s) == 'table' then have_tables = true elseif type(s) == 'userdata' then @@ -493,7 +519,7 @@ exports.combine_selectors = function(_, selectors, delimiter) local tbl = {} local res = {} - for i,s in ipairs(selectors) do + for i, s in ipairs(selectors) do if type(s) == 'string' then rawset(tbl, i, fun.duplicate(s)) elseif type(s) == 'userdata' then @@ -505,7 +531,7 @@ exports.combine_selectors = function(_, selectors, delimiter) end fun.each(function(...) - table.insert(res, table.concat({...}, delimiter)) + table.insert(res, table.concat({ ... }, delimiter)) end, fun.zip(lua_util.unpack(tbl))) return res @@ -520,7 +546,7 @@ exports.flatten_selectors = function(_, selectors, _) local res = {} local function fill(tbl) - for _,s in ipairs(tbl) do + for _, s in ipairs(tbl) do if type(s) == 'string' then rawset(res, #res + 1, s) elseif type(s) == 'userdata' then @@ -553,7 +579,7 @@ exports.kv_table_from_pairs = function(log_obj, selectors, _) tbl_len) return end - for i=1,tbl_len,2 do + for i = 1, tbl_len, 2 do local k = tostring(tbl[i]) local v = tbl[i + 1] if type(v) == 'string' then @@ -561,7 +587,9 @@ exports.kv_table_from_pairs = function(log_obj, selectors, _) elseif type(v) == 'userdata' then res[k] = tostring(v) else - res[k] = fun.totable(fun.map(function(elt) return tostring(elt) end, v)) + res[k] = fun.totable(fun.map(function(elt) + return tostring(elt) + end, v)) end end end @@ -605,7 +633,7 @@ exports.create_selector_closure = function(cfg, selector_str, delimiter, flatten end local function display_selectors(tbl) - return fun.tomap(fun.map(function(k,v) + return fun.tomap(fun.map(function(k, v) return k, fun.tomap(fun.filter(function(kk, vv) return type(vv) ~= 'function' end, v)) diff --git a/lualib/lua_selectors/transforms.lua b/lualib/lua_selectors/transforms.lua index 55e1bffb7..4b839de28 100644 --- a/lualib/lua_selectors/transforms.lua +++ b/lualib/lua_selectors/transforms.lua @@ -36,7 +36,7 @@ local transform_function = { }, ['map_type'] = 'string', ['process'] = function(inp, _) - return inp:lower(),'string' + return inp:lower(), 'string' end, ['description'] = 'Returns the lowercased string', }, @@ -57,7 +57,7 @@ local transform_function = { ['list'] = true, }, ['process'] = function(inp, t) - return fun.head(inp),pure_type(t) + return fun.head(inp), pure_type(t) end, ['description'] = 'Returns the first element', }, @@ -67,7 +67,7 @@ local transform_function = { ['list'] = true, }, ['process'] = function(inp, t) - return fun.nth(fun.length(inp), inp),pure_type(t) + return fun.nth(fun.length(inp), inp), pure_type(t) end, ['description'] = 'Returns the last element', }, @@ -77,30 +77,30 @@ local transform_function = { ['list'] = true, }, ['process'] = function(inp, t, args) - return fun.nth(args[1] or 1, inp),pure_type(t) + return fun.nth(args[1] or 1, inp), pure_type(t) end, ['description'] = 'Returns the nth element', - ['args_schema'] = {ts.number + ts.string / tonumber} + ['args_schema'] = { ts.number + ts.string / tonumber } }, ['take_n'] = { ['types'] = { ['list'] = true, }, ['process'] = function(inp, t, args) - return fun.take_n(args[1] or 1, inp),t + return fun.take_n(args[1] or 1, inp), t end, ['description'] = 'Returns the n first elements', - ['args_schema'] = {ts.number + ts.string / tonumber} + ['args_schema'] = { ts.number + ts.string / tonumber } }, ['drop_n'] = { ['types'] = { ['list'] = true, }, ['process'] = function(inp, t, args) - return fun.drop_n(args[1] or 1, inp),t + return fun.drop_n(args[1] or 1, inp), t end, ['description'] = 'Returns list without the first n elements', - ['args_schema'] = {ts.number + ts.string / tonumber} + ['args_schema'] = { ts.number + ts.string / tonumber } }, -- Joins strings into a single string using separator in the argument ['join'] = { @@ -111,7 +111,7 @@ local transform_function = { return table.concat(fun.totable(inp), args[1] or ''), 'string' end, ['description'] = 'Joins strings into a single string using separator in the argument', - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, -- Joins strings into a set of strings using N elements and a separator in the argument ['join_nth'] = { @@ -124,13 +124,13 @@ local transform_function = { local inp_t = fun.totable(inp) local res = {} - for i=1, #inp_t, step do + for i = 1, #inp_t, step do table.insert(res, table.concat(inp_t, sep, i, i + step)) end - return res,'string_list' + return res, 'string_list' end, ['description'] = 'Joins strings into a set of strings using N elements and a separator in the argument', - ['args_schema'] = {ts.number + ts.string / tonumber, ts.string:is_optional()} + ['args_schema'] = { ts.number + ts.string / tonumber, ts.string:is_optional() } }, -- Joins tables into a table of strings ['join_tables'] = { @@ -139,10 +139,12 @@ local transform_function = { }, ['process'] = function(inp, _, args) local sep = args[1] or '' - return fun.map(function(t) return table.concat(t, sep) end, inp), 'string_list' + return fun.map(function(t) + return table.concat(t, sep) + end, inp), 'string_list' end, ['description'] = 'Joins tables into a table of strings', - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, -- Sort strings ['sort'] = { @@ -166,7 +168,9 @@ local transform_function = { tmp[val] = true end, inp) - return fun.map(function(k, _) return k end, tmp), t + return fun.map(function(k, _) + return k + end, tmp), t end, ['description'] = 'Returns a list of unique elements (using a hash table)', }, @@ -177,7 +181,7 @@ local transform_function = { }, ['map_type'] = 'string', ['process'] = function(inp, _, args) - return common.create_digest(inp, args),'string' + return common.create_digest(inp, args), 'string' end, ['description'] = [[Create a digest from a string. The first argument is encoding (`hex`, `base32` (and forms `bleach32`, `rbase32`), `base64`), @@ -197,8 +201,8 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, return inp:sub(start_pos, end_pos), 'string' end, ['description'] = 'Extracts substring; the first argument is start, the second is the last (like in Lua)', - ['args_schema'] = {(ts.number + ts.string / tonumber):is_optional(), - (ts.number + ts.string / tonumber):is_optional()} + ['args_schema'] = { (ts.number + ts.string / tonumber):is_optional(), + (ts.number + ts.string / tonumber):is_optional() } }, -- Prepends a string or a strings list ['prepend'] = { @@ -257,13 +261,13 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, end end flatten_table(res) - return flattened_table,'string_list' + return flattened_table, 'string_list' end return nil end, ['description'] = 'Regexp matching, returns all matches flattened in a single list', - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Returns a value if it exists in some map (or acts like a `filter` function) ['filter_map'] = { @@ -282,13 +286,13 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, local res = map:get_key(inp) if res then - return inp,t + return inp, t end return nil end, ['description'] = 'Returns a value if it exists in some map (or acts like a `filter` function)', - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Returns a value if it exists in some map (or acts like a `filter` function) ['except_map'] = { @@ -307,13 +311,13 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, local res = map:get_key(inp) if not res then - return inp,t + return inp, t end return nil end, ['description'] = 'Returns a value if it does not exists in some map (or acts like a `except` function)', - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Returns a value from some map corresponding to some key (or acts like a `map` function) ['apply_map'] = { @@ -332,13 +336,13 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, local res = map:get_key(inp) if res then - return res,t + return res, t end return nil end, ['description'] = 'Returns a value from some map corresponding to some key (or acts like a `map` function)', - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Drops input value and return values from function's arguments or an empty string ['id'] = { @@ -349,12 +353,12 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, ['map_type'] = 'string', ['process'] = function(_, _, args) if args[1] and args[2] then - return fun.map(tostring, args),'string_list' + return fun.map(tostring, args), 'string_list' elseif args[1] then - return args[1],'string' + return args[1], 'string' end - return '','string' + return '', 'string' end, ['description'] = 'Drops input value and return values from function\'s arguments or an empty string', ['args_schema'] = (ts.string + ts.array_of(ts.string)):is_optional() @@ -366,14 +370,14 @@ the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, ['map_type'] = 'string', ['process'] = function(inp, _, args) if inp == args[1] then - return inp,'string' + return inp, 'string' end return nil end, ['description'] = [[Boolean function equal. Returns either nil or its argument if input is equal to argument]], - ['args_schema'] = {ts.string} + ['args_schema'] = { ts.string } }, -- Boolean function in, returns either nil or its input if input is in args list ['in'] = { @@ -382,7 +386,11 @@ Returns either nil or its argument if input is equal to argument]], }, ['map_type'] = 'string', ['process'] = function(inp, t, args) - for _,a in ipairs(args) do if a == inp then return inp,t end end + for _, a in ipairs(args) do + if a == inp then + return inp, t + end + end return nil end, ['description'] = [[Boolean function in. @@ -395,8 +403,12 @@ Returns either nil or its input if input is in args list]], }, ['map_type'] = 'string', ['process'] = function(inp, t, args) - for _,a in ipairs(args) do if a == inp then return nil end end - return inp,t + for _, a in ipairs(args) do + if a == inp then + return nil + end + end + return inp, t end, ['description'] = [[Boolean function not in. Returns either nil or its input if input is not in args list]], @@ -411,12 +423,12 @@ Returns either nil or its input if input is not in args list]], if inp then return nil else - return (args[1] or 'true'),'string' + return (args[1] or 'true'), 'string' end end, ['description'] = [[Inverses input. Empty string comes the first argument or 'true', non-empty string comes nil]], - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, ['ipmask'] = { ['types'] = { @@ -436,19 +448,19 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], if ip:get_version() == 4 then local mask = tonumber(args[1]) - return ip:apply_mask(mask):to_string(),'string' + return ip:apply_mask(mask):to_string(), 'string' else -- IPv6 takes the second argument or the first one... local mask_str = args[2] or args[1] local mask = tonumber(mask_str) - return ip:apply_mask(mask):to_string(),'string' + return ip:apply_mask(mask):to_string(), 'string' end end, ['description'] = 'Applies mask to IP address.' .. - ' The first argument is the mask for IPv4 addresses, the second is the mask for IPv6 addresses.', - ['args_schema'] = {(ts.number + ts.string / tonumber), - (ts.number + ts.string / tonumber):is_optional()} + ' The first argument is the mask for IPv4 addresses, the second is the mask for IPv6 addresses.', + ['args_schema'] = { (ts.number + ts.string / tonumber), + (ts.number + ts.string / tonumber):is_optional() } }, -- Returns the string(s) with all non ascii chars replaced ['to_ascii'] = { @@ -460,16 +472,16 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], ['process'] = function(inp, _, args) if type(inp) == 'table' then return fun.map( - function(s) - return string.gsub(tostring(s), '[\128-\255]', args[1] or '?') - end, inp), 'string_list' + function(s) + return string.gsub(tostring(s), '[\128-\255]', args[1] or '?') + end, inp), 'string_list' else return string.gsub(tostring(inp), '[\128-\255]', '?'), 'string' end end, ['description'] = 'Returns the string with all non-ascii bytes replaced with the character ' .. - 'given as second argument or `?`', - ['args_schema'] = {ts.string:is_optional()} + 'given as second argument or `?`', + ['args_schema'] = { ts.string:is_optional() } }, -- Extracts tld from a hostname ['get_tld'] = { @@ -478,7 +490,7 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], }, ['map_type'] = 'string', ['process'] = function(inp, _, _) - return rspamd_util.get_tld(inp),'string' + return rspamd_util.get_tld(inp), 'string' end, ['description'] = 'Extracts tld from a hostname represented as a string', ['args_schema'] = {} @@ -498,7 +510,7 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], return rspamd_util.pack(string.rep(fmt, #res), lua_util.unpack(res)), 'string' end, ['description'] = 'Converts a list of strings to numbers & returns a packed string', - ['args_schema'] = {ts.string:is_optional()} + ['args_schema'] = { ts.string:is_optional() } }, -- Filter nils from a list ['filter_string_nils'] = { @@ -506,7 +518,9 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], ['string_list'] = true }, ['process'] = function(inp, _, _) - return fun.filter(function(val) return type(val) == 'string' and val ~= 'nil' end, inp), 'string_list' + return fun.filter(function(val) + return type(val) == 'string' and val ~= 'nil' + end, inp), 'string_list' end, ['description'] = 'Removes all nils from a list of strings (when converted implicitly)', ['args_schema'] = {} @@ -518,12 +532,14 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], }, ['process'] = function(inp, _, args) local res = {} - for _,arg in ipairs(args) do + for _, arg in ipairs(args) do local meth = inp[arg] local ret = meth(inp) - if ret then table.insert(res, tostring(ret)) end + if ret then + table.insert(res, tostring(ret)) + end end - return res,'string_list' + return res, 'string_list' end, ['description'] = 'Apply a list of method calls to the userdata object', } diff --git a/lualib/lua_settings.lua b/lualib/lua_settings.lua index feeb03d28..d6d24d67e 100644 --- a/lualib/lua_settings.lua +++ b/lualib/lua_settings.lua @@ -37,7 +37,7 @@ local function register_settings_cb(from_postload) default_symbols = fun.totable(fun.filter(function(_, v) return not v.allowed_ids or #v.allowed_ids == 0 or v.flags.explicit_disable - end,all_symbols)) + end, all_symbols)) local explicit_symbols = lua_util.keys(fun.filter(function(k, v) return v.flags.explicit_disable @@ -45,7 +45,7 @@ local function register_settings_cb(from_postload) local symnames = lua_util.list_to_hash(lua_util.keys(all_symbols)) - for _,set in pairs(known_ids) do + for _, set in pairs(known_ids) do local s = set.settings.apply or {} set.symbols = lua_util.shallowcopy(symnames) local enabled_symbols = {} @@ -58,18 +58,18 @@ local function register_settings_cb(from_postload) -- Remove all symbols from set.symbols aside of explicit_disable symbols set.symbols = lua_util.list_to_hash(explicit_symbols) seen_enabled = true - for _,sym in ipairs(s.symbols_enabled) do + for _, sym in ipairs(s.symbols_enabled) do enabled_symbols[sym] = true set.symbols[sym] = true end end if s.groups_enabled then seen_enabled = true - for _,gr in ipairs(s.groups_enabled) do + for _, gr in ipairs(s.groups_enabled) do local syms = rspamd_config:get_group_symbols(gr) if syms then - for _,sym in ipairs(syms) do + for _, sym in ipairs(syms) do enabled_symbols[sym] = true set.symbols[sym] = true end @@ -80,18 +80,18 @@ local function register_settings_cb(from_postload) -- Disabled map if s.symbols_disabled then seen_disabled = true - for _,sym in ipairs(s.symbols_disabled) do + for _, sym in ipairs(s.symbols_disabled) do disabled_symbols[sym] = true set.symbols[sym] = false end end if s.groups_disabled then seen_disabled = true - for _,gr in ipairs(s.groups_disabled) do + for _, gr in ipairs(s.groups_disabled) do local syms = rspamd_config:get_group_symbols(gr) if syms then - for _,sym in ipairs(syms) do + for _, sym in ipairs(syms) do disabled_symbols[sym] = true set.symbols[sym] = false end @@ -100,8 +100,12 @@ local function register_settings_cb(from_postload) end -- Deal with complexity to avoid mess in C - if not seen_enabled then enabled_symbols = nil end - if not seen_disabled then disabled_symbols = nil end + if not seen_enabled then + enabled_symbols = nil + end + if not seen_disabled then + disabled_symbols = nil + end if enabled_symbols or disabled_symbols then -- Specify what symbols are really enabled for this settings id @@ -118,9 +122,9 @@ local function register_settings_cb(from_postload) end -- We now iterate over all symbols and check for allowed_ids/forbidden_ids - for k,v in pairs(all_symbols) do + for k, v in pairs(all_symbols) do if v.allowed_ids and not v.flags.explicit_disable then - for _,id in ipairs(v.allowed_ids) do + for _, id in ipairs(v.allowed_ids) do if known_ids[id] then local set = known_ids[id] if not set.has_specific_symbols then @@ -134,7 +138,7 @@ local function register_settings_cb(from_postload) end end if v.forbidden_ids then - for _,id in ipairs(v.forbidden_ids) do + for _, id in ipairs(v.forbidden_ids) do if known_ids[id] then local set = known_ids[id] if not set.has_specific_symbols then @@ -150,8 +154,10 @@ local function register_settings_cb(from_postload) end -- Now we create lists of symbols for each settings and digest - for _,set in pairs(known_ids) do - set.symbols = lua_util.keys(fun.filter(function(_, v) return v end, set.symbols)) + for _, set in pairs(known_ids) do + set.symbols = lua_util.keys(fun.filter(function(_, v) + return v + end, set.symbols)) table.sort(set.symbols) set.digest = lua_util.table_digest(set.symbols) end @@ -188,7 +194,7 @@ local function transform_settings_maybe(settings, name) if not apply.scores then apply.scores = {} end - for k,v in pairs(senabled) do + for k, v in pairs(senabled) do if tonumber(v) then -- Move to symbols as well apply.scores[k] = tonumber(v) @@ -205,7 +211,7 @@ local function transform_settings_maybe(settings, name) if apply.symbols then -- Check if added symbols are enabled - for k,v in pairs(apply.symbols) do + for k, v in pairs(apply.symbols) do local s -- Check if we have ["sym1", "sym2" ...] or {"sym1": xx, "sym2": yy} if type(k) == 'string' then @@ -251,7 +257,9 @@ local function register_settings_id(str, settings, from_postload) if not from_postload and not post_init_added then -- Use high priority to ensure that settings are initialised early but not before all -- plugins are loaded - rspamd_config:add_post_init(function () register_settings_cb(true) end, 150) + rspamd_config:add_post_init(function() + register_settings_cb(true) + end, 150) rspamd_config:add_config_unload(function() if post_init_added then known_ids = {} @@ -268,7 +276,6 @@ end exports.register_settings_id = register_settings_id - local function settings_by_id(id) if not post_init_performed then register_settings_cb(false) @@ -276,7 +283,6 @@ local function settings_by_id(id) return known_ids[id] end - exports.settings_by_id = settings_by_id exports.all_settings = function() if not post_init_performed then diff --git a/lualib/lua_smtp.lua b/lualib/lua_smtp.lua index 33c69159f..3c403497c 100644 --- a/lualib/lua_smtp.lua +++ b/lualib/lua_smtp.lua @@ -47,7 +47,7 @@ local function sendmail(opts, message, callback) wantcode = wantcode or '2' if merr then callback(false, string.format('error on stage %s: %s', - stage, merr)) + stage, merr)) if conn then conn:close() end @@ -108,9 +108,9 @@ local function sendmail(opts, message, callback) local function data_done_cb(merr, mdata) if no_error_read(merr, mdata, '3') then if type(message) == 'string' or type(message) == 'userdata' then - conn:add_write(pre_quit_cb, {message, CRLF.. '.' .. CRLF}) + conn:add_write(pre_quit_cb, { message, CRLF .. '.' .. CRLF }) else - table.insert(message, CRLF.. '.' .. CRLF) + table.insert(message, CRLF .. '.' .. CRLF) conn:add_write(pre_quit_cb, message) end end @@ -124,7 +124,7 @@ local function sendmail(opts, message, callback) -- RCPT phase local next_recipient local function rcpt_done_cb_gen(i) - return function (merr, mdata) + return function(merr, mdata) if no_error_read(merr, mdata) then if i == #opts.recipients then conn:add_write(data_cb, 'DATA' .. CRLF) @@ -136,7 +136,7 @@ local function sendmail(opts, message, callback) end local function rcpt_cb_gen(i) - return function (merr, _) + return function(merr, _) if no_error_write(merr, '2') then conn:add_read(rcpt_done_cb_gen(i), CRLF) end @@ -178,12 +178,12 @@ local function sendmail(opts, message, callback) if no_error_read(err, data) then stage = 'helo' conn:add_write(hello_cb, string.format('HELO %s%s', - opts.helo, CRLF)) + opts.helo, CRLF)) end end if type(opts.recipients) == 'string' then - opts.recipients = {opts.recipients} + opts.recipients = { opts.recipients } end local tcp_opts = lua_util.shallowcopy(opts) diff --git a/lualib/lua_stat.lua b/lualib/lua_stat.lua index e7111735a..a0f3303ad 100644 --- a/lualib/lua_stat.lua +++ b/lualib/lua_stat.lua @@ -29,7 +29,7 @@ local exports = {} local N = "stat_tools" -- luacheck: ignore (maybe unused) -- Performs synchronous conversion of redis schema -local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expire) +local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expire) -- Old schema is the following one: -- Keys are named <symbol>[<user>] @@ -40,7 +40,7 @@ local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expi -- So we can expire individual records, measure most popular elements by zranges, -- add new fields, such as tokens etc - local res,conn = lua_redis.redis_connect_sync(redis_params, true) + local res, conn = lua_redis.redis_connect_sync(redis_params, true) if not res then logger.errx("cannot connect to redis server") @@ -79,7 +79,7 @@ end return nconverted ]] - conn:add_cmd('EVAL', {lua_script, '3', symbol_spam, 'S', tostring(expire)}) + conn:add_cmd('EVAL', { lua_script, '3', symbol_spam, 'S', tostring(expire) }) local ret ret, res = conn:exec() @@ -90,7 +90,7 @@ return nconverted logger.messagex('converted %s elements from symbol %s', res, symbol_spam) end - conn:add_cmd('EVAL', {lua_script, '3', symbol_ham, 'H', tostring(expire)}) + conn:add_cmd('EVAL', { lua_script, '3', symbol_ham, 'H', tostring(expire) }) ret, res = conn:exec() if not ret then @@ -118,15 +118,15 @@ for _,k in ipairs(keys) do end ]] - conn:add_cmd('EVAL', {lua_script, '2', symbol_spam, 'learns_spam'}) - ret,res = conn:exec() + conn:add_cmd('EVAL', { lua_script, '2', symbol_spam, 'learns_spam' }) + ret, res = conn:exec() if not ret then logger.errx('error converting metadata for symbol %s: %s', symbol_spam, res) return false end - conn:add_cmd('EVAL', {lua_script, '2', symbol_ham, 'learns_ham'}) + conn:add_cmd('EVAL', { lua_script, '2', symbol_ham, 'learns_ham' }) ret, res = conn:exec() if not ret then @@ -150,8 +150,8 @@ exports.convert_bayes_schema = convert_bayes_schema -- learn_cache_ham - name for sqlite database with ham learn cache -- reset_previous - if true, then the old database is flushed (slow) local function convert_sqlite_to_redis(redis_params, - sqlite_db_spam, sqlite_db_ham, symbol_spam, symbol_ham, - learn_cache_db, expire, reset_previous) + sqlite_db_spam, sqlite_db_ham, symbol_spam, symbol_ham, + learn_cache_db, expire, reset_previous) local nusers = 0 local lim = 1000 -- Update each 1000 tokens local users_map = {} @@ -168,7 +168,7 @@ local function convert_sqlite_to_redis(redis_params, return false end - local res,conn = lua_redis.redis_connect_sync(redis_params, true) + local res, conn = lua_redis.redis_connect_sync(redis_params, true) if not res then logger.errx("cannot connect to redis server") @@ -187,19 +187,19 @@ for _,prefix in ipairs(members) do end ]] -- Common keys - for _,sym in ipairs({symbol_spam, symbol_ham}) do + for _, sym in ipairs({ symbol_spam, symbol_ham }) do logger.messagex('Cleaning up old data for %s', sym) - conn:add_cmd('EVAL', {script, '1', sym}) + conn:add_cmd('EVAL', { script, '1', sym }) conn:exec() - conn:add_cmd('DEL', {sym .. "_version"}) - conn:add_cmd('DEL', {sym .. "_keys"}) + conn:add_cmd('DEL', { sym .. "_version" }) + conn:add_cmd('DEL', { sym .. "_keys" }) conn:exec() end if learn_cache_db then -- Cleanup learned_cache logger.messagex('Cleaning up old data learned cache') - conn:add_cmd('DEL', {"learned_ids"}) + conn:add_cmd('DEL', { "learned_ids" }) conn:exec() end end @@ -240,16 +240,16 @@ end if is_spam then hash_key = 'S' end - for _,tok in ipairs(tokens) do + for _, tok in ipairs(tokens) do -- tok schema: -- tok[1] = token_id (uint64 represented as a string) -- tok[2] = token value (number) -- tok[3] = user_map[user_id] or '' local rkey = string.format('%s%s_%s', prefix, tok[3], tok[1]) - conn:add_cmd('HINCRBYFLOAT', {rkey, hash_key, tostring(tok[2])}) + conn:add_cmd('HINCRBYFLOAT', { rkey, hash_key, tostring(tok[2]) }) if expire and expire ~= 0 then - conn:add_cmd('EXPIRE', {rkey, tostring(expire)}) + conn:add_cmd('EXPIRE', { rkey, tostring(expire) }) end end @@ -268,13 +268,13 @@ end user = users_map[row.user] end - table.insert(tokens, {row.token, row.value, user}) + table.insert(tokens, { row.token, row.value, user }) num = num + 1 total = total + 1 if num > lim then -- TODO: we use the default 'RS' prefix, it can be false in case of -- classifiers with labels - local ret,err_str = send_batch(tokens, 'RS') + local ret, err_str = send_batch(tokens, 'RS') if not ret then logger.errx('Cannot send tokens to the redis server: ' .. err_str) db:sql('COMMIT;') @@ -289,7 +289,7 @@ end end -- Last batch if #tokens > 0 then - local ret,err_str = send_batch(tokens, 'RS') + local ret, err_str = send_batch(tokens, 'RS') if not ret then logger.errx('Cannot send tokens to the redis server: ' .. err_str) db:sql('COMMIT;') @@ -312,19 +312,19 @@ end learns_elt = "learns_spam" end - for id,learned in pairs(learns) do + for id, learned in pairs(learns) do local user = users_map[id] - if not conn:add_cmd('HSET', {'RS' .. user, learns_elt, learned}) then + if not conn:add_cmd('HSET', { 'RS' .. user, learns_elt, learned }) then logger.errx('Cannot update learns for user: ' .. user) return false end - if not conn:add_cmd('SADD', {symbol .. '_keys', 'RS' .. user}) then + if not conn:add_cmd('SADD', { symbol .. '_keys', 'RS' .. user }) then logger.errx('Cannot update learns for user: ' .. user) return false end end -- Set version - conn:add_cmd('SET', {symbol..'_version', '2'}) + conn:add_cmd('SET', { symbol .. '_version', '2' }) return conn:exec() end @@ -361,7 +361,7 @@ end is_spam = '1' end - if not conn:add_cmd('HSET', {'learned_ids', digest, is_spam}) then + if not conn:add_cmd('HSET', { 'learned_ids', digest, is_spam }) then logger.errx('Cannot add hash: ' .. digest) ret = false else @@ -376,7 +376,7 @@ end if ret then logger.messagex('Converted %s cached items from sqlite3 learned cache to redis', - total) + total) else logger.errx('Error occurred during sending data to redis') end @@ -422,7 +422,7 @@ local function load_sqlite_config(cfg) end local statfiles = cls.statfile - for _,stf in ipairs(statfiles) do + for _, stf in ipairs(statfiles) do local path = (stf.file or stf.path or stf.db or stf.dbname) local symbol = stf.symbol or 'undefined' @@ -460,8 +460,10 @@ local function load_sqlite_config(cfg) if classifier then if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.bayes then cls = cls.bayes end + for _, cls in ipairs(classifier) do + if cls.bayes then + cls = cls.bayes + end if cls.backend and cls.backend == 'sqlite3' then parse_classifier(cls) end @@ -470,7 +472,7 @@ local function load_sqlite_config(cfg) if classifier.bayes then classifier = classifier.bayes if classifier[1] then - for _,cls in ipairs(classifier) do + for _, cls in ipairs(classifier) do if cls.backend and cls.backend == 'sqlite3' then parse_classifier(cls) end @@ -512,7 +514,7 @@ local function redis_classifier_from_sqlite(sqlite_classifier, expire) result.expire = expire end - return {classifier = {bayes = result}} + return { classifier = { bayes = result } } end exports.redis_classifier_from_sqlite = redis_classifier_from_sqlite @@ -548,7 +550,7 @@ local function process_stat_config(cfg) -- Postprocess classify_headers local classify_headers_parsed = {} - for _,v in ipairs(res_config.classify_headers) do + for _, v in ipairs(res_config.classify_headers) do local s1, s2 = v:match("^([A-Z])[^%-]+%-([A-Z]).*$") local hname @@ -567,7 +569,7 @@ local function process_stat_config(cfg) if classify_headers_parsed[hname] then table.insert(classify_headers_parsed[hname], v) else - classify_headers_parsed[hname] = {v} + classify_headers_parsed[hname] = { v } end end @@ -585,7 +587,7 @@ local function get_mime_stat_tokens(task, res, i) local empty_html = false local online_text = false - for _,part in ipairs(parts) do + for _, part in ipairs(parts) do local fname = part:get_filename() local sz = part:get_length() @@ -692,8 +694,8 @@ local function get_headers_stat_tokens(task, cf, res, i) end ]]-- - for k,hdrs in pairs(cf.classify_headers_parsed) do - for _,hname in ipairs(hdrs) do + for k, hdrs in pairs(cf.classify_headers_parsed) do + for _, hname in ipairs(hdrs) do local value = task:get_header(hname) if value then @@ -719,7 +721,7 @@ end local function get_meta_stat_tokens(task, res, i) local day_and_hour = os.date('%u:%H', - task:get_date{format = 'message', gmt = true}) + task:get_date { format = 'message', gmt = true }) rawset(res, i, string.format("#dt:%s", day_and_hour)) lua_util.debugm("bayes", task, "added day_of_week token: %s", res[i]) @@ -737,7 +739,7 @@ local function get_meta_stat_tokens(task, res, i) local trace = task:get_symbol('DKIM_TRACE') local dkim_opts = trace[1]['options'] if dkim_opts then - for _,o in ipairs(dkim_opts) do + for _, o in ipairs(dkim_opts) do local check_res = string.sub(o, -1) local domain = string.sub(o, 1, -3) @@ -752,8 +754,7 @@ local function get_meta_stat_tokens(task, res, i) if aur then local spf = aur:match('spf=([a-z]+)') - local dkim,dkim_domain = aur:match('dkim=([a-z]+) header.d=([a-z.%-]+)') - + local dkim, dkim_domain = aur:match('dkim=([a-z]+) header.d=([a-z.%-]+)') if spf then table.insert(pol, 's=' .. spf) @@ -807,7 +808,7 @@ local function get_stat_tokens(task, cf) if cf.classify_images then local images = task:get_images() or E - for _,img in ipairs(images) do + for _, img in ipairs(images) do rawset(res, i, "image") i = i + 1 rawset(res, i, tostring(img:get_height())) @@ -838,10 +839,10 @@ local function get_stat_tokens(task, cf) end if cf.classify_urls then - local urls = lua_util.extract_specific_urls{task = task, limit = 5, esld_limit = 1} + local urls = lua_util.extract_specific_urls { task = task, limit = 5, esld_limit = 1 } if urls then - for _,u in ipairs(urls) do + for _, u in ipairs(urls) do rawset(res, i, string.format("#u:%s", u:get_tld())) lua_util.debugm("bayes", task, "added url token: %s", res[i]) diff --git a/lualib/lua_tcp_sync.lua b/lualib/lua_tcp_sync.lua index a85a3f974..f8e60446f 100644 --- a/lualib/lua_tcp_sync.lua +++ b/lualib/lua_tcp_sync.lua @@ -4,9 +4,9 @@ local lua_util = require "lua_util" local exports = {} local N = 'tcp_sync' -local tcp_sync = {_conn = nil, _data = '', _eof = false, _addr = ''} +local tcp_sync = { _conn = nil, _data = '', _eof = false, _addr = '' } local metatable = { - __tostring = function (self) + __tostring = function(self) return "class {tcp_sync connect to: " .. self._addr .. "}" end } @@ -66,7 +66,7 @@ end -- --]] function tcp_sync:read_until(pattern) - repeat + repeat local pos_start, pos_end = self._data:find(pattern, 1, true) if pos_start then local data = self._data:sub(1, pos_start - 1) @@ -196,7 +196,7 @@ function tcp_sync:shutdown() return self._conn:shutdown() end -exports.connect = function (args) +exports.connect = function(args) local is_ok, connection = rspamd_tcp.connect_sync(args) if not is_ok then return is_ok, connection diff --git a/lualib/lua_urls_compose.lua b/lualib/lua_urls_compose.lua index 5216721b9..11134215a 100644 --- a/lualib/lua_urls_compose.lua +++ b/lualib/lua_urls_compose.lua @@ -39,8 +39,8 @@ local function process_url(self, log_obj, url_tld, url_host) lua_util.debugm(N, log_obj, 'found compose tld for %s (host = %s)', url_tld, url_host) - for _,excl in ipairs(tld_elt.except_rules) do - local matched,ret = excl[2](url_tld, url_host) + for _, excl in ipairs(tld_elt.except_rules) do + local matched, ret = excl[2](url_tld, url_host) if matched then lua_util.debugm(N, log_obj, 'found compose exclusion for %s (%s) -> %s', url_host, excl[1], ret) @@ -55,7 +55,7 @@ local function process_url(self, log_obj, url_tld, url_host) if matches then local lua_pat_idx = math.huge - for m,_ in pairs(matches) do + for m, _ in pairs(matches) do if m < lua_pat_idx then lua_pat_idx = m end @@ -63,7 +63,7 @@ local function process_url(self, log_obj, url_tld, url_host) if #tld_elt.compose_rules >= lua_pat_idx then local lua_pat = tld_elt.compose_rules[lua_pat_idx] - local matched,ret = lua_pat[2](url_tld, url_host) + local matched, ret = lua_pat[2](url_tld, url_host) if not matched then lua_util.debugm(N, log_obj, 'NOT found compose inclusion for %s (%s) -> %s', @@ -85,8 +85,8 @@ local function process_url(self, log_obj, url_tld, url_host) end else -- Match one by one - for _,lua_pat in ipairs(tld_elt.compose_rules) do - local matched,ret = lua_pat[2](url_tld, url_host) + for _, lua_pat in ipairs(tld_elt.compose_rules) do + local matched, ret = lua_pat[2](url_tld, url_host) if matched then lua_util.debugm(N, log_obj, 'found compose inclusion for %s (%s) -> %s', url_host, lua_pat[1], ret) @@ -128,7 +128,7 @@ local function include_elt_gen(pat) return function(_, host) local matches = pat:search(host, false, true) if matches then - return true,matches[1][2] + return true, matches[1][2] end return false @@ -139,7 +139,7 @@ local function exclude_elt_gen(pat) pat = rspamd_regexp.create(tld_pattern_transform(pat)) return function(tld, host) if pat:search(host) then - return true,tld + return true, tld end return false @@ -150,22 +150,26 @@ local function compose_map_cb(self, map_text) local lpeg = require "lpeg" local singleline_comment = lpeg.P '#' * (1 - lpeg.S '\r\n\f') ^ 0 - local comments_strip_grammar = lpeg.C((1 - lpeg.P '#') ^ 1) * lpeg.S(' \t')^0 * singleline_comment^0 + local comments_strip_grammar = lpeg.C((1 - lpeg.P '#') ^ 1) * lpeg.S(' \t') ^ 0 * singleline_comment ^ 0 local function process_tld_rule(tld_elt, l) if l:sub(1, 1) == '!' then -- Exclusion elt - table.insert(tld_elt.except_rules, {l, exclude_elt_gen(l:sub(2))}) + table.insert(tld_elt.except_rules, { l, exclude_elt_gen(l:sub(2)) }) else - table.insert(tld_elt.compose_rules, {l, include_elt_gen(l)}) + table.insert(tld_elt.compose_rules, { l, include_elt_gen(l) }) end end local function process_map_line(l) -- Skip empty lines and comments - if #l == 0 then return end + if #l == 0 then + return + end l = comments_strip_grammar:match(l) - if not l or #l == 0 then return end + if not l or #l == 0 then + return + end -- Get TLD local tld = rspamd_util.get_tld(l) @@ -195,12 +199,12 @@ local function compose_map_cb(self, map_text) end local multipattern_threshold = 1 - for tld,tld_elt in pairs(self.tlds) do + for tld, tld_elt in pairs(self.tlds) do -- Sort patterns to have longest labels before shortest ones, -- so we can ensure that they match before table.sort(tld_elt.compose_rules, function(e1, e2) - local _,ndots1 = string.gsub(e1[1], '(%.)', '') - local _,ndots2 = string.gsub(e2[1], '(%.)', '') + local _, ndots1 = string.gsub(e1[1], '(%.)', '') + local _, ndots2 = string.gsub(e2[1], '(%.)', '') return ndots1 > ndots2 end) @@ -237,11 +241,13 @@ exports.add_composition_map = function(cfg, map_obj) tlds = {}, } - map = cfg:add_map{ + map = cfg:add_map { type = 'callback', description = 'URL compose map', url = map_obj, - callback = function(input) compose_map_cb(ret, input) end, + callback = function(input) + compose_map_cb(ret, input) + end, opaque_data = true, } diff --git a/lualib/lua_verdict.lua b/lualib/lua_verdict.lua index 8f3f23ea9..6ce99e660 100644 --- a/lualib/lua_verdict.lua +++ b/lualib/lua_verdict.lua @@ -32,7 +32,7 @@ local function default_verdict_function(task) if result then if result.passthrough then - return 'passthrough',nil + return 'passthrough', nil end local score = result.score @@ -40,21 +40,21 @@ local function default_verdict_function(task) local action = result.action if action == 'reject' and result.npositive > 1 then - return 'spam',score + return 'spam', score elseif action == 'no action' then if score < 0 or result.nnegative > 3 then - return 'ham',score + return 'ham', score end else -- All colors of junk if action == 'add header' or action == 'rewrite subject' then if result.npositive > 2 then - return 'junk',score + return 'junk', score end end end - return 'uncertain',score + return 'uncertain', score end end @@ -120,8 +120,8 @@ end exports.set_verdict_table = function(verdict_tbl, what) assert(type(verdict_tbl) == 'table' and - type(verdict_tbl.callback) == 'function' and - type(verdict_tbl.possible_verdicts) == 'table') + type(verdict_tbl.callback) == 'function' and + type(verdict_tbl.possible_verdicts) == 'table') if not what then -- Default verdict @@ -195,7 +195,7 @@ end exports.adjust_passthrough_action = function(task) local action = task:get_metric_action() if action == 'soft reject' then - local has_pr,_,_,module = task:has_pre_result() + local has_pr, _, _, module = task:has_pre_result() if has_pr and module then action = module diff --git a/lualib/plugins/dmarc.lua b/lualib/plugins/dmarc.lua index 24efb5078..6acf982c3 100644 --- a/lualib/plugins/dmarc.lua +++ b/lualib/plugins/dmarc.lua @@ -73,7 +73,7 @@ exports.default_settings = { -- Returns a key used to be inserted into dmarc report sample -exports.dmarc_report = function (task, settings, data) +exports.dmarc_report = function(task, settings, data) local rspamd_lua_utils = require "lua_util" local E = {} @@ -100,16 +100,15 @@ exports.dmarc_report = function (task, settings, data) local res = table.concat({ ip, data.spf_ok, data.dkim_ok, disposition_to_return, (data.sampled_out and 'sampled_out' or ''), data.domain, - dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, data.spf_domain, data.spf_result}, ',') + dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, data.spf_domain, data.spf_result }, ',') return res end - exports.gen_munging_callback = function(munging_opts, settings) local rspamd_util = require "rspamd_util" local lua_mime = require "lua_mime" - return function (task) + return function(task) if munging_opts.mitigate_allow_only then if not task:has_symbol(settings.symbols.allow) then lua_util.debugm(N, task, 'skip munging, no %s symbol', @@ -131,11 +130,11 @@ exports.gen_munging_callback = function(munging_opts, settings) end end if munging_opts.mitigate_strict_only then - local s = task:get_symbol(settings.symbols.allow) or {[1] = {}} + local s = task:get_symbol(settings.symbols.allow) or { [1] = {} } local sopts = s[1].options or {} local seen_strict - for _,o in ipairs(sopts) do + for _, o in ipairs(sopts) do if o == 'reject' or o == 'quarantine' then seen_strict = true break @@ -150,7 +149,7 @@ exports.gen_munging_callback = function(munging_opts, settings) end end if munging_opts.munge_map_condition then - local accepted,trace = munging_opts.munge_map_condition:process(task) + local accepted, trace = munging_opts.munge_map_condition:process(task) if not accepted then lua_util.debugm(task, 'skip munging, maps condition not satisfied: (%s)', trace) @@ -159,10 +158,10 @@ exports.gen_munging_callback = function(munging_opts, settings) end end -- Now, look for domain for munging - local mr = task:get_recipients({ 'mime', 'orig'}) + local mr = task:get_recipients({ 'mime', 'orig' }) local rcpt_found if mr then - for _,r in ipairs(mr) do + for _, r in ipairs(mr) do if r.domain and munging_opts.list_map:get_key(r.addr) then rcpt_found = r break @@ -176,7 +175,7 @@ exports.gen_munging_callback = function(munging_opts, settings) return end - local from = task:get_from({ 'mime', 'orig'}) + local from = task:get_from({ 'mime', 'orig' }) if not from or not from[1] then lua_util.debugm(task, 'skip munging, from is bad') @@ -204,7 +203,7 @@ exports.gen_munging_callback = function(munging_opts, settings) local add_hdrs = { ['From'] = { order = 1, value = hdr_encoded }, } - local remove_hdrs = {['From'] = 0} + local remove_hdrs = { ['From'] = 0 } local nreply = from.addr if munging_opts.reply_goes_to_list then @@ -222,9 +221,9 @@ exports.gen_munging_callback = function(munging_opts, settings) remove_hdrs['Reply-To'] = 1 end - add_hdrs['Reply-To'] = {order = 0, value = nreply} + add_hdrs['Reply-To'] = { order = 0, value = nreply } - add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded} + add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded } lua_mime.modify_headers(task, { remove = remove_hdrs, add = add_hdrs @@ -239,12 +238,12 @@ end local function gen_dmarc_grammar() local lpeg = require "lpeg" lpeg.locale(lpeg) - local space = lpeg.space^0 - local name = lpeg.C(lpeg.alpha^1) * space - local sep = space * (lpeg.S("\\;") * space) + (lpeg.space^1) - local value = lpeg.C(lpeg.P(lpeg.graph - sep)^1) - local pair = lpeg.Cg(name * "=" * space * value) * sep^-1 - local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset) + local space = lpeg.space ^ 0 + local name = lpeg.C(lpeg.alpha ^ 1) * space + local sep = space * (lpeg.S("\\;") * space) + (lpeg.space ^ 1) + local value = lpeg.C(lpeg.P(lpeg.graph - sep) ^ 1) + local pair = lpeg.Cg(name * "=" * space * value) * sep ^ -1 + local list = lpeg.Cf(lpeg.Ct("") * pair ^ 0, rawset) local version = lpeg.P("v") * space * lpeg.P("=") * space * lpeg.P("DMARC1") local record = version * sep * list @@ -297,7 +296,7 @@ local function dmarc_check_record(log_obj, record, is_tld) result.strict_dkim = true elseif dkim_pol ~= 'r' then failed_policy = 'adkim tag has invalid value: ' .. dkim_pol - return false,failed_policy + return false, failed_policy end end @@ -307,7 +306,7 @@ local function dmarc_check_record(log_obj, record, is_tld) result.strict_spf = true elseif spf_pol ~= 'r' then failed_policy = 'aspf tag has invalid value: ' .. spf_pol - return false,failed_policy + return false, failed_policy end end @@ -319,7 +318,7 @@ local function dmarc_check_record(log_obj, record, is_tld) result.dmarc_policy = 'quarantine' elseif (policy ~= 'none') then failed_policy = 'p tag has invalid value: ' .. policy - return false,failed_policy + return false, failed_policy end end @@ -336,7 +335,7 @@ local function dmarc_check_record(log_obj, record, is_tld) result.dmarc_policy = 'none' elseif (subdomain_policy ~= 'none') then failed_policy = 'sp tag has invalid value: ' .. subdomain_policy - return false,failed_policy + return false, failed_policy end end result.pct = elts['pct'] @@ -349,7 +348,7 @@ local function dmarc_check_record(log_obj, record, is_tld) end result.raw_elts = elts else - return false,false -- Ignore garbage + return false, false -- Ignore garbage end return true, result diff --git a/lualib/plugins/neural.lua b/lualib/plugins/neural.lua index 05dace489..6e88ef21c 100644 --- a/lualib/plugins/neural.lua +++ b/lualib/plugins/neural.lua @@ -96,7 +96,6 @@ local module_config = rspamd_config:get_all_opt(N) settings = lua_util.override_defaults(settings, module_config) local redis_params = lua_redis.parse_redis_server('neural') - local redis_lua_script_vectors_len = "neural_train_size.lua" local redis_lua_script_maybe_invalidate = "neural_maybe_invalidate.lua" local redis_lua_script_maybe_lock = "neural_maybe_lock.lua" @@ -106,17 +105,17 @@ local redis_script_id = {} local function load_scripts() redis_script_id.vectors_len = lua_redis.load_redis_script_from_file(redis_lua_script_vectors_len, - redis_params) + redis_params) redis_script_id.maybe_invalidate = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_invalidate, - redis_params) + redis_params) redis_script_id.maybe_lock = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_lock, - redis_params) + redis_params) redis_script_id.save_unlock = lua_redis.load_redis_script_from_file(redis_lua_script_save_unlock, - redis_params) + redis_params) end local function create_ann(n, nlayers, rule) - -- We ignore number of layers so far when using kann + -- We ignore number of layers so far when using kann local nhidden = math.floor(n * (rule.hidden_layer_mult or 1.0) + 1.0) local t = rspamd_kann.layer.input(n) t = rspamd_kann.transform.relu(t) @@ -146,7 +145,7 @@ local function learn_pca(inputs, max_inputs) -- scatter matrix is not filled with eigenvectors lua_util.debugm(N, 'eigenvalues: %s', eigenvals) local w = rspamd_tensor.new(2, max_inputs, #scatter_matrix[1]) - for i=1,max_inputs do + for i = 1, max_inputs do w[i] = scatter_matrix[#scatter_matrix - i + 1] end @@ -172,15 +171,19 @@ local function get_roc_thresholds(ann, inputs, outputs, alpha, beta) local a = {} local b = {} - for i=1,n do + for i = 1, n do r[i] = i end - local cmp = function(p, q) return p < q end + local cmp = function(p, q) + return p < q + end - table.sort(r, function(p, q) return cmp(x[p], x[q]) end) + table.sort(r, function(p, q) + return cmp(x[p], x[q]) + end) - for i=1,n do + for i = 1, n do a[i] = x[r[i]] b[i] = y[r[i]] end @@ -190,89 +193,89 @@ local function get_roc_thresholds(ann, inputs, outputs, alpha, beta) local function get_scores(nn, input_vectors) local scores = {} - for i=1,#inputs do + for i = 1, #inputs do local score = nn:apply1(input_vectors[i], nn.pca)[1] - scores[#scores+1] = score + scores[#scores + 1] = score end return scores end local fpr = {} - local fnr = {} - local scores = get_scores(ann, inputs) - - scores, outputs = sort_relative(scores, outputs) - - local n_samples = #outputs - local n_spam = 0 - local n_ham = 0 - local ham_count_ahead = {} - local spam_count_ahead = {} - local ham_count_behind = {} - local spam_count_behind = {} - - ham_count_ahead[n_samples + 1] = 0 - spam_count_ahead[n_samples + 1] = 0 - - for i=n_samples,1,-1 do - - if outputs[i][1] == 0 then - n_ham = n_ham + 1 - ham_count_ahead[i] = 1 - spam_count_ahead[i] = 0 - else - n_spam = n_spam + 1 - ham_count_ahead[i] = 0 - spam_count_ahead[i] = 1 - end - - ham_count_ahead[i] = ham_count_ahead[i] + ham_count_ahead[i + 1] - spam_count_ahead[i] = spam_count_ahead[i] + spam_count_ahead[i + 1] - end - - for i=1,n_samples do + local fnr = {} + local scores = get_scores(ann, inputs) + + scores, outputs = sort_relative(scores, outputs) + + local n_samples = #outputs + local n_spam = 0 + local n_ham = 0 + local ham_count_ahead = {} + local spam_count_ahead = {} + local ham_count_behind = {} + local spam_count_behind = {} + + ham_count_ahead[n_samples + 1] = 0 + spam_count_ahead[n_samples + 1] = 0 + + for i = n_samples, 1, -1 do + + if outputs[i][1] == 0 then + n_ham = n_ham + 1 + ham_count_ahead[i] = 1 + spam_count_ahead[i] = 0 + else + n_spam = n_spam + 1 + ham_count_ahead[i] = 0 + spam_count_ahead[i] = 1 + end + + ham_count_ahead[i] = ham_count_ahead[i] + ham_count_ahead[i + 1] + spam_count_ahead[i] = spam_count_ahead[i] + spam_count_ahead[i + 1] + end + + for i = 1, n_samples do if outputs[i][1] == 0 then - ham_count_behind[i] = 1 - spam_count_behind[i] = 0 - else - ham_count_behind[i] = 0 - spam_count_behind[i] = 1 - end - - if i ~= 1 then - ham_count_behind[i] = ham_count_behind[i] + ham_count_behind[i - 1] - spam_count_behind[i] = spam_count_behind[i] + spam_count_behind[i - 1] - end - end - - for i=1,n_samples do - fpr[i] = 0 - fnr[i] = 0 - - if (ham_count_ahead[i + 1] + ham_count_behind[i]) ~= 0 then - fpr[i] = ham_count_ahead[i + 1] / (ham_count_ahead[i + 1] + ham_count_behind[i]) - end - - if (spam_count_behind[i] + spam_count_ahead[i + 1]) ~= 0 then - fnr[i] = spam_count_behind[i] / (spam_count_behind[i] + spam_count_ahead[i + 1]) - end - end - - local p = n_spam / (n_spam + n_ham) - - local cost = {} - local min_cost_idx = 0 - local min_cost = math.huge - for i=1,n_samples do - cost[i] = ((1 - p) * alpha * fpr[i]) + (p * beta * fnr[i]) - if min_cost >= cost[i] then - min_cost = cost[i] - min_cost_idx = i - end - end - - return scores[min_cost_idx] + ham_count_behind[i] = 1 + spam_count_behind[i] = 0 + else + ham_count_behind[i] = 0 + spam_count_behind[i] = 1 + end + + if i ~= 1 then + ham_count_behind[i] = ham_count_behind[i] + ham_count_behind[i - 1] + spam_count_behind[i] = spam_count_behind[i] + spam_count_behind[i - 1] + end + end + + for i = 1, n_samples do + fpr[i] = 0 + fnr[i] = 0 + + if (ham_count_ahead[i + 1] + ham_count_behind[i]) ~= 0 then + fpr[i] = ham_count_ahead[i + 1] / (ham_count_ahead[i + 1] + ham_count_behind[i]) + end + + if (spam_count_behind[i] + spam_count_ahead[i + 1]) ~= 0 then + fnr[i] = spam_count_behind[i] / (spam_count_behind[i] + spam_count_ahead[i + 1]) + end + end + + local p = n_spam / (n_spam + n_ham) + + local cost = {} + local min_cost_idx = 0 + local min_cost = math.huge + for i = 1, n_samples do + cost[i] = ((1 - p) * alpha * fpr[i]) + (p * beta * fnr[i]) + if min_cost >= cost[i] then + min_cost = cost[i] + min_cost_idx = i + end + end + + return scores[min_cost_idx] end -- This function is intended to extend lock for ANN during training @@ -299,7 +302,7 @@ local function register_lock_extender(rule, set, ev_base, ann_key) true, -- is write redis_lock_extend_cb, --callback 'HINCRBY', -- command - {ann_key, 'lock', '30'} + { ann_key, 'lock', '30' } ) else lua_util.debugm(N, rspamd_config, "stop lock extension as learning_spawned is false") @@ -337,7 +340,8 @@ local function can_push_train_vector(rule, task, learn_type, nspam, nham) end end return true - else -- Enough learns + else + -- Enough learns rspamd_logger.infox(task, 'skip %s sample to keep spam/ham balance; too many spam samples: %s', learn_type, nspam) @@ -403,7 +407,7 @@ end -- Closure generator for unlock function local function gen_unlock_cb(rule, set, ann_key) - return function (err) + return function(err) if err then rspamd_logger.errx(rspamd_config, 'cannot unlock ANN %s:%s at %s from redis: %s', rule.prefix, set.name, ann_key, err) @@ -426,7 +430,7 @@ local function redis_ann_prefix(rule, settings_name) -- We also need to count metatokens: local n = meta_functions.version return string.format('%s%d_%s_%d_%s', - settings.prefix, plugin_ver, rule.prefix, n, settings_name) + settings.prefix, plugin_ver, rule.prefix, n, settings_name) end -- This function receives training vectors, checks them, spawn learning and saves ANN in Redis @@ -449,7 +453,7 @@ local function spawn_train(params) -- Used to show parsed vectors in a convenient format (for debugging only) local function debug_vec(t) local ret = {} - for i,v in ipairs(t) do + for i, v in ipairs(t) do if v ~= 0 then ret[#ret + 1] = string.format('%d=%.2f', i, v) end @@ -462,14 +466,14 @@ local function spawn_train(params) -- KANN automatically shuffles those samples -- 1.0 is used for spam and -1.0 is used for ham -- It implies that output layer can express that (e.g. tanh output) - for _,e in ipairs(params.spam_vec) do + for _, e in ipairs(params.spam_vec) do inputs[#inputs + 1] = e - outputs[#outputs + 1] = {1.0} + outputs[#outputs + 1] = { 1.0 } --rspamd_logger.debugm(N, rspamd_config, 'spam vector: %s', debug_vec(e)) end - for _,e in ipairs(params.ham_vec) do + for _, e in ipairs(params.ham_vec) do inputs[#inputs + 1] = e - outputs[#outputs + 1] = {-1.0} + outputs[#outputs + 1] = { -1.0 } --rspamd_logger.debugm(N, rspamd_config, 'ham vector: %s', debug_vec(e)) end @@ -486,7 +490,7 @@ local function spawn_train(params) rspamd_logger.errx(rspamd_config, 'ANN %s:%s: train error: observed nan in error cost!; value cost = %s', params.rule.prefix, params.set.name, value_cost) - for i,e in ipairs(inputs) do + for i, e in ipairs(inputs) do lua_util.debugm(N, rspamd_config, 'train vector %s -> %s', debug_vec(e), outputs[i][1]) end @@ -515,7 +519,7 @@ local function spawn_train(params) lua_util.debugm(N, rspamd_config, "start neural train for ANN %s:%s", params.rule.prefix, params.set.name) - local ret,err = pcall(train_ann.train1, train_ann, + local ret, err = pcall(train_ann.train1, train_ann, inputs, outputs, { lr = params.rule.train.learning_rate, max_epoch = params.rule.train.max_iterations, @@ -536,19 +540,19 @@ local function spawn_train(params) local roc_thresholds = {} if params.rule.roc_enabled then local spam_threshold = get_roc_thresholds(train_ann, - inputs, - outputs, - 1 - params.rule.roc_misclassification_cost, - params.rule.roc_misclassification_cost) + inputs, + outputs, + 1 - params.rule.roc_misclassification_cost, + params.rule.roc_misclassification_cost) local ham_threshold = get_roc_thresholds(train_ann, - inputs, - outputs, - params.rule.roc_misclassification_cost, - 1 - params.rule.roc_misclassification_cost) - roc_thresholds = {spam_threshold, ham_threshold} + inputs, + outputs, + params.rule.roc_misclassification_cost, + 1 - params.rule.roc_misclassification_cost) + roc_thresholds = { spam_threshold, ham_threshold } rspamd_logger.messagex("ROC thresholds: (spam_threshold: %s, ham_threshold: %s)", - roc_thresholds[1], roc_thresholds[2]) + roc_thresholds[1], roc_thresholds[2]) end if not seen_nan then @@ -585,7 +589,7 @@ local function spawn_train(params) false, -- is write gen_unlock_cb(params.rule, params.set, params.ann_key), --callback 'HDEL', -- command - {params.ann_key, 'lock'} + { params.ann_key, 'lock' } ) else rspamd_logger.infox(rspamd_config, 'saved ANN %s:%s to redis: %s', @@ -605,7 +609,7 @@ local function spawn_train(params) true, -- is write gen_unlock_cb(params.rule, params.set, params.ann_key), --callback 'HDEL', -- command - {params.ann_key, 'lock'} + { params.ann_key, 'lock' } ) else local parser = ucl.parser() @@ -653,17 +657,17 @@ local function spawn_train(params) params.set.ann.redis_key, params.ann_key) lua_redis.exec_redis_script(redis_script_id.save_unlock, - {ev_base = params.ev_base, is_write = true}, + { ev_base = params.ev_base, is_write = true }, redis_save_cb, - {profile.redis_key, - redis_ann_prefix(params.rule, params.set.name), - ann_data, - profile_serialized, - tostring(params.rule.ann_expire), - tostring(os.time()), - params.ann_key, -- old key to unlock... - roc_thresholds_serialized, - pca_data, + { profile.redis_key, + redis_ann_prefix(params.rule, params.set.name), + ann_data, + profile_serialized, + tostring(params.rule.ann_expire), + tostring(os.time()), + params.ann_key, -- old key to unlock... + roc_thresholds_serialized, + pca_data, }) end end @@ -672,7 +676,7 @@ local function spawn_train(params) fill_set_ann(params.set, params.ann_key) end - params.worker:spawn_process{ + params.worker:spawn_process { func = train, on_complete = ann_trained, proctitle = string.format("ANN train for %s/%s", params.rule.prefix, params.set.name), @@ -695,7 +699,9 @@ local function process_rules_settings() -- Ensure that we have an array... lua_util.debugm(N, rspamd_config, "use static profile for %s (%s): %s", rule.prefix, selt.name, profile) - if not profile[1] then profile = lua_util.keys(profile) end + if not profile[1] then + profile = lua_util.keys(profile) + end selt.symbols = profile else lua_util.debugm(N, rspamd_config, "use dynamic cfg based profile for %s (%s)", @@ -758,7 +764,7 @@ local function process_rules_settings() }) end - for k,rule in pairs(settings.rules) do + for k, rule in pairs(settings.rules) do if not rule.allowed_settings then rule.allowed_settings = {} elseif rule.allowed_settings == 'all' then @@ -788,7 +794,7 @@ local function process_rules_settings() -- Now, for each allowed settings, we store sorted symbols + digest -- We set table rule.settings[id] -> { name = name, symbols = symbols, digest = digest } - for s,_ in pairs(rule.allowed_settings) do + for s, _ in pairs(rule.allowed_settings) do -- Here, we have a name, set of symbols and local settings_id = s if type(settings_id) ~= 'number' then @@ -802,7 +808,7 @@ local function process_rules_settings() } process_settings_elt(rule, nelt) - for id,ex in pairs(rule.settings) do + for id, ex in pairs(rule.settings) do if type(ex) == 'table' then if nelt and lua_util.distance_sorted(ex.symbols, nelt.symbols) == 0 then -- Equal symbols, add reference @@ -829,7 +835,9 @@ local function get_rule_settings(task, rule) local sid = task:get_settings_id() or -1 local set = rule.settings[sid] - if not set then return nil end + if not set then + return nil + end while type(set) == 'number' do -- Reference to another settings! @@ -843,10 +851,10 @@ local function result_to_vector(task, profile) if not profile.zeros then -- Fill zeros vector local zeros = {} - for i=1,meta_functions.count_metatokens() do + for i = 1, meta_functions.count_metatokens() do zeros[i] = 0.0 end - for _,_ in ipairs(profile.symbols) do + for _, _ in ipairs(profile.symbols) do zeros[#zeros + 1] = 0.0 end profile.zeros = zeros @@ -855,7 +863,7 @@ local function result_to_vector(task, profile) local vec = lua_util.shallowcopy(profile.zeros) local mt = meta_functions.rspamd_gen_metatokens(task) - for i,v in ipairs(mt) do + for i, v in ipairs(mt) do vec[i] = v end diff --git a/lualib/plugins/rbl.lua b/lualib/plugins/rbl.lua index baa019f41..bff53f9ba 100644 --- a/lualib/plugins/rbl.lua +++ b/lualib/plugins/rbl.lua @@ -21,24 +21,24 @@ local lua_util = require "lua_util" -- Common RBL plugin definitions local check_types = { - from = { - connfilter = true, - }, - received = {}, - helo = { - connfilter = true, - }, - urls = {}, - content_urls = {}, - emails = {}, - replyto = {}, - dkim = {}, - rdns = { - connfilter = true, - }, - selector = { - require_argument = true, - }, + from = { + connfilter = true, + }, + received = {}, + helo = { + connfilter = true, + }, + urls = {}, + content_urls = {}, + emails = {}, + replyto = {}, + dkim = {}, + rdns = { + connfilter = true, + }, + selector = { + require_argument = true, + }, } local default_options = { @@ -93,8 +93,8 @@ local rule_schema_tbl = { exclude_private_ips = ts.boolean:is_optional(), exclude_users = ts.boolean:is_optional(), from = ts.boolean:is_optional(), - hash = ts.one_of{"sha1", "sha256", "sha384", "sha512", "md5", "blake2"}:is_optional(), - hash_format = ts.one_of{"hex", "base32", "base64"}:is_optional(), + hash = ts.one_of { "sha1", "sha256", "sha384", "sha512", "md5", "blake2" }:is_optional(), + hash_format = ts.one_of { "hex", "base32", "base64" }:is_optional(), hash_len = (ts.integer + ts.string / tonumber):is_optional(), helo = ts.boolean:is_optional(), ignore_default = ts.boolean:is_optional(), -- alias @@ -120,14 +120,16 @@ local rule_schema_tbl = { replyto = ts.boolean:is_optional(), requests_limit = (ts.integer + ts.string / tonumber):is_optional(), require_symbols = ( - ts.array_of(ts.string) + (ts.string / function(s) return {s} end) + ts.array_of(ts.string) + (ts.string / function(s) + return { s } + end) ):is_optional(), resolve_ip = ts.boolean:is_optional(), return_bits = return_bits_schema:is_optional(), return_codes = return_codes_schema:is_optional(), returnbits = return_bits_schema:is_optional(), returncodes = return_codes_schema:is_optional(), - selector = ts.one_of{ts.string, ts.table}:is_optional(), + selector = ts.one_of { ts.string, ts.table }:is_optional(), selector_flatten = ts.boolean:is_optional(), symbol = ts.string:is_optional(), symbols_prefixes = ts.map_of(ts.string, ts.string):is_optional(), @@ -137,7 +139,9 @@ local rule_schema_tbl = { urls = ts.boolean:is_optional(), whitelist = lua_maps.map_schema:is_optional(), whitelist_exception = ( - ts.array_of(ts.string) + (ts.string / function(s) return {s} end) + ts.array_of(ts.string) + (ts.string / function(s) + return { s } + end) ):is_optional(), checks = ts.array_of(ts.one_of(lua_util.keys(check_types))):is_optional(), exclude_checks = ts.array_of(ts.one_of(lua_util.keys(check_types))):is_optional(), @@ -148,13 +152,13 @@ local function convert_checks(rule) if rule.checks then local all_connfilter = true local exclude_checks = lua_util.list_to_hash(rule.exclude_checks or {}) - for _,check in ipairs(rule.checks) do + for _, check in ipairs(rule.checks) do if not exclude_checks[check] then local check_type = check_types[check] if check_type.require_argument then if not rule[check] then rspamd_logger.errx(rspamd_config, 'rbl rule %s has check %s which requires an argument', - rule.symbol, check) + rule.symbol, check) return nil end end @@ -167,12 +171,12 @@ local function convert_checks(rule) if not check_type then rspamd_logger.errx(rspamd_config, 'rbl rule %s has invalid check type: %s', - rule.symbol, check) + rule.symbol, check) return nil end else rspamd_logger.infox(rspamd_config, 'disable check %s in %s: excluded explicitly', - check, rule.symbol) + check, rule.symbol) end end rule.connfilter = all_connfilter @@ -180,7 +184,7 @@ local function convert_checks(rule) -- Now check if we have any check enabled at all local check_found = false - for k,_ in pairs(check_types) do + for k, _ in pairs(check_types) do if type(rule[k]) ~= 'nil' then check_found = true break @@ -199,7 +203,7 @@ end -- Add default boolean flags to the schema -for def_k,_ in pairs(default_options) do +for def_k, _ in pairs(default_options) do rule_schema_tbl[def_k:sub(#('default_') + 1)] = ts.boolean:is_optional() end diff --git a/lualib/redis_scripts/neural_maybe_invalidate.lua b/lualib/redis_scripts/neural_maybe_invalidate.lua index c54871717..517fa019d 100644 --- a/lualib/redis_scripts/neural_maybe_invalidate.lua +++ b/lualib/redis_scripts/neural_maybe_invalidate.lua @@ -8,7 +8,7 @@ local lim = tonumber(KEYS[2]) if card > lim then local to_delete = redis.call('ZRANGE', KEYS[1], 0, card - lim - 1) if to_delete then - for _,k in ipairs(to_delete) do + for _, k in ipairs(to_delete) do local tb = cjson.decode(k) if type(tb) == 'table' and type(tb.redis_key) == 'string' then redis.call('DEL', tb.redis_key) diff --git a/lualib/redis_scripts/neural_maybe_lock.lua b/lualib/redis_scripts/neural_maybe_lock.lua index 7b5c6a60f..f705115b0 100644 --- a/lualib/redis_scripts/neural_maybe_lock.lua +++ b/lualib/redis_scripts/neural_maybe_lock.lua @@ -11,7 +11,7 @@ if locked then locked = tonumber(locked) local expire = tonumber(KEYS[3]) if now > locked and (now - locked) < expire then - return {tostring(locked), redis.call('HGET', KEYS[1], 'hostname') or 'unknown'} + return { tostring(locked), redis.call('HGET', KEYS[1], 'hostname') or 'unknown' } end end redis.call('HSET', KEYS[1], 'lock', tostring(now)) diff --git a/lualib/redis_scripts/neural_train_size.lua b/lualib/redis_scripts/neural_train_size.lua index 5a00ae3fc..45ad6a931 100644 --- a/lualib/redis_scripts/neural_train_size.lua +++ b/lualib/redis_scripts/neural_train_size.lua @@ -13,8 +13,12 @@ local nspam = 0 local nham = 0 local ret = redis.call('SCARD', prefix .. '_spam_set') -if ret then nspam = tonumber(ret) end +if ret then + nspam = tonumber(ret) +end ret = redis.call('SCARD', prefix .. '_ham_set') -if ret then nham = tonumber(ret) end +if ret then + nham = tonumber(ret) +end -return {nspam,nham}
\ No newline at end of file +return { nspam, nham }
\ No newline at end of file diff --git a/lualib/redis_scripts/ratelimit_check.lua b/lualib/redis_scripts/ratelimit_check.lua index 1c2b32a69..d39cdf148 100644 --- a/lualib/redis_scripts/ratelimit_check.lua +++ b/lualib/redis_scripts/ratelimit_check.lua @@ -34,36 +34,46 @@ if not last then -- New bucket redis.call('HMSET', prefix, 'l', tostring(now), 'b', '0', 'dr', '10000', 'db', '10000', 'p', tostring(nrcpt)) redis.call('EXPIRE', prefix, KEYS[5]) - return {0, '0', '1', '1', '0'} + return { 0, '0', '1', '1', '0' } end last = tonumber(last) -local burst,pending = unpack(redis.call('HMGET', prefix, 'b', 'p')) -burst,pending = tonumber(burst or '0'),tonumber(pending or '0') +local burst, pending = unpack(redis.call('HMGET', prefix, 'b', 'p')) +burst, pending = tonumber(burst or '0'), tonumber(pending or '0') -- Sanity to avoid races -if burst < 0 then burst = 0 end -if pending < 0 then pending = 0 end +if burst < 0 then + burst = 0 +end +if pending < 0 then + pending = 0 +end pending = pending + nrcpt -- this message -- Perform leak if burst + pending > 0 then -- If we have any time passed if burst > 0 and last < now then dynr = tonumber(redis.call('HGET', prefix, 'dr')) / 10000.0 - if dynr == 0 then dynr = 0.0001 end + if dynr == 0 then + dynr = 0.0001 + end leak_rate = leak_rate * dynr leaked = ((now - last) * leak_rate) - if leaked > burst then leaked = burst end + if leaked > burst then + leaked = burst + end burst = burst - leaked redis.call('HINCRBYFLOAT', prefix, 'b', -(leaked)) redis.call('HSET', prefix, 'l', tostring(now)) end dynb = tonumber(redis.call('HGET', prefix, 'db')) / 10000.0 - if dynb == 0 then dynb = 0.0001 end + if dynb == 0 then + dynb = 0.0001 + end burst = burst + pending if burst > 0 and burst > max_burst * dynb then - return {1, tostring(burst - pending), tostring(dynr), tostring(dynb), tostring(leaked)} + return { 1, tostring(burst - pending), tostring(dynr), tostring(dynb), tostring(leaked) } end -- Increase pending if we allow ratelimit redis.call('HINCRBY', prefix, 'p', nrcpt) @@ -72,4 +82,4 @@ else redis.call('HMSET', prefix, 'b', '0', 'p', tostring(nrcpt)) end -return {0, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked)}
\ No newline at end of file +return { 0, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked) }
\ No newline at end of file diff --git a/lualib/redis_scripts/ratelimit_cleanup_pending.lua b/lualib/redis_scripts/ratelimit_cleanup_pending.lua index 67ae634f1..698a3ec19 100644 --- a/lualib/redis_scripts/ratelimit_cleanup_pending.lua +++ b/lualib/redis_scripts/ratelimit_cleanup_pending.lua @@ -19,7 +19,11 @@ end -- 2. Update the pending values based on the number of recipients (requests) local pending = redis.call('HGET', prefix, 'p') pending = tonumber(pending or '0') -if pending < nrcpt then pending = 0 else pending = pending - nrcpt end +if pending < nrcpt then + pending = 0 +else + pending = pending - nrcpt +end -- 3. Set the updated values back to Redis and update the expiration time for the bucket redis.call('HMSET', prefix, 'p', tostring(pending), 'l', KEYS[2]) diff --git a/lualib/redis_scripts/ratelimit_update.lua b/lualib/redis_scripts/ratelimit_update.lua index f08a250ea..caee8fb31 100644 --- a/lualib/redis_scripts/ratelimit_update.lua +++ b/lualib/redis_scripts/ratelimit_update.lua @@ -19,7 +19,7 @@ if not last then -- 2. Initialize a new bucket if the last hit time is not found (must not happen) redis.call('HMSET', prefix, 'l', tostring(now), 'b', tostring(nrcpt), 'dr', '10000', 'db', '10000', 'p', '0') redis.call('EXPIRE', prefix, KEYS[7]) - return {1, 1, 1} + return { 1, 1, 1 } end -- 3. Update the dynamic rate multiplier based on input parameters @@ -72,14 +72,22 @@ if max_db > 1 then end -- 5. Update the burst and pending values based on the number of recipients (requests) -local burst,pending = unpack(redis.call('HMGET', prefix, 'b', 'p')) -burst,pending = tonumber(burst or '0'),tonumber(pending or '0') -if burst < 0 then burst = nrcpt else burst = burst + nrcpt end -if pending < nrcpt then pending = 0 else pending = pending - nrcpt end +local burst, pending = unpack(redis.call('HMGET', prefix, 'b', 'p')) +burst, pending = tonumber(burst or '0'), tonumber(pending or '0') +if burst < 0 then + burst = nrcpt +else + burst = burst + nrcpt +end +if pending < nrcpt then + pending = 0 +else + pending = pending - nrcpt +end -- 6. Set the updated values back to Redis and update the expiration time for the bucket redis.call('HMSET', prefix, 'b', tostring(burst), 'p', tostring(pending), 'l', KEYS[2]) redis.call('EXPIRE', prefix, KEYS[7]) -- 7. Return the updated burst value, dynamic rate multiplier, and dynamic burst multiplier -return {tostring(burst), tostring(dr), tostring(db)}
\ No newline at end of file +return { tostring(burst), tostring(dr), tostring(db) }
\ No newline at end of file diff --git a/lualib/rescore_utility.lua b/lualib/rescore_utility.lua index 195ae4364..01e5aabcb 100644 --- a/lualib/rescore_utility.lua +++ b/lualib/rescore_utility.lua @@ -11,7 +11,7 @@ function utility.get_all_symbols(logs, ignore_symbols) for _, line in pairs(logs) do line = lua_util.rspamd_str_split(line, " ") - for i=4,(#line-1) do + for i = 4, (#line - 1) do line[i] = line[i]:gsub("%s+", "") if not symbols_set[line[i]] then symbols_set[line[i]] = true @@ -41,57 +41,57 @@ function utility.read_log_file(file) local fname = string.gsub(file, "(.*/)(.*)", "%2") for line in fd:lines() do - local start,stop = string.find(line, fname .. ':') + local start, stop = string.find(line, fname .. ':') if start and stop then table.insert(lines, string.sub(line, 1, start)) table.insert(messages, string.sub(line, stop + 1, -1)) end -end + end io.close(fd) - return lines,messages + return lines, messages end function utility.get_all_logs(dirs) -- Reads all log files in the directory and returns a list of logs. if type(dirs) == 'string' then - dirs = {dirs} + dirs = { dirs } end local all_logs = {} local all_messages = {} - for _,dir in ipairs(dirs) do + for _, dir in ipairs(dirs) do if dir:sub(-1, -1) == "/" then dir = dir:sub(1, -2) local files = rspamd_util.glob(dir .. "/*.log") for _, file in pairs(files) do - local logs,messages = utility.read_log_file(file) - for i=1,#logs do + local logs, messages = utility.read_log_file(file) + for i = 1, #logs do table.insert(all_logs, logs[i]) table.insert(all_messages, messages[i]) end end else - local logs,messages = utility.read_log_file(dir) - for i=1,#logs do + local logs, messages = utility.read_log_file(dir) + for i = 1, #logs do table.insert(all_logs, logs[i]) table.insert(all_messages, messages[i]) end end end - return all_logs,all_messages + return all_logs, all_messages end function utility.get_all_symbol_scores(conf, ignore_symbols) local symbols = conf:get_symbols_scores() return fun.tomap(fun.map(function(name, elt) - return name,elt['score'] + return name, elt['score'] end, fun.filter(function(name, elt) return not ignore_symbols[name] end, symbols))) @@ -158,7 +158,7 @@ function utility.generate_statistics_from_logs(logs, messages, threshold) true_negatives = true_negatives + 1 end - for j=4, (#log-1) do + for j = 4, (#log - 1) do if all_symbols_stats[log[j]] == nil then all_symbols_stats[log[j]] = { name = message, @@ -180,8 +180,8 @@ function utility.generate_statistics_from_logs(logs, messages, threshold) -- Find slowest message if ((tonumber(log[#log]) or 0) > file_stats.slowest) then - file_stats.slowest = tonumber(log[#log]) - file_stats.slowest_file = message + file_stats.slowest = tonumber(log[#log]) + file_stats.slowest_file = message end end end diff --git a/lualib/rspamadm/clickhouse.lua b/lualib/rspamadm/clickhouse.lua index 6bdc612d8..b22d8007c 100644 --- a/lualib/rspamadm/clickhouse.lua +++ b/lualib/rspamadm/clickhouse.lua @@ -66,62 +66,62 @@ parser:flag '--use-https' :argname('use_https') local neural_profile = parser:command 'neural_profile' - :description 'Generate symbols profile using data from Clickhouse' + :description 'Generate symbols profile using data from Clickhouse' neural_profile:option '-w --where' - :description 'WHERE clause for Clickhouse query' - :argname('where') + :description 'WHERE clause for Clickhouse query' + :argname('where') neural_profile:flag '-j --json' - :description 'Write output as JSON' - :argname('json') + :description 'Write output as JSON' + :argname('json') neural_profile:option '--days' - :description 'Number of days to collect stats for' - :argname('days') - :default('7') + :description 'Number of days to collect stats for' + :argname('days') + :default('7') neural_profile:option '--limit -l' - :description 'Maximum rows to fetch per day' - :argname('limit') + :description 'Maximum rows to fetch per day' + :argname('limit') neural_profile:option '--settings-id' - :description 'Settings ID to query' - :argname('settings_id') - :default('') + :description 'Settings ID to query' + :argname('settings_id') + :default('') local neural_train = parser:command 'neural_train' - :description 'Train neural using data from Clickhouse' + :description 'Train neural using data from Clickhouse' neural_train:option '--days' - :description 'Number of days to query data for' - :argname('days') - :default('7') + :description 'Number of days to query data for' + :argname('days') + :default('7') neural_train:option '--column-name-digest' - :description 'Name of neural profile digest column in Clickhouse' - :argname('column_name_digest') - :default('NeuralDigest') + :description 'Name of neural profile digest column in Clickhouse' + :argname('column_name_digest') + :default('NeuralDigest') neural_train:option '--column-name-vector' - :description 'Name of neural training vector column in Clickhouse' - :argname('column_name_vector') - :default('NeuralMpack') + :description 'Name of neural training vector column in Clickhouse' + :argname('column_name_vector') + :default('NeuralMpack') neural_train:option '--limit -l' - :description 'Maximum rows to fetch per day' - :argname('limit') + :description 'Maximum rows to fetch per day' + :argname('limit') neural_train:option '--profile -p' - :description 'Profile to use for training' - :argname('profile') - :default('default') + :description 'Profile to use for training' + :argname('profile') + :default('default') neural_train:option '--rule -r' - :description 'Rule to train' - :argname('rule') - :default('default') + :description 'Rule to train' + :argname('rule') + :default('default') neural_train:option '--spam -s' - :description 'WHERE clause to use for spam' - :argname('spam') - :default("Action == 'reject'") + :description 'WHERE clause to use for spam' + :argname('spam') + :default("Action == 'reject'") neural_train:option '--ham -h' - :description 'WHERE clause to use for ham' - :argname('ham') - :default('Score < 0') + :description 'WHERE clause to use for ham' + :argname('ham') + :default('Score < 0') neural_train:option '--url -u' - :description 'URL to use for training' - :argname('url') - :default('http://127.0.0.1:11334/plugins/neural/learn') + :description 'URL to use for training' + :argname('url') + :default('http://127.0.0.1:11334/plugins/neural/learn') local http_params = { config = rspamd_config, @@ -131,14 +131,14 @@ local http_params = { } local function load_config(config_file) - local _r,err = rspamd_config:load_ucl(config_file) + local _r, err = rspamd_config:load_ucl(config_file) if not _r then rspamd_logger.errx('cannot load %s: %s', config_file, err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', config_file, err) os.exit(1) @@ -196,7 +196,7 @@ local function get_excluded_symbols(known_symbols, correlations, seen_total) elseif not all_symbols[k] then remove[k] = 'nonexistent symbol' else - for fl,_ in pairs(all_symbols[k].flags or {}) do + for fl, _ in pairs(all_symbols[k].flags or {}) do if skip_flags[fl] then remove[k] = fl .. ' symbol' break @@ -238,7 +238,7 @@ local function handle_neural_profile(args) local nsym = #r['Symbols.Names'] - for i = 1,nsym do + for i = 1, nsym do local sym = r['Symbols.Names'][i] local t = known_symbols[sym] if not t then @@ -266,8 +266,8 @@ local function handle_neural_profile(args) end -- Fill correlations - for i = 1,nsym do - for j = 1,nsym do + for i = 1, nsym do + for j = 1, nsym do if i ~= j then local sym = r['Symbols.Names'][i] local inner_sym_name = r['Symbols.Names'][j] @@ -342,11 +342,11 @@ end local function post_neural_training(url, rule, spam_rows, ham_rows) -- Prepare JSON payload local payload = ucl.to_format( - { - ham_vec = ham_rows, - rule = rule, - spam_vec = spam_rows, - }, 'json') + { + ham_vec = ham_rows, + rule = rule, + spam_vec = spam_rows, + }, 'json') -- POST the payload local err, response = rspamd_http.request({ @@ -423,11 +423,11 @@ local function handle_neural_train(args) limit = string.format(' LIMIT %d', num_limit) -- Contains leading space end -- Prepare query elements - local conditions = {string.format("%s = '%s'", args.column_name_digest, symbols_digest)} + local conditions = { string.format("%s = '%s'", args.column_name_digest, symbols_digest) } local query_fmt = 'SELECT %s FROM rspamd WHERE %s%s' -- Run queries - for _, the_where in ipairs({args.ham, args.spam}) do + for _, the_where in ipairs({ args.ham, args.spam }) do -- Inform callback which group of vectors we're collecting this_where = the_where table.insert(conditions, the_where) -- should be 2nd from last condition @@ -437,7 +437,7 @@ local function handle_neural_train(args) if this_where == args.ham then if not want_ham then break - end + end else if not want_spam then break diff --git a/lualib/rspamadm/configgraph.lua b/lualib/rspamadm/configgraph.lua index 75eb2d940..07f14a9ca 100644 --- a/lualib/rspamadm/configgraph.lua +++ b/lualib/rspamadm/configgraph.lua @@ -31,7 +31,6 @@ parser:option "-c --config" parser:flag "-a --all" :description('Show all nodes, not just existing ones') - local function process_filename(fname) local cdir = rspamd_paths['CONFDIR'] .. '/' fname = fname:gsub(cdir, '') @@ -40,8 +39,8 @@ end local function output_dot(opts, nodes, adjacency) rspamd_logger.messagex("digraph rspamd {") - for k,node in pairs(nodes) do - local attrs = {"shape=box"} + for k, node in pairs(nodes) do + local attrs = { "shape=box" } local skip = false if node.exists then if node.priority >= 10 then @@ -62,7 +61,7 @@ local function output_dot(opts, nodes, adjacency) table.concat(attrs, ',')) end end - for _,adj in ipairs(adjacency) do + for _, adj in ipairs(adjacency) do local attrs = {} local skip = false @@ -95,7 +94,7 @@ local function load_config_traced(opts) local nodes = {} local function maybe_match_glob(file) - for _,gl in ipairs(glob_traces) do + for _, gl in ipairs(glob_traces) do if gl.re:match(file) then return gl end @@ -151,7 +150,7 @@ local function load_config_traced(opts) end end - local _r,err = rspamd_config:load_ucl(opts['config'], trace_func) + local _r, err = rspamd_config:load_ucl(opts['config'], trace_func) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) @@ -160,7 +159,6 @@ local function load_config_traced(opts) output_dot(opts, nodes, adjacency) end - local function handler(args) local res = parser:parse(args) diff --git a/lualib/rspamadm/confighelp.lua b/lualib/rspamadm/confighelp.lua index 1eecb9238..38b26b6fc 100644 --- a/lualib/rspamadm/confighelp.lua +++ b/lualib/rspamadm/confighelp.lua @@ -16,11 +16,11 @@ local parser = argparse() parser:argument "path":args "*" :description('Optional config paths') parser:flag "--no-color" - :description "Disable coloured output" + :description "Disable coloured output" parser:flag "--short" - :description "Show only option names" + :description "Show only option names" parser:flag "--no-examples" - :description "Do not show examples (implied by --short)" + :description "Do not show examples (implied by --short)" local function maybe_print_color(key) if not opts['no-color'] then @@ -84,10 +84,10 @@ local function print_help(key, value, tabs) if type(value['required']) == 'boolean' then if value['required'] then print(string.format('%s\tRequired: %s', tabs, - maybe_print_color(tostring(value['required'])))) + maybe_print_color(tostring(value['required'])))) else print(string.format('%s\tRequired: %s', tabs, - tostring(value['required']))) + tostring(value['required']))) end end if value['default'] then @@ -116,7 +116,7 @@ return function(args, res) local sorted = sort_values(res) - for _,v in ipairs(sorted) do + for _, v in ipairs(sorted) do print_help(v['key'], v['value'], '') print('') end diff --git a/lualib/rspamadm/configwizard.lua b/lualib/rspamadm/configwizard.lua index c79d73b97..27c358e3b 100644 --- a/lualib/rspamadm/configwizard.lua +++ b/lualib/rspamadm/configwizard.lua @@ -74,19 +74,29 @@ local function ask_yes_no(greet, default) local reply = rspamd_util.readline(greet) - if not reply then os.exit(0) end - if #reply == 0 then reply = def_str end + if not reply then + os.exit(0) + end + if #reply == 0 then + reply = def_str + end reply = reply:lower() - if reply == 'y' or reply == 'yes' then return true end + if reply == 'y' or reply == 'yes' then + return true + end return false end local function readline_default(greet, def_value) local reply = rspamd_util.readline(greet) - if not reply then os.exit(0) end + if not reply then + os.exit(0) + end - if #reply == 0 then return def_value end + if #reply == 0 then + return def_value + end return reply end @@ -95,7 +105,7 @@ local function readline_expire() local expire = '100d' repeat expire = readline_default("Expire time for new tokens [" .. expire .. "]: ", - expire) + expire) expire = lua_util.parse_time_interval(expire) if not expire then @@ -118,9 +128,9 @@ end local function print_changes(changes) local function print_change(k, c, where) printf('File: %s, changes list:', highlight(local_conf .. '/' - .. where .. '/'.. k)) + .. where .. '/' .. k)) - for ek,ev in pairs(c) do + for ek, ev in pairs(c) do printf("%s => %s", highlight(ek), rspamd_logger.slog("%s", ev)) end end @@ -144,7 +154,7 @@ local function apply_changes(changes) end local function apply_change(k, c, where) - local fname = local_conf .. '/' .. where .. '/'.. k + local fname = local_conf .. '/' .. where .. '/' .. k if not rspamd_util.file_exists(fname) then printf("Create file %s", highlight(fname)) @@ -181,7 +191,6 @@ local function apply_changes(changes) end end - local function setup_controller(controller, changes) printf("Setup %s and controller worker:", highlight("WebUI")) @@ -208,13 +217,13 @@ local function setup_redis(cfg, changes) printf("%s servers are not set:", highlight("Redis")) printf("The following modules will be enabled if you add Redis servers:") - for k,_ in pairs(rspamd_plugins_state.disabled_redis) do + for k, _ in pairs(rspamd_plugins_state.disabled_redis) do printf("\t* %s", highlight(k)) end if ask_yes_no("Do you wish to set Redis servers?", true) then local read_servers = readline_default("Input read only servers separated by `,` [default: localhost]: ", - "localhost") + "localhost") local rs = parse_servers(read_servers) if rs and #rs > 0 then @@ -263,7 +272,9 @@ end local function setup_dkim_signing(cfg, changes) -- Remove the trailing slash of a pathname, if present. local function remove_trailing_slash(path) - if string.sub(path, -1) ~= "/" then return path end + if string.sub(path, -1) ~= "/" then + return path + end return string.sub(path, 1, string.len(path) - 1) end @@ -281,7 +292,7 @@ local function setup_dkim_signing(cfg, changes) local use_esld local sign_domain = 'pet luacheck' - local defined_auth_types = {'header', 'envelope', 'auth', 'recipient'} + local defined_auth_types = { 'header', 'envelope', 'auth', 'recipient' } if sign_type == '4' then repeat @@ -318,7 +329,9 @@ local function setup_dkim_signing(cfg, changes) sign_authenticated = true end - if fun.any(function(s) return s == sign_domain end, defined_auth_types) then + if fun.any(function(s) + return s == sign_domain + end, defined_auth_types) then -- Allow mismatch allow_mismatch = ask_yes_no( string.format('Allow data %s, e.g. if mime from domain is not equal to authenticated user domain? ', @@ -337,7 +350,7 @@ local function setup_dkim_signing(cfg, changes) local dkim_keys_dir = rspamd_paths["DBDIR"] .. "/dkim/" local prompt = string.format("Enter output directory for the keys [default: %s]: ", - highlight(dkim_keys_dir)) + highlight(dkim_keys_dir)) dkim_keys_dir = remove_trailing_slash(readline_default(prompt, dkim_keys_dir)) local ret, err = rspamd_util.mkdir(dkim_keys_dir, true) @@ -349,7 +362,7 @@ local function setup_dkim_signing(cfg, changes) local function print_domains() printf("Domains configured:") - for k,v in pairs(domains) do + for k, v in pairs(domains) do printf("Domain: %s, selector: %s, privkey: %s", highlight(k), v.selector, v.privkey) end @@ -370,13 +383,15 @@ local function setup_dkim_signing(cfg, changes) until #domain ~= 0 local selector = readline_default("Enter selector [default: dkim]: ", 'dkim') - if not selector then selector = 'dkim' end + if not selector then + selector = 'dkim' + end local privkey_file = string.format("%s/%s.%s.key", dkim_keys_dir, domain, selector) if not rspamd_util.file_exists(privkey_file) then if ask_yes_no("Do you want to create privkey " .. highlight(privkey_file), - true) then + true) then local pubkey_file = privkey_file .. ".pub" local rspamd_cryptobox = require "rspamd_cryptobox" local sk, pk = rspamd_cryptobox.generate_keypair("rsa", 2048) @@ -402,7 +417,7 @@ local function setup_dkim_signing(cfg, changes) } until not ask_yes_no("Do you wish to add another DKIM domain?") - changes.l['dkim_signing.conf'] = {domain = domains} + changes.l['dkim_signing.conf'] = { domain = domains } local res_tbl = changes.l['dkim_signing.conf'] if sign_networks then @@ -426,7 +441,7 @@ local function check_redis_classifier(cls, changes) local symbol_spam, symbol_ham -- Load symbols from statfiles local statfiles = cls.statfile - for _,stf in ipairs(statfiles) do + for _, stf in ipairs(statfiles) do local symbol = stf.symbol or 'undefined' local spam @@ -484,12 +499,14 @@ local function check_redis_classifier(cls, changes) end local function get_version(conn) - conn:add_cmd("SMEMBERS", {"RS_keys"}) + conn:add_cmd("SMEMBERS", { "RS_keys" }) - local ret,members = conn:exec() + local ret, members = conn:exec() -- Empty db - if not ret or #members == 0 then return false,0 end + if not ret or #members == 0 then + return false, 0 + end -- We still need to check versions local lua_script = [[ @@ -502,10 +519,10 @@ end return ver ]] - conn:add_cmd('EVAL', {lua_script, '1', 'RS'}) - local _,ver = conn:exec() + conn:add_cmd('EVAL', { lua_script, '1', 'RS' }) + local _, ver = conn:exec() - return true,tonumber(ver) + return true, tonumber(ver) end local function check_expire(conn) @@ -522,21 +539,23 @@ end return ttl ]] - conn:add_cmd('EVAL', {lua_script, '0'}) - local _,ttl = conn:exec() + conn:add_cmd('EVAL', { lua_script, '0' }) + local _, ttl = conn:exec() return tonumber(ttl) end - local res,conn = lua_redis.redis_connect_sync(parsed_redis, true) + local res, conn = lua_redis.redis_connect_sync(parsed_redis, true) if not res then printf("Cannot connect to Redis server") return false end if not cls.new_schema then - local r,ver = get_version(conn) - if not r then return false end + local r, ver = get_version(conn) + if not r then + return false + end if ver ~= 2 then if not ver then printf('Key "RS_version" has not been found in Redis for %s/%s', @@ -562,17 +581,19 @@ return ttl end end else - local r,ver = get_version(conn) - if not r then return false end + local r, ver = get_version(conn) + if not r then + return false + end if ver ~= 2 then printf("You have configured new schema for %s/%s but your DB has old version: %s", - symbol_spam, symbol_ham, ver) + symbol_spam, symbol_ham, ver) try_convert(false) else printf( 'You have configured new schema for %s/%s and your DB already has new layout (v. %s).' .. ' DB conversion is not needed.', - symbol_spam, symbol_ham, ver) + symbol_spam, symbol_ham, ver) end end end @@ -584,7 +605,7 @@ local function setup_statistic(cfg, changes) if not redis_params then printf('You have %d sqlite classifiers, but you have no Redis servers being set', - #sqlite_configs) + #sqlite_configs) return false end @@ -596,7 +617,7 @@ local function setup_statistic(cfg, changes) local reset_previous = ask_yes_no("Reset previous data?") if ask_yes_no('Do you wish to convert them to Redis?', true) then - for _,cls in ipairs(sqlite_configs) do + for _, cls in ipairs(sqlite_configs) do if rspamd_util.file_exists(cls.db_spam) and rspamd_util.file_exists(cls.db_ham) then if not lua_stat_tools.convert_sqlite_to_redis(parsed_redis, cls.db_spam, cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, expire, @@ -634,8 +655,10 @@ local function setup_statistic(cfg, changes) if classifier then if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.bayes then cls = cls.bayes end + for _, cls in ipairs(classifier) do + if cls.bayes then + cls = cls.bayes + end if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, changes) end @@ -645,7 +668,7 @@ local function setup_statistic(cfg, changes) classifier = classifier.bayes if classifier[1] then - for _,cls in ipairs(classifier) do + for _, cls in ipairs(classifier) do if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, changes) end @@ -663,22 +686,24 @@ end local function find_worker(cfg, wtype) if cfg.worker then - for k,s in pairs(cfg.worker) do + for k, s in pairs(cfg.worker) do if type(k) == 'number' and type(s) == 'table' then - if s[wtype] then return s[wtype] end + if s[wtype] then + return s[wtype] + end end if type(s) == 'table' and s.type and s.type == wtype then return s end - if type(k) == 'string' and k == wtype then return s end + if type(k) == 'string' and k == wtype then + return s + end end end return nil end - - return { handler = function(cmd_args) local changes = { @@ -698,14 +723,14 @@ return { local opts = parser:parse(cmd_args) local args = opts['checks'] or {} - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -721,13 +746,13 @@ return { if #args > 0 then interactive_start = false - for _,arg in ipairs(args) do + for _, arg in ipairs(args) do if arg == 'all' then checks = all_checks elseif arg == 'list' then printf(highlight(rspamd_logo)) printf('Available modules') - for _,c in ipairs(all_checks) do + for _, c in ipairs(all_checks) do printf('- %s', c) end return @@ -740,7 +765,7 @@ return { end local function has_check(check) - for _,c in ipairs(checks) do + for _, c in ipairs(checks) do if c == check then return true end @@ -791,8 +816,12 @@ return { end local nchanges = 0 - for _,_ in pairs(changes.l) do nchanges = nchanges + 1 end - for _,_ in pairs(changes.o) do nchanges = nchanges + 1 end + for _, _ in pairs(changes.l) do + nchanges = nchanges + 1 + end + for _, _ in pairs(changes.o) do + nchanges = nchanges + 1 + end if nchanges > 0 then print_changes(changes) diff --git a/lualib/rspamadm/cookie.lua b/lualib/rspamadm/cookie.lua index bc4df9de2..7e0526a7e 100644 --- a/lualib/rspamadm/cookie.lua +++ b/lualib/rspamadm/cookie.lua @@ -44,7 +44,9 @@ parser:argument "cookie":args "?" local function gen_cookie(args, key) local cr = require "rspamd_cryptobox" - if not args.cookie then return end + if not args.cookie then + return + end local function encrypt() if #args.cookie > 31 then @@ -67,7 +69,7 @@ local function gen_cookie(args, key) extracted_cookie = args.cookie end - local dec_cookie,ts = cr.decrypt_cookie(key, extracted_cookie) + local dec_cookie, ts = cr.decrypt_cookie(key, extracted_cookie) if dec_cookie then if args.timestamp then @@ -79,7 +81,7 @@ local function gen_cookie(args, key) print('cannot decrypt cookie') os.exit(1) end - end + end if args.decrypt then decrypt() @@ -96,8 +98,10 @@ local function handler(args) end if res.key then - local pattern = {'^'} - for i=1,32 do pattern[i + 1] = '[a-zA-Z0-9]' end + local pattern = { '^' } + for i = 1, 32 do + pattern[i + 1] = '[a-zA-Z0-9]' + end pattern[34] = '$' if not res.key:match(table.concat(pattern, '')) then diff --git a/lualib/rspamadm/corpus_test.lua b/lualib/rspamadm/corpus_test.lua index 939c3c0fa..0e63f9f6f 100644 --- a/lualib/rspamadm/corpus_test.lua +++ b/lualib/rspamadm/corpus_test.lua @@ -177,10 +177,9 @@ local function handler(args) rspamd_logger.messagex("Messages/sec: %s", (total_msgs / elapsed_time)) end - return { name = 'corpustest', - aliases = {'corpus_test', 'corpus'}, + aliases = { 'corpus_test', 'corpus' }, handler = handler, description = parser._description } diff --git a/lualib/rspamadm/dkim_keygen.lua b/lualib/rspamadm/dkim_keygen.lua index 0d0ebe5a6..3d51903dc 100644 --- a/lualib/rspamadm/dkim_keygen.lua +++ b/lualib/rspamadm/dkim_keygen.lua @@ -32,39 +32,39 @@ parser:option '-k --privkey' :description 'Save private key to file instead of printing it to stdout' parser:option '-b --bits' :convert(function(input) - local n = tonumber(input) - if not n or n < 512 or n > 4096 then - return nil - end - return n - end) + local n = tonumber(input) + if not n or n < 512 or n > 4096 then + return nil + end + return n +end) :description 'Generate an RSA key with the specified number of bits (512 to 4096)' :default(1024) parser:option '-t --type' :description 'Key type: RSA, ED25519 or ED25119-seed' :convert { - ['rsa'] = 'rsa', - ['RSA'] = 'rsa', - ['ed25519'] = 'ed25519', - ['ED25519'] = 'ed25519', - ['ed25519-seed'] = 'ed25519-seed', - ['ED25519-seed'] = 'ed25519-seed', - } + ['rsa'] = 'rsa', + ['RSA'] = 'rsa', + ['ed25519'] = 'ed25519', + ['ED25519'] = 'ed25519', + ['ed25519-seed'] = 'ed25519-seed', + ['ED25519-seed'] = 'ed25519-seed', +} :default 'rsa' parser:option '-o --output' :description 'Output public key in the following format: dns, dnskey or plain' :convert { - ['dns'] = 'dns', - ['plain'] = 'plain', - ['dnskey'] = 'dnskey', - } + ['dns'] = 'dns', + ['plain'] = 'plain', + ['dnskey'] = 'dnskey', +} :default 'dns' parser:option '--priv-output' :description 'Output private key in the following format: PEM or DER (for RSA)' :convert { - ['pem'] = 'pem', - ['der'] = 'der', - } + ['pem'] = 'pem', + ['der'] = 'der', +} :default 'pem' parser:flag '-f --force' :description 'Force overwrite of existing files' @@ -83,7 +83,6 @@ local function split_string(input, max_length) return pieces end - local function print_public_key_dns(opts, base64_pk) local key_type = opts.type == 'rsa' and 'rsa' or 'ed25519' if #base64_pk < 255 - 2 then @@ -93,7 +92,7 @@ local function print_public_key_dns(opts, base64_pk) -- Split it by parts local parts = split_string(base64_pk) io.write(string.format('%s._domainkey IN TXT ( "v=DKIM1; k=%s; "\n', opts.selector, key_type)) - for i,part in ipairs(parts) do + for i, part in ipairs(parts) do if i == 1 then io.write(string.format('\t"p=%s"\n', part)) else @@ -121,7 +120,7 @@ end local function gen_rsa_key(opts) local rsa = require "rspamd_rsa" - local sk,pk = rsa.keypair(opts.bits or 1024) + local sk, pk = rsa.keypair(opts.bits or 1024) if opts.privkey then if opts.force then os.remove(opts.privkey) @@ -135,7 +134,7 @@ local function gen_rsa_key(opts) end local function gen_eddsa_key(opts) - local sk,pk = rspamd_cryptobox.gen_dkim_keypair(opts.type) + local sk, pk = rspamd_cryptobox.gen_dkim_keypair(opts.type) if opts.privkey and opts.force then os.remove(opts.privkey) @@ -156,7 +155,9 @@ end local function handler(args) local opts = parser:parse(args) - if not opts then os.exit(1) end + if not opts then + os.exit(1) + end if opts.type == 'rsa' then gen_rsa_key(opts) @@ -167,7 +168,7 @@ end return { name = 'dkim_keygen', - aliases = {'dkimkeygen'}, + aliases = { 'dkimkeygen' }, handler = handler, description = parser._description } diff --git a/lualib/rspamadm/dmarc_report.lua b/lualib/rspamadm/dmarc_report.lua index c81be13cf..65b1da158 100644 --- a/lualib/rspamadm/dmarc_report.lua +++ b/lualib/rspamadm/dmarc_report.lua @@ -46,9 +46,9 @@ parser:flag "-n --no-opt" :description "Do not reset reporting data/send reports" parser:argument "date" - :description "Date to process (today by default)" - :argname "<YYYYMMDD>" - :args "*" + :description "Date to process (today by default)" + :argname "<YYYYMMDD>" + :args "*" parser:option "-b --batch-size" :description "Send reports in batches up to <batch-size> messages" :argname "<number>" @@ -102,14 +102,14 @@ local redis_attrs = { local pool local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -118,11 +118,9 @@ end -- Concat elements using redis_keys.join_char local function redis_prefix(...) - return table.concat({...}, dmarc_settings.reporting.redis_keys.join_char) + return table.concat({ ... }, dmarc_settings.reporting.redis_keys.join_char) end - - local function get_rua(rep_key) local parts = lua_util.str_split(rep_key, dmarc_settings.reporting.redis_keys.join_char) @@ -144,8 +142,8 @@ local function get_domain(rep_key) end local function gen_uuid() - local template ='xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' - return string.gsub(template, '[xy]', function (c) + local template = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' + return string.gsub(template, '[xy]', function(c) local v = (c == 'x') and math.random(0, 0xf) or math.random(8, 0xb) return string.format('%x', v) end) @@ -159,7 +157,7 @@ local function gen_xml_grammar() local quot = lpeg.P('"') / '"' local apos = lpeg.P("'") / ''' local special = lt + gt + amp + quot + apos - local grammar = lpeg.Cs((special + 1)^0) + local grammar = lpeg.Cs((special + 1) ^ 0) return grammar end @@ -250,7 +248,7 @@ local function entry_to_xml(data) </auth_results> </record> ]] - return lua_util.jinja_template(xml_template, {data = data}, true) + return lua_util.jinja_template(xml_template, { data = data }, true) end -- Process a report entry stored in Redis splitting it to a lua table @@ -273,7 +271,7 @@ local function process_report_entry(data, score) if dkim_data and dkim_data ~= '' then local dkim_elts = lua_util.str_split(dkim_data, '|') for _, d in ipairs(dkim_elts) do - table.insert(row.dkim_results, {domain = d, result = result}) + table.insert(row.dkim_results, { domain = d, result = result }) end end end @@ -313,7 +311,7 @@ local function process_rua(dmarc_domain, rua) logger.errx('cannot resolve %s: %s; exclude %s', resolve_str, results, rua_part) else local found = false - for _,t in ipairs(results) do + for _, t in ipairs(results) do if string.match(t, 'v=DMARC1') then found = true break @@ -350,7 +348,7 @@ local function validate_reporting_domain(reporting_domain) config = rspamd_config, session = rspamadm_session, type = 'txt', - name = '_dmarc.' .. dmarc_domain , + name = '_dmarc.' .. dmarc_domain, }) if not is_ok or not results then @@ -358,8 +356,8 @@ local function validate_reporting_domain(reporting_domain) return nil end - for _,r in ipairs(results) do - local processed,rec = dmarc_common.dmarc_check_record(rspamd_config, r, false) + for _, r in ipairs(results) do + local processed, rec = dmarc_common.dmarc_check_record(rspamd_config, r, false) if processed and rec.rua then -- We need to check or alter rua if needed local processed_rua = process_rua(dmarc_domain, rec.rua) @@ -385,7 +383,7 @@ end -- Returns a list of recipients from a table as a string processing elements if needed local function rcpt_list(tbl, func) local res = {} - for _,r in ipairs(tbl) do + for _, r in ipairs(tbl) do if func then table.insert(res, func(r)) else @@ -431,14 +429,14 @@ local function send_reports_by_smtp(opts, reports, finish_cb) local nreports = math.min(#reports - cur_batch + 1, opts.batch_size) local next_start = cur_batch + nreports lua_util.debugm(N, 'send data for %s domains (from %s to %s)', - nreports, cur_batch, next_start-1) + nreports, cur_batch, next_start - 1) -- Shared across all closures local gen_args = { cont_func = send_data_in_batches, nreports = nreports, next_start = next_start } - for i=cur_batch,next_start-1 do + for i = cur_batch, next_start - 1 do local report = reports[i] local send_opts = { ev_base = rspamadm_ev_base, @@ -475,7 +473,7 @@ local function prepare_report(opts, start_time, end_time, rep_key) end local ret, results = lua_redis.request(redis_params, redis_attrs, - {'EXISTS', rep_key}) + { 'EXISTS', rep_key }) if not ret or not results or results == 0 then return nil @@ -484,7 +482,7 @@ local function prepare_report(opts, start_time, end_time, rep_key) -- Rename report key to avoid races if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'RENAME', rep_key, rep_key .. '_processing'}) + { 'RENAME', rep_key, rep_key .. '_processing' }) rep_key = rep_key .. '_processing' end @@ -494,7 +492,7 @@ local function prepare_report(opts, start_time, end_time, rep_key) if not dmarc_record then if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', rep_key}) + { 'DEL', rep_key }) end logger.messagex('Cannot process reports for domain %s; invalid dmarc record', reporting_domain) return nil @@ -502,11 +500,11 @@ local function prepare_report(opts, start_time, end_time, rep_key) -- Get all reports for a domain ret, results = lua_redis.request(redis_params, redis_attrs, - {'ZRANGE', rep_key, '0', '-1', 'WITHSCORES'}) + { 'ZRANGE', rep_key, '0', '-1', 'WITHSCORES' }) local report_entries = {} table.insert(report_entries, report_header(reporting_domain, start_time, end_time, dmarc_record)) - for i=1,#results,2 do + for i = 1, #results, 2 do local xml_record = entry_to_xml(process_report_entry(results[i], results[i + 1])) table.insert(report_entries, xml_record) end @@ -542,24 +540,23 @@ local function prepare_report(opts, start_time, end_time, rep_key) local rfooter = lua_util.jinja_template(report_footer, { uuid = uuid, }, true) - local message = rspamd_text.fromtable{ + local message = rspamd_text.fromtable { (rhead:gsub("\n", "\r\n")), rspamd_util.encode_base64(rspamd_util.gzip_compress(xml_to_compress), 73), rfooter:gsub("\n", "\r\n"), } - lua_util.debugm(N, 'got final message: %s', message) if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', rep_key}) + { 'DEL', rep_key }) end local report_rcpts = lua_util.str_split(rcpt_string, ',') if report_settings.bcc_addrs then - for _,b in ipairs(report_settings.bcc_addrs) do + for _, b in ipairs(report_settings.bcc_addrs) do table.insert(report_rcpts, b) end end @@ -574,7 +571,7 @@ end local function process_report_date(opts, start_time, end_time, date) local idx_key = redis_prefix(dmarc_settings.reporting.redis_keys.index_prefix, date) local ret, results = lua_redis.request(redis_params, redis_attrs, - {'EXISTS', idx_key}) + { 'EXISTS', idx_key }) if not ret or not results or results == 0 then logger.messagex('No reports for %s', date) @@ -584,24 +581,24 @@ local function process_report_date(opts, start_time, end_time, date) -- Rename index key to avoid races if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'RENAME', idx_key, idx_key .. '_processing'}) + { 'RENAME', idx_key, idx_key .. '_processing' }) idx_key = idx_key .. '_processing' end ret, results = lua_redis.request(redis_params, redis_attrs, - {'SMEMBERS', idx_key}) + { 'SMEMBERS', idx_key }) if not ret or not results then -- Remove bad key if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', idx_key}) + { 'DEL', idx_key }) end logger.messagex('Cannot get reports for %s', date) return {} end local reports = {} - for _,rep in ipairs(results) do + for _, rep in ipairs(results) do local report = prepare_report(opts, start_time, end_time, rep) if report then @@ -614,7 +611,7 @@ local function process_report_date(opts, start_time, end_time, date) -- Remove processed key if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', idx_key}) + { 'DEL', idx_key }) end return reports @@ -669,14 +666,14 @@ local function handler(args) os.exit(1) end - for _, e in ipairs({'email', 'domain', 'org_name'}) do + for _, e in ipairs({ 'email', 'domain', 'org_name' }) do if not dmarc_settings.reporting[e] then logger.errx('Missing required setting: dmarc.reporting.%s', e) return end end - local ret,results = lua_redis.request(redis_params, redis_attrs, { + local ret, results = lua_redis.request(redis_params, redis_attrs, { 'GET', 'rspamd_dmarc_last_collection' }) @@ -696,14 +693,14 @@ local function handler(args) local ndates = 0 local nreports = 0 local all_reports = {} - for _,date in ipairs(opts.date) do + for _, date in ipairs(opts.date) do lua_util.debugm(N, 'Process date %s', date) local reports_for_date = process_report_date(opts, start_time, start_collection, date) if #reports_for_date > 0 then ndates = ndates + 1 nreports = nreports + #reports_for_date - for _,r in ipairs(reports_for_date) do + for _, r in ipairs(reports_for_date) do table.insert(all_reports, r) end end @@ -718,8 +715,8 @@ local function handler(args) ndates, nreports, nsuccess, nfail) end lua_redis.request(redis_params, redis_attrs, - {'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2, - tostring(start_collection)}) + { 'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2, + tostring(start_collection) }) else logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed', ndates, nreports, nsuccess, nfail) @@ -736,7 +733,7 @@ end return { name = 'dmarc_report', - aliases = {'dmarc_reporting'}, + aliases = { 'dmarc_reporting' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/dns_tool.lua b/lualib/rspamadm/dns_tool.lua index e3ffb29dd..3eb09a82b 100644 --- a/lualib/rspamadm/dns_tool.lua +++ b/lualib/rspamadm/dns_tool.lua @@ -33,7 +33,7 @@ parser:option "-c --config" :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf") local spf = parser:command "spf" - :description "Extracts spf records" + :description "Extracts spf records" spf:mutex( spf:option "-d --domain" :description "Domain to use" @@ -69,14 +69,14 @@ local function red(str) end local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -109,7 +109,7 @@ local function spf_handler(opts) os.exit(1) end elseif opts.domain then - task:set_from('smtp', {user = 'user', domain = opts.domain}) + task:set_from('smtp', { user = 'user', domain = opts.domain }) else io.stderr:write('Neither domain nor from specified\n') os.exit(1) @@ -128,7 +128,9 @@ local function spf_handler(opts) end local function display_spf_results(elt, colored) - local dec = function(e) return e end + local dec = function(e) + return e + end local policy_decode = function(e) if e == rspamd_spf.policy.fail then return 'reject' @@ -144,12 +146,18 @@ local function spf_handler(opts) end if colored then - dec = function(e) return highlight(e) end + dec = function(e) + return highlight(e) + end - if elt.result == rspamd_spf.policy.pass then - dec = function(e) return green(e) end - elseif elt.result == rspamd_spf.policy.fail then - dec = function(e) return red(e) end + if elt.result == rspamd_spf.policy.pass then + dec = function(e) + return green(e) + end + elseif elt.result == rspamd_spf.policy.fail then + dec = function(e) + return red(e) + end end end @@ -182,7 +190,7 @@ local function spf_handler(opts) if result then printf('SPF record for %s; digest: %s', highlight(opts.domain or opts.from), highlight(record:get_digest())) - for _,elt in ipairs(record:get_elts()) do + for _, elt in ipairs(record:get_elts()) do if result and error_or_addr and elt.str and elt.str == error_or_addr.str then printf("%s", highlight('*** Matched ***')) display_spf_results(elt, true) @@ -218,7 +226,7 @@ end return { name = 'dnstool', - aliases = {'dns', 'dns_tool'}, + aliases = { 'dns', 'dns_tool' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/fuzzy_convert.lua b/lualib/rspamadm/fuzzy_convert.lua index 2d473ca46..a31baa4e2 100644 --- a/lualib/rspamadm/fuzzy_convert.lua +++ b/lualib/rspamadm/fuzzy_convert.lua @@ -13,13 +13,13 @@ local function connect_redis(server, password, db) end if password then - ret = conn:add_cmd('AUTH', {password}) + ret = conn:add_cmd('AUTH', { password }) if not ret then return nil, 'Cannot queue command' end end if db then - ret = conn:add_cmd('SELECT', {db}) + ret = conn:add_cmd('SELECT', { db }) if not ret then return nil, 'Cannot queue command' end @@ -126,7 +126,7 @@ local function update_counters(total, redis_host, redis_password, redis_db) return true end -return function (_, res) +return function(_, res) local db = sqlite3.open(res['source_db']) local shingles = {} local digests = {} @@ -152,11 +152,11 @@ return function (_, res) local expire_in = math.floor(now - row.time + res['expiry']) if expire_in >= 1 then - table.insert(digests, {row.digest, row.flag, row.value, expire_in}) + table.insert(digests, { row.digest, row.flag, row.value, expire_in }) num_batch_digests = num_batch_digests + 1 total_digests = total_digests + 1 for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do - table.insert(shingles, {srow.value, srow.number, expire_in, row.digest}) + table.insert(shingles, { srow.value, srow.number, expire_in, row.digest }) total_shingles = total_shingles + 1 num_batch_shingles = num_batch_shingles + 1 end @@ -188,8 +188,8 @@ return function (_, res) end local message = string.format( - 'Migrated %d digests and %d shingles', - total_digests, total_shingles + 'Migrated %d digests and %d shingles', + total_digests, total_shingles ) if not update_counters(total_digests, res['redis_host'], redis_password, redis_db) then message = message .. ' but failed to update counters' diff --git a/lualib/rspamadm/fuzzy_stat.lua b/lualib/rspamadm/fuzzy_stat.lua index d0c14d213..ef8a5de08 100644 --- a/lualib/rspamadm/fuzzy_stat.lua +++ b/lualib/rspamadm/fuzzy_stat.lua @@ -18,14 +18,14 @@ parser:flag "-n --number" parser:option "--sort" :description "Sort order" :convert { - checked = "checked", - matched = "matched", - errors = "errors", - name = "name" - } + checked = "checked", + matched = "matched", + errors = "errors", + name = "name" +} local function add_data(target, src) - for k,v in pairs(src) do + for k, v in pairs(src) do if type(v) == 'number' then if target[k] then target[k] = target[k] + v @@ -33,17 +33,25 @@ local function add_data(target, src) target[k] = v end elseif k == 'ips' then - if not target['ips'] then target['ips'] = {} end + if not target['ips'] then + target['ips'] = {} + end -- Iterate over IPs - for ip,st in pairs(v) do - if not target['ips'][ip] then target['ips'][ip] = {} end + for ip, st in pairs(v) do + if not target['ips'][ip] then + target['ips'][ip] = {} + end add_data(target['ips'][ip], st) end elseif k == 'flags' then - if not target['flags'] then target['flags'] = {} end + if not target['flags'] then + target['flags'] = {} + end -- Iterate over Flags - for flag,st in pairs(v) do - if not target['flags'][flag] then target['flags'][flag] = {} end + for flag, st in pairs(v) do + if not target['flags'][flag] then + target['flags'][flag] = {} + end add_data(target['flags'][flag], st) end elseif k == 'keypair' then @@ -109,8 +117,8 @@ end -- Sort by checked local function sort_hash_table(tbl, sort_opts, key_key) local res = {} - for k,v in pairs(tbl) do - table.insert(res, {[key_key] = k, data = v}) + for k, v in pairs(tbl) do + table.insert(res, { [key_key] = k, data = v }) end local function sort_order(elt) @@ -145,12 +153,12 @@ local function add_result(dst, src, k) if type(src) == 'table' then if type(dst) == 'number' then -- Convert dst to table - dst = {dst} + dst = { dst } elseif type(dst) == 'nil' then dst = {} end - for i,v in ipairs(src) do + for i, v in ipairs(src) do if dst[i] and k ~= 'fuzzy_stored' then dst[i] = dst[i] + v else @@ -193,7 +201,7 @@ local function print_result(r) end if type(r) == 'table' then local res = {} - for i,num in ipairs(r) do + for i, num in ipairs(r) do res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num)) end @@ -210,7 +218,7 @@ return function(args, res) opts = parser:parse(args) if wrk then - for _,pr in pairs(wrk) do + for _, pr in pairs(wrk) do -- processes cycle if pr['data'] then local id = pr['id'] @@ -225,7 +233,7 @@ return function(args, res) end -- General stats - for k,v in pairs(pr['data']) do + for k, v in pairs(pr['data']) do if k ~= 'keys' and k ~= 'errors_ips' then res_db[k] = add_result(res_db[k], v, k) elseif k == 'errors_ips' then @@ -233,7 +241,7 @@ return function(args, res) if not res_db['errors_ips'] then res_db['errors_ips'] = {} end - for ip,nerrors in pairs(v) do + for ip, nerrors in pairs(v) do if not res_db['errors_ips'][ip] then res_db['errors_ips'][ip] = nerrors else @@ -250,7 +258,7 @@ return function(args, res) res_db['keys'] = res_keys end -- Go through keys in input - for k,elts in pairs(pr['data']['keys']) do + for k, elts in pairs(pr['data']['keys']) do -- keys cycle if not res_keys[k] then res_keys[k] = {} @@ -259,7 +267,7 @@ return function(args, res) add_data(res_keys[k], elts) if elts['ips'] then - for ip,v in pairs(elts['ips']) do + for ip, v in pairs(elts['ips']) do if not res_ips[ip] then res_ips[ip] = {} end @@ -274,10 +282,10 @@ return function(args, res) end -- General stats - for db,st in pairs(res_databases) do + for db, st in pairs(res_databases) do print(string.format('Statistics for storage %s', db)) - for k,v in pairs(st) do + for k, v in pairs(st) do if k ~= 'keys' and k ~= 'errors_ips' then print(string.format('%s: %s', k, print_result(v))) end @@ -305,7 +313,7 @@ return function(args, res) print('\tIPs stat:') local sorted_ips = sort_hash_table(key_stat['ips'], opts, 'ip') - for _,v in ipairs(sorted_ips) do + for _, v in ipairs(sorted_ips) do print(string.format('\t%s', v['ip'])) print_stat(v['data'], '\t\t') print('') @@ -315,7 +323,7 @@ return function(args, res) if key_stat.flags then print('') print('\tFlags stat:') - for flag,v in pairs(key_stat.flags) do + for flag, v in pairs(key_stat.flags) do print(string.format('\t[%s]:', flag)) -- Remove irrelevant fields v.checked = nil diff --git a/lualib/rspamadm/grep.lua b/lualib/rspamadm/grep.lua index 426b93c7a..6ed05691e 100644 --- a/lualib/rspamadm/grep.lua +++ b/lualib/rspamadm/grep.lua @@ -80,7 +80,7 @@ local function handler(args) if search_str and not sensitive then search_str = string.lower(search_str) end - local inputs = res['input'] or {'stdin'} + local inputs = res['input'] or { 'stdin' } for _, n in ipairs(inputs) do local h, err @@ -115,7 +115,7 @@ local function handler(args) if buffer[hash] then table.insert(buffer[hash], line) else - buffer[hash] = {line} + buffer[hash] = { line } end end end diff --git a/lualib/rspamadm/keypair.lua b/lualib/rspamadm/keypair.lua index 27dfd59a3..f0716a22f 100644 --- a/lualib/rspamadm/keypair.lua +++ b/lualib/rspamadm/keypair.lua @@ -84,9 +84,9 @@ verify:mutex( :argname "<file>" ) verify:argument "file" - :description "File to verify" - :argname "<file>" - :args "*" + :description "File to verify" + :argname "<file>" + :args "*" verify:flag "-n --nist" :description "Uses nistp curves (P256)" verify:option "-s --suffix" @@ -143,15 +143,15 @@ decrypt:flag "-r --rm" -- Default command is generate, so duplicate options to be compatible parser:flag "-s --sign" - :description "Generates a sign keypair instead of the encryption one" + :description "Generates a sign keypair instead of the encryption one" parser:flag "-n --nist" - :description "Uses nistp curves (P256)" + :description "Uses nistp curves (P256)" parser:mutex( parser:flag "-j --json" - :description "Output JSON instead of UCL", + :description "Output JSON instead of UCL", parser:flag "-u --ucl" - :description "Output UCL" - :default(true) + :description "Output UCL" + :default(true) ) parser:option "-o --output" :description "Write keypair to file" @@ -174,10 +174,16 @@ local function ask_yes_no(greet, default) local reply = rspamd_util.readline(greet) - if not reply then os.exit(0) end - if #reply == 0 then reply = def_str end + if not reply then + os.exit(0) + end + if #reply == 0 then + reply = def_str + end reply = reply:lower() - if reply == 'y' or reply == 'yes' then return true end + if reply == 'y' or reply == 'yes' then + return true + end return false end @@ -221,7 +227,7 @@ end local function sign_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to sign') @@ -231,7 +237,7 @@ local function sign_handler(opts) end local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -243,7 +249,7 @@ local function sign_handler(opts) fatal("cannot load keypair: " .. opts.keypair) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local sig = rspamd_crypto.sign_file(kp, fname) if not sig then @@ -264,7 +270,7 @@ end local function verify_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to verify') @@ -275,7 +281,7 @@ local function verify_handler(opts) if opts.keypair then local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -290,10 +296,14 @@ local function verify_handler(opts) pk = kp:pk() alg = kp:alg() elseif opts.pubkey then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg) elseif opts.pubstr then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg) end @@ -303,7 +313,7 @@ local function verify_handler(opts) local valid = true - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local sig_fname = string.format('%s.%s', fname, opts.suffix or 'sig') local sig = rspamd_signature.load(sig_fname, alg) @@ -330,7 +340,7 @@ end local function encrypt_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to sign') @@ -341,7 +351,7 @@ local function encrypt_handler(opts) if opts.keypair then local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -356,10 +366,14 @@ local function encrypt_handler(opts) pk = kp:pk() alg = kp:alg() elseif opts.pubkey then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg) elseif opts.pubstr then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg) end @@ -367,7 +381,7 @@ local function encrypt_handler(opts) fatal("cannot load keypair: " .. opts.keypair) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local enc = rspamd_crypto.encrypt_file(pk, fname, alg) if not enc then @@ -404,7 +418,7 @@ end local function decrypt_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to decrypt') @@ -414,7 +428,7 @@ local function decrypt_handler(opts) end local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -426,7 +440,7 @@ local function decrypt_handler(opts) fatal("cannot load keypair: " .. opts.keypair) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local decrypted = rspamd_crypto.decrypt_file(kp, fname) if not decrypted then @@ -488,7 +502,7 @@ end return { name = 'keypair', - aliases = {'kp', 'key'}, + aliases = { 'kp', 'key' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 5c1cebac5..0b0605582 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -44,7 +44,7 @@ parser:mutex( parser:flag "-U --ucl" :description "UCL output", parser:flag "-M --messagepack" - :description "MessagePack output" + :description "MessagePack output" ) parser:flag "-C --compact" :description "Use compact format" @@ -67,12 +67,12 @@ extract:option "-o --output" :description "Output format ('raw', 'content', 'oneline', 'decoded', 'decoded_utf')" :argname("<type>") :convert { - raw = "raw", - content = "content", - oneline = "content_oneline", - decoded = "raw_parsed", - decoded_utf = "raw_utf" - } + raw = "raw", + content = "content", + oneline = "content_oneline", + decoded = "raw_parsed", + decoded_utf = "raw_utf" +} :default "content" extract:flag "-w --words" :description "Extracts words" @@ -86,16 +86,15 @@ extract:option "-F --words-format" :description "Words format ('stem', 'norm', 'raw', 'full')" :argname("<type>") :convert { - stem = "stem", - norm = "norm", - raw = "raw", - full = "full", - } + stem = "stem", + norm = "norm", + raw = "raw", + full = "full", +} :default "stem" - local stat = parser:command "stat st s" - :description "Extracts statistical data from MIME messages" + :description "Extracts statistical data from MIME messages" stat:argument "file" :description "File to process" :argname "<file>" @@ -123,7 +122,7 @@ urls:mutex( urls:flag "-H --host" :description "Get hosts only", urls:flag "-f --full" - :description "Show piecewise urls as processed by Rspamd" + :description "Show piecewise urls as processed by Rspamd" ) urls:flag "-u --unique" @@ -136,7 +135,7 @@ urls:flag "-r --reverse" :description "Reverse sort order" local modify = parser:command "modify mod m" - :description "Modifies MIME message" + :description "Modifies MIME message" modify:argument "file" :description "File to process" :argname "<file>" @@ -162,11 +161,11 @@ modify:option "-H --html-footer" :argname "<file>" local sign = parser:command "sign" - :description "Performs DKIM signing" + :description "Performs DKIM signing" sign:argument "file" - :description "File to process" - :argname "<file>" - :args "+" + :description "File to process" + :argname "<file>" + :args "+" sign:option "-d --domain" :description "Use specific domain" @@ -184,17 +183,17 @@ sign:option "-t type" :description "ARC or DKIM signing" :argname("<arc|dkim>") :convert { - ['arc'] = 'arc', - ['dkim'] = 'dkim', - } + ['arc'] = 'arc', + ['dkim'] = 'dkim', +} :default 'dkim' sign:option "-o --output" :description "Output format" :argname("<message|signature>") :convert { - ['message'] = 'message', - ['signature'] = 'signature', - } + ['message'] = 'message', + ['signature'] = 'signature', +} :default 'message' local dump = parser:command "dump" @@ -213,21 +212,21 @@ dump:mutex( :description "MessagePack output" ) dump:flag "-s --split" - :description "Split the output file contents such that no content is embedded" + :description "Split the output file contents such that no content is embedded" dump:option "-o --outdir" - :description "Output directory" - :argname("<directory>") + :description "Output directory" + :argname("<directory>") local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -239,7 +238,7 @@ local function load_task(opts, fname) fname = '-' end - local res,task = rspamd_task.load_from_file(fname, rspamd_config) + local res, task = rspamd_task.load_from_file(fname, rspamd_config) if not res then parser:error(string.format('cannot read message from %s: %s', fname, @@ -266,9 +265,15 @@ end local function output_fmt(opts) local fmt = 'json' - if opts.compact then fmt = 'json-compact' end - if opts.ucl then fmt = 'ucl' end - if opts.messagepack then fmt = 'msgpack' end + if opts.compact then + fmt = 'json-compact' + end + if opts.ucl then + fmt = 'ucl' + end + if opts.messagepack then + fmt = 'msgpack' + end return fmt end @@ -320,20 +325,20 @@ local function extract_handler(opts) if not opts.json and not opts.ucl then table.insert(out, rspamd_logger.slog('Part: %s: %s, language: %s, size: %s (%s raw), words: %s', - part:get_mimepart():get_digest():sub(1,8), - t, - part:get_language(), - part:get_length(), part:get_raw_length(), - part:get_words_count())) + part:get_mimepart():get_digest():sub(1, 8), + t, + part:get_language(), + part:get_length(), part:get_raw_length(), + part:get_words_count())) table.insert(out, rspamd_logger.slog('Stats: %s', - fun.foldl(function(acc, k, v) - if acc ~= '' then - return string.format('%s, %s:%s', acc, k, v) - else - return string.format('%s:%s', k,v) - end - end, '', part:get_stats()))) + fun.foldl(function(acc, k, v) + if acc ~= '' then + return string.format('%s, %s:%s', acc, k, v) + else + return string.format('%s:%s', k, v) + end + end, '', part:get_stats()))) end end end @@ -342,11 +347,11 @@ local function extract_handler(opts) if opts.part then if not opts.json and not opts.ucl then - local mtype,msubtype = part:get_type() - local det_mtype,det_msubtype = part:get_detected_type() + local mtype, msubtype = part:get_type() + local det_mtype, det_msubtype = part:get_detected_type() table.insert(out, rspamd_logger.slog('Mime Part: %s: %s/%s (%s/%s detected), filename: %s (%s detected ext), size: %s', - part:get_digest():sub(1,8), + part:get_digest():sub(1, 8), mtype, msubtype, det_mtype, det_msubtype, part:get_filename(), @@ -378,7 +383,7 @@ local function extract_handler(opts) end end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) out_elts[fname] = {} @@ -396,10 +401,12 @@ local function extract_handler(opts) if opts.text or opts.html then local mp = task:get_parts() or {} - for _,mime_part in ipairs(mp) do + for _, mime_part in ipairs(mp) do local how = opts.output local part - if mime_part:is_text() then part = mime_part:get_text() end + if mime_part:is_text() then + part = mime_part:get_text() + end if part and opts.text and not part:is_html() then maybe_print_text_part_info(part, out_elts[fname]) @@ -445,24 +452,25 @@ local function extract_handler(opts) end hc:foreach_tag('any', function(tag) - local elt = {} - local ex = tag:get_extra() - elt.tag = tag:get_type() - if ex then - elt.extra = ex - end - local content = tag:get_content() - if content then - elt.content = tostring(content) - end - local style = tag:get_style() - if style then - elt.style = style - end - table.insert(res, elt) + local elt = {} + local ex = tag:get_extra() + elt.tag = tag:get_type() + if ex then + elt.extra = ex + end + local content = tag:get_content() + if content then + elt.content = tostring(content) + end + local style = tag:get_style() + if style then + elt.style = style + end + table.insert(res, elt) end) table.insert(out_elts[fname], res) - else -- opts.structure + else + -- opts.structure table.insert(out_elts[fname], tostring(part:get_content(how))) end if opts.invisible then @@ -485,7 +493,9 @@ local function extract_handler(opts) print_elts(out_elts, opts, process_func) -- To avoid use after free we postpone tasks destruction - for _,task in ipairs(tasks) do task:destroy() end + for _, task in ipairs(tasks) do + task:destroy() + end end local function stat_handler(opts) @@ -498,7 +508,7 @@ local function stat_handler(opts) local process_func - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) out_elts[fname] = {} @@ -514,7 +524,9 @@ local function stat_handler(opts) process_func = function(e) return string.format('%s (%d): "%s"+"%s", [%s]', e.data, e.win, e.t1 or "", e.t2 or "", table.concat(fun.totable( - fun.map(function(k) return k end, e.flags)), ",")) + fun.map(function(k) + return k + end, e.flags)), ",")) end elseif opts.fuzzy then local parts = task:get_parts() or {} @@ -523,7 +535,7 @@ local function stat_handler(opts) local ret = string.format('part: %s(%s): %s', e.type, e.file or "", e.digest) if opts.shingles and e.shingles then local sgl = {} - for _,s in ipairs(e.shingles) do + for _, s in ipairs(e.shingles) do table.insert(sgl, string.format('%s: %s+%s+%s', s[1], s[2], s[3], s[4])) end @@ -531,26 +543,26 @@ local function stat_handler(opts) end return ret end - for _,part in ipairs(parts) do + for _, part in ipairs(parts) do if not part:is_multipart() then local text = part:get_text() if text then - local digest,shingles = text:get_fuzzy_hashes(task:get_mempool()) + local digest, shingles = text:get_fuzzy_hashes(task:get_mempool()) table.insert(out_elts[fname], { digest = digest, shingles = shingles, type = string.format('%s/%s', - ({part:get_type()})[1], - ({part:get_type()})[2]) + ({ part:get_type() })[1], + ({ part:get_type() })[2]) }) else table.insert(out_elts[fname], { digest = part:get_digest(), file = part:get_filename(), type = string.format('%s/%s', - ({part:get_type()})[1], - ({part:get_type()})[2]) + ({ part:get_type() })[1], + ({ part:get_type() })[2]) }) end end @@ -568,9 +580,11 @@ local function urls_handler(opts) rspamd_url.init(rspamd_config:get_tld_path()) local out_elts = {} - if opts.json then rspamd_logger.messagex('[') end + if opts.json then + rspamd_logger.messagex('[') + end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do out_elts[fname] = {} local task = load_task(opts, fname) local elts = {} @@ -605,7 +619,7 @@ local function urls_handler(opts) end end - for _,u in ipairs(task:get_urls(true)) do + for _, u in ipairs(task:get_urls(true)) do process_url(u) end @@ -672,12 +686,11 @@ local function urls_handler(opts) end end - - for s,u in lua_util.spairs(elts, sfunc) do + for s, u in lua_util.spairs(elts, sfunc) do process_elt(s, u) end else - for s,u in pairs(elts) do + for s, u in pairs(elts) do process_elt(s, u) end end @@ -723,7 +736,7 @@ local function modify_handler(opts) html_footer = read_file(opts['html_footer']) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) local newline_s = newline(task) local seen_cte @@ -732,14 +745,14 @@ local function modify_handler(opts) local out = {} -- Start with headers local function process_headers_cb(name, hdr) - for _,h in ipairs(opts['remove_header']) do + for _, h in ipairs(opts['remove_header']) do if name:match(h) then return end end - for _,h in ipairs(opts['rewrite_header']) do - local hname,hpattern = h:match('^([^=]+)=(.+)$') + for _, h in ipairs(opts['rewrite_header']) do + local hname, hpattern = h:match('^([^=]+)=(.+)$') if hname == name then local new_value = string.format(hpattern, hdr.decoded) new_value = string.format('%s:%s%s', @@ -769,10 +782,10 @@ local function modify_handler(opts) out[#out + 1] = hdr.raw:gsub('\r?\n?$', '') end - task:headers_foreach(process_headers_cb, {full = true}) + task:headers_foreach(process_headers_cb, { full = true }) - for _,h in ipairs(opts['add_header']) do - local hname,hvalue = h:match('^([^=]+)=(.+)$') + for _, h in ipairs(opts['add_header']) do + local hname, hvalue = h:match('^([^=]+)=(.+)$') if hname and hvalue then out[#out + 1] = string.format('%s: %s', hname, @@ -789,14 +802,14 @@ local function modify_handler(opts) out[#out + 1] = '' if rewrite.out then - for _,o in ipairs(rewrite.out) do + for _, o in ipairs(rewrite.out) do out[#out + 1] = o end else - out[#out + 1] = {task:get_rawbody(), false} + out[#out + 1] = { task:get_rawbody(), false } end - for _,o in ipairs(out) do + for _, o in ipairs(out) do if type(o) == 'string' then io.write(o) io.write(newline_s) @@ -844,7 +857,7 @@ local function sign_handler(opts) os.exit(1) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) local ctx = lua_dkim.create_sign_context(task, sign_key, nil, opts.algorithm) @@ -917,7 +930,7 @@ local function write_dump_content(dump_content, fname, extension, outdir) end if dump_content:save_in_file(outpath) then wrote_filepath = outpath - io.write(wrote_filepath.."\n") + io.write(wrote_filepath .. "\n") else io.stderr:write(string.format("Unable to save dump content to file: %s\n", outpath)) end @@ -954,7 +967,7 @@ local function dump_handler(opts) load_config(opts) rspamd_url.init(rspamd_config:get_tld_path()) - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) local data, extension = get_dump_content(task, opts, fname) write_dump_content(data, fname, extension, opts.outdir) @@ -969,7 +982,7 @@ local function handler(args) local command = opts.command if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } elseif type(opts.file) == 'none' then opts.file = {} end @@ -993,7 +1006,7 @@ end return { name = 'mime', - aliases = {'mime_tool'}, + aliases = { 'mime_tool' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/neural_test.lua b/lualib/rspamadm/neural_test.lua index cb056b97a..31d21a990 100644 --- a/lualib/rspamadm/neural_test.lua +++ b/lualib/rspamadm/neural_test.lua @@ -4,9 +4,9 @@ local lua_util = require "lua_util" local ucl = require "ucl" local parser = argparse() - :name "rspamadm neural_test" - :description "Test the neural network with labelled dataset" - :help_description_margin(32) + :name "rspamadm neural_test" + :description "Test the neural network with labelled dataset" + :help_description_margin(32) parser:option "-c --config" :description "Path to config file" @@ -40,26 +40,24 @@ parser:option '--rule' :description 'Rule to test' :argname('<rule>') - local HAM = "HAM" local SPAM = "SPAM" local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) end end - local function scan_email(rspamc_path, host, n_parallel, path, timeout) local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s", @@ -128,13 +126,13 @@ end local function get_stats_from_scan_results(results, rules) local rule_stats = {} - for rule,_ in pairs(rules) do - rule_stats[rule] = {tp = 0, tn = 0, fp = 0, fn = 0} + for rule, _ in pairs(rules) do + rule_stats[rule] = { tp = 0, tn = 0, fp = 0, fn = 0 } end - for _,result in ipairs(results) do - for _,symbol in ipairs(result["symbols"]) do - for name,rule in pairs(rules) do + for _, result in ipairs(results) do + for _, symbol in ipairs(result["symbols"]) do + for name, rule in pairs(rules) do if rule.symbol_spam and rule.symbol_spam == symbol then if result.type == HAM then rule_stats[name].fp = rule_stats[name].fp + 1 @@ -152,7 +150,7 @@ local function get_stats_from_scan_results(results, rules) end end - for rule,_ in pairs(rules) do + for rule, _ in pairs(rules) do rule_stats[rule].fpr = rule_stats[rule].fp / (rule_stats[rule].fp + rule_stats[rule].tn) rule_stats[rule].fnr = rule_stats[rule].fn / (rule_stats[rule].fn + rule_stats[rule].tp) end @@ -222,10 +220,9 @@ local function handler(args) end - return { name = "neuraltest", - aliases = {"neural_test"}, + aliases = { "neural_test" }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/publicsuffix.lua b/lualib/rspamadm/publicsuffix.lua index f8e1bac93..96bf0699e 100644 --- a/lualib/rspamadm/publicsuffix.lua +++ b/lualib/rspamadm/publicsuffix.lua @@ -31,17 +31,17 @@ parser:option '-c --config' :default(rspamd_paths['CONFDIR'] .. '/rspamd.conf') parser:command 'compile' - :description 'Compile publicsuffix list if needed' + :description 'Compile publicsuffix list if needed' local function load_config(config_file) - local _r,err = rspamd_config:load_ucl(config_file) + local _r, err = rspamd_config:load_ucl(config_file) if not _r then rspamd_logger.errx('cannot load %s: %s', config_file, err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', config_file, err) os.exit(1) @@ -67,7 +67,6 @@ local function handler(args) load_config(cmd_opts.config_file) - if cmd_opts.command == 'compile' then compile_handler(cmd_opts) else diff --git a/lualib/rspamadm/stat_convert.lua b/lualib/rspamadm/stat_convert.lua index 3b54826aa..62a19a221 100644 --- a/lualib/rspamadm/stat_convert.lua +++ b/lualib/rspamadm/stat_convert.lua @@ -4,7 +4,7 @@ local ucl = require "ucl" local logger = require "rspamd_logger" local lua_util = require "lua_util" -return function (_, res) +return function(_, res) local redis_params = lua_redis.try_load_redis_servers(res.redis, nil) if res.expire then res.expire = lua_util.parse_time_interval(res.expire) @@ -22,7 +22,7 @@ return function (_, res) return false end - for _,cls in ipairs(sqlite_params) do + for _, cls in ipairs(sqlite_params) do if not stat_tools.convert_sqlite_to_redis(redis_params, cls.db_spam, cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, res.expire, res.reset_previous) then @@ -33,6 +33,6 @@ return function (_, res) logger.messagex('Converted classifier to the from sqlite to redis') logger.messagex('Suggested configuration:') logger.messagex(ucl.to_format(stat_tools.redis_classifier_from_sqlite(cls, res.expire), - 'config')) + 'config')) end end diff --git a/lualib/rspamadm/statistics_dump.lua b/lualib/rspamadm/statistics_dump.lua index 0ed921f6a..6bc045850 100644 --- a/lualib/rspamadm/statistics_dump.lua +++ b/lualib/rspamadm/statistics_dump.lua @@ -68,31 +68,31 @@ restore:argument "file" :argname "<file>" :args "*" restore:option "-b --batch-size" - :description "Number of entires to process at once" - :argname("<elts>") - :convert(tonumber) - :default(1000) + :description "Number of entires to process at once" + :argname("<elts>") + :convert(tonumber) + :default(1000) restore:option "-m --mode" :description "Number of entires to process at once" :argname("<append|subtract|replace>") :convert { - ['append'] = 'append', - ['subtract'] = 'subtract', - ['replace'] = 'replace', - } + ['append'] = 'append', + ['subtract'] = 'subtract', + ['replace'] = 'replace', +} :default 'append' restore:flag "-n --no-operation" - :description "Only show redis commands to be issued" + :description "Only show redis commands to be issued" local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -128,9 +128,9 @@ local function check_redis_classifier(cls, cfg) local statfiles = cls.statfile if statfiles[1] then - for _,stf in ipairs(statfiles) do + for _, stf in ipairs(statfiles) do if not stf.symbol then - for k,v in pairs(stf) do + for k, v in pairs(stf) do check_statfile_table(v, k) end else @@ -138,7 +138,7 @@ local function check_redis_classifier(cls, cfg) end end else - for stn,stf in pairs(statfiles) do + for stn, stf in pairs(statfiles) do check_statfile_table(stf, stn) end end @@ -168,7 +168,7 @@ end local function redis_map_zip(ar) local data = {} - for j=1,#ar,2 do + for j = 1, #ar, 2 do data[ar[j]] = ar[j + 1] end @@ -178,7 +178,7 @@ end -- Used to clear tables local clear_fcn = table.clear or function(tbl) local keys = lua_util.keys(tbl) - for _,k in ipairs(keys) do + for _, k in ipairs(keys) do tbl[k] = nil end end @@ -197,7 +197,7 @@ local function dump_out(out, opts, last) compress_ctx:stream(rspamd_text.fromtable(out), 'flush'):write() end else - for _,o in ipairs(out) do + for _, o in ipairs(out) do io.write(o) end end @@ -213,7 +213,7 @@ local function dump_cdb(out, opts, last, pattern) out.cdb_builder:add('_lrnham_', rspamd_i64.fromstring(results.learns_ham or '0')) end - for _,o in ipairs(results.elts) do + for _, o in ipairs(results.elts) do out.cdb_builder:add(o.key, o.value) end @@ -227,9 +227,9 @@ local function dump_pattern(conn, pattern, opts, out, key) local cursor = 0 repeat - conn:add_cmd('SCAN', {tostring(cursor), - 'MATCH', pattern, - 'COUNT', tostring(opts.batch_size)}) + conn:add_cmd('SCAN', { tostring(cursor), + 'MATCH', pattern, + 'COUNT', tostring(opts.batch_size) }) local ret, results = conn:exec() if not ret then @@ -242,26 +242,26 @@ local function dump_pattern(conn, pattern, opts, out, key) local elts = results[2] local tokens = {} - for _,e in ipairs(elts) do - conn:add_cmd('HGETALL', {e}) + for _, e in ipairs(elts) do + conn:add_cmd('HGETALL', { e }) end -- This function returns many results, each for each command -- So if we have batch 1000, then we would have 1000 tables in form -- [result, {hash_content}] - local all_results = {conn:exec()} + local all_results = { conn:exec() } - for i=1,#all_results,2 do + for i = 1, #all_results, 2 do local r, hash_content = all_results[i], all_results[i + 1] if r then -- List to a hash map local data = redis_map_zip(hash_content) - tokens[#tokens + 1] = {key = elts[(i + 1)/2], data = data} + tokens[#tokens + 1] = { key = elts[(i + 1) / 2], data = data } end end -- Output keeping track of the commas - for i,d in ipairs(tokens) do + for i, d in ipairs(tokens) do if cursor == 0 and i == #tokens or not opts.json then if opts.cdb then table.insert(out[key].elts, { @@ -302,8 +302,8 @@ end local function dump_handler(opts) local patterns_seen = {} - for _,cls in ipairs(classifiers) do - local res,conn = lua_redis.redis_connect_sync(cls.redis_params, false) + for _, cls in ipairs(classifiers) do + local res, conn = lua_redis.redis_connect_sync(cls.redis_params, false) if not res then rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params) @@ -314,7 +314,7 @@ local function dump_handler(opts) local function check_keys(sym) local sym_keys_pattern = string.format("%s_keys", sym) conn:add_cmd('SMEMBERS', { sym_keys_pattern }) - local ret,keys = conn:exec() + local ret, keys = conn:exec() if not ret then rspamd_logger.errx("cannot execute command to get keys: %s", keys) @@ -325,11 +325,11 @@ local function dump_handler(opts) out[#out + 1] = string.format('"%s": %s\n', sym_keys_pattern, ucl.to_format(keys, 'json-compact')) end - for _,k in ipairs(keys) do + for _, k in ipairs(keys) do local pat = string.format('%s*_*', k) if not patterns_seen[pat] then - conn:add_cmd('HGETALL', {k}) - local _ret,additional_keys = conn:exec() + conn:add_cmd('HGETALL', { k }) + local _ret, additional_keys = conn:exec() if _ret then if opts.json then @@ -359,19 +359,19 @@ local function dump_handler(opts) end local function obj_to_redis_arguments(obj, opts, cmd_pipe) - local key,value = next(obj) + local key, value = next(obj) if type(key) == 'string' then if type(value) == 'table' then if not value[1] then if opts.mode == 'replace' then local cmd = 'HMSET' - local params = {key} - for k,v in pairs(value) do + local params = { key } + for k, v in pairs(value) do table.insert(params, k) table.insert(params, v) end - table.insert(cmd_pipe, {cmd, params}) + table.insert(cmd_pipe, { cmd, params }) else local cmd = 'HINCRBYFLOAT' local mult = 1.0 @@ -379,19 +379,19 @@ local function obj_to_redis_arguments(obj, opts, cmd_pipe) mult = (-mult) end - for k,v in pairs(value) do + for k, v in pairs(value) do if tonumber(v) then v = tonumber(v) - table.insert(cmd_pipe, {cmd, {key, k, tostring(v * mult)}}) + table.insert(cmd_pipe, { cmd, { key, k, tostring(v * mult) } }) else - table.insert(cmd_pipe, {'HSET', {key, k, v}}) + table.insert(cmd_pipe, { 'HSET', { key, k, v } }) end end end else -- Numeric table of elements (e.g. _keys) - it is actually a set in Redis - for _,elt in ipairs(value) do - table.insert(cmd_pipe, {'SADD', {key, elt}}) + for _, elt in ipairs(value) do + table.insert(cmd_pipe, { 'SADD', { key, elt } }) end end end @@ -403,17 +403,17 @@ end local function execute_batch(batch, conns, opts) local cmd_pipe = {} - for _,cmd in ipairs(batch) do + for _, cmd in ipairs(batch) do obj_to_redis_arguments(cmd, opts, cmd_pipe) end if opts.no_operation then - for _,cmd in ipairs(cmd_pipe) do + for _, cmd in ipairs(cmd_pipe) do rspamd_logger.messagex('%s %s', cmd[1], table.concat(cmd[2], ' ')) end else for _, conn in ipairs(conns) do - for _,cmd in ipairs(cmd_pipe) do + for _, cmd in ipairs(cmd_pipe) do local is_ok, err = conn:add_cmd(cmd[1], cmd[2]) if not is_ok then @@ -427,11 +427,11 @@ local function execute_batch(batch, conns, opts) end local function restore_handler(opts) - local files = opts.file or {'-'} + local files = opts.file or { '-' } local conns = {} - for _,cls in ipairs(classifiers) do - local res,conn = lua_redis.redis_connect_sync(cls.redis_params, true) + for _, cls in ipairs(classifiers) do + local res, conn = lua_redis.redis_connect_sync(cls.redis_params, true) if not res then rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params) @@ -443,7 +443,7 @@ local function restore_handler(opts) local batch = {} - for _,f in ipairs(files) do + for _, f in ipairs(files) do local fd if f ~= '-' then fd = io.open(f, 'r') @@ -454,7 +454,7 @@ local function restore_handler(opts) for line in io.lines() do local ucl_parser = ucl.parser() local res, err - res,err = ucl_parser:parse_string(line) + res, err = ucl_parser:parse_string(line) if not res then rspamd_logger.errx("%s: cannot read line %s: %s", f, cur_line, err) @@ -470,7 +470,9 @@ local function restore_handler(opts) end end - if fd then fd:close() end + if fd then + fd:close() + end end if #batch > 0 then @@ -492,8 +494,10 @@ local function handler(args) if classifier then if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.bayes then cls = cls.bayes end + for _, cls in ipairs(classifier) do + if cls.bayes then + cls = cls.bayes + end if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, obj) end @@ -503,7 +507,7 @@ local function handler(args) classifier = classifier.bayes if classifier[1] then - for _,cls in ipairs(classifier) do + for _, cls in ipairs(classifier) do if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, obj) end @@ -518,7 +522,7 @@ local function handler(args) end if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } elseif type(opts.file) == 'none' then opts.file = {} end @@ -534,7 +538,7 @@ end return { name = 'statistics_dump', - aliases = {'stat_dump', 'bayes_dump'}, + aliases = { 'stat_dump', 'bayes_dump' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/template.lua b/lualib/rspamadm/template.lua index bb799f730..ca1779aa8 100644 --- a/lualib/rspamadm/template.lua +++ b/lualib/rspamadm/template.lua @@ -42,17 +42,17 @@ parser:mutex( :description "Store files with the new suffix" :argname "<suffix>", parser:flag "-i --inplace" - :description "Replace input file(s)" + :description "Replace input file(s)" ) local lua_util = require "lua_util" local function set_env(opts, env) if opts.env then - for _,fname in ipairs(opts.env) do + for _, fname in ipairs(opts.env) do for kv in assert(io.open(fname)):lines() do if not kv:match('%s*#.*') then - local k,v = kv:match('([^=%s]+)%s*=%s*(.+)') + local k, v = kv:match('([^=%s]+)%s*=%s*(.+)') if k and v then env[k] = v @@ -65,14 +65,14 @@ local function set_env(opts, env) end if opts.lua_env then - for _,fname in ipairs(opts.env) do - local ret,res_or_err = pcall(loadfile(fname)) + for _, fname in ipairs(opts.env) do + local ret, res_or_err = pcall(loadfile(fname)) if not ret then io.write(string.format('cannot load %s: %s\n', fname, res_or_err)) else if type(res_or_err) == 'table' then - for k,v in pairs(res_or_err) do + for k, v in pairs(res_or_err) do env[k] = lua_util.deepcopy(v) end else @@ -100,8 +100,10 @@ local function handler(args) local env = {} set_env(opts, env) - if not opts.file or #opts.file == 0 then opts.file = {'-'} end - for _,fname in ipairs(opts.file) do + if not opts.file or #opts.file == 0 then + opts.file = { '-' } + end + for _, fname in ipairs(opts.file) do local content = read_file(fname) local res = lua_util.jinja_template(content, env, opts.no_vars) diff --git a/lualib/rspamadm/vault.lua b/lualib/rspamadm/vault.lua index 2c7d5abfe..840e504e0 100644 --- a/lualib/rspamadm/vault.lua +++ b/lualib/rspamadm/vault.lua @@ -43,28 +43,27 @@ parser:option "-o --output" :description "Output format ('ucl', 'json', 'json-compact', 'yaml')" :argname("<type>") :convert { - ucl = "ucl", - json = "json", - ['json-compact'] = "json-compact", - yaml = "yaml", - } - :default "ucl" + ucl = "ucl", + json = "json", + ['json-compact'] = "json-compact", + yaml = "yaml", +} + :default "ucl" parser:command "list ls l" - :description "List elements in the vault" + :description "List elements in the vault" local show = parser:command "show get" - :description "Extract element from the vault" + :description "Extract element from the vault" show:argument "domain" - :description "Domain to create key for" - :args "+" + :description "Domain to create key for" + :args "+" local delete = parser:command "delete del rm remove" - :description "Delete element from the vault" + :description "Delete element from the vault" delete:argument "domain" - :description "Domain to create delete key(s) for" - :args "+" - + :description "Domain to create delete key(s) for" + :args "+" local newkey = parser:command "newkey new create" :description "Add new key to the vault" @@ -77,10 +76,10 @@ newkey:option "-s --selector" newkey:option "-A --algorithm" :argname("<type>") :convert { - rsa = "rsa", - ed25519 = "ed25519", - eddsa = "ed25519", - } + rsa = "rsa", + ed25519 = "ed25519", + eddsa = "ed25519", +} :default "rsa" newkey:option "-b --bits" :argname("<nbits>") @@ -137,18 +136,18 @@ end local function parse_vault_reply(data) local p = ucl.parser() - local res,parser_err = p:parse_string(data) + local res, parser_err = p:parse_string(data) if not res then - return nil,parser_err + return nil, parser_err else - return p:get_object(),nil + return p:get_object(), nil end end local function maybe_print_vault_data(opts, data, func) if data then - local res,parser_err = parse_vault_reply(data) + local res, parser_err = parse_vault_reply(data) if not res then printf('vault reply for cannot be parsed: %s', parser_err) @@ -169,9 +168,9 @@ local function print_dkim_txt_record(b64, selector, alg) local prefix = string.format("v=DKIM1; k=%s; p=", alg) b64 = prefix .. b64 if #b64 < 255 then - labels = {'"' .. b64 .. '"'} + labels = { '"' .. b64 .. '"' } else - for sl=1,#b64,256 do + for sl = 1, #b64, 256 do table.insert(labels, '"' .. b64:sub(sl, sl + 255) .. '"') end end @@ -182,7 +181,7 @@ end local function show_handler(opts, domain) local uri = vault_url(opts, domain) - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -206,7 +205,7 @@ end local function delete_handler(opts, domain) local uri = vault_url(opts, domain) - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -229,7 +228,7 @@ end local function list_handler(opts) local uri = vault_url(opts) - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -258,7 +257,7 @@ end local function create_and_push_key(opts, domain, existing) local uri = vault_url(opts, domain) - local sk,pk = genkey(opts) + local sk, pk = genkey(opts) local res = { selectors = { @@ -274,7 +273,7 @@ local function create_and_push_key(opts, domain, existing) } } - for _,sel in ipairs(existing) do + for _, sel in ipairs(existing) do res.selectors[#res.selectors + 1] = sel end @@ -282,7 +281,7 @@ local function create_and_push_key(opts, domain, existing) res.selectors[1].valid_end = os.time() + opts.expire * 3600 * 24 end - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -303,7 +302,7 @@ local function create_and_push_key(opts, domain, existing) maybe_print_vault_data(opts, data.content) os.exit(1) else - maybe_printf(opts,'stored key for: %s, selector: %s', domain, opts.selector) + maybe_printf(opts, 'stored key for: %s, selector: %s', domain, opts.selector) maybe_printf(opts, 'please place the corresponding public key as following:') if opts.silent then @@ -322,7 +321,7 @@ local function newkey_handler(opts, domain) os.date("!%Y%m%d")) end - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -335,7 +334,7 @@ local function newkey_handler(opts, domain) } if is_http_error(err, data) or not data.content then - create_and_push_key(opts, domain,{}) + create_and_push_key(opts, domain, {}) else -- Key exists local rep = parse_vault_reply(data.content) @@ -348,11 +347,11 @@ local function newkey_handler(opts, domain) local elts = rep.data.selectors if not elts then - create_and_push_key(opts, domain,{}) + create_and_push_key(opts, domain, {}) os.exit(0) end - for _,sel in ipairs(elts) do + for _, sel in ipairs(elts) do if sel.alg == opts.algorithm then printf('key with the specific algorithm %s is already presented at %s selector for %s domain', opts.algorithm, sel.selector, domain) @@ -370,7 +369,7 @@ local function roll_handler(opts, domain) selectors = {} } - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -414,7 +413,7 @@ local function roll_handler(opts, domain) table.insert(nkeys[sel.alg], sel) end - for _,sel in ipairs(elts) do + for _, sel in ipairs(elts) do if sel.valid_end and sel.valid_end < os.time() then if not opts.remove_expired then insert_key(sel, false) @@ -428,7 +427,7 @@ local function roll_handler(opts, domain) end -- Now we need to ensure that all but one selectors have either expired or just a single key - for alg,keys in pairs(nkeys) do + for alg, keys in pairs(nkeys) do table.sort(keys, function(k1, k2) if k1.valid_end and k2.valid_end then return k1.valid_end > k2.valid_end @@ -441,8 +440,8 @@ local function roll_handler(opts, domain) end) -- Exclude the key with the highest expiration date and examine the rest if not (#keys == 1 or fun.all(function(k) - return k.valid_end and k.valid_end < os.time() - end, fun.tail(keys))) then + return k.valid_end and k.valid_end < os.time() + end, fun.tail(keys))) then printf('bad keys list for %s and %s algorithm', domain, alg) fun.each(function(k) if not k.valid_end then @@ -459,7 +458,7 @@ local function roll_handler(opts, domain) if not opts.remove_expired then -- OK to process -- Insert keys for each algorithm in pairs <old_key(s)>, <new_key> - local sk,pk = genkey({algorithm = alg, bits = keys[1].bits}) + local sk, pk = genkey({ algorithm = alg, bits = keys[1].bits }) local selector = string.format('%s-%s', alg, os.date("!%Y%m%d")) @@ -482,14 +481,14 @@ local function roll_handler(opts, domain) table.insert(res.selectors, nelt) end - for _,k in ipairs(keys) do + for _, k in ipairs(keys) do table.insert(res.selectors, k) end end end -- We can now store res in the vault - err,data = rspamd_http.request{ + err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -510,9 +509,9 @@ local function roll_handler(opts, domain) maybe_print_vault_data(opts, data.content) os.exit(1) else - for _,key in ipairs(res.selectors) do - if not key.valid_end or key.valid_end > os.time() + opts.ttl * 3600 * 24 then - maybe_printf(opts,'rolled key for: %s, new selector: %s', domain, key.selector) + for _, key in ipairs(res.selectors) do + if not key.valid_end or key.valid_end > os.time() + opts.ttl * 3600 * 24 then + maybe_printf(opts, 'rolled key for: %s, new selector: %s', domain, key.selector) maybe_printf(opts, 'please place the corresponding public key as following:') if opts.silent then @@ -553,13 +552,21 @@ local function handler(args) if command == 'list' then list_handler(opts) elseif command == 'show' then - fun.each(function(d) show_handler(opts, d) end, opts.domain) + fun.each(function(d) + show_handler(opts, d) + end, opts.domain) elseif command == 'newkey' then - fun.each(function(d) newkey_handler(opts, d) end, opts.domain) + fun.each(function(d) + newkey_handler(opts, d) + end, opts.domain) elseif command == 'roll' then - fun.each(function(d) roll_handler(opts, d) end, opts.domain) + fun.each(function(d) + roll_handler(opts, d) + end, opts.domain) elseif command == 'delete' then - fun.each(function(d) delete_handler(opts, d) end, opts.domain) + fun.each(function(d) + delete_handler(opts, d) + end, opts.domain) else parser:error(string.format('command %s is not implemented', command)) end |