diff options
author | Vsevolod Stakhov <vsevolod@rspamd.com> | 2023-08-07 11:41:28 +0100 |
---|---|---|
committer | Vsevolod Stakhov <vsevolod@rspamd.com> | 2023-08-07 11:41:28 +0100 |
commit | 662145d0554de5e769b92dab2d41173a98adcee5 (patch) | |
tree | ec28311a0bce6181f248ba7b50304293ad764e44 /lualib/rspamadm | |
parent | bbd88232db43d18f5e0de5a6502848d4074621c5 (diff) | |
download | rspamd-662145d0554de5e769b92dab2d41173a98adcee5.tar.gz rspamd-662145d0554de5e769b92dab2d41173a98adcee5.zip |
[Minor] Reformat all Lua code, no functional changes
Diffstat (limited to 'lualib/rspamadm')
-rw-r--r-- | lualib/rspamadm/clickhouse.lua | 108 | ||||
-rw-r--r-- | lualib/rspamadm/configgraph.lua | 12 | ||||
-rw-r--r-- | lualib/rspamadm/confighelp.lua | 12 | ||||
-rw-r--r-- | lualib/rspamadm/configwizard.lua | 135 | ||||
-rw-r--r-- | lualib/rspamadm/cookie.lua | 14 | ||||
-rw-r--r-- | lualib/rspamadm/corpus_test.lua | 3 | ||||
-rw-r--r-- | lualib/rspamadm/dkim_keygen.lua | 53 | ||||
-rw-r--r-- | lualib/rspamadm/dmarc_report.lua | 81 | ||||
-rw-r--r-- | lualib/rspamadm/dns_tool.lua | 32 | ||||
-rw-r--r-- | lualib/rspamadm/fuzzy_convert.lua | 14 | ||||
-rw-r--r-- | lualib/rspamadm/fuzzy_stat.lua | 60 | ||||
-rw-r--r-- | lualib/rspamadm/grep.lua | 4 | ||||
-rw-r--r-- | lualib/rspamadm/keypair.lua | 70 | ||||
-rw-r--r-- | lualib/rspamadm/mime.lua | 213 | ||||
-rw-r--r-- | lualib/rspamadm/neural_test.lua | 27 | ||||
-rw-r--r-- | lualib/rspamadm/publicsuffix.lua | 7 | ||||
-rw-r--r-- | lualib/rspamadm/stat_convert.lua | 6 | ||||
-rw-r--r-- | lualib/rspamadm/statistics_dump.lua | 116 | ||||
-rw-r--r-- | lualib/rspamadm/template.lua | 18 | ||||
-rw-r--r-- | lualib/rspamadm/vault.lua | 107 |
20 files changed, 586 insertions, 506 deletions
diff --git a/lualib/rspamadm/clickhouse.lua b/lualib/rspamadm/clickhouse.lua index 6bdc612d8..b22d8007c 100644 --- a/lualib/rspamadm/clickhouse.lua +++ b/lualib/rspamadm/clickhouse.lua @@ -66,62 +66,62 @@ parser:flag '--use-https' :argname('use_https') local neural_profile = parser:command 'neural_profile' - :description 'Generate symbols profile using data from Clickhouse' + :description 'Generate symbols profile using data from Clickhouse' neural_profile:option '-w --where' - :description 'WHERE clause for Clickhouse query' - :argname('where') + :description 'WHERE clause for Clickhouse query' + :argname('where') neural_profile:flag '-j --json' - :description 'Write output as JSON' - :argname('json') + :description 'Write output as JSON' + :argname('json') neural_profile:option '--days' - :description 'Number of days to collect stats for' - :argname('days') - :default('7') + :description 'Number of days to collect stats for' + :argname('days') + :default('7') neural_profile:option '--limit -l' - :description 'Maximum rows to fetch per day' - :argname('limit') + :description 'Maximum rows to fetch per day' + :argname('limit') neural_profile:option '--settings-id' - :description 'Settings ID to query' - :argname('settings_id') - :default('') + :description 'Settings ID to query' + :argname('settings_id') + :default('') local neural_train = parser:command 'neural_train' - :description 'Train neural using data from Clickhouse' + :description 'Train neural using data from Clickhouse' neural_train:option '--days' - :description 'Number of days to query data for' - :argname('days') - :default('7') + :description 'Number of days to query data for' + :argname('days') + :default('7') neural_train:option '--column-name-digest' - :description 'Name of neural profile digest column in Clickhouse' - :argname('column_name_digest') - :default('NeuralDigest') + :description 'Name of neural profile digest column in Clickhouse' + :argname('column_name_digest') + :default('NeuralDigest') neural_train:option '--column-name-vector' - :description 'Name of neural training vector column in Clickhouse' - :argname('column_name_vector') - :default('NeuralMpack') + :description 'Name of neural training vector column in Clickhouse' + :argname('column_name_vector') + :default('NeuralMpack') neural_train:option '--limit -l' - :description 'Maximum rows to fetch per day' - :argname('limit') + :description 'Maximum rows to fetch per day' + :argname('limit') neural_train:option '--profile -p' - :description 'Profile to use for training' - :argname('profile') - :default('default') + :description 'Profile to use for training' + :argname('profile') + :default('default') neural_train:option '--rule -r' - :description 'Rule to train' - :argname('rule') - :default('default') + :description 'Rule to train' + :argname('rule') + :default('default') neural_train:option '--spam -s' - :description 'WHERE clause to use for spam' - :argname('spam') - :default("Action == 'reject'") + :description 'WHERE clause to use for spam' + :argname('spam') + :default("Action == 'reject'") neural_train:option '--ham -h' - :description 'WHERE clause to use for ham' - :argname('ham') - :default('Score < 0') + :description 'WHERE clause to use for ham' + :argname('ham') + :default('Score < 0') neural_train:option '--url -u' - :description 'URL to use for training' - :argname('url') - :default('http://127.0.0.1:11334/plugins/neural/learn') + :description 'URL to use for training' + :argname('url') + :default('http://127.0.0.1:11334/plugins/neural/learn') local http_params = { config = rspamd_config, @@ -131,14 +131,14 @@ local http_params = { } local function load_config(config_file) - local _r,err = rspamd_config:load_ucl(config_file) + local _r, err = rspamd_config:load_ucl(config_file) if not _r then rspamd_logger.errx('cannot load %s: %s', config_file, err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', config_file, err) os.exit(1) @@ -196,7 +196,7 @@ local function get_excluded_symbols(known_symbols, correlations, seen_total) elseif not all_symbols[k] then remove[k] = 'nonexistent symbol' else - for fl,_ in pairs(all_symbols[k].flags or {}) do + for fl, _ in pairs(all_symbols[k].flags or {}) do if skip_flags[fl] then remove[k] = fl .. ' symbol' break @@ -238,7 +238,7 @@ local function handle_neural_profile(args) local nsym = #r['Symbols.Names'] - for i = 1,nsym do + for i = 1, nsym do local sym = r['Symbols.Names'][i] local t = known_symbols[sym] if not t then @@ -266,8 +266,8 @@ local function handle_neural_profile(args) end -- Fill correlations - for i = 1,nsym do - for j = 1,nsym do + for i = 1, nsym do + for j = 1, nsym do if i ~= j then local sym = r['Symbols.Names'][i] local inner_sym_name = r['Symbols.Names'][j] @@ -342,11 +342,11 @@ end local function post_neural_training(url, rule, spam_rows, ham_rows) -- Prepare JSON payload local payload = ucl.to_format( - { - ham_vec = ham_rows, - rule = rule, - spam_vec = spam_rows, - }, 'json') + { + ham_vec = ham_rows, + rule = rule, + spam_vec = spam_rows, + }, 'json') -- POST the payload local err, response = rspamd_http.request({ @@ -423,11 +423,11 @@ local function handle_neural_train(args) limit = string.format(' LIMIT %d', num_limit) -- Contains leading space end -- Prepare query elements - local conditions = {string.format("%s = '%s'", args.column_name_digest, symbols_digest)} + local conditions = { string.format("%s = '%s'", args.column_name_digest, symbols_digest) } local query_fmt = 'SELECT %s FROM rspamd WHERE %s%s' -- Run queries - for _, the_where in ipairs({args.ham, args.spam}) do + for _, the_where in ipairs({ args.ham, args.spam }) do -- Inform callback which group of vectors we're collecting this_where = the_where table.insert(conditions, the_where) -- should be 2nd from last condition @@ -437,7 +437,7 @@ local function handle_neural_train(args) if this_where == args.ham then if not want_ham then break - end + end else if not want_spam then break diff --git a/lualib/rspamadm/configgraph.lua b/lualib/rspamadm/configgraph.lua index 75eb2d940..07f14a9ca 100644 --- a/lualib/rspamadm/configgraph.lua +++ b/lualib/rspamadm/configgraph.lua @@ -31,7 +31,6 @@ parser:option "-c --config" parser:flag "-a --all" :description('Show all nodes, not just existing ones') - local function process_filename(fname) local cdir = rspamd_paths['CONFDIR'] .. '/' fname = fname:gsub(cdir, '') @@ -40,8 +39,8 @@ end local function output_dot(opts, nodes, adjacency) rspamd_logger.messagex("digraph rspamd {") - for k,node in pairs(nodes) do - local attrs = {"shape=box"} + for k, node in pairs(nodes) do + local attrs = { "shape=box" } local skip = false if node.exists then if node.priority >= 10 then @@ -62,7 +61,7 @@ local function output_dot(opts, nodes, adjacency) table.concat(attrs, ',')) end end - for _,adj in ipairs(adjacency) do + for _, adj in ipairs(adjacency) do local attrs = {} local skip = false @@ -95,7 +94,7 @@ local function load_config_traced(opts) local nodes = {} local function maybe_match_glob(file) - for _,gl in ipairs(glob_traces) do + for _, gl in ipairs(glob_traces) do if gl.re:match(file) then return gl end @@ -151,7 +150,7 @@ local function load_config_traced(opts) end end - local _r,err = rspamd_config:load_ucl(opts['config'], trace_func) + local _r, err = rspamd_config:load_ucl(opts['config'], trace_func) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) @@ -160,7 +159,6 @@ local function load_config_traced(opts) output_dot(opts, nodes, adjacency) end - local function handler(args) local res = parser:parse(args) diff --git a/lualib/rspamadm/confighelp.lua b/lualib/rspamadm/confighelp.lua index 1eecb9238..38b26b6fc 100644 --- a/lualib/rspamadm/confighelp.lua +++ b/lualib/rspamadm/confighelp.lua @@ -16,11 +16,11 @@ local parser = argparse() parser:argument "path":args "*" :description('Optional config paths') parser:flag "--no-color" - :description "Disable coloured output" + :description "Disable coloured output" parser:flag "--short" - :description "Show only option names" + :description "Show only option names" parser:flag "--no-examples" - :description "Do not show examples (implied by --short)" + :description "Do not show examples (implied by --short)" local function maybe_print_color(key) if not opts['no-color'] then @@ -84,10 +84,10 @@ local function print_help(key, value, tabs) if type(value['required']) == 'boolean' then if value['required'] then print(string.format('%s\tRequired: %s', tabs, - maybe_print_color(tostring(value['required'])))) + maybe_print_color(tostring(value['required'])))) else print(string.format('%s\tRequired: %s', tabs, - tostring(value['required']))) + tostring(value['required']))) end end if value['default'] then @@ -116,7 +116,7 @@ return function(args, res) local sorted = sort_values(res) - for _,v in ipairs(sorted) do + for _, v in ipairs(sorted) do print_help(v['key'], v['value'], '') print('') end diff --git a/lualib/rspamadm/configwizard.lua b/lualib/rspamadm/configwizard.lua index c79d73b97..27c358e3b 100644 --- a/lualib/rspamadm/configwizard.lua +++ b/lualib/rspamadm/configwizard.lua @@ -74,19 +74,29 @@ local function ask_yes_no(greet, default) local reply = rspamd_util.readline(greet) - if not reply then os.exit(0) end - if #reply == 0 then reply = def_str end + if not reply then + os.exit(0) + end + if #reply == 0 then + reply = def_str + end reply = reply:lower() - if reply == 'y' or reply == 'yes' then return true end + if reply == 'y' or reply == 'yes' then + return true + end return false end local function readline_default(greet, def_value) local reply = rspamd_util.readline(greet) - if not reply then os.exit(0) end + if not reply then + os.exit(0) + end - if #reply == 0 then return def_value end + if #reply == 0 then + return def_value + end return reply end @@ -95,7 +105,7 @@ local function readline_expire() local expire = '100d' repeat expire = readline_default("Expire time for new tokens [" .. expire .. "]: ", - expire) + expire) expire = lua_util.parse_time_interval(expire) if not expire then @@ -118,9 +128,9 @@ end local function print_changes(changes) local function print_change(k, c, where) printf('File: %s, changes list:', highlight(local_conf .. '/' - .. where .. '/'.. k)) + .. where .. '/' .. k)) - for ek,ev in pairs(c) do + for ek, ev in pairs(c) do printf("%s => %s", highlight(ek), rspamd_logger.slog("%s", ev)) end end @@ -144,7 +154,7 @@ local function apply_changes(changes) end local function apply_change(k, c, where) - local fname = local_conf .. '/' .. where .. '/'.. k + local fname = local_conf .. '/' .. where .. '/' .. k if not rspamd_util.file_exists(fname) then printf("Create file %s", highlight(fname)) @@ -181,7 +191,6 @@ local function apply_changes(changes) end end - local function setup_controller(controller, changes) printf("Setup %s and controller worker:", highlight("WebUI")) @@ -208,13 +217,13 @@ local function setup_redis(cfg, changes) printf("%s servers are not set:", highlight("Redis")) printf("The following modules will be enabled if you add Redis servers:") - for k,_ in pairs(rspamd_plugins_state.disabled_redis) do + for k, _ in pairs(rspamd_plugins_state.disabled_redis) do printf("\t* %s", highlight(k)) end if ask_yes_no("Do you wish to set Redis servers?", true) then local read_servers = readline_default("Input read only servers separated by `,` [default: localhost]: ", - "localhost") + "localhost") local rs = parse_servers(read_servers) if rs and #rs > 0 then @@ -263,7 +272,9 @@ end local function setup_dkim_signing(cfg, changes) -- Remove the trailing slash of a pathname, if present. local function remove_trailing_slash(path) - if string.sub(path, -1) ~= "/" then return path end + if string.sub(path, -1) ~= "/" then + return path + end return string.sub(path, 1, string.len(path) - 1) end @@ -281,7 +292,7 @@ local function setup_dkim_signing(cfg, changes) local use_esld local sign_domain = 'pet luacheck' - local defined_auth_types = {'header', 'envelope', 'auth', 'recipient'} + local defined_auth_types = { 'header', 'envelope', 'auth', 'recipient' } if sign_type == '4' then repeat @@ -318,7 +329,9 @@ local function setup_dkim_signing(cfg, changes) sign_authenticated = true end - if fun.any(function(s) return s == sign_domain end, defined_auth_types) then + if fun.any(function(s) + return s == sign_domain + end, defined_auth_types) then -- Allow mismatch allow_mismatch = ask_yes_no( string.format('Allow data %s, e.g. if mime from domain is not equal to authenticated user domain? ', @@ -337,7 +350,7 @@ local function setup_dkim_signing(cfg, changes) local dkim_keys_dir = rspamd_paths["DBDIR"] .. "/dkim/" local prompt = string.format("Enter output directory for the keys [default: %s]: ", - highlight(dkim_keys_dir)) + highlight(dkim_keys_dir)) dkim_keys_dir = remove_trailing_slash(readline_default(prompt, dkim_keys_dir)) local ret, err = rspamd_util.mkdir(dkim_keys_dir, true) @@ -349,7 +362,7 @@ local function setup_dkim_signing(cfg, changes) local function print_domains() printf("Domains configured:") - for k,v in pairs(domains) do + for k, v in pairs(domains) do printf("Domain: %s, selector: %s, privkey: %s", highlight(k), v.selector, v.privkey) end @@ -370,13 +383,15 @@ local function setup_dkim_signing(cfg, changes) until #domain ~= 0 local selector = readline_default("Enter selector [default: dkim]: ", 'dkim') - if not selector then selector = 'dkim' end + if not selector then + selector = 'dkim' + end local privkey_file = string.format("%s/%s.%s.key", dkim_keys_dir, domain, selector) if not rspamd_util.file_exists(privkey_file) then if ask_yes_no("Do you want to create privkey " .. highlight(privkey_file), - true) then + true) then local pubkey_file = privkey_file .. ".pub" local rspamd_cryptobox = require "rspamd_cryptobox" local sk, pk = rspamd_cryptobox.generate_keypair("rsa", 2048) @@ -402,7 +417,7 @@ local function setup_dkim_signing(cfg, changes) } until not ask_yes_no("Do you wish to add another DKIM domain?") - changes.l['dkim_signing.conf'] = {domain = domains} + changes.l['dkim_signing.conf'] = { domain = domains } local res_tbl = changes.l['dkim_signing.conf'] if sign_networks then @@ -426,7 +441,7 @@ local function check_redis_classifier(cls, changes) local symbol_spam, symbol_ham -- Load symbols from statfiles local statfiles = cls.statfile - for _,stf in ipairs(statfiles) do + for _, stf in ipairs(statfiles) do local symbol = stf.symbol or 'undefined' local spam @@ -484,12 +499,14 @@ local function check_redis_classifier(cls, changes) end local function get_version(conn) - conn:add_cmd("SMEMBERS", {"RS_keys"}) + conn:add_cmd("SMEMBERS", { "RS_keys" }) - local ret,members = conn:exec() + local ret, members = conn:exec() -- Empty db - if not ret or #members == 0 then return false,0 end + if not ret or #members == 0 then + return false, 0 + end -- We still need to check versions local lua_script = [[ @@ -502,10 +519,10 @@ end return ver ]] - conn:add_cmd('EVAL', {lua_script, '1', 'RS'}) - local _,ver = conn:exec() + conn:add_cmd('EVAL', { lua_script, '1', 'RS' }) + local _, ver = conn:exec() - return true,tonumber(ver) + return true, tonumber(ver) end local function check_expire(conn) @@ -522,21 +539,23 @@ end return ttl ]] - conn:add_cmd('EVAL', {lua_script, '0'}) - local _,ttl = conn:exec() + conn:add_cmd('EVAL', { lua_script, '0' }) + local _, ttl = conn:exec() return tonumber(ttl) end - local res,conn = lua_redis.redis_connect_sync(parsed_redis, true) + local res, conn = lua_redis.redis_connect_sync(parsed_redis, true) if not res then printf("Cannot connect to Redis server") return false end if not cls.new_schema then - local r,ver = get_version(conn) - if not r then return false end + local r, ver = get_version(conn) + if not r then + return false + end if ver ~= 2 then if not ver then printf('Key "RS_version" has not been found in Redis for %s/%s', @@ -562,17 +581,19 @@ return ttl end end else - local r,ver = get_version(conn) - if not r then return false end + local r, ver = get_version(conn) + if not r then + return false + end if ver ~= 2 then printf("You have configured new schema for %s/%s but your DB has old version: %s", - symbol_spam, symbol_ham, ver) + symbol_spam, symbol_ham, ver) try_convert(false) else printf( 'You have configured new schema for %s/%s and your DB already has new layout (v. %s).' .. ' DB conversion is not needed.', - symbol_spam, symbol_ham, ver) + symbol_spam, symbol_ham, ver) end end end @@ -584,7 +605,7 @@ local function setup_statistic(cfg, changes) if not redis_params then printf('You have %d sqlite classifiers, but you have no Redis servers being set', - #sqlite_configs) + #sqlite_configs) return false end @@ -596,7 +617,7 @@ local function setup_statistic(cfg, changes) local reset_previous = ask_yes_no("Reset previous data?") if ask_yes_no('Do you wish to convert them to Redis?', true) then - for _,cls in ipairs(sqlite_configs) do + for _, cls in ipairs(sqlite_configs) do if rspamd_util.file_exists(cls.db_spam) and rspamd_util.file_exists(cls.db_ham) then if not lua_stat_tools.convert_sqlite_to_redis(parsed_redis, cls.db_spam, cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, expire, @@ -634,8 +655,10 @@ local function setup_statistic(cfg, changes) if classifier then if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.bayes then cls = cls.bayes end + for _, cls in ipairs(classifier) do + if cls.bayes then + cls = cls.bayes + end if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, changes) end @@ -645,7 +668,7 @@ local function setup_statistic(cfg, changes) classifier = classifier.bayes if classifier[1] then - for _,cls in ipairs(classifier) do + for _, cls in ipairs(classifier) do if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, changes) end @@ -663,22 +686,24 @@ end local function find_worker(cfg, wtype) if cfg.worker then - for k,s in pairs(cfg.worker) do + for k, s in pairs(cfg.worker) do if type(k) == 'number' and type(s) == 'table' then - if s[wtype] then return s[wtype] end + if s[wtype] then + return s[wtype] + end end if type(s) == 'table' and s.type and s.type == wtype then return s end - if type(k) == 'string' and k == wtype then return s end + if type(k) == 'string' and k == wtype then + return s + end end end return nil end - - return { handler = function(cmd_args) local changes = { @@ -698,14 +723,14 @@ return { local opts = parser:parse(cmd_args) local args = opts['checks'] or {} - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -721,13 +746,13 @@ return { if #args > 0 then interactive_start = false - for _,arg in ipairs(args) do + for _, arg in ipairs(args) do if arg == 'all' then checks = all_checks elseif arg == 'list' then printf(highlight(rspamd_logo)) printf('Available modules') - for _,c in ipairs(all_checks) do + for _, c in ipairs(all_checks) do printf('- %s', c) end return @@ -740,7 +765,7 @@ return { end local function has_check(check) - for _,c in ipairs(checks) do + for _, c in ipairs(checks) do if c == check then return true end @@ -791,8 +816,12 @@ return { end local nchanges = 0 - for _,_ in pairs(changes.l) do nchanges = nchanges + 1 end - for _,_ in pairs(changes.o) do nchanges = nchanges + 1 end + for _, _ in pairs(changes.l) do + nchanges = nchanges + 1 + end + for _, _ in pairs(changes.o) do + nchanges = nchanges + 1 + end if nchanges > 0 then print_changes(changes) diff --git a/lualib/rspamadm/cookie.lua b/lualib/rspamadm/cookie.lua index bc4df9de2..7e0526a7e 100644 --- a/lualib/rspamadm/cookie.lua +++ b/lualib/rspamadm/cookie.lua @@ -44,7 +44,9 @@ parser:argument "cookie":args "?" local function gen_cookie(args, key) local cr = require "rspamd_cryptobox" - if not args.cookie then return end + if not args.cookie then + return + end local function encrypt() if #args.cookie > 31 then @@ -67,7 +69,7 @@ local function gen_cookie(args, key) extracted_cookie = args.cookie end - local dec_cookie,ts = cr.decrypt_cookie(key, extracted_cookie) + local dec_cookie, ts = cr.decrypt_cookie(key, extracted_cookie) if dec_cookie then if args.timestamp then @@ -79,7 +81,7 @@ local function gen_cookie(args, key) print('cannot decrypt cookie') os.exit(1) end - end + end if args.decrypt then decrypt() @@ -96,8 +98,10 @@ local function handler(args) end if res.key then - local pattern = {'^'} - for i=1,32 do pattern[i + 1] = '[a-zA-Z0-9]' end + local pattern = { '^' } + for i = 1, 32 do + pattern[i + 1] = '[a-zA-Z0-9]' + end pattern[34] = '$' if not res.key:match(table.concat(pattern, '')) then diff --git a/lualib/rspamadm/corpus_test.lua b/lualib/rspamadm/corpus_test.lua index 939c3c0fa..0e63f9f6f 100644 --- a/lualib/rspamadm/corpus_test.lua +++ b/lualib/rspamadm/corpus_test.lua @@ -177,10 +177,9 @@ local function handler(args) rspamd_logger.messagex("Messages/sec: %s", (total_msgs / elapsed_time)) end - return { name = 'corpustest', - aliases = {'corpus_test', 'corpus'}, + aliases = { 'corpus_test', 'corpus' }, handler = handler, description = parser._description } diff --git a/lualib/rspamadm/dkim_keygen.lua b/lualib/rspamadm/dkim_keygen.lua index 0d0ebe5a6..3d51903dc 100644 --- a/lualib/rspamadm/dkim_keygen.lua +++ b/lualib/rspamadm/dkim_keygen.lua @@ -32,39 +32,39 @@ parser:option '-k --privkey' :description 'Save private key to file instead of printing it to stdout' parser:option '-b --bits' :convert(function(input) - local n = tonumber(input) - if not n or n < 512 or n > 4096 then - return nil - end - return n - end) + local n = tonumber(input) + if not n or n < 512 or n > 4096 then + return nil + end + return n +end) :description 'Generate an RSA key with the specified number of bits (512 to 4096)' :default(1024) parser:option '-t --type' :description 'Key type: RSA, ED25519 or ED25119-seed' :convert { - ['rsa'] = 'rsa', - ['RSA'] = 'rsa', - ['ed25519'] = 'ed25519', - ['ED25519'] = 'ed25519', - ['ed25519-seed'] = 'ed25519-seed', - ['ED25519-seed'] = 'ed25519-seed', - } + ['rsa'] = 'rsa', + ['RSA'] = 'rsa', + ['ed25519'] = 'ed25519', + ['ED25519'] = 'ed25519', + ['ed25519-seed'] = 'ed25519-seed', + ['ED25519-seed'] = 'ed25519-seed', +} :default 'rsa' parser:option '-o --output' :description 'Output public key in the following format: dns, dnskey or plain' :convert { - ['dns'] = 'dns', - ['plain'] = 'plain', - ['dnskey'] = 'dnskey', - } + ['dns'] = 'dns', + ['plain'] = 'plain', + ['dnskey'] = 'dnskey', +} :default 'dns' parser:option '--priv-output' :description 'Output private key in the following format: PEM or DER (for RSA)' :convert { - ['pem'] = 'pem', - ['der'] = 'der', - } + ['pem'] = 'pem', + ['der'] = 'der', +} :default 'pem' parser:flag '-f --force' :description 'Force overwrite of existing files' @@ -83,7 +83,6 @@ local function split_string(input, max_length) return pieces end - local function print_public_key_dns(opts, base64_pk) local key_type = opts.type == 'rsa' and 'rsa' or 'ed25519' if #base64_pk < 255 - 2 then @@ -93,7 +92,7 @@ local function print_public_key_dns(opts, base64_pk) -- Split it by parts local parts = split_string(base64_pk) io.write(string.format('%s._domainkey IN TXT ( "v=DKIM1; k=%s; "\n', opts.selector, key_type)) - for i,part in ipairs(parts) do + for i, part in ipairs(parts) do if i == 1 then io.write(string.format('\t"p=%s"\n', part)) else @@ -121,7 +120,7 @@ end local function gen_rsa_key(opts) local rsa = require "rspamd_rsa" - local sk,pk = rsa.keypair(opts.bits or 1024) + local sk, pk = rsa.keypair(opts.bits or 1024) if opts.privkey then if opts.force then os.remove(opts.privkey) @@ -135,7 +134,7 @@ local function gen_rsa_key(opts) end local function gen_eddsa_key(opts) - local sk,pk = rspamd_cryptobox.gen_dkim_keypair(opts.type) + local sk, pk = rspamd_cryptobox.gen_dkim_keypair(opts.type) if opts.privkey and opts.force then os.remove(opts.privkey) @@ -156,7 +155,9 @@ end local function handler(args) local opts = parser:parse(args) - if not opts then os.exit(1) end + if not opts then + os.exit(1) + end if opts.type == 'rsa' then gen_rsa_key(opts) @@ -167,7 +168,7 @@ end return { name = 'dkim_keygen', - aliases = {'dkimkeygen'}, + aliases = { 'dkimkeygen' }, handler = handler, description = parser._description } diff --git a/lualib/rspamadm/dmarc_report.lua b/lualib/rspamadm/dmarc_report.lua index c81be13cf..65b1da158 100644 --- a/lualib/rspamadm/dmarc_report.lua +++ b/lualib/rspamadm/dmarc_report.lua @@ -46,9 +46,9 @@ parser:flag "-n --no-opt" :description "Do not reset reporting data/send reports" parser:argument "date" - :description "Date to process (today by default)" - :argname "<YYYYMMDD>" - :args "*" + :description "Date to process (today by default)" + :argname "<YYYYMMDD>" + :args "*" parser:option "-b --batch-size" :description "Send reports in batches up to <batch-size> messages" :argname "<number>" @@ -102,14 +102,14 @@ local redis_attrs = { local pool local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -118,11 +118,9 @@ end -- Concat elements using redis_keys.join_char local function redis_prefix(...) - return table.concat({...}, dmarc_settings.reporting.redis_keys.join_char) + return table.concat({ ... }, dmarc_settings.reporting.redis_keys.join_char) end - - local function get_rua(rep_key) local parts = lua_util.str_split(rep_key, dmarc_settings.reporting.redis_keys.join_char) @@ -144,8 +142,8 @@ local function get_domain(rep_key) end local function gen_uuid() - local template ='xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' - return string.gsub(template, '[xy]', function (c) + local template = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' + return string.gsub(template, '[xy]', function(c) local v = (c == 'x') and math.random(0, 0xf) or math.random(8, 0xb) return string.format('%x', v) end) @@ -159,7 +157,7 @@ local function gen_xml_grammar() local quot = lpeg.P('"') / '"' local apos = lpeg.P("'") / ''' local special = lt + gt + amp + quot + apos - local grammar = lpeg.Cs((special + 1)^0) + local grammar = lpeg.Cs((special + 1) ^ 0) return grammar end @@ -250,7 +248,7 @@ local function entry_to_xml(data) </auth_results> </record> ]] - return lua_util.jinja_template(xml_template, {data = data}, true) + return lua_util.jinja_template(xml_template, { data = data }, true) end -- Process a report entry stored in Redis splitting it to a lua table @@ -273,7 +271,7 @@ local function process_report_entry(data, score) if dkim_data and dkim_data ~= '' then local dkim_elts = lua_util.str_split(dkim_data, '|') for _, d in ipairs(dkim_elts) do - table.insert(row.dkim_results, {domain = d, result = result}) + table.insert(row.dkim_results, { domain = d, result = result }) end end end @@ -313,7 +311,7 @@ local function process_rua(dmarc_domain, rua) logger.errx('cannot resolve %s: %s; exclude %s', resolve_str, results, rua_part) else local found = false - for _,t in ipairs(results) do + for _, t in ipairs(results) do if string.match(t, 'v=DMARC1') then found = true break @@ -350,7 +348,7 @@ local function validate_reporting_domain(reporting_domain) config = rspamd_config, session = rspamadm_session, type = 'txt', - name = '_dmarc.' .. dmarc_domain , + name = '_dmarc.' .. dmarc_domain, }) if not is_ok or not results then @@ -358,8 +356,8 @@ local function validate_reporting_domain(reporting_domain) return nil end - for _,r in ipairs(results) do - local processed,rec = dmarc_common.dmarc_check_record(rspamd_config, r, false) + for _, r in ipairs(results) do + local processed, rec = dmarc_common.dmarc_check_record(rspamd_config, r, false) if processed and rec.rua then -- We need to check or alter rua if needed local processed_rua = process_rua(dmarc_domain, rec.rua) @@ -385,7 +383,7 @@ end -- Returns a list of recipients from a table as a string processing elements if needed local function rcpt_list(tbl, func) local res = {} - for _,r in ipairs(tbl) do + for _, r in ipairs(tbl) do if func then table.insert(res, func(r)) else @@ -431,14 +429,14 @@ local function send_reports_by_smtp(opts, reports, finish_cb) local nreports = math.min(#reports - cur_batch + 1, opts.batch_size) local next_start = cur_batch + nreports lua_util.debugm(N, 'send data for %s domains (from %s to %s)', - nreports, cur_batch, next_start-1) + nreports, cur_batch, next_start - 1) -- Shared across all closures local gen_args = { cont_func = send_data_in_batches, nreports = nreports, next_start = next_start } - for i=cur_batch,next_start-1 do + for i = cur_batch, next_start - 1 do local report = reports[i] local send_opts = { ev_base = rspamadm_ev_base, @@ -475,7 +473,7 @@ local function prepare_report(opts, start_time, end_time, rep_key) end local ret, results = lua_redis.request(redis_params, redis_attrs, - {'EXISTS', rep_key}) + { 'EXISTS', rep_key }) if not ret or not results or results == 0 then return nil @@ -484,7 +482,7 @@ local function prepare_report(opts, start_time, end_time, rep_key) -- Rename report key to avoid races if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'RENAME', rep_key, rep_key .. '_processing'}) + { 'RENAME', rep_key, rep_key .. '_processing' }) rep_key = rep_key .. '_processing' end @@ -494,7 +492,7 @@ local function prepare_report(opts, start_time, end_time, rep_key) if not dmarc_record then if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', rep_key}) + { 'DEL', rep_key }) end logger.messagex('Cannot process reports for domain %s; invalid dmarc record', reporting_domain) return nil @@ -502,11 +500,11 @@ local function prepare_report(opts, start_time, end_time, rep_key) -- Get all reports for a domain ret, results = lua_redis.request(redis_params, redis_attrs, - {'ZRANGE', rep_key, '0', '-1', 'WITHSCORES'}) + { 'ZRANGE', rep_key, '0', '-1', 'WITHSCORES' }) local report_entries = {} table.insert(report_entries, report_header(reporting_domain, start_time, end_time, dmarc_record)) - for i=1,#results,2 do + for i = 1, #results, 2 do local xml_record = entry_to_xml(process_report_entry(results[i], results[i + 1])) table.insert(report_entries, xml_record) end @@ -542,24 +540,23 @@ local function prepare_report(opts, start_time, end_time, rep_key) local rfooter = lua_util.jinja_template(report_footer, { uuid = uuid, }, true) - local message = rspamd_text.fromtable{ + local message = rspamd_text.fromtable { (rhead:gsub("\n", "\r\n")), rspamd_util.encode_base64(rspamd_util.gzip_compress(xml_to_compress), 73), rfooter:gsub("\n", "\r\n"), } - lua_util.debugm(N, 'got final message: %s', message) if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', rep_key}) + { 'DEL', rep_key }) end local report_rcpts = lua_util.str_split(rcpt_string, ',') if report_settings.bcc_addrs then - for _,b in ipairs(report_settings.bcc_addrs) do + for _, b in ipairs(report_settings.bcc_addrs) do table.insert(report_rcpts, b) end end @@ -574,7 +571,7 @@ end local function process_report_date(opts, start_time, end_time, date) local idx_key = redis_prefix(dmarc_settings.reporting.redis_keys.index_prefix, date) local ret, results = lua_redis.request(redis_params, redis_attrs, - {'EXISTS', idx_key}) + { 'EXISTS', idx_key }) if not ret or not results or results == 0 then logger.messagex('No reports for %s', date) @@ -584,24 +581,24 @@ local function process_report_date(opts, start_time, end_time, date) -- Rename index key to avoid races if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'RENAME', idx_key, idx_key .. '_processing'}) + { 'RENAME', idx_key, idx_key .. '_processing' }) idx_key = idx_key .. '_processing' end ret, results = lua_redis.request(redis_params, redis_attrs, - {'SMEMBERS', idx_key}) + { 'SMEMBERS', idx_key }) if not ret or not results then -- Remove bad key if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', idx_key}) + { 'DEL', idx_key }) end logger.messagex('Cannot get reports for %s', date) return {} end local reports = {} - for _,rep in ipairs(results) do + for _, rep in ipairs(results) do local report = prepare_report(opts, start_time, end_time, rep) if report then @@ -614,7 +611,7 @@ local function process_report_date(opts, start_time, end_time, date) -- Remove processed key if not opts.no_opt then lua_redis.request(redis_params, redis_attrs, - {'DEL', idx_key}) + { 'DEL', idx_key }) end return reports @@ -669,14 +666,14 @@ local function handler(args) os.exit(1) end - for _, e in ipairs({'email', 'domain', 'org_name'}) do + for _, e in ipairs({ 'email', 'domain', 'org_name' }) do if not dmarc_settings.reporting[e] then logger.errx('Missing required setting: dmarc.reporting.%s', e) return end end - local ret,results = lua_redis.request(redis_params, redis_attrs, { + local ret, results = lua_redis.request(redis_params, redis_attrs, { 'GET', 'rspamd_dmarc_last_collection' }) @@ -696,14 +693,14 @@ local function handler(args) local ndates = 0 local nreports = 0 local all_reports = {} - for _,date in ipairs(opts.date) do + for _, date in ipairs(opts.date) do lua_util.debugm(N, 'Process date %s', date) local reports_for_date = process_report_date(opts, start_time, start_collection, date) if #reports_for_date > 0 then ndates = ndates + 1 nreports = nreports + #reports_for_date - for _,r in ipairs(reports_for_date) do + for _, r in ipairs(reports_for_date) do table.insert(all_reports, r) end end @@ -718,8 +715,8 @@ local function handler(args) ndates, nreports, nsuccess, nfail) end lua_redis.request(redis_params, redis_attrs, - {'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2, - tostring(start_collection)}) + { 'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2, + tostring(start_collection) }) else logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed', ndates, nreports, nsuccess, nfail) @@ -736,7 +733,7 @@ end return { name = 'dmarc_report', - aliases = {'dmarc_reporting'}, + aliases = { 'dmarc_reporting' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/dns_tool.lua b/lualib/rspamadm/dns_tool.lua index e3ffb29dd..3eb09a82b 100644 --- a/lualib/rspamadm/dns_tool.lua +++ b/lualib/rspamadm/dns_tool.lua @@ -33,7 +33,7 @@ parser:option "-c --config" :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf") local spf = parser:command "spf" - :description "Extracts spf records" + :description "Extracts spf records" spf:mutex( spf:option "-d --domain" :description "Domain to use" @@ -69,14 +69,14 @@ local function red(str) end local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -109,7 +109,7 @@ local function spf_handler(opts) os.exit(1) end elseif opts.domain then - task:set_from('smtp', {user = 'user', domain = opts.domain}) + task:set_from('smtp', { user = 'user', domain = opts.domain }) else io.stderr:write('Neither domain nor from specified\n') os.exit(1) @@ -128,7 +128,9 @@ local function spf_handler(opts) end local function display_spf_results(elt, colored) - local dec = function(e) return e end + local dec = function(e) + return e + end local policy_decode = function(e) if e == rspamd_spf.policy.fail then return 'reject' @@ -144,12 +146,18 @@ local function spf_handler(opts) end if colored then - dec = function(e) return highlight(e) end + dec = function(e) + return highlight(e) + end - if elt.result == rspamd_spf.policy.pass then - dec = function(e) return green(e) end - elseif elt.result == rspamd_spf.policy.fail then - dec = function(e) return red(e) end + if elt.result == rspamd_spf.policy.pass then + dec = function(e) + return green(e) + end + elseif elt.result == rspamd_spf.policy.fail then + dec = function(e) + return red(e) + end end end @@ -182,7 +190,7 @@ local function spf_handler(opts) if result then printf('SPF record for %s; digest: %s', highlight(opts.domain or opts.from), highlight(record:get_digest())) - for _,elt in ipairs(record:get_elts()) do + for _, elt in ipairs(record:get_elts()) do if result and error_or_addr and elt.str and elt.str == error_or_addr.str then printf("%s", highlight('*** Matched ***')) display_spf_results(elt, true) @@ -218,7 +226,7 @@ end return { name = 'dnstool', - aliases = {'dns', 'dns_tool'}, + aliases = { 'dns', 'dns_tool' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/fuzzy_convert.lua b/lualib/rspamadm/fuzzy_convert.lua index 2d473ca46..a31baa4e2 100644 --- a/lualib/rspamadm/fuzzy_convert.lua +++ b/lualib/rspamadm/fuzzy_convert.lua @@ -13,13 +13,13 @@ local function connect_redis(server, password, db) end if password then - ret = conn:add_cmd('AUTH', {password}) + ret = conn:add_cmd('AUTH', { password }) if not ret then return nil, 'Cannot queue command' end end if db then - ret = conn:add_cmd('SELECT', {db}) + ret = conn:add_cmd('SELECT', { db }) if not ret then return nil, 'Cannot queue command' end @@ -126,7 +126,7 @@ local function update_counters(total, redis_host, redis_password, redis_db) return true end -return function (_, res) +return function(_, res) local db = sqlite3.open(res['source_db']) local shingles = {} local digests = {} @@ -152,11 +152,11 @@ return function (_, res) local expire_in = math.floor(now - row.time + res['expiry']) if expire_in >= 1 then - table.insert(digests, {row.digest, row.flag, row.value, expire_in}) + table.insert(digests, { row.digest, row.flag, row.value, expire_in }) num_batch_digests = num_batch_digests + 1 total_digests = total_digests + 1 for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do - table.insert(shingles, {srow.value, srow.number, expire_in, row.digest}) + table.insert(shingles, { srow.value, srow.number, expire_in, row.digest }) total_shingles = total_shingles + 1 num_batch_shingles = num_batch_shingles + 1 end @@ -188,8 +188,8 @@ return function (_, res) end local message = string.format( - 'Migrated %d digests and %d shingles', - total_digests, total_shingles + 'Migrated %d digests and %d shingles', + total_digests, total_shingles ) if not update_counters(total_digests, res['redis_host'], redis_password, redis_db) then message = message .. ' but failed to update counters' diff --git a/lualib/rspamadm/fuzzy_stat.lua b/lualib/rspamadm/fuzzy_stat.lua index d0c14d213..ef8a5de08 100644 --- a/lualib/rspamadm/fuzzy_stat.lua +++ b/lualib/rspamadm/fuzzy_stat.lua @@ -18,14 +18,14 @@ parser:flag "-n --number" parser:option "--sort" :description "Sort order" :convert { - checked = "checked", - matched = "matched", - errors = "errors", - name = "name" - } + checked = "checked", + matched = "matched", + errors = "errors", + name = "name" +} local function add_data(target, src) - for k,v in pairs(src) do + for k, v in pairs(src) do if type(v) == 'number' then if target[k] then target[k] = target[k] + v @@ -33,17 +33,25 @@ local function add_data(target, src) target[k] = v end elseif k == 'ips' then - if not target['ips'] then target['ips'] = {} end + if not target['ips'] then + target['ips'] = {} + end -- Iterate over IPs - for ip,st in pairs(v) do - if not target['ips'][ip] then target['ips'][ip] = {} end + for ip, st in pairs(v) do + if not target['ips'][ip] then + target['ips'][ip] = {} + end add_data(target['ips'][ip], st) end elseif k == 'flags' then - if not target['flags'] then target['flags'] = {} end + if not target['flags'] then + target['flags'] = {} + end -- Iterate over Flags - for flag,st in pairs(v) do - if not target['flags'][flag] then target['flags'][flag] = {} end + for flag, st in pairs(v) do + if not target['flags'][flag] then + target['flags'][flag] = {} + end add_data(target['flags'][flag], st) end elseif k == 'keypair' then @@ -109,8 +117,8 @@ end -- Sort by checked local function sort_hash_table(tbl, sort_opts, key_key) local res = {} - for k,v in pairs(tbl) do - table.insert(res, {[key_key] = k, data = v}) + for k, v in pairs(tbl) do + table.insert(res, { [key_key] = k, data = v }) end local function sort_order(elt) @@ -145,12 +153,12 @@ local function add_result(dst, src, k) if type(src) == 'table' then if type(dst) == 'number' then -- Convert dst to table - dst = {dst} + dst = { dst } elseif type(dst) == 'nil' then dst = {} end - for i,v in ipairs(src) do + for i, v in ipairs(src) do if dst[i] and k ~= 'fuzzy_stored' then dst[i] = dst[i] + v else @@ -193,7 +201,7 @@ local function print_result(r) end if type(r) == 'table' then local res = {} - for i,num in ipairs(r) do + for i, num in ipairs(r) do res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num)) end @@ -210,7 +218,7 @@ return function(args, res) opts = parser:parse(args) if wrk then - for _,pr in pairs(wrk) do + for _, pr in pairs(wrk) do -- processes cycle if pr['data'] then local id = pr['id'] @@ -225,7 +233,7 @@ return function(args, res) end -- General stats - for k,v in pairs(pr['data']) do + for k, v in pairs(pr['data']) do if k ~= 'keys' and k ~= 'errors_ips' then res_db[k] = add_result(res_db[k], v, k) elseif k == 'errors_ips' then @@ -233,7 +241,7 @@ return function(args, res) if not res_db['errors_ips'] then res_db['errors_ips'] = {} end - for ip,nerrors in pairs(v) do + for ip, nerrors in pairs(v) do if not res_db['errors_ips'][ip] then res_db['errors_ips'][ip] = nerrors else @@ -250,7 +258,7 @@ return function(args, res) res_db['keys'] = res_keys end -- Go through keys in input - for k,elts in pairs(pr['data']['keys']) do + for k, elts in pairs(pr['data']['keys']) do -- keys cycle if not res_keys[k] then res_keys[k] = {} @@ -259,7 +267,7 @@ return function(args, res) add_data(res_keys[k], elts) if elts['ips'] then - for ip,v in pairs(elts['ips']) do + for ip, v in pairs(elts['ips']) do if not res_ips[ip] then res_ips[ip] = {} end @@ -274,10 +282,10 @@ return function(args, res) end -- General stats - for db,st in pairs(res_databases) do + for db, st in pairs(res_databases) do print(string.format('Statistics for storage %s', db)) - for k,v in pairs(st) do + for k, v in pairs(st) do if k ~= 'keys' and k ~= 'errors_ips' then print(string.format('%s: %s', k, print_result(v))) end @@ -305,7 +313,7 @@ return function(args, res) print('\tIPs stat:') local sorted_ips = sort_hash_table(key_stat['ips'], opts, 'ip') - for _,v in ipairs(sorted_ips) do + for _, v in ipairs(sorted_ips) do print(string.format('\t%s', v['ip'])) print_stat(v['data'], '\t\t') print('') @@ -315,7 +323,7 @@ return function(args, res) if key_stat.flags then print('') print('\tFlags stat:') - for flag,v in pairs(key_stat.flags) do + for flag, v in pairs(key_stat.flags) do print(string.format('\t[%s]:', flag)) -- Remove irrelevant fields v.checked = nil diff --git a/lualib/rspamadm/grep.lua b/lualib/rspamadm/grep.lua index 426b93c7a..6ed05691e 100644 --- a/lualib/rspamadm/grep.lua +++ b/lualib/rspamadm/grep.lua @@ -80,7 +80,7 @@ local function handler(args) if search_str and not sensitive then search_str = string.lower(search_str) end - local inputs = res['input'] or {'stdin'} + local inputs = res['input'] or { 'stdin' } for _, n in ipairs(inputs) do local h, err @@ -115,7 +115,7 @@ local function handler(args) if buffer[hash] then table.insert(buffer[hash], line) else - buffer[hash] = {line} + buffer[hash] = { line } end end end diff --git a/lualib/rspamadm/keypair.lua b/lualib/rspamadm/keypair.lua index 27dfd59a3..f0716a22f 100644 --- a/lualib/rspamadm/keypair.lua +++ b/lualib/rspamadm/keypair.lua @@ -84,9 +84,9 @@ verify:mutex( :argname "<file>" ) verify:argument "file" - :description "File to verify" - :argname "<file>" - :args "*" + :description "File to verify" + :argname "<file>" + :args "*" verify:flag "-n --nist" :description "Uses nistp curves (P256)" verify:option "-s --suffix" @@ -143,15 +143,15 @@ decrypt:flag "-r --rm" -- Default command is generate, so duplicate options to be compatible parser:flag "-s --sign" - :description "Generates a sign keypair instead of the encryption one" + :description "Generates a sign keypair instead of the encryption one" parser:flag "-n --nist" - :description "Uses nistp curves (P256)" + :description "Uses nistp curves (P256)" parser:mutex( parser:flag "-j --json" - :description "Output JSON instead of UCL", + :description "Output JSON instead of UCL", parser:flag "-u --ucl" - :description "Output UCL" - :default(true) + :description "Output UCL" + :default(true) ) parser:option "-o --output" :description "Write keypair to file" @@ -174,10 +174,16 @@ local function ask_yes_no(greet, default) local reply = rspamd_util.readline(greet) - if not reply then os.exit(0) end - if #reply == 0 then reply = def_str end + if not reply then + os.exit(0) + end + if #reply == 0 then + reply = def_str + end reply = reply:lower() - if reply == 'y' or reply == 'yes' then return true end + if reply == 'y' or reply == 'yes' then + return true + end return false end @@ -221,7 +227,7 @@ end local function sign_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to sign') @@ -231,7 +237,7 @@ local function sign_handler(opts) end local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -243,7 +249,7 @@ local function sign_handler(opts) fatal("cannot load keypair: " .. opts.keypair) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local sig = rspamd_crypto.sign_file(kp, fname) if not sig then @@ -264,7 +270,7 @@ end local function verify_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to verify') @@ -275,7 +281,7 @@ local function verify_handler(opts) if opts.keypair then local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -290,10 +296,14 @@ local function verify_handler(opts) pk = kp:pk() alg = kp:alg() elseif opts.pubkey then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg) elseif opts.pubstr then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg) end @@ -303,7 +313,7 @@ local function verify_handler(opts) local valid = true - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local sig_fname = string.format('%s.%s', fname, opts.suffix or 'sig') local sig = rspamd_signature.load(sig_fname, alg) @@ -330,7 +340,7 @@ end local function encrypt_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to sign') @@ -341,7 +351,7 @@ local function encrypt_handler(opts) if opts.keypair then local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -356,10 +366,14 @@ local function encrypt_handler(opts) pk = kp:pk() alg = kp:alg() elseif opts.pubkey then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg) elseif opts.pubstr then - if opts.nist then alg = 'nist' end + if opts.nist then + alg = 'nist' + end pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg) end @@ -367,7 +381,7 @@ local function encrypt_handler(opts) fatal("cannot load keypair: " .. opts.keypair) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local enc = rspamd_crypto.encrypt_file(pk, fname, alg) if not enc then @@ -404,7 +418,7 @@ end local function decrypt_handler(opts) if opts.file then if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } end else parser:error('no files to decrypt') @@ -414,7 +428,7 @@ local function decrypt_handler(opts) end local ucl_parser = ucl.parser() - local res,err = ucl_parser:parse_file(opts.keypair) + local res, err = ucl_parser:parse_file(opts.keypair) if not res then fatal(string.format('cannot load %s: %s', opts.keypair, err)) @@ -426,7 +440,7 @@ local function decrypt_handler(opts) fatal("cannot load keypair: " .. opts.keypair) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local decrypted = rspamd_crypto.decrypt_file(kp, fname) if not decrypted then @@ -488,7 +502,7 @@ end return { name = 'keypair', - aliases = {'kp', 'key'}, + aliases = { 'kp', 'key' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 5c1cebac5..0b0605582 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -44,7 +44,7 @@ parser:mutex( parser:flag "-U --ucl" :description "UCL output", parser:flag "-M --messagepack" - :description "MessagePack output" + :description "MessagePack output" ) parser:flag "-C --compact" :description "Use compact format" @@ -67,12 +67,12 @@ extract:option "-o --output" :description "Output format ('raw', 'content', 'oneline', 'decoded', 'decoded_utf')" :argname("<type>") :convert { - raw = "raw", - content = "content", - oneline = "content_oneline", - decoded = "raw_parsed", - decoded_utf = "raw_utf" - } + raw = "raw", + content = "content", + oneline = "content_oneline", + decoded = "raw_parsed", + decoded_utf = "raw_utf" +} :default "content" extract:flag "-w --words" :description "Extracts words" @@ -86,16 +86,15 @@ extract:option "-F --words-format" :description "Words format ('stem', 'norm', 'raw', 'full')" :argname("<type>") :convert { - stem = "stem", - norm = "norm", - raw = "raw", - full = "full", - } + stem = "stem", + norm = "norm", + raw = "raw", + full = "full", +} :default "stem" - local stat = parser:command "stat st s" - :description "Extracts statistical data from MIME messages" + :description "Extracts statistical data from MIME messages" stat:argument "file" :description "File to process" :argname "<file>" @@ -123,7 +122,7 @@ urls:mutex( urls:flag "-H --host" :description "Get hosts only", urls:flag "-f --full" - :description "Show piecewise urls as processed by Rspamd" + :description "Show piecewise urls as processed by Rspamd" ) urls:flag "-u --unique" @@ -136,7 +135,7 @@ urls:flag "-r --reverse" :description "Reverse sort order" local modify = parser:command "modify mod m" - :description "Modifies MIME message" + :description "Modifies MIME message" modify:argument "file" :description "File to process" :argname "<file>" @@ -162,11 +161,11 @@ modify:option "-H --html-footer" :argname "<file>" local sign = parser:command "sign" - :description "Performs DKIM signing" + :description "Performs DKIM signing" sign:argument "file" - :description "File to process" - :argname "<file>" - :args "+" + :description "File to process" + :argname "<file>" + :args "+" sign:option "-d --domain" :description "Use specific domain" @@ -184,17 +183,17 @@ sign:option "-t type" :description "ARC or DKIM signing" :argname("<arc|dkim>") :convert { - ['arc'] = 'arc', - ['dkim'] = 'dkim', - } + ['arc'] = 'arc', + ['dkim'] = 'dkim', +} :default 'dkim' sign:option "-o --output" :description "Output format" :argname("<message|signature>") :convert { - ['message'] = 'message', - ['signature'] = 'signature', - } + ['message'] = 'message', + ['signature'] = 'signature', +} :default 'message' local dump = parser:command "dump" @@ -213,21 +212,21 @@ dump:mutex( :description "MessagePack output" ) dump:flag "-s --split" - :description "Split the output file contents such that no content is embedded" + :description "Split the output file contents such that no content is embedded" dump:option "-o --outdir" - :description "Output directory" - :argname("<directory>") + :description "Output directory" + :argname("<directory>") local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -239,7 +238,7 @@ local function load_task(opts, fname) fname = '-' end - local res,task = rspamd_task.load_from_file(fname, rspamd_config) + local res, task = rspamd_task.load_from_file(fname, rspamd_config) if not res then parser:error(string.format('cannot read message from %s: %s', fname, @@ -266,9 +265,15 @@ end local function output_fmt(opts) local fmt = 'json' - if opts.compact then fmt = 'json-compact' end - if opts.ucl then fmt = 'ucl' end - if opts.messagepack then fmt = 'msgpack' end + if opts.compact then + fmt = 'json-compact' + end + if opts.ucl then + fmt = 'ucl' + end + if opts.messagepack then + fmt = 'msgpack' + end return fmt end @@ -320,20 +325,20 @@ local function extract_handler(opts) if not opts.json and not opts.ucl then table.insert(out, rspamd_logger.slog('Part: %s: %s, language: %s, size: %s (%s raw), words: %s', - part:get_mimepart():get_digest():sub(1,8), - t, - part:get_language(), - part:get_length(), part:get_raw_length(), - part:get_words_count())) + part:get_mimepart():get_digest():sub(1, 8), + t, + part:get_language(), + part:get_length(), part:get_raw_length(), + part:get_words_count())) table.insert(out, rspamd_logger.slog('Stats: %s', - fun.foldl(function(acc, k, v) - if acc ~= '' then - return string.format('%s, %s:%s', acc, k, v) - else - return string.format('%s:%s', k,v) - end - end, '', part:get_stats()))) + fun.foldl(function(acc, k, v) + if acc ~= '' then + return string.format('%s, %s:%s', acc, k, v) + else + return string.format('%s:%s', k, v) + end + end, '', part:get_stats()))) end end end @@ -342,11 +347,11 @@ local function extract_handler(opts) if opts.part then if not opts.json and not opts.ucl then - local mtype,msubtype = part:get_type() - local det_mtype,det_msubtype = part:get_detected_type() + local mtype, msubtype = part:get_type() + local det_mtype, det_msubtype = part:get_detected_type() table.insert(out, rspamd_logger.slog('Mime Part: %s: %s/%s (%s/%s detected), filename: %s (%s detected ext), size: %s', - part:get_digest():sub(1,8), + part:get_digest():sub(1, 8), mtype, msubtype, det_mtype, det_msubtype, part:get_filename(), @@ -378,7 +383,7 @@ local function extract_handler(opts) end end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) out_elts[fname] = {} @@ -396,10 +401,12 @@ local function extract_handler(opts) if opts.text or opts.html then local mp = task:get_parts() or {} - for _,mime_part in ipairs(mp) do + for _, mime_part in ipairs(mp) do local how = opts.output local part - if mime_part:is_text() then part = mime_part:get_text() end + if mime_part:is_text() then + part = mime_part:get_text() + end if part and opts.text and not part:is_html() then maybe_print_text_part_info(part, out_elts[fname]) @@ -445,24 +452,25 @@ local function extract_handler(opts) end hc:foreach_tag('any', function(tag) - local elt = {} - local ex = tag:get_extra() - elt.tag = tag:get_type() - if ex then - elt.extra = ex - end - local content = tag:get_content() - if content then - elt.content = tostring(content) - end - local style = tag:get_style() - if style then - elt.style = style - end - table.insert(res, elt) + local elt = {} + local ex = tag:get_extra() + elt.tag = tag:get_type() + if ex then + elt.extra = ex + end + local content = tag:get_content() + if content then + elt.content = tostring(content) + end + local style = tag:get_style() + if style then + elt.style = style + end + table.insert(res, elt) end) table.insert(out_elts[fname], res) - else -- opts.structure + else + -- opts.structure table.insert(out_elts[fname], tostring(part:get_content(how))) end if opts.invisible then @@ -485,7 +493,9 @@ local function extract_handler(opts) print_elts(out_elts, opts, process_func) -- To avoid use after free we postpone tasks destruction - for _,task in ipairs(tasks) do task:destroy() end + for _, task in ipairs(tasks) do + task:destroy() + end end local function stat_handler(opts) @@ -498,7 +508,7 @@ local function stat_handler(opts) local process_func - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) out_elts[fname] = {} @@ -514,7 +524,9 @@ local function stat_handler(opts) process_func = function(e) return string.format('%s (%d): "%s"+"%s", [%s]', e.data, e.win, e.t1 or "", e.t2 or "", table.concat(fun.totable( - fun.map(function(k) return k end, e.flags)), ",")) + fun.map(function(k) + return k + end, e.flags)), ",")) end elseif opts.fuzzy then local parts = task:get_parts() or {} @@ -523,7 +535,7 @@ local function stat_handler(opts) local ret = string.format('part: %s(%s): %s', e.type, e.file or "", e.digest) if opts.shingles and e.shingles then local sgl = {} - for _,s in ipairs(e.shingles) do + for _, s in ipairs(e.shingles) do table.insert(sgl, string.format('%s: %s+%s+%s', s[1], s[2], s[3], s[4])) end @@ -531,26 +543,26 @@ local function stat_handler(opts) end return ret end - for _,part in ipairs(parts) do + for _, part in ipairs(parts) do if not part:is_multipart() then local text = part:get_text() if text then - local digest,shingles = text:get_fuzzy_hashes(task:get_mempool()) + local digest, shingles = text:get_fuzzy_hashes(task:get_mempool()) table.insert(out_elts[fname], { digest = digest, shingles = shingles, type = string.format('%s/%s', - ({part:get_type()})[1], - ({part:get_type()})[2]) + ({ part:get_type() })[1], + ({ part:get_type() })[2]) }) else table.insert(out_elts[fname], { digest = part:get_digest(), file = part:get_filename(), type = string.format('%s/%s', - ({part:get_type()})[1], - ({part:get_type()})[2]) + ({ part:get_type() })[1], + ({ part:get_type() })[2]) }) end end @@ -568,9 +580,11 @@ local function urls_handler(opts) rspamd_url.init(rspamd_config:get_tld_path()) local out_elts = {} - if opts.json then rspamd_logger.messagex('[') end + if opts.json then + rspamd_logger.messagex('[') + end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do out_elts[fname] = {} local task = load_task(opts, fname) local elts = {} @@ -605,7 +619,7 @@ local function urls_handler(opts) end end - for _,u in ipairs(task:get_urls(true)) do + for _, u in ipairs(task:get_urls(true)) do process_url(u) end @@ -672,12 +686,11 @@ local function urls_handler(opts) end end - - for s,u in lua_util.spairs(elts, sfunc) do + for s, u in lua_util.spairs(elts, sfunc) do process_elt(s, u) end else - for s,u in pairs(elts) do + for s, u in pairs(elts) do process_elt(s, u) end end @@ -723,7 +736,7 @@ local function modify_handler(opts) html_footer = read_file(opts['html_footer']) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) local newline_s = newline(task) local seen_cte @@ -732,14 +745,14 @@ local function modify_handler(opts) local out = {} -- Start with headers local function process_headers_cb(name, hdr) - for _,h in ipairs(opts['remove_header']) do + for _, h in ipairs(opts['remove_header']) do if name:match(h) then return end end - for _,h in ipairs(opts['rewrite_header']) do - local hname,hpattern = h:match('^([^=]+)=(.+)$') + for _, h in ipairs(opts['rewrite_header']) do + local hname, hpattern = h:match('^([^=]+)=(.+)$') if hname == name then local new_value = string.format(hpattern, hdr.decoded) new_value = string.format('%s:%s%s', @@ -769,10 +782,10 @@ local function modify_handler(opts) out[#out + 1] = hdr.raw:gsub('\r?\n?$', '') end - task:headers_foreach(process_headers_cb, {full = true}) + task:headers_foreach(process_headers_cb, { full = true }) - for _,h in ipairs(opts['add_header']) do - local hname,hvalue = h:match('^([^=]+)=(.+)$') + for _, h in ipairs(opts['add_header']) do + local hname, hvalue = h:match('^([^=]+)=(.+)$') if hname and hvalue then out[#out + 1] = string.format('%s: %s', hname, @@ -789,14 +802,14 @@ local function modify_handler(opts) out[#out + 1] = '' if rewrite.out then - for _,o in ipairs(rewrite.out) do + for _, o in ipairs(rewrite.out) do out[#out + 1] = o end else - out[#out + 1] = {task:get_rawbody(), false} + out[#out + 1] = { task:get_rawbody(), false } end - for _,o in ipairs(out) do + for _, o in ipairs(out) do if type(o) == 'string' then io.write(o) io.write(newline_s) @@ -844,7 +857,7 @@ local function sign_handler(opts) os.exit(1) end - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) local ctx = lua_dkim.create_sign_context(task, sign_key, nil, opts.algorithm) @@ -917,7 +930,7 @@ local function write_dump_content(dump_content, fname, extension, outdir) end if dump_content:save_in_file(outpath) then wrote_filepath = outpath - io.write(wrote_filepath.."\n") + io.write(wrote_filepath .. "\n") else io.stderr:write(string.format("Unable to save dump content to file: %s\n", outpath)) end @@ -954,7 +967,7 @@ local function dump_handler(opts) load_config(opts) rspamd_url.init(rspamd_config:get_tld_path()) - for _,fname in ipairs(opts.file) do + for _, fname in ipairs(opts.file) do local task = load_task(opts, fname) local data, extension = get_dump_content(task, opts, fname) write_dump_content(data, fname, extension, opts.outdir) @@ -969,7 +982,7 @@ local function handler(args) local command = opts.command if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } elseif type(opts.file) == 'none' then opts.file = {} end @@ -993,7 +1006,7 @@ end return { name = 'mime', - aliases = {'mime_tool'}, + aliases = { 'mime_tool' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/neural_test.lua b/lualib/rspamadm/neural_test.lua index cb056b97a..31d21a990 100644 --- a/lualib/rspamadm/neural_test.lua +++ b/lualib/rspamadm/neural_test.lua @@ -4,9 +4,9 @@ local lua_util = require "lua_util" local ucl = require "ucl" local parser = argparse() - :name "rspamadm neural_test" - :description "Test the neural network with labelled dataset" - :help_description_margin(32) + :name "rspamadm neural_test" + :description "Test the neural network with labelled dataset" + :help_description_margin(32) parser:option "-c --config" :description "Path to config file" @@ -40,26 +40,24 @@ parser:option '--rule' :description 'Rule to test' :argname('<rule>') - local HAM = "HAM" local SPAM = "SPAM" local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) end end - local function scan_email(rspamc_path, host, n_parallel, path, timeout) local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s", @@ -128,13 +126,13 @@ end local function get_stats_from_scan_results(results, rules) local rule_stats = {} - for rule,_ in pairs(rules) do - rule_stats[rule] = {tp = 0, tn = 0, fp = 0, fn = 0} + for rule, _ in pairs(rules) do + rule_stats[rule] = { tp = 0, tn = 0, fp = 0, fn = 0 } end - for _,result in ipairs(results) do - for _,symbol in ipairs(result["symbols"]) do - for name,rule in pairs(rules) do + for _, result in ipairs(results) do + for _, symbol in ipairs(result["symbols"]) do + for name, rule in pairs(rules) do if rule.symbol_spam and rule.symbol_spam == symbol then if result.type == HAM then rule_stats[name].fp = rule_stats[name].fp + 1 @@ -152,7 +150,7 @@ local function get_stats_from_scan_results(results, rules) end end - for rule,_ in pairs(rules) do + for rule, _ in pairs(rules) do rule_stats[rule].fpr = rule_stats[rule].fp / (rule_stats[rule].fp + rule_stats[rule].tn) rule_stats[rule].fnr = rule_stats[rule].fn / (rule_stats[rule].fn + rule_stats[rule].tp) end @@ -222,10 +220,9 @@ local function handler(args) end - return { name = "neuraltest", - aliases = {"neural_test"}, + aliases = { "neural_test" }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/publicsuffix.lua b/lualib/rspamadm/publicsuffix.lua index f8e1bac93..96bf0699e 100644 --- a/lualib/rspamadm/publicsuffix.lua +++ b/lualib/rspamadm/publicsuffix.lua @@ -31,17 +31,17 @@ parser:option '-c --config' :default(rspamd_paths['CONFDIR'] .. '/rspamd.conf') parser:command 'compile' - :description 'Compile publicsuffix list if needed' + :description 'Compile publicsuffix list if needed' local function load_config(config_file) - local _r,err = rspamd_config:load_ucl(config_file) + local _r, err = rspamd_config:load_ucl(config_file) if not _r then rspamd_logger.errx('cannot load %s: %s', config_file, err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', config_file, err) os.exit(1) @@ -67,7 +67,6 @@ local function handler(args) load_config(cmd_opts.config_file) - if cmd_opts.command == 'compile' then compile_handler(cmd_opts) else diff --git a/lualib/rspamadm/stat_convert.lua b/lualib/rspamadm/stat_convert.lua index 3b54826aa..62a19a221 100644 --- a/lualib/rspamadm/stat_convert.lua +++ b/lualib/rspamadm/stat_convert.lua @@ -4,7 +4,7 @@ local ucl = require "ucl" local logger = require "rspamd_logger" local lua_util = require "lua_util" -return function (_, res) +return function(_, res) local redis_params = lua_redis.try_load_redis_servers(res.redis, nil) if res.expire then res.expire = lua_util.parse_time_interval(res.expire) @@ -22,7 +22,7 @@ return function (_, res) return false end - for _,cls in ipairs(sqlite_params) do + for _, cls in ipairs(sqlite_params) do if not stat_tools.convert_sqlite_to_redis(redis_params, cls.db_spam, cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, res.expire, res.reset_previous) then @@ -33,6 +33,6 @@ return function (_, res) logger.messagex('Converted classifier to the from sqlite to redis') logger.messagex('Suggested configuration:') logger.messagex(ucl.to_format(stat_tools.redis_classifier_from_sqlite(cls, res.expire), - 'config')) + 'config')) end end diff --git a/lualib/rspamadm/statistics_dump.lua b/lualib/rspamadm/statistics_dump.lua index 0ed921f6a..6bc045850 100644 --- a/lualib/rspamadm/statistics_dump.lua +++ b/lualib/rspamadm/statistics_dump.lua @@ -68,31 +68,31 @@ restore:argument "file" :argname "<file>" :args "*" restore:option "-b --batch-size" - :description "Number of entires to process at once" - :argname("<elts>") - :convert(tonumber) - :default(1000) + :description "Number of entires to process at once" + :argname("<elts>") + :convert(tonumber) + :default(1000) restore:option "-m --mode" :description "Number of entires to process at once" :argname("<append|subtract|replace>") :convert { - ['append'] = 'append', - ['subtract'] = 'subtract', - ['replace'] = 'replace', - } + ['append'] = 'append', + ['subtract'] = 'subtract', + ['replace'] = 'replace', +} :default 'append' restore:flag "-n --no-operation" - :description "Only show redis commands to be issued" + :description "Only show redis commands to be issued" local function load_config(opts) - local _r,err = rspamd_config:load_ucl(opts['config']) + local _r, err = rspamd_config:load_ucl(opts['config']) if not _r then rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) os.exit(1) end - _r,err = rspamd_config:parse_rcl({'logging', 'worker'}) + _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) if not _r then rspamd_logger.errx('cannot process %s: %s', opts['config'], err) os.exit(1) @@ -128,9 +128,9 @@ local function check_redis_classifier(cls, cfg) local statfiles = cls.statfile if statfiles[1] then - for _,stf in ipairs(statfiles) do + for _, stf in ipairs(statfiles) do if not stf.symbol then - for k,v in pairs(stf) do + for k, v in pairs(stf) do check_statfile_table(v, k) end else @@ -138,7 +138,7 @@ local function check_redis_classifier(cls, cfg) end end else - for stn,stf in pairs(statfiles) do + for stn, stf in pairs(statfiles) do check_statfile_table(stf, stn) end end @@ -168,7 +168,7 @@ end local function redis_map_zip(ar) local data = {} - for j=1,#ar,2 do + for j = 1, #ar, 2 do data[ar[j]] = ar[j + 1] end @@ -178,7 +178,7 @@ end -- Used to clear tables local clear_fcn = table.clear or function(tbl) local keys = lua_util.keys(tbl) - for _,k in ipairs(keys) do + for _, k in ipairs(keys) do tbl[k] = nil end end @@ -197,7 +197,7 @@ local function dump_out(out, opts, last) compress_ctx:stream(rspamd_text.fromtable(out), 'flush'):write() end else - for _,o in ipairs(out) do + for _, o in ipairs(out) do io.write(o) end end @@ -213,7 +213,7 @@ local function dump_cdb(out, opts, last, pattern) out.cdb_builder:add('_lrnham_', rspamd_i64.fromstring(results.learns_ham or '0')) end - for _,o in ipairs(results.elts) do + for _, o in ipairs(results.elts) do out.cdb_builder:add(o.key, o.value) end @@ -227,9 +227,9 @@ local function dump_pattern(conn, pattern, opts, out, key) local cursor = 0 repeat - conn:add_cmd('SCAN', {tostring(cursor), - 'MATCH', pattern, - 'COUNT', tostring(opts.batch_size)}) + conn:add_cmd('SCAN', { tostring(cursor), + 'MATCH', pattern, + 'COUNT', tostring(opts.batch_size) }) local ret, results = conn:exec() if not ret then @@ -242,26 +242,26 @@ local function dump_pattern(conn, pattern, opts, out, key) local elts = results[2] local tokens = {} - for _,e in ipairs(elts) do - conn:add_cmd('HGETALL', {e}) + for _, e in ipairs(elts) do + conn:add_cmd('HGETALL', { e }) end -- This function returns many results, each for each command -- So if we have batch 1000, then we would have 1000 tables in form -- [result, {hash_content}] - local all_results = {conn:exec()} + local all_results = { conn:exec() } - for i=1,#all_results,2 do + for i = 1, #all_results, 2 do local r, hash_content = all_results[i], all_results[i + 1] if r then -- List to a hash map local data = redis_map_zip(hash_content) - tokens[#tokens + 1] = {key = elts[(i + 1)/2], data = data} + tokens[#tokens + 1] = { key = elts[(i + 1) / 2], data = data } end end -- Output keeping track of the commas - for i,d in ipairs(tokens) do + for i, d in ipairs(tokens) do if cursor == 0 and i == #tokens or not opts.json then if opts.cdb then table.insert(out[key].elts, { @@ -302,8 +302,8 @@ end local function dump_handler(opts) local patterns_seen = {} - for _,cls in ipairs(classifiers) do - local res,conn = lua_redis.redis_connect_sync(cls.redis_params, false) + for _, cls in ipairs(classifiers) do + local res, conn = lua_redis.redis_connect_sync(cls.redis_params, false) if not res then rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params) @@ -314,7 +314,7 @@ local function dump_handler(opts) local function check_keys(sym) local sym_keys_pattern = string.format("%s_keys", sym) conn:add_cmd('SMEMBERS', { sym_keys_pattern }) - local ret,keys = conn:exec() + local ret, keys = conn:exec() if not ret then rspamd_logger.errx("cannot execute command to get keys: %s", keys) @@ -325,11 +325,11 @@ local function dump_handler(opts) out[#out + 1] = string.format('"%s": %s\n', sym_keys_pattern, ucl.to_format(keys, 'json-compact')) end - for _,k in ipairs(keys) do + for _, k in ipairs(keys) do local pat = string.format('%s*_*', k) if not patterns_seen[pat] then - conn:add_cmd('HGETALL', {k}) - local _ret,additional_keys = conn:exec() + conn:add_cmd('HGETALL', { k }) + local _ret, additional_keys = conn:exec() if _ret then if opts.json then @@ -359,19 +359,19 @@ local function dump_handler(opts) end local function obj_to_redis_arguments(obj, opts, cmd_pipe) - local key,value = next(obj) + local key, value = next(obj) if type(key) == 'string' then if type(value) == 'table' then if not value[1] then if opts.mode == 'replace' then local cmd = 'HMSET' - local params = {key} - for k,v in pairs(value) do + local params = { key } + for k, v in pairs(value) do table.insert(params, k) table.insert(params, v) end - table.insert(cmd_pipe, {cmd, params}) + table.insert(cmd_pipe, { cmd, params }) else local cmd = 'HINCRBYFLOAT' local mult = 1.0 @@ -379,19 +379,19 @@ local function obj_to_redis_arguments(obj, opts, cmd_pipe) mult = (-mult) end - for k,v in pairs(value) do + for k, v in pairs(value) do if tonumber(v) then v = tonumber(v) - table.insert(cmd_pipe, {cmd, {key, k, tostring(v * mult)}}) + table.insert(cmd_pipe, { cmd, { key, k, tostring(v * mult) } }) else - table.insert(cmd_pipe, {'HSET', {key, k, v}}) + table.insert(cmd_pipe, { 'HSET', { key, k, v } }) end end end else -- Numeric table of elements (e.g. _keys) - it is actually a set in Redis - for _,elt in ipairs(value) do - table.insert(cmd_pipe, {'SADD', {key, elt}}) + for _, elt in ipairs(value) do + table.insert(cmd_pipe, { 'SADD', { key, elt } }) end end end @@ -403,17 +403,17 @@ end local function execute_batch(batch, conns, opts) local cmd_pipe = {} - for _,cmd in ipairs(batch) do + for _, cmd in ipairs(batch) do obj_to_redis_arguments(cmd, opts, cmd_pipe) end if opts.no_operation then - for _,cmd in ipairs(cmd_pipe) do + for _, cmd in ipairs(cmd_pipe) do rspamd_logger.messagex('%s %s', cmd[1], table.concat(cmd[2], ' ')) end else for _, conn in ipairs(conns) do - for _,cmd in ipairs(cmd_pipe) do + for _, cmd in ipairs(cmd_pipe) do local is_ok, err = conn:add_cmd(cmd[1], cmd[2]) if not is_ok then @@ -427,11 +427,11 @@ local function execute_batch(batch, conns, opts) end local function restore_handler(opts) - local files = opts.file or {'-'} + local files = opts.file or { '-' } local conns = {} - for _,cls in ipairs(classifiers) do - local res,conn = lua_redis.redis_connect_sync(cls.redis_params, true) + for _, cls in ipairs(classifiers) do + local res, conn = lua_redis.redis_connect_sync(cls.redis_params, true) if not res then rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params) @@ -443,7 +443,7 @@ local function restore_handler(opts) local batch = {} - for _,f in ipairs(files) do + for _, f in ipairs(files) do local fd if f ~= '-' then fd = io.open(f, 'r') @@ -454,7 +454,7 @@ local function restore_handler(opts) for line in io.lines() do local ucl_parser = ucl.parser() local res, err - res,err = ucl_parser:parse_string(line) + res, err = ucl_parser:parse_string(line) if not res then rspamd_logger.errx("%s: cannot read line %s: %s", f, cur_line, err) @@ -470,7 +470,9 @@ local function restore_handler(opts) end end - if fd then fd:close() end + if fd then + fd:close() + end end if #batch > 0 then @@ -492,8 +494,10 @@ local function handler(args) if classifier then if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.bayes then cls = cls.bayes end + for _, cls in ipairs(classifier) do + if cls.bayes then + cls = cls.bayes + end if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, obj) end @@ -503,7 +507,7 @@ local function handler(args) classifier = classifier.bayes if classifier[1] then - for _,cls in ipairs(classifier) do + for _, cls in ipairs(classifier) do if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, obj) end @@ -518,7 +522,7 @@ local function handler(args) end if type(opts.file) == 'string' then - opts.file = {opts.file} + opts.file = { opts.file } elseif type(opts.file) == 'none' then opts.file = {} end @@ -534,7 +538,7 @@ end return { name = 'statistics_dump', - aliases = {'stat_dump', 'bayes_dump'}, + aliases = { 'stat_dump', 'bayes_dump' }, handler = handler, description = parser._description }
\ No newline at end of file diff --git a/lualib/rspamadm/template.lua b/lualib/rspamadm/template.lua index bb799f730..ca1779aa8 100644 --- a/lualib/rspamadm/template.lua +++ b/lualib/rspamadm/template.lua @@ -42,17 +42,17 @@ parser:mutex( :description "Store files with the new suffix" :argname "<suffix>", parser:flag "-i --inplace" - :description "Replace input file(s)" + :description "Replace input file(s)" ) local lua_util = require "lua_util" local function set_env(opts, env) if opts.env then - for _,fname in ipairs(opts.env) do + for _, fname in ipairs(opts.env) do for kv in assert(io.open(fname)):lines() do if not kv:match('%s*#.*') then - local k,v = kv:match('([^=%s]+)%s*=%s*(.+)') + local k, v = kv:match('([^=%s]+)%s*=%s*(.+)') if k and v then env[k] = v @@ -65,14 +65,14 @@ local function set_env(opts, env) end if opts.lua_env then - for _,fname in ipairs(opts.env) do - local ret,res_or_err = pcall(loadfile(fname)) + for _, fname in ipairs(opts.env) do + local ret, res_or_err = pcall(loadfile(fname)) if not ret then io.write(string.format('cannot load %s: %s\n', fname, res_or_err)) else if type(res_or_err) == 'table' then - for k,v in pairs(res_or_err) do + for k, v in pairs(res_or_err) do env[k] = lua_util.deepcopy(v) end else @@ -100,8 +100,10 @@ local function handler(args) local env = {} set_env(opts, env) - if not opts.file or #opts.file == 0 then opts.file = {'-'} end - for _,fname in ipairs(opts.file) do + if not opts.file or #opts.file == 0 then + opts.file = { '-' } + end + for _, fname in ipairs(opts.file) do local content = read_file(fname) local res = lua_util.jinja_template(content, env, opts.no_vars) diff --git a/lualib/rspamadm/vault.lua b/lualib/rspamadm/vault.lua index 2c7d5abfe..840e504e0 100644 --- a/lualib/rspamadm/vault.lua +++ b/lualib/rspamadm/vault.lua @@ -43,28 +43,27 @@ parser:option "-o --output" :description "Output format ('ucl', 'json', 'json-compact', 'yaml')" :argname("<type>") :convert { - ucl = "ucl", - json = "json", - ['json-compact'] = "json-compact", - yaml = "yaml", - } - :default "ucl" + ucl = "ucl", + json = "json", + ['json-compact'] = "json-compact", + yaml = "yaml", +} + :default "ucl" parser:command "list ls l" - :description "List elements in the vault" + :description "List elements in the vault" local show = parser:command "show get" - :description "Extract element from the vault" + :description "Extract element from the vault" show:argument "domain" - :description "Domain to create key for" - :args "+" + :description "Domain to create key for" + :args "+" local delete = parser:command "delete del rm remove" - :description "Delete element from the vault" + :description "Delete element from the vault" delete:argument "domain" - :description "Domain to create delete key(s) for" - :args "+" - + :description "Domain to create delete key(s) for" + :args "+" local newkey = parser:command "newkey new create" :description "Add new key to the vault" @@ -77,10 +76,10 @@ newkey:option "-s --selector" newkey:option "-A --algorithm" :argname("<type>") :convert { - rsa = "rsa", - ed25519 = "ed25519", - eddsa = "ed25519", - } + rsa = "rsa", + ed25519 = "ed25519", + eddsa = "ed25519", +} :default "rsa" newkey:option "-b --bits" :argname("<nbits>") @@ -137,18 +136,18 @@ end local function parse_vault_reply(data) local p = ucl.parser() - local res,parser_err = p:parse_string(data) + local res, parser_err = p:parse_string(data) if not res then - return nil,parser_err + return nil, parser_err else - return p:get_object(),nil + return p:get_object(), nil end end local function maybe_print_vault_data(opts, data, func) if data then - local res,parser_err = parse_vault_reply(data) + local res, parser_err = parse_vault_reply(data) if not res then printf('vault reply for cannot be parsed: %s', parser_err) @@ -169,9 +168,9 @@ local function print_dkim_txt_record(b64, selector, alg) local prefix = string.format("v=DKIM1; k=%s; p=", alg) b64 = prefix .. b64 if #b64 < 255 then - labels = {'"' .. b64 .. '"'} + labels = { '"' .. b64 .. '"' } else - for sl=1,#b64,256 do + for sl = 1, #b64, 256 do table.insert(labels, '"' .. b64:sub(sl, sl + 255) .. '"') end end @@ -182,7 +181,7 @@ end local function show_handler(opts, domain) local uri = vault_url(opts, domain) - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -206,7 +205,7 @@ end local function delete_handler(opts, domain) local uri = vault_url(opts, domain) - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -229,7 +228,7 @@ end local function list_handler(opts) local uri = vault_url(opts) - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -258,7 +257,7 @@ end local function create_and_push_key(opts, domain, existing) local uri = vault_url(opts, domain) - local sk,pk = genkey(opts) + local sk, pk = genkey(opts) local res = { selectors = { @@ -274,7 +273,7 @@ local function create_and_push_key(opts, domain, existing) } } - for _,sel in ipairs(existing) do + for _, sel in ipairs(existing) do res.selectors[#res.selectors + 1] = sel end @@ -282,7 +281,7 @@ local function create_and_push_key(opts, domain, existing) res.selectors[1].valid_end = os.time() + opts.expire * 3600 * 24 end - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -303,7 +302,7 @@ local function create_and_push_key(opts, domain, existing) maybe_print_vault_data(opts, data.content) os.exit(1) else - maybe_printf(opts,'stored key for: %s, selector: %s', domain, opts.selector) + maybe_printf(opts, 'stored key for: %s, selector: %s', domain, opts.selector) maybe_printf(opts, 'please place the corresponding public key as following:') if opts.silent then @@ -322,7 +321,7 @@ local function newkey_handler(opts, domain) os.date("!%Y%m%d")) end - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -335,7 +334,7 @@ local function newkey_handler(opts, domain) } if is_http_error(err, data) or not data.content then - create_and_push_key(opts, domain,{}) + create_and_push_key(opts, domain, {}) else -- Key exists local rep = parse_vault_reply(data.content) @@ -348,11 +347,11 @@ local function newkey_handler(opts, domain) local elts = rep.data.selectors if not elts then - create_and_push_key(opts, domain,{}) + create_and_push_key(opts, domain, {}) os.exit(0) end - for _,sel in ipairs(elts) do + for _, sel in ipairs(elts) do if sel.alg == opts.algorithm then printf('key with the specific algorithm %s is already presented at %s selector for %s domain', opts.algorithm, sel.selector, domain) @@ -370,7 +369,7 @@ local function roll_handler(opts, domain) selectors = {} } - local err,data = rspamd_http.request{ + local err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -414,7 +413,7 @@ local function roll_handler(opts, domain) table.insert(nkeys[sel.alg], sel) end - for _,sel in ipairs(elts) do + for _, sel in ipairs(elts) do if sel.valid_end and sel.valid_end < os.time() then if not opts.remove_expired then insert_key(sel, false) @@ -428,7 +427,7 @@ local function roll_handler(opts, domain) end -- Now we need to ensure that all but one selectors have either expired or just a single key - for alg,keys in pairs(nkeys) do + for alg, keys in pairs(nkeys) do table.sort(keys, function(k1, k2) if k1.valid_end and k2.valid_end then return k1.valid_end > k2.valid_end @@ -441,8 +440,8 @@ local function roll_handler(opts, domain) end) -- Exclude the key with the highest expiration date and examine the rest if not (#keys == 1 or fun.all(function(k) - return k.valid_end and k.valid_end < os.time() - end, fun.tail(keys))) then + return k.valid_end and k.valid_end < os.time() + end, fun.tail(keys))) then printf('bad keys list for %s and %s algorithm', domain, alg) fun.each(function(k) if not k.valid_end then @@ -459,7 +458,7 @@ local function roll_handler(opts, domain) if not opts.remove_expired then -- OK to process -- Insert keys for each algorithm in pairs <old_key(s)>, <new_key> - local sk,pk = genkey({algorithm = alg, bits = keys[1].bits}) + local sk, pk = genkey({ algorithm = alg, bits = keys[1].bits }) local selector = string.format('%s-%s', alg, os.date("!%Y%m%d")) @@ -482,14 +481,14 @@ local function roll_handler(opts, domain) table.insert(res.selectors, nelt) end - for _,k in ipairs(keys) do + for _, k in ipairs(keys) do table.insert(res.selectors, k) end end end -- We can now store res in the vault - err,data = rspamd_http.request{ + err, data = rspamd_http.request { config = rspamd_config, ev_base = rspamadm_ev_base, session = rspamadm_session, @@ -510,9 +509,9 @@ local function roll_handler(opts, domain) maybe_print_vault_data(opts, data.content) os.exit(1) else - for _,key in ipairs(res.selectors) do - if not key.valid_end or key.valid_end > os.time() + opts.ttl * 3600 * 24 then - maybe_printf(opts,'rolled key for: %s, new selector: %s', domain, key.selector) + for _, key in ipairs(res.selectors) do + if not key.valid_end or key.valid_end > os.time() + opts.ttl * 3600 * 24 then + maybe_printf(opts, 'rolled key for: %s, new selector: %s', domain, key.selector) maybe_printf(opts, 'please place the corresponding public key as following:') if opts.silent then @@ -553,13 +552,21 @@ local function handler(args) if command == 'list' then list_handler(opts) elseif command == 'show' then - fun.each(function(d) show_handler(opts, d) end, opts.domain) + fun.each(function(d) + show_handler(opts, d) + end, opts.domain) elseif command == 'newkey' then - fun.each(function(d) newkey_handler(opts, d) end, opts.domain) + fun.each(function(d) + newkey_handler(opts, d) + end, opts.domain) elseif command == 'roll' then - fun.each(function(d) roll_handler(opts, d) end, opts.domain) + fun.each(function(d) + roll_handler(opts, d) + end, opts.domain) elseif command == 'delete' then - fun.each(function(d) delete_handler(opts, d) end, opts.domain) + fun.each(function(d) + delete_handler(opts, d) + end, opts.domain) else parser:error(string.format('command %s is not implemented', command)) end |