diff options
Diffstat (limited to 'lualib')
| -rw-r--r-- | lualib/lua_mime.lua | 18 | ||||
| -rw-r--r-- | lualib/rspamadm/mime.lua | 54 | ||||
| -rw-r--r-- | lualib/rspamadm/statistics_dump.lua | 71 |
3 files changed, 127 insertions, 16 deletions
diff --git a/lualib/lua_mime.lua b/lualib/lua_mime.lua index f84cc4d4e..a57c2181e 100644 --- a/lualib/lua_mime.lua +++ b/lualib/lua_mime.lua @@ -581,6 +581,24 @@ exports.message_to_ucl = function(task) result.size = task:get_size() result.digest = task:get_digest() + result.headers = task:get_headers(true) or {} + + local parts = task:get_parts() or {} + result.parts = {} + for _,part in ipairs(parts) do + local l = part:get_length() + if l > 0 then + local p = {} + p.size = l + p.type = string.format('%s/%s', part:get_type()) + p.detected_type = string.format('%s/%s', part:get_detected_type()) + p.filename = part:get_filename() + p.content = part:get_content() + p.headers = part:get_headers(true) or {} + table.insert(result.parts, p) + end + end + return result end diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 0faeeca78..0864af3fb 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -41,7 +41,9 @@ parser:mutex( parser:flag "-j --json" :description "JSON output", parser:flag "-U --ucl" - :description "UCL output" + :description "UCL output", + parser:flag "-M --messagepack" + :description "MessagePack output" ) parser:flag "-C --compact" :description "Use compact format" @@ -194,6 +196,22 @@ sign:option "-o --output" } :default 'message' +local dump = parser:command "dump" + :description "Dumps a raw message in different formats" +dump:argument "file" + :description "File to process" + :argname "<file>" + :args "+" +-- Duplicate format for convenience +dump:mutex( + parser:flag "-j --json" + :description "JSON output", + parser:flag "-U --ucl" + :description "UCL output", + parser:flag "-M --messagepack" + :description "MessagePack output" +) + local function load_config(opts) local _r,err = rspamd_config:load_ucl(opts['config']) @@ -239,16 +257,22 @@ local function maybe_print_fname(opts, fname) end end +local function output_fmt(opts) + local fmt = 'json' + if opts.compact then fmt = 'json-compact' end + if opts.ucl then fmt = 'ucl' end + if opts.messagepack then fmt = 'msgpack' end + + return fmt +end + -- Print elements in form -- filename -> table of elements local function print_elts(elts, opts, func) local fun = require "fun" if opts.json or opts.ucl then - local fmt = 'json' - if opts.compact then fmt = 'json-compact' end - if opts.ucl then fmt = 'ucl' end - io.write(ucl.to_format(elts, fmt)) + io.write(ucl.to_format(elts, output_fmt(opts))) else fun.each(function(fname, elt) @@ -849,6 +873,24 @@ local function sign_handler(opts) end end +local function dump_handler(opts) + load_config(opts) + rspamd_url.init(rspamd_config:get_tld_path()) + + for _,fname in ipairs(opts.file) do + local task = load_task(opts, fname) + + if opts.ucl or opts.json or opts.messagepack then + local ucl_object = lua_mime.message_to_ucl(task) + io.write(ucl.to_format(ucl_object, output_fmt(opts))) + else + task:get_content():save_in_file(1) + end + + task:destroy() -- No automatic dtor + end +end + local function handler(args) local opts = parser:parse(args) @@ -870,6 +912,8 @@ local function handler(args) modify_handler(opts) elseif command == 'sign' then sign_handler(opts) + elseif command == 'dump' then + dump_handler(opts) else parser:error('command %s is not implemented', command) end diff --git a/lualib/rspamadm/statistics_dump.lua b/lualib/rspamadm/statistics_dump.lua index 7b1dc581e..aa461a92a 100644 --- a/lualib/rspamadm/statistics_dump.lua +++ b/lualib/rspamadm/statistics_dump.lua @@ -19,6 +19,10 @@ local rspamd_logger = require "rspamd_logger" local argparse = require "argparse" local rspamd_zstd = require "rspamd_zstd" local rspamd_text = require "rspamd_text" +local rspamd_util = require "rspamd_util" +local rspamd_cdb = require "rspamd_cdb" +local lua_util = require "lua_util" +local rspamd_i64 = require "rspamd_int64" local ucl = require "ucl" local N = "statistics_dump" @@ -41,8 +45,12 @@ parser:option "-c --config" -- Extract subcommand local dump = parser:command "dump d" :description "Dump bayes statistics" -dump:flag "-j --json" - :description "Json output" +dump:mutex( + dump:flag "-j --json" + :description "Json output", + dump:flag "-C --cdb" + :description "CDB output" +) dump:flag "-c --compress" :description "Compress output" dump:option "-b --batch-size" @@ -51,6 +59,7 @@ dump:option "-b --batch-size" :convert(tonumber) :default(1000) + -- Restore local restore = parser:command "restore r" :description "Restore bayes statistics" @@ -166,10 +175,12 @@ local function redis_map_zip(ar) return data end --- Used to clear plain numeric tables +-- Used to clear tables local clear_fcn = table.clear or function(tbl) - local l = #tbl - for i=1,l do tbl[i] = nil end + local keys = lua_util.keys(tbl) + for _,k in ipairs(keys) do + tbl[k] = nil + end end local compress_ctx @@ -192,7 +203,27 @@ local function dump_out(out, opts, last) end end -local function dump_pattern(conn, pattern, opts, out) +local function dump_cdb(out, opts, last, pattern) + local results = out[pattern] + + if not out.cdb_builder then + -- First invocation + out.cdb_builder = rspamd_cdb.build(string.format('%s.cdb', pattern)) + out.cdb_builder:add('_lrnspam', rspamd_i64.fromstring(results.learns_spam or '0')) + out.cdb_builder:add('_lrnham_', rspamd_i64.fromstring(results.learns_ham or '0')) + end + + for _,o in ipairs(results.elts) do + out.cdb_builder:add(o.key, o.value) + end + + if last then + out.cdb_builder:finalize() + out.cdb_builder = nil + end +end + +local function dump_pattern(conn, pattern, opts, out, key) local cursor = 0 repeat @@ -232,8 +263,16 @@ local function dump_pattern(conn, pattern, opts, out) -- Output keeping track of the commas for i,d in ipairs(tokens) do if cursor == 0 and i == #tokens or not opts.json then - out[#out + 1] = rspamd_logger.slog('"%s": %s\n', d.key, - ucl.to_format(d.data, "json-compact")) + if opts.cdb then + table.insert(out[key].elts, { + key = rspamd_i64.fromstring(string.match(d.key, '%d+')), + value = rspamd_util.pack('ff', tonumber(d.data["S"] or '0') or 0, + tonumber(d.data["H"] or '0')) + }) + else + out[#out + 1] = rspamd_logger.slog('"%s": %s\n', d.key, + ucl.to_format(d.data, "json-compact")) + end else out[#out + 1] = rspamd_logger.slog('"%s": %s,\n', d.key, ucl.to_format(d.data, "json-compact")) @@ -247,8 +286,15 @@ local function dump_pattern(conn, pattern, opts, out) -- Do not write the last chunk of out as it will be processed afterwards if not cursor == 0 then - dump_out(out, opts, false) - clear_fcn(out) + if opts.cdb then + dump_out(out, opts, false) + clear_fcn(out) + else + dump_cdb(out, opts, false, key) + out[key].elts = {} + end + elseif opts.cdb then + dump_cdb(out, opts, true, key) end until cursor == 0 @@ -289,11 +335,14 @@ local function dump_handler(opts) if opts.json then out[#out + 1] = string.format('{"pattern": "%s", "meta": %s, "elts": {\n', k, ucl.to_format(redis_map_zip(additional_keys), 'json-compact')) + elseif opts.cdb then + out[k] = redis_map_zip(additional_keys) + out[k].elts = {} else out[#out + 1] = string.format('"%s": %s\n', k, ucl.to_format(redis_map_zip(additional_keys), 'json-compact')) end - dump_pattern(conn, pat, opts, out) + dump_pattern(conn, pat, opts, out, k) patterns_seen[pat] = true end end |