From 239d6fc79857a9282885e94eb344a6fa15c451e8 Mon Sep 17 00:00:00 2001 From: Jason Stephenson Date: Thu, 21 Jul 2022 11:57:57 +0100 Subject: Implement writing to directory --- lualib/rspamadm/mime.lua | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index d2e9c052d..5f063666d 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -209,8 +209,13 @@ dump:mutex( parser:flag "-U --ucl" :description "UCL output", parser:flag "-M --messagepack" - :description "MessagePack output" + :description "MessagePack output", + parser:flag "-E --extract" + :description "Always extract content to disk" ) +dump:option "-o --outdir" + :description "Output directory" + :argname("") local function load_config(opts) local _r,err = rspamd_config:load_ucl(opts['config']) @@ -873,18 +878,51 @@ local function sign_handler(opts) end end +--Strips directories and .extensions from a filename/path +local function filename_only(filename) + stripped_filename = filename:match(".*%/([^%.]*)") + if not stripped_filename then + stripped_filename = filename:match("(.*)%.") + end + return stripped_filename +end + local function dump_handler(opts) load_config(opts) rspamd_url.init(rspamd_config:get_tld_path()) for _,fname in ipairs(opts.file) do + local task = load_task(opts, fname) + local data_to_write = nil + local out_extension = nil if opts.ucl or opts.json or opts.messagepack then local ucl_object = lua_mime.message_to_ucl(task) - io.write(ucl.to_format(ucl_object, output_fmt(opts))) + out_extension = output_fmt(opts) + data_to_write = ucl.to_format(ucl_object, out_extension) else - task:get_content():save_in_file(1) + out_extension = "mime" + data_to_write = tostring(task:get_content()) + end + + if opts.outdir then + if opts.outdir:sub(-1) ~= "/" then + opts.outdir = opts.outdir .. "/" + end + + local outpath = string.format("%s%s.%s", opts.outdir, filename_only(fname), out_extension) + local outfile = io.open(outpath, "w") + + if outfile then + outfile:write(data_to_write) + outfile:close() + io.write(outpath.."\n") + else + io.stderr:write(string.format("Unable to open: %s\n", outpath)) + end + else + io.write(data_to_write) end task:destroy() -- No automatic dtor -- cgit v1.2.3 From c2559dc9fa65e05bf1e2aac0c56897d144184a22 Mon Sep 17 00:00:00 2001 From: Jason Stephenson Date: Thu, 21 Jul 2022 12:20:42 +0100 Subject: Refactor before adding support to extract option --- lualib/rspamadm/mime.lua | 75 +++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 5f063666d..96aed5ca4 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -878,52 +878,55 @@ local function sign_handler(opts) end end ---Strips directories and .extensions from a filename/path -local function filename_only(filename) - stripped_filename = filename:match(".*%/([^%.]*)") - if not stripped_filename then - stripped_filename = filename:match("(.*)%.") +--Strips directories and .extensions (if present) from a filepath +local function filename_only(filepath) + filename = filepath:match(".*%/([^%.]*)") + if not filename then + filename = filepath:match("(.*)%.") end - return stripped_filename + return filename end -local function dump_handler(opts) - load_config(opts) - rspamd_url.init(rspamd_config:get_tld_path()) +local function get_dump_content(task, opts) + if opts.ucl or opts.json or opts.messagepack then + local ucl_object = lua_mime.message_to_ucl(task) + local extension = output_fmt(opts) + return ucl.to_format(ucl_object, out_extension), extension + end + return tostring(task:get_content()), "mime" +end - for _,fname in ipairs(opts.file) do +--Write the dump content to file or standard out +local function write_dump_content(data, fname, extension, outdir) + if outdir then + if outdir:sub(-1) ~= "/" then + outdir = outdir .. "/" + end - local task = load_task(opts, fname) + local outpath = string.format("%s%s.%s", outdir, filename_only(fname), extension) + local outfile = io.open(outpath, "w") - local data_to_write = nil - local out_extension = nil - if opts.ucl or opts.json or opts.messagepack then - local ucl_object = lua_mime.message_to_ucl(task) - out_extension = output_fmt(opts) - data_to_write = ucl.to_format(ucl_object, out_extension) + if outfile then + outfile:write(data) + outfile:close() + io.write(outpath.."\n") else - out_extension = "mime" - data_to_write = tostring(task:get_content()) + io.stderr:write(string.format("Unable to open: %s\n", outpath)) end + else + io.write(data) + end +end - if opts.outdir then - if opts.outdir:sub(-1) ~= "/" then - opts.outdir = opts.outdir .. "/" - end +local function dump_handler(opts) + load_config(opts) + rspamd_url.init(rspamd_config:get_tld_path()) - local outpath = string.format("%s%s.%s", opts.outdir, filename_only(fname), out_extension) - local outfile = io.open(outpath, "w") - - if outfile then - outfile:write(data_to_write) - outfile:close() - io.write(outpath.."\n") - else - io.stderr:write(string.format("Unable to open: %s\n", outpath)) - end - else - io.write(data_to_write) - end + for _,fname in ipairs(opts.file) do + local task = load_task(opts, fname) + + local data, extension = get_dump_content(task, opts) + write_dump_content(data, fname, extension, opts.outdir) task:destroy() -- No automatic dtor end -- cgit v1.2.3 From 6460f4088b8f8fcaa167ee0cef3505d7ded9d6ec Mon Sep 17 00:00:00 2001 From: Jason Stephenson Date: Fri, 22 Jul 2022 11:08:30 +0100 Subject: Add support for --split --- lualib/rspamadm/mime.lua | 69 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 96aed5ca4..19409eb88 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -209,10 +209,11 @@ dump:mutex( parser:flag "-U --ucl" :description "UCL output", parser:flag "-M --messagepack" - :description "MessagePack output", - parser:flag "-E --extract" - :description "Always extract content to disk" + :description "MessagePack output" ) +dump:flag "-s --split" + :description "Split the output file contents such that no content is embedded" + dump:option "-o --outdir" :description "Output directory" :argname("") @@ -878,26 +879,28 @@ local function sign_handler(opts) end end ---Strips directories and .extensions (if present) from a filepath +-- Strips directories and .extensions (if present) from a filepath + -- very_simple + -- /home/very_simple.eml + -- very_simple.eml + -- very_simple.example.eml + -- /home/very_simple + -- home/very_simple + -- ./home/very_simple + -- ../home/very_simple.eml +--All the above end up as very_simple local function filename_only(filepath) - filename = filepath:match(".*%/([^%.]*)") + filename = filepath:match(".*%/([^%.]+)") if not filename then - filename = filepath:match("(.*)%.") + filename = filepath:match("([^%.]+)") end return filename end -local function get_dump_content(task, opts) - if opts.ucl or opts.json or opts.messagepack then - local ucl_object = lua_mime.message_to_ucl(task) - local extension = output_fmt(opts) - return ucl.to_format(ucl_object, out_extension), extension - end - return tostring(task:get_content()), "mime" -end - --Write the dump content to file or standard out -local function write_dump_content(data, fname, extension, outdir) +local function write_dump_content(dump_content, fname, extension, outdir) + + wrote_filepath = nil if outdir then if outdir:sub(-1) ~= "/" then outdir = outdir .. "/" @@ -907,15 +910,43 @@ local function write_dump_content(data, fname, extension, outdir) local outfile = io.open(outpath, "w") if outfile then - outfile:write(data) + outfile:write(dump_content) outfile:close() io.write(outpath.."\n") + wrote_filepath = outpath else io.stderr:write(string.format("Unable to open: %s\n", outpath)) end else - io.write(data) + io.write(dump_content) + end + + return wrote_filepath +end + +-- Get the formatted ucl (split or unsplit) or the raw task content +local function get_dump_content(task, opts, fname) + if opts.ucl or opts.json or opts.messagepack then + local ucl_object = lua_mime.message_to_ucl(task) + + --Split out the content field into separate raws and update the ucl + if opts.split then + for i, part in ipairs(ucl_object.parts) do + if part.content then + local part_filename = string.format("%s-part%d", filename_only(fname), i) + local part_path = write_dump_content(tostring(part.content), part_filename, "raw", opts.outdir) + if part_path then + part.content = ucl.null + part.content_path = part_path + end + end + end + end + + local extension = output_fmt(opts) + return ucl.to_format(ucl_object, extension), extension end + return tostring(task:get_content()), "mime" end local function dump_handler(opts) @@ -925,7 +956,7 @@ local function dump_handler(opts) for _,fname in ipairs(opts.file) do local task = load_task(opts, fname) - local data, extension = get_dump_content(task, opts) + local data, extension = get_dump_content(task, opts, fname) write_dump_content(data, fname, extension, opts.outdir) task:destroy() -- No automatic dtor -- cgit v1.2.3 From 3158d56ea48f8c90ff0274f76041c052e580de4b Mon Sep 17 00:00:00 2001 From: Jason Stephenson Date: Fri, 22 Jul 2022 14:29:08 +0100 Subject: luacheck fixes --- lualib/rspamadm/mime.lua | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 19409eb88..44140829b 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -890,7 +890,7 @@ end -- ../home/very_simple.eml --All the above end up as very_simple local function filename_only(filepath) - filename = filepath:match(".*%/([^%.]+)") + local filename = filepath:match(".*%/([^%.]+)") if not filename then filename = filepath:match("([^%.]+)") end @@ -899,8 +899,7 @@ end --Write the dump content to file or standard out local function write_dump_content(dump_content, fname, extension, outdir) - - wrote_filepath = nil + local wrote_filepath = nil if outdir then if outdir:sub(-1) ~= "/" then outdir = outdir .. "/" @@ -920,7 +919,7 @@ local function write_dump_content(dump_content, fname, extension, outdir) else io.write(dump_content) end - +luacheck return wrote_filepath end @@ -928,8 +927,7 @@ end local function get_dump_content(task, opts, fname) if opts.ucl or opts.json or opts.messagepack then local ucl_object = lua_mime.message_to_ucl(task) - - --Split out the content field into separate raws and update the ucl + -- Split out the content field into separate raws and update the ucl if opts.split then for i, part in ipairs(ucl_object.parts) do if part.content then @@ -955,7 +953,6 @@ local function dump_handler(opts) for _,fname in ipairs(opts.file) do local task = load_task(opts, fname) - local data, extension = get_dump_content(task, opts, fname) write_dump_content(data, fname, extension, opts.outdir) -- cgit v1.2.3 From 1db23e2b9e8ea83f1caa97f493ba17df306e4018 Mon Sep 17 00:00:00 2001 From: Jason Stephenson Date: Fri, 22 Jul 2022 14:35:24 +0100 Subject: Fix silly mistake --- lualib/rspamadm/mime.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 44140829b..782524545 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -919,7 +919,6 @@ local function write_dump_content(dump_content, fname, extension, outdir) else io.write(dump_content) end -luacheck return wrote_filepath end -- cgit v1.2.3 From 265bfde0a3088b23f6e7b7476fd4cf02dd4ea8be Mon Sep 17 00:00:00 2001 From: Jason Stephenson Date: Sun, 24 Jul 2022 19:28:11 +0100 Subject: Change comment to asserts. String type check before tostring call --- lualib/rspamadm/mime.lua | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 782524545..2bff5f07c 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -880,15 +880,6 @@ local function sign_handler(opts) end -- Strips directories and .extensions (if present) from a filepath - -- very_simple - -- /home/very_simple.eml - -- very_simple.eml - -- very_simple.example.eml - -- /home/very_simple - -- home/very_simple - -- ./home/very_simple - -- ../home/very_simple.eml ---All the above end up as very_simple local function filename_only(filepath) local filename = filepath:match(".*%/([^%.]+)") if not filename then @@ -897,6 +888,16 @@ local function filename_only(filepath) return filename end +assert(filename_only("very_simple") == "very_simple") +assert(filename_only("/home/very_simple.eml") == "very_simple") +assert(filename_only("very_simple.eml") == "very_simple") +assert(filename_only("very_simple.example.eml") == "very_simple") +assert(filename_only("/home/very_simple") == "very_simple") +assert(filename_only("home/very_simple") == "very_simple") +assert(filename_only("./home/very_simple") == "very_simple") +assert(filename_only("../home/very_simple.eml") == "very_simple") +assert(filename_only("/home/dir.with.dots/very_simple.eml") == "very_simple") + --Write the dump content to file or standard out local function write_dump_content(dump_content, fname, extension, outdir) local wrote_filepath = nil @@ -931,7 +932,12 @@ local function get_dump_content(task, opts, fname) for i, part in ipairs(ucl_object.parts) do if part.content then local part_filename = string.format("%s-part%d", filename_only(fname), i) - local part_path = write_dump_content(tostring(part.content), part_filename, "raw", opts.outdir) + local part_path = nil + if type(part.content) == "string" then + part_path = write_dump_content(part.content, part_filename, "raw", opts.outdir) + else + part_path = write_dump_content(tostring(part.content), part_filename, "raw", opts.outdir) + end if part_path then part.content = ucl.null part.content_path = part_path @@ -943,7 +949,14 @@ local function get_dump_content(task, opts, fname) local extension = output_fmt(opts) return ucl.to_format(ucl_object, extension), extension end - return tostring(task:get_content()), "mime" + + local content = task:get_content() + if type(content) == "string" then + return content, "mime" + else + return tostring(content), "mime" + end + end local function dump_handler(opts) -- cgit v1.2.3 From 2b6fdae55cd09bc2183f3e30e6b816792c53cfce Mon Sep 17 00:00:00 2001 From: Jason Stephenson Date: Sun, 24 Jul 2022 19:47:32 +0100 Subject: fix whitespace --- lualib/rspamadm/mime.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 2bff5f07c..13b022def 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -949,14 +949,12 @@ local function get_dump_content(task, opts, fname) local extension = output_fmt(opts) return ucl.to_format(ucl_object, extension), extension end - local content = task:get_content() if type(content) == "string" then return content, "mime" else return tostring(content), "mime" end - end local function dump_handler(opts) -- cgit v1.2.3 From 8fe5e7d31887f743718e5307692dadd56e7d5431 Mon Sep 17 00:00:00 2001 From: Jason Stephenson Date: Mon, 25 Jul 2022 09:26:48 +0100 Subject: get_dump_content to return content directly without converting userdata to string. write_dump_content to convert string to userdata (rspamd_text) and use save_in_file. --- lualib/rspamadm/mime.lua | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index 13b022def..b09ab57bf 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -18,6 +18,7 @@ local argparse = require "argparse" local ansicolors = require "ansicolors" local rspamd_util = require "rspamd_util" local rspamd_task = require "rspamd_task" +local rspamd_text = require "rspamd_text" local rspamd_logger = require "rspamd_logger" local lua_meta = require "lua_meta" local rspamd_url = require "rspamd_url" @@ -900,6 +901,10 @@ assert(filename_only("/home/dir.with.dots/very_simple.eml") == "very_simple") --Write the dump content to file or standard out local function write_dump_content(dump_content, fname, extension, outdir) + if type(dump_content) == "string" then + dump_content = rspamd_text.fromstring(dump_content) + end + local wrote_filepath = nil if outdir then if outdir:sub(-1) ~= "/" then @@ -907,18 +912,18 @@ local function write_dump_content(dump_content, fname, extension, outdir) end local outpath = string.format("%s%s.%s", outdir, filename_only(fname), extension) - local outfile = io.open(outpath, "w") - - if outfile then - outfile:write(dump_content) - outfile:close() - io.write(outpath.."\n") + if rspamd_util.file_exists(outpath) then + os.remove(outpath) + end + + if dump_content:save_in_file(outpath) then wrote_filepath = outpath + io.write(wrote_filepath.."\n") else - io.stderr:write(string.format("Unable to open: %s\n", outpath)) + io.stderr:write(string.format("Unable to save dump content to file: %s\n", outpath)) end else - io.write(dump_content) + dump_content:save_in_file(1) end return wrote_filepath end @@ -932,12 +937,7 @@ local function get_dump_content(task, opts, fname) for i, part in ipairs(ucl_object.parts) do if part.content then local part_filename = string.format("%s-part%d", filename_only(fname), i) - local part_path = nil - if type(part.content) == "string" then - part_path = write_dump_content(part.content, part_filename, "raw", opts.outdir) - else - part_path = write_dump_content(tostring(part.content), part_filename, "raw", opts.outdir) - end + local part_path = write_dump_content(part.content, part_filename, "raw", opts.outdir) if part_path then part.content = ucl.null part.content_path = part_path @@ -945,16 +945,10 @@ local function get_dump_content(task, opts, fname) end end end - local extension = output_fmt(opts) return ucl.to_format(ucl_object, extension), extension end - local content = task:get_content() - if type(content) == "string" then - return content, "mime" - else - return tostring(content), "mime" - end + return task:get_content(), "mime" end local function dump_handler(opts) -- cgit v1.2.3 From a308d6f77e0e683321eccdc9c64f7175361101ab Mon Sep 17 00:00:00 2001 From: Jason Stephenson Date: Mon, 25 Jul 2022 09:35:27 +0100 Subject: whitespace.. --- lualib/rspamadm/mime.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/lualib/rspamadm/mime.lua b/lualib/rspamadm/mime.lua index b09ab57bf..5c1cebac5 100644 --- a/lualib/rspamadm/mime.lua +++ b/lualib/rspamadm/mime.lua @@ -915,7 +915,6 @@ local function write_dump_content(dump_content, fname, extension, outdir) if rspamd_util.file_exists(outpath) then os.remove(outpath) end - if dump_content:save_in_file(outpath) then wrote_filepath = outpath io.write(wrote_filepath.."\n") -- cgit v1.2.3