From 0d13c724df534012de7e41c5d5b743a7d9314431 Mon Sep 17 00:00:00 2001 From: rm-minus-rf Date: Tue, 14 Apr 2020 11:42:28 +0200 Subject: [Feature] allow variables in force_actions messages See https://github.com/rspamd/rspamd/issues/3335 for details. --- src/plugins/lua/force_actions.lua | 63 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) (limited to 'src/plugins') diff --git a/src/plugins/lua/force_actions.lua b/src/plugins/lua/force_actions.lua index 108c0b76e..8e645290e 100644 --- a/src/plugins/lua/force_actions.lua +++ b/src/plugins/lua/force_actions.lua @@ -29,6 +29,7 @@ local lua_util = require "lua_util" local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash" local rspamd_expression = require "rspamd_expression" local rspamd_logger = require "rspamd_logger" +local lua_selectors = require "lua_selectors" local function gen_cb(expr, act, pool, message, subject, raction, honor, limit, least) @@ -63,6 +64,66 @@ local function gen_cb(expr, act, pool, message, subject, raction, honor, limit, return function(task) + local function fill_vars(repl, var_class, var_payload) + -- fill template vars prefixed with + -- 'selector::' => fill with value extracted by a selector + -- 'symbol::' => fill with matching symbol option string(s) + -- 'task::' => fill with matching task attribute + -- (only queue_id and uid allowed) + + if var_class == "selector" then + local selector = lua_selectors.create_selector_closure(rspamd_config, var_payload, '', true) + if not selector then + rspamd_logger.errx(rspamd_config, 'could not create selector [%1]', var_payload) + return "((could not create selector))" + end + local extracted = selector(task) + if extracted then + -- replace non ascii chars + if type(extracted) == 'table' then + extracted = table.concat(extracted, ',') + end + extracted = string.gsub(extracted, '[\128-\255]', '?') + else + rspamd_logger.errx(rspamd_config, 'could not extract value with selector [%1]', var_payload) + extracted = '((error extracting value))' + end + return extracted + end + + if var_class == "symbol" then + local symb_opts + local symb_strs = {} + if task:has_symbol(var_payload) then + local symb_tbl = task:get_symbol(var_payload) + for _,symb in ipairs(symb_tbl) do + local symb_opts = symb.options + if symb_opts then + -- replace non ascii chars + symb_strs[#symb_strs+1] = string.gsub(table.concat(symb_opts, ','), '[\128-\255]', '?') + end + end + symb_str = table.concat(symb_strs, ',') + else + symb_str = '((symbol not found))' + end + return symb_str + end + + -- NOTE-TO-VSTAKHOV: would it make sense to export task:get_queue_id and task:get_uid as selector data definition? + if var_class == "task" then + local attr_val = '((unknown task attribute))' + if var_payload == 'queue_id' then + attr_val = task:get_queue_id() + elseif var_payload == 'uid' then + attr_val = task:get_uid() + end + return attr_val + end + + end + + local cact = task:get_metric_action('default') if cact == act then return false @@ -83,6 +144,8 @@ local function gen_cb(expr, act, pool, message, subject, raction, honor, limit, if least then flags = "least" end if type(message) == 'string' then + -- fill vars in return message + message = string.gsub(message, '(${(.-)::(.-)})', fill_vars) task:set_pre_result(act, message, N, nil, nil, flags) else -- cgit v1.2.3 From 834279171c3ec8c8a355a9e438f07a06f5b16325 Mon Sep 17 00:00:00 2001 From: rm-minus-rf Date: Fri, 17 Apr 2020 07:43:56 +0200 Subject: [Feature] add queueid, uid, messageid and specific symbols to selectors [Minor] use only selectors to fill vars in force_actions message --- lualib/lua_selectors/extractors.lua | 50 +++++++++++++++++++++++++- lualib/lua_selectors/transforms.lua | 14 +++++++- src/plugins/lua/force_actions.lua | 71 +++++++++---------------------------- 3 files changed, 79 insertions(+), 56 deletions(-) (limited to 'src/plugins') diff --git a/lualib/lua_selectors/extractors.lua b/lualib/lua_selectors/extractors.lua index 625af435c..635972226 100644 --- a/lualib/lua_selectors/extractors.lua +++ b/lualib/lua_selectors/extractors.lua @@ -388,6 +388,54 @@ The first argument must be header name.]], ]], ['args_schema'] = { ts.one_of { 'stem', 'raw', 'norm', 'full' }:is_optional()}, }, + -- Get queue ID + ['queueid'] = { + ['get_value'] = function(task) + local queueid = task:get_queue_id() + if queueid then return queueid,'string' end + return nil + end, + ['description'] = [[Get queue ID]], + }, + -- Get ID of the task being processed + ['uid'] = { + ['get_value'] = function(task) + local uid = task:get_uid() + if uid then return uid,'string' end + return nil + end, + ['description'] = [[Get ID of the task being processed]], + }, + -- Get message ID of the task being processed + ['messageid'] = { + ['get_value'] = function(task) + local mid = task:get_message_id() + if mid then return mid,'string' end + return nil + end, + ['description'] = [[Get message ID]], + }, + -- Get specific symbol + ['symbol'] = { + ['get_value'] = function(task, args) + local symbol = task:get_symbol(args[1]) + if args[2] and symbol then + if args[2] == 'options' then + -- concat options tables to avoid table representation strings produced by implicit conversion + return fun.map(function(r) return table.concat(r[args[2]], ', ') end, symbol), 'string_list' + elseif args[2] == 'score' then + -- only userdata_list seems to work for scores + return fun.map(function(r) return r[args[2]] end, symbol), 'userdata_list' + else + return fun.map(function(r) return r[args[2]] end, symbol), 'string_list' + end + end + return symbol,'table_list' + end, + ['description'] = [[Get specific symbol. The first argument must be the symbol name. If no second argument is specified, returns a list of symbol tables. Otherwise the second argument specifies the attribute which is returned as list (`options`, `score` or `group`)]], + ['args_schema'] = {ts.string, ts.one_of{'options','score','group'}:is_optional()} + }, + } -return extractors \ No newline at end of file +return extractors diff --git a/lualib/lua_selectors/transforms.lua b/lualib/lua_selectors/transforms.lua index b0c912deb..be896126d 100644 --- a/lualib/lua_selectors/transforms.lua +++ b/lualib/lua_selectors/transforms.lua @@ -413,8 +413,20 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], ['args_schema'] = {(ts.number + ts.string / tonumber), (ts.number + ts.string / tonumber):is_optional()} }, + -- Returns the string with all non ascii chars replaced + ['to_ascii'] = { + ['types'] = { + ['string'] = true, + }, + ['map_type'] = 'string', + ['process'] = function(inp, _) + return string.gsub(inp, '[\128-\255]', '?'), 'string' + end, + ['description'] = 'Returns the string with all non-ascii bytes replaced with `?`', + }, + } transform_function.match = transform_function.regexp -return transform_function \ No newline at end of file +return transform_function diff --git a/src/plugins/lua/force_actions.lua b/src/plugins/lua/force_actions.lua index 8e645290e..caa03da64 100644 --- a/src/plugins/lua/force_actions.lua +++ b/src/plugins/lua/force_actions.lua @@ -64,66 +64,29 @@ local function gen_cb(expr, act, pool, message, subject, raction, honor, limit, return function(task) - local function fill_vars(repl, var_class, var_payload) - -- fill template vars prefixed with - -- 'selector::' => fill with value extracted by a selector - -- 'symbol::' => fill with matching symbol option string(s) - -- 'task::' => fill with matching task attribute - -- (only queue_id and uid allowed) - - if var_class == "selector" then - local selector = lua_selectors.create_selector_closure(rspamd_config, var_payload, '', true) + local function process_message_selectors(repl, selector_expr) + -- create/reuse selector to extract value for this placeholder + local selector = selector_cache[selector_expr] + if not selector then + selector_cache[selector_expr] = lua_selectors.create_selector_closure(rspamd_config, selector_expr, '', true) + selector = selector_cache[selector_expr] if not selector then - rspamd_logger.errx(rspamd_config, 'could not create selector [%1]', var_payload) + rspamd_logger.errx(task, 'could not create selector [%1]', selector_expr) return "((could not create selector))" end - local extracted = selector(task) - if extracted then - -- replace non ascii chars - if type(extracted) == 'table' then - extracted = table.concat(extracted, ',') - end - extracted = string.gsub(extracted, '[\128-\255]', '?') - else - rspamd_logger.errx(rspamd_config, 'could not extract value with selector [%1]', var_payload) - extracted = '((error extracting value))' - end - return extracted end - - if var_class == "symbol" then - local symb_opts - local symb_strs = {} - if task:has_symbol(var_payload) then - local symb_tbl = task:get_symbol(var_payload) - for _,symb in ipairs(symb_tbl) do - local symb_opts = symb.options - if symb_opts then - -- replace non ascii chars - symb_strs[#symb_strs+1] = string.gsub(table.concat(symb_opts, ','), '[\128-\255]', '?') - end - end - symb_str = table.concat(symb_strs, ',') - else - symb_str = '((symbol not found))' + local extracted = selector(task) + if extracted then + if type(extracted) == 'table' then + extracted = table.concat(extracted, ',') end - return symb_str - end - - -- NOTE-TO-VSTAKHOV: would it make sense to export task:get_queue_id and task:get_uid as selector data definition? - if var_class == "task" then - local attr_val = '((unknown task attribute))' - if var_payload == 'queue_id' then - attr_val = task:get_queue_id() - elseif var_payload == 'uid' then - attr_val = task:get_uid() - end - return attr_val + else + rspamd_logger.errx(task, 'could not extract value with selector [%1]', selector_expr) + extracted = '((error extracting value))' end - + return extracted end - local cact = task:get_metric_action('default') if cact == act then return false @@ -144,8 +107,8 @@ local function gen_cb(expr, act, pool, message, subject, raction, honor, limit, if least then flags = "least" end if type(message) == 'string' then - -- fill vars in return message - message = string.gsub(message, '(${(.-)::(.-)})', fill_vars) + -- process selector expressions in the message + message = string.gsub(message, '(${(.-)})', process_message_selectors) task:set_pre_result(act, message, N, nil, nil, flags) else -- cgit v1.2.3 From e1e23ddef268163cec9e1d1fafbfda0b36917e77 Mon Sep 17 00:00:00 2001 From: rm-minus-rf Date: Fri, 17 Apr 2020 08:22:56 +0200 Subject: [Fix] add missing selector_cache declaration --- src/plugins/lua/force_actions.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'src/plugins') diff --git a/src/plugins/lua/force_actions.lua b/src/plugins/lua/force_actions.lua index caa03da64..e86892f6a 100644 --- a/src/plugins/lua/force_actions.lua +++ b/src/plugins/lua/force_actions.lua @@ -23,6 +23,7 @@ end local E = {} local N = 'force_actions' +local selector_cache = {} local fun = require "fun" local lua_util = require "lua_util" -- cgit v1.2.3 From 00b2ecaa3b1ec5629a1c37781af80b7afa23913b Mon Sep 17 00:00:00 2001 From: rm-minus-rf Date: Sat, 18 Apr 2020 00:17:29 +0200 Subject: [Minor] selectors: cleaner symbol extractor 'symbol' extractor returns a table 'to_ascii' transform can handle tables --- lualib/lua_selectors/extractors.lua | 17 ++++------------- lualib/lua_selectors/transforms.lua | 15 +++++++++++---- src/plugins/lua/force_actions.lua | 1 - 3 files changed, 15 insertions(+), 18 deletions(-) (limited to 'src/plugins') diff --git a/lualib/lua_selectors/extractors.lua b/lualib/lua_selectors/extractors.lua index 635972226..695371e79 100644 --- a/lualib/lua_selectors/extractors.lua +++ b/lualib/lua_selectors/extractors.lua @@ -419,21 +419,12 @@ The first argument must be header name.]], ['symbol'] = { ['get_value'] = function(task, args) local symbol = task:get_symbol(args[1]) - if args[2] and symbol then - if args[2] == 'options' then - -- concat options tables to avoid table representation strings produced by implicit conversion - return fun.map(function(r) return table.concat(r[args[2]], ', ') end, symbol), 'string_list' - elseif args[2] == 'score' then - -- only userdata_list seems to work for scores - return fun.map(function(r) return r[args[2]] end, symbol), 'userdata_list' - else - return fun.map(function(r) return r[args[2]] end, symbol), 'string_list' - end + if symbol then + return symbol[1],'table' end - return symbol,'table_list' end, - ['description'] = [[Get specific symbol. The first argument must be the symbol name. If no second argument is specified, returns a list of symbol tables. Otherwise the second argument specifies the attribute which is returned as list (`options`, `score` or `group`)]], - ['args_schema'] = {ts.string, ts.one_of{'options','score','group'}:is_optional()} + ['description'] = [[Get specific symbol. The first argument must be the symbol name. Returns the symbol table. See task:get_symbol()]], + ['args_schema'] = {ts.string} }, } diff --git a/lualib/lua_selectors/transforms.lua b/lualib/lua_selectors/transforms.lua index be896126d..b1088aa75 100644 --- a/lualib/lua_selectors/transforms.lua +++ b/lualib/lua_selectors/transforms.lua @@ -413,16 +413,23 @@ Empty string comes the first argument or 'true', non-empty string comes nil]], ['args_schema'] = {(ts.number + ts.string / tonumber), (ts.number + ts.string / tonumber):is_optional()} }, - -- Returns the string with all non ascii chars replaced + -- Returns the string(s) with all non ascii chars replaced ['to_ascii'] = { ['types'] = { ['string'] = true, + ['list'] = true, }, ['map_type'] = 'string', - ['process'] = function(inp, _) - return string.gsub(inp, '[\128-\255]', '?'), 'string' + ['process'] = function(inp, _, args) + if type(inp) == 'table' then + return fun.map( function(s) return string.gsub(tostring(s), '[\128-\255]', args[1] or '?') end , inp), 'string_list' + else + return string.gsub(tostring(inp), '[\128-\255]', '?'), 'string' + end + end, - ['description'] = 'Returns the string with all non-ascii bytes replaced with `?`', + ['description'] = 'Returns the string with all non-ascii bytes replaced with the character given as second argument or `?`', + ['args_schema'] = {ts.string:is_optional()} }, } diff --git a/src/plugins/lua/force_actions.lua b/src/plugins/lua/force_actions.lua index e86892f6a..eda87ead0 100644 --- a/src/plugins/lua/force_actions.lua +++ b/src/plugins/lua/force_actions.lua @@ -111,7 +111,6 @@ local function gen_cb(expr, act, pool, message, subject, raction, honor, limit, -- process selector expressions in the message message = string.gsub(message, '(${(.-)})', process_message_selectors) task:set_pre_result(act, message, N, nil, nil, flags) - else task:set_pre_result(act, nil, N, nil, nil, flags) end -- cgit v1.2.3