local opts = rspamd_config:get_all_opt('dcc') | local opts = rspamd_config:get_all_opt('dcc') | ||||
local logger = require "rspamd_logger" | local logger = require "rspamd_logger" | ||||
local tcp = require "rspamd_tcp" | local tcp = require "rspamd_tcp" | ||||
require "fun" () | |||||
local fun = require "fun" | |||||
local function check_dcc (task) | local function check_dcc (task) | ||||
-- Connection | -- Connection | ||||
local envrcpt = 'test@example.com' | local envrcpt = 'test@example.com' | ||||
local rcpts = task:get_recipients(); | local rcpts = task:get_recipients(); | ||||
if rcpts then | if rcpts then | ||||
local r = table.concat(totable(map(function(rcpt) | |||||
local r = table.concat(fun.totable(map(function(rcpt) | |||||
return rcpt['addr'] end, | return rcpt['addr'] end, | ||||
rcpts)), '\n') | rcpts)), '\n') | ||||
if r then | if r then |
local rspamd_redis = require 'rspamd_redis' | local rspamd_redis = require 'rspamd_redis' | ||||
local redis_params | local redis_params | ||||
local ucl = require "ucl" | local ucl = require "ucl" | ||||
require "fun" () | |||||
local fun = require "fun" | |||||
local settings = { | local settings = { | ||||
redis_key = "dynamic_conf", | redis_key = "dynamic_conf", | ||||
end | end | ||||
if not addr then | if not addr then | ||||
logger.errx(task, 'cannot select server to make redis request') | |||||
rspamd_logger.errx(cfg, 'cannot select server to make redis request') | |||||
end | end | ||||
local options = { | local options = { | ||||
end | end | ||||
local function apply_dynamic_actions(cfg, acts) | local function apply_dynamic_actions(cfg, acts) | ||||
each(function(k, v) | |||||
fun.each(function(k, v) | |||||
if type(v) == 'table' then | if type(v) == 'table' then | ||||
v['name'] = k | v['name'] = k | ||||
if not v['priority'] then | if not v['priority'] then | ||||
priority = settings.priority | priority = settings.priority | ||||
}) | }) | ||||
end | end | ||||
end, filter(function(k, v) | |||||
end, fun.filter(function(k, v) | |||||
local act = rspamd_config:get_metric_action(k) | local act = rspamd_config:get_metric_action(k) | ||||
if (act and alpha_cmp(act, v)) or cur_settings.updates.actions[k] then | if (act and alpha_cmp(act, v)) or cur_settings.updates.actions[k] then | ||||
return false | return false | ||||
end | end | ||||
local function apply_dynamic_scores(cfg, sc) | local function apply_dynamic_scores(cfg, sc) | ||||
each(function(k, v) | |||||
fun.each(function(k, v) | |||||
if type(v) == 'table' then | if type(v) == 'table' then | ||||
v['name'] = k | v['name'] = k | ||||
if not v['priority'] then | if not v['priority'] then | ||||
priority = settings.priority | priority = settings.priority | ||||
}) | }) | ||||
end | end | ||||
end, filter(function(k, v) | |||||
end, fun.filter(function(k, v) | |||||
-- Select elts with scores that are different from local ones | -- Select elts with scores that are different from local ones | ||||
local sym = rspamd_config:get_metric_symbol(k) | local sym = rspamd_config:get_metric_symbol(k) | ||||
if (sym and alpha_cmp(sym.score, v)) or cur_settings.updates.symbols[k] then | if (sym and alpha_cmp(sym.score, v)) or cur_settings.updates.symbols[k] then | ||||
end | end | ||||
if data['symbols_enabled'] then | if data['symbols_enabled'] then | ||||
each(function(i, v) | |||||
fun.each(function(i, v) | |||||
cfg:enable_symbol(v) | cfg:enable_symbol(v) | ||||
end, data['symbols_enabled']) | end, data['symbols_enabled']) | ||||
end | end | ||||
if data['symbols_disabled'] then | if data['symbols_disabled'] then | ||||
each(function(i, v) | |||||
fun.each(function(i, v) | |||||
cfg:disable_symbol(v) | cfg:disable_symbol(v) | ||||
end, data['symbols_disabled']) | end, data['symbols_disabled']) | ||||
end | end | ||||
if not cur_settings.data.scores then | if not cur_settings.data.scores then | ||||
cur_settings.data.scores = {} | cur_settings.data.scores = {} | ||||
end | end | ||||
each(function(k, v) | |||||
fun.each(function(k, v) | |||||
cur_settings.data.scores[k] = v | cur_settings.data.scores[k] = v | ||||
end, | end, | ||||
filter(function(k,v) | |||||
fun.filter(function(k,v) | |||||
if cur_settings.updates.symbols[k] then | if cur_settings.updates.symbols[k] then | ||||
return false | return false | ||||
end | end | ||||
if not cur_settings.data.actions then | if not cur_settings.data.actions then | ||||
cur_settings.data.actions = {} | cur_settings.data.actions = {} | ||||
end | end | ||||
each(function(k, v) | |||||
fun.each(function(k, v) | |||||
cur_settings.data.actions[k] = v | cur_settings.data.actions[k] = v | ||||
end, | end, | ||||
filter(function(k,v) | |||||
fun.filter(function(k,v) | |||||
if cur_settings.updates.actions[k] then | if cur_settings.updates.actions[k] then | ||||
return false | return false | ||||
end | end |
local rspamd_util = require "rspamd_util" | local rspamd_util = require "rspamd_util" | ||||
local fann_symbol_spam = 'FANN_SPAM' | local fann_symbol_spam = 'FANN_SPAM' | ||||
local fann_symbol_ham = 'FANN_HAM' | local fann_symbol_ham = 'FANN_HAM' | ||||
require "fun" () | |||||
local fun = require "fun" | |||||
local ucl = require "ucl" | local ucl = require "ucl" | ||||
local module_log_id = 0x100 | local module_log_id = 0x100 | ||||
end | end | ||||
if not addr then | if not addr then | ||||
logger.errx(task, 'cannot select server to make redis request') | |||||
rspamd_logger.errx(cfg, 'cannot select server to make redis request') | |||||
end | end | ||||
local options = { | local options = { | ||||
local matched_symbols = {} | local matched_symbols = {} | ||||
local n = rspamd_config:get_symbols_count() | local n = rspamd_config:get_symbols_count() | ||||
each(function(s, score) | |||||
fun.each(function(s, score) | |||||
matched_symbols[s + 1] = rspamd_util.tanh(score) | matched_symbols[s + 1] = rspamd_util.tanh(score) | ||||
end, zip(syms, scores)) | |||||
end, fun.zip(syms, scores)) | |||||
for i=1,n do | for i=1,n do | ||||
if matched_symbols[i] then | if matched_symbols[i] then | ||||
local function can_train_cb(err, data) | local function can_train_cb(err, data) | ||||
if not err and tonumber(data) > 0 then | if not err and tonumber(data) > 0 then | ||||
local learn_data = symbols_to_fann_vector( | local learn_data = symbols_to_fann_vector( | ||||
map(function(r) return r[1] end, results), | |||||
map(function(r) return r[2] end, results) | |||||
fun.map(function(r) return r[1] end, results), | |||||
fun.map(function(r) return r[2] end, results) | |||||
) | ) | ||||
-- Add filtered meta tokens | -- Add filtered meta tokens | ||||
each(function(e) table.insert(learn_data, e) end, extra) | |||||
fun.each(function(e) table.insert(learn_data, e) end, extra) | |||||
local str = rspamd_util.zstd_compress(table.concat(learn_data, ';')) | local str = rspamd_util.zstd_compress(table.concat(learn_data, ';')) | ||||
redis_make_request(ev_base, | redis_make_request(ev_base, | ||||
if err then | if err then | ||||
rspamd_logger.errx(rspamd_config, 'cannot get FANNS list from redis: %s', err) | rspamd_logger.errx(rspamd_config, 'cannot get FANNS list from redis: %s', err) | ||||
elseif type(data) == 'table' then | elseif type(data) == 'table' then | ||||
each(function(i, elt) | |||||
fun.each(function(i, elt) | |||||
local redis_len_cb = function(err, data) | local redis_len_cb = function(err, data) | ||||
if err then | if err then | ||||
rspamd_logger.errx(rspamd_config, 'cannot get FANN trains %s from redis: %s', elt, err) | rspamd_logger.errx(rspamd_config, 'cannot get FANN trains %s from redis: %s', elt, err) | ||||
if err then | if err then | ||||
rspamd_logger.errx(rspamd_config, 'cannot get FANNS list from redis: %s', err) | rspamd_logger.errx(rspamd_config, 'cannot get FANNS list from redis: %s', err) | ||||
elseif type(data) == 'table' then | elseif type(data) == 'table' then | ||||
each(function(i, elt) | |||||
fun.each(function(i, elt) | |||||
local redis_update_cb = function(err, data) | local redis_update_cb = function(err, data) | ||||
if err then | if err then | ||||
rspamd_logger.errx(rspamd_config, 'cannot get FANN version %s from redis: %s', elt, err) | rspamd_logger.errx(rspamd_config, 'cannot get FANN version %s from redis: %s', elt, err) | ||||
local ret = cfg:register_worker_script("log_helper", | local ret = cfg:register_worker_script("log_helper", | ||||
function(score, req_score, results, cf, id, extra, ev_base) | function(score, req_score, results, cf, id, extra, ev_base) | ||||
-- map (snd x) (filter (fst x == module_id) extra) | -- map (snd x) (filter (fst x == module_id) extra) | ||||
local extra_fann = map(function(e) return e[2] end, | |||||
filter(function(e) return e[1] == module_log_id end, extra)) | |||||
local extra_fann = fun.map(function(e) return e[2] end, | |||||
fun.filter(function(e) return e[1] == module_log_id end, extra)) | |||||
if use_settings then | if use_settings then | ||||
fann_train_callback(score, req_score, results, cf, | fann_train_callback(score, req_score, results, cf, | ||||
tostring(id), opts['train'], extra_fann, ev_base) | tostring(id), opts['train'], extra_fann, ev_base) | ||||
-- This is needed to pass extra tokens from worker to log_helper | -- This is needed to pass extra tokens from worker to log_helper | ||||
rspamd_plugins["fann_score"] = { | rspamd_plugins["fann_score"] = { | ||||
log_callback = function(task) | log_callback = function(task) | ||||
return totable(map( | |||||
return fun.totable(fun.map( | |||||
function(tok) return {module_log_id, tok} end, | function(tok) return {module_log_id, tok} end, | ||||
gen_metatokens(task))) | gen_metatokens(task))) | ||||
end | end |
local rspamd_expression = require "rspamd_expression" | local rspamd_expression = require "rspamd_expression" | ||||
local rspamd_redis = require "rspamd_redis" | local rspamd_redis = require "rspamd_redis" | ||||
local redis_params | local redis_params | ||||
require "fun" () | |||||
local fun = require "fun" | |||||
local urls = {} | local urls = {} | ||||
local function match_list(r, ls, fields) | local function match_list(r, ls, fields) | ||||
if ls then | if ls then | ||||
if fields then | if fields then | ||||
each(function(e) | |||||
fun.each(function(e) | |||||
local match = e[fields[1]] | local match = e[fields[1]] | ||||
if match then | if match then | ||||
if fields[2] then | if fields[2] then | ||||
end | end | ||||
end, ls) | end, ls) | ||||
else | else | ||||
each(function(e) match_rule(r, e) end, ls) | |||||
fun.each(function(e) match_rule(r, e) end, ls) | |||||
end | end | ||||
end | end | ||||
end | end | ||||
local atoms = {} | local atoms = {} | ||||
local function parse_atom(str) | local function parse_atom(str) | ||||
local atom = table.concat(totable(take_while(function(c) | |||||
local atom = table.concat(fun.totable(fun.take_while(function(c) | |||||
if string.find(', \t()><+!|&\n', c) then | if string.find(', \t()><+!|&\n', c) then | ||||
return false | return false | ||||
end | end | ||||
return true | return true | ||||
end, iter(str))), '') | |||||
end, fun.iter(str))), '') | |||||
table.insert(atoms, atom) | table.insert(atoms, atom) | ||||
return atom | return atom | ||||
end | end | ||||
if expression then | if expression then | ||||
newrule['expression'] = expression | newrule['expression'] = expression | ||||
each(function(v) | |||||
fun.each(function(v) | |||||
rspamd_logger.debugx(rspamd_config, 'add dependency %s -> %s', | rspamd_logger.debugx(rspamd_config, 'add dependency %s -> %s', | ||||
newrule['symbol'], v) | newrule['symbol'], v) | ||||
rspamd_config:register_dependency(newrule['symbol'], v) | rspamd_config:register_dependency(newrule['symbol'], v) | ||||
end | end | ||||
end | end | ||||
-- add fake symbol to check all maps inside a single callback | -- add fake symbol to check all maps inside a single callback | ||||
each(function(rule) | |||||
fun.each(function(rule) | |||||
local id = rspamd_config:register_symbol({ | local id = rspamd_config:register_symbol({ | ||||
type = 'normal', | type = 'normal', | ||||
name = rule['symbol'], | name = rule['symbol'], | ||||
if rule['symbols'] then | if rule['symbols'] then | ||||
-- Find allowed symbols by this map | -- Find allowed symbols by this map | ||||
rule['symbols_set'] = {} | rule['symbols_set'] = {} | ||||
each(function(s) | |||||
fun.each(function(s) | |||||
rspamd_config:register_symbol({ | rspamd_config:register_symbol({ | ||||
type = 'virtual', | type = 'virtual', | ||||
name = s, | name = s, | ||||
}) | }) | ||||
end | end | ||||
end, | end, | ||||
filter(function(r) return not r['prefilter'] end, rules)) | |||||
fun.filter(function(r) return not r['prefilter'] end, rules)) | |||||
each(function(r) | |||||
fun.each(function(r) | |||||
rspamd_config:register_symbol({ | rspamd_config:register_symbol({ | ||||
type = 'prefilter', | type = 'prefilter', | ||||
name = r['symbol'], | name = r['symbol'], | ||||
callback = gen_multimap_callback(r), | callback = gen_multimap_callback(r), | ||||
}) | }) | ||||
end, | end, | ||||
filter(function(r) return r['prefilter'] end, rules)) | |||||
fun.filter(function(r) return r['prefilter'] end, rules)) | |||||
end | end |
local rspamd_tcp = require "rspamd_tcp" | local rspamd_tcp = require "rspamd_tcp" | ||||
local rspamd_redis = require "rspamd_redis" | local rspamd_redis = require "rspamd_redis" | ||||
local rspamd_util = require "rspamd_util" | local rspamd_util = require "rspamd_util" | ||||
require "fun" () | |||||
local fun = require "fun" | |||||
local settings = { | local settings = { | ||||
timeout = 1.0, -- connect timeout | timeout = 1.0, -- connect timeout | ||||
local valid = false | local valid = false | ||||
local function check_results(mxes) | local function check_results(mxes) | ||||
if all(function(k, elt) return elt.checked end, mxes) then | |||||
if fun.all(function(k, elt) return elt.checked end, mxes) then | |||||
-- Save cache | -- Save cache | ||||
local key = settings.key_prefix .. mx_domain | local key = settings.key_prefix .. mx_domain | ||||
local function redis_cache_cb(err, data) | local function redis_cache_cb(err, data) | ||||
) | ) | ||||
else | else | ||||
local valid_mx = {} | local valid_mx = {} | ||||
each(function(k, mx) | |||||
fun.each(function(k, mx) | |||||
table.insert(valid_mx, k) | table.insert(valid_mx, k) | ||||
end, filter(function (k, elt) return elt.working end, mxes)) | |||||
end, fun.filter(function (k, elt) return elt.working end, mxes)) | |||||
task:insert_result(settings.symbol_good_mx, 1.0, valid_mx) | task:insert_result(settings.symbol_good_mx, 1.0, valid_mx) | ||||
local ret,_,_ = rspamd_redis_make_request(task, | local ret,_,_ = rspamd_redis_make_request(task, | ||||
redis_params, -- connect params | redis_params, -- connect params |
-- This plugin implements dynamic updates for rspamd | -- This plugin implements dynamic updates for rspamd | ||||
local ucl = require "ucl" | local ucl = require "ucl" | ||||
require "fun" () | |||||
local fun = require "fun" | |||||
local rspamd_logger = require "rspamd_logger" | local rspamd_logger = require "rspamd_logger" | ||||
local updates_priority = 2 | local updates_priority = 2 | ||||
local rspamd_config = rspamd_config | local rspamd_config = rspamd_config | ||||
local maps = {} | local maps = {} | ||||
local function process_symbols(obj, priority) | local function process_symbols(obj, priority) | ||||
each(function(sym, score) | |||||
fun.each(function(sym, score) | |||||
rspamd_config:set_metric_symbol({ | rspamd_config:set_metric_symbol({ | ||||
name = sym, | name = sym, | ||||
score = score, | score = score, | ||||
end | end | ||||
local function process_actions(obj, priority) | local function process_actions(obj, priority) | ||||
each(function(act, score) | |||||
fun.each(function(act, score) | |||||
rspamd_config:set_metric_action({ | rspamd_config:set_metric_action({ | ||||
action = act, | action = act, | ||||
score = score, | score = score, | ||||
end | end | ||||
local function process_rules(obj) | local function process_rules(obj) | ||||
each(function(key, code) | |||||
fun.each(function(key, code) | |||||
local f = loadstring(code) | local f = loadstring(code) | ||||
if f then | if f then | ||||
f() | f() | ||||
local section = rspamd_config:get_all_opt("rspamd_update") | local section = rspamd_config:get_all_opt("rspamd_update") | ||||
if section then | if section then | ||||
local trusted_key | local trusted_key | ||||
each(function(k, elt) | |||||
fun.each(function(k, elt) | |||||
if k == 'priority' then | if k == 'priority' then | ||||
updates_priority = tonumber(elt) | updates_priority = tonumber(elt) | ||||
elseif k == 'key' then | elseif k == 'key' then | ||||
end | end | ||||
end, section) | end, section) | ||||
each(function(k, map) | |||||
fun.each(function(k, map) | |||||
-- Check sanity for maps | -- Check sanity for maps | ||||
local proto = map:get_proto() | local proto = map:get_proto() | ||||
if (proto == 'http' or proto == 'https') and not map:get_sign_key() then | if (proto == 'http' or proto == 'https') and not map:get_sign_key() then |
local rspamd_ip = require "rspamd_ip" | local rspamd_ip = require "rspamd_ip" | ||||
local rspamd_regexp = require "rspamd_regexp" | local rspamd_regexp = require "rspamd_regexp" | ||||
local ucl = require "ucl" | local ucl = require "ucl" | ||||
require "fun" () | |||||
local fun = require "fun" | |||||
-- Checks for overrided settings within query params and returns 'true' if | -- Checks for overrided settings within query params and returns 'true' if | ||||
-- settings are overrided | -- settings are overrided | ||||
end | end | ||||
if rule['symbols'] then | if rule['symbols'] then | ||||
-- Add symbols, specified in the settings | -- Add symbols, specified in the settings | ||||
each(function(val) | |||||
fun.each(function(val) | |||||
task:insert_result(val, 1.0) | task:insert_result(val, 1.0) | ||||
end, rule['symbols']) | end, rule['symbols']) | ||||
end | end | ||||
settings_ids = {} | settings_ids = {} | ||||
for k,v in pairs(settings) do settings[k]={} end | for k,v in pairs(settings) do settings[k]={} end | ||||
-- fill new settings by priority | -- fill new settings by priority | ||||
for_each(function(k, v) | |||||
fun.for_each(function(k, v) | |||||
local pri = get_priority(v) | local pri = get_priority(v) | ||||
if pri > max_pri then max_pri = pri end | if pri > max_pri then max_pri = pri end | ||||
if not settings[pri] then | if not settings[pri] then | ||||
end | end | ||||
end | end | ||||
each(function(id, h) | |||||
fun.each(function(id, h) | |||||
rspamd_config:register_symbol({ | rspamd_config:register_symbol({ | ||||
name = 'REDIS_SETTINGS' .. tostring(id), | name = 'REDIS_SETTINGS' .. tostring(id), | ||||
type = 'prefilter', | type = 'prefilter', |
local rspamd_mempool = require "rspamd_mempool" | local rspamd_mempool = require "rspamd_mempool" | ||||
local rspamd_trie = require "rspamd_trie" | local rspamd_trie = require "rspamd_trie" | ||||
local util = require "rspamd_util" | local util = require "rspamd_util" | ||||
require "fun" () | |||||
local fun = require "fun" | |||||
-- Known plugins | -- Known plugins | ||||
local known_plugins = { | local known_plugins = { | ||||
ordinary = false | ordinary = false | ||||
end | end | ||||
each(function(func) | |||||
fun.each(function(func) | |||||
if func == 'addr' then | if func == 'addr' then | ||||
cur_param['function'] = function(str) | cur_param['function'] = function(str) | ||||
local addr_parsed = util.parse_addr(str) | local addr_parsed = util.parse_addr(str) | ||||
rspamd_logger.warnx(rspamd_config, 'Function %1 is not supported in %2', | rspamd_logger.warnx(rspamd_config, 'Function %1 is not supported in %2', | ||||
func, cur_rule['symbol']) | func, cur_rule['symbol']) | ||||
end | end | ||||
end, tail(args)) | |||||
end, fun.tail(args)) | |||||
local function split_hdr_param(param, headers) | local function split_hdr_param(param, headers) | ||||
for i,h in ipairs(headers) do | for i,h in ipairs(headers) do | ||||
end | end | ||||
local function words_to_re(words, start) | local function words_to_re(words, start) | ||||
return table.concat(totable(drop_n(start, words)), " "); | |||||
return table.concat(fun.totable(fun.drop_n(start, words)), " "); | |||||
end | end | ||||
local function process_tflags(rule, flags) | local function process_tflags(rule, flags) | ||||
each(function(flag) | |||||
fun.each(function(flag) | |||||
if flag == 'publish' then | if flag == 'publish' then | ||||
rule['publish'] = true | rule['publish'] = true | ||||
elseif flag == 'multiple' then | elseif flag == 'multiple' then | ||||
elseif flag == 'nice' then | elseif flag == 'nice' then | ||||
rule['nice'] = true | rule['nice'] = true | ||||
end | end | ||||
end, drop_n(1, flags)) | |||||
end, fun.drop_n(1, flags)) | |||||
if rule['re'] then | if rule['re'] then | ||||
if rule['maxhits'] then | if rule['maxhits'] then | ||||
if string.match(l, '^ifplugin') then | if string.match(l, '^ifplugin') then | ||||
local ls = split(l) | local ls = split(l) | ||||
if not any(function(pl) | |||||
if not fun.any(function(pl) | |||||
if pl == ls[2] then return true end | if pl == ls[2] then return true end | ||||
return false | return false | ||||
end, known_plugins) then | end, known_plugins) then | ||||
end | end | ||||
elseif string.match(l, '^if !plugin%(') then | elseif string.match(l, '^if !plugin%(') then | ||||
local pname = string.match(l, '^if !plugin%(([A-Za-z:]+)%)') | local pname = string.match(l, '^if !plugin%(([A-Za-z:]+)%)') | ||||
if any(function(pl) | |||||
if fun.any(function(pl) | |||||
if pl == pname then return true end | if pl == pname then return true end | ||||
return false | return false | ||||
end, known_plugins) then | end, known_plugins) then | ||||
local slash = string.find(l, '/') | local slash = string.find(l, '/') | ||||
-- Skip comments | -- Skip comments | ||||
words = totable(take_while( | |||||
local words = fun.totable(fun.take_while( | |||||
function(w) return string.sub(w, 1, 1) ~= '#' end, | function(w) return string.sub(w, 1, 1) ~= '#' end, | ||||
filter(function(w) | |||||
fun.filter(function(w) | |||||
return w ~= "" end, | return w ~= "" end, | ||||
iter(split(l))))) | |||||
fun.iter(split(l))))) | |||||
if words[1] == "header" or words[1] == 'mimeheader' then | if words[1] == "header" or words[1] == 'mimeheader' then | ||||
-- header SYMBOL Header ~= /regexp/ | -- header SYMBOL Header ~= /regexp/ | ||||
local unset_comp = string.find(cur_rule['re_expr'], '%s+%[if%-unset:') | local unset_comp = string.find(cur_rule['re_expr'], '%s+%[if%-unset:') | ||||
if unset_comp then | if unset_comp then | ||||
-- We have optional part that needs to be processed | -- We have optional part that needs to be processed | ||||
unset = string.match(string.sub(cur_rule['re_expr'], unset_comp), | |||||
local unset = string.match(string.sub(cur_rule['re_expr'], unset_comp), | |||||
'%[if%-unset:%s*([^%]%s]+)]') | '%[if%-unset:%s*([^%]%s]+)]') | ||||
cur_rule['unset'] = unset | cur_rule['unset'] = unset | ||||
-- Cut it down | -- Cut it down | ||||
}) | }) | ||||
end | end | ||||
else | else | ||||
h['mime'] = cur_rule[mime] | |||||
h['mime'] = cur_rule['mime'] | |||||
if cur_rule['mime'] then | if cur_rule['mime'] then | ||||
rspamd_config:register_regexp({ | rspamd_config:register_regexp({ | ||||
re = cur_rule['re'], | re = cur_rule['re'], | ||||
elseif words[1] == "score" then | elseif words[1] == "score" then | ||||
scores[words[2]] = parse_score(words) | scores[words[2]] = parse_score(words) | ||||
elseif words[1] == 'freemail_domains' then | elseif words[1] == 'freemail_domains' then | ||||
each(function(dom) | |||||
fun.each(function(dom) | |||||
table.insert(freemail_domains, '@' .. dom) | table.insert(freemail_domains, '@' .. dom) | ||||
end, drop_n(1, words)) | |||||
end, fun.drop_n(1, words)) | |||||
elseif words[1] == 'blacklist_from' then | elseif words[1] == 'blacklist_from' then | ||||
sa_lists['from_blacklist'][words[2]] = 1 | sa_lists['from_blacklist'][words[2]] = 1 | ||||
sa_lists['elts'] = sa_lists['elts'] + 1 | sa_lists['elts'] = sa_lists['elts'] + 1 | ||||
elseif words[1] == 'replace_post' then | elseif words[1] == 'replace_post' then | ||||
process_replace(words, replace['post']) | process_replace(words, replace['post']) | ||||
elseif words[1] == 'replace_rules' then | elseif words[1] == 'replace_rules' then | ||||
each(function(r) table.insert(replace['rules'], r) end, | |||||
drop_n(1, words)) | |||||
fun.each(function(r) table.insert(replace['rules'], r) end, | |||||
fun.drop_n(1, words)) | |||||
end | end | ||||
end)() | end)() | ||||
end | end | ||||
-- Now check all valid rules and add the according rspamd rules | -- Now check all valid rules and add the according rspamd rules | ||||
local function calculate_score(sym, rule) | local function calculate_score(sym, rule) | ||||
if all(function(c) return c == '_' end, take_n(2, iter(sym))) then | |||||
if fun.all(function(c) return c == '_' end, fun.take_n(2, fun.iter(sym))) then | |||||
return 0.0 | return 0.0 | ||||
end | end | ||||
local function check_specific_tag(prefix, s, tbl) | local function check_specific_tag(prefix, s, tbl) | ||||
local replacement = nil | local replacement = nil | ||||
local ret = s | local ret = s | ||||
each(function(n, t) | |||||
fun.each(function(n, t) | |||||
local ns,matches = string.gsub(s, string.format("<%s%s>", prefix, n), "") | local ns,matches = string.gsub(s, string.format("<%s%s>", prefix, n), "") | ||||
if matches > 0 then | if matches > 0 then | ||||
replacement = t | replacement = t | ||||
local function replace_all_tags(s) | local function replace_all_tags(s) | ||||
local str, matches | local str, matches | ||||
str = s | str = s | ||||
each(function(n, t) | |||||
fun.each(function(n, t) | |||||
str,matches = string.gsub(str, string.format("<%s>", n), | str,matches = string.gsub(str, string.format("<%s>", n), | ||||
string.format("%s%s%s", pre, t, post)) | string.format("%s%s%s", pre, t, post)) | ||||
end, replace['tags']) | end, replace['tags']) | ||||
end | end | ||||
local function parse_atom(str) | local function parse_atom(str) | ||||
local atom = table.concat(totable(take_while(function(c) | |||||
local atom = table.concat(fun.totable(fun.take_while(function(c) | |||||
if string.find(', \t()><+!|&\n', c) then | if string.find(', \t()><+!|&\n', c) then | ||||
return false | return false | ||||
end | end | ||||
return true | return true | ||||
end, iter(str))), '') | |||||
end, fun.iter(str))), '') | |||||
return atom | return atom | ||||
end | end | ||||
local ntags = {} | local ntags = {} | ||||
local function rec_replace_tags(tag, tagv) | local function rec_replace_tags(tag, tagv) | ||||
if ntags[tag] then return ntags[tag] end | if ntags[tag] then return ntags[tag] end | ||||
each(function(n, t) | |||||
fun.each(function(n, t) | |||||
if n ~= tag then | if n ~= tag then | ||||
local s, matches = string.gsub(tagv, string.format("<%s>", n), t) | local s, matches = string.gsub(tagv, string.format("<%s>", n), t) | ||||
if matches > 0 then | if matches > 0 then | ||||
return ntags[tag] | return ntags[tag] | ||||
end | end | ||||
each(function(n, t) | |||||
fun.each(function(n, t) | |||||
rec_replace_tags(n, t) | rec_replace_tags(n, t) | ||||
end, replace['tags']) | end, replace['tags']) | ||||
each(function(n, t) | |||||
fun.each(function(n, t) | |||||
replace['tags'][n] = t | replace['tags'][n] = t | ||||
end, ntags) | end, ntags) | ||||
each(function(r) | |||||
fun.each(function(r) | |||||
local rule = rules[r] | local rule = rules[r] | ||||
if rule['re_expr'] and rule['re'] then | if rule['re_expr'] and rule['re'] then | ||||
end | end | ||||
end, replace['rules']) | end, replace['rules']) | ||||
each(function(key, score) | |||||
fun.each(function(key, score) | |||||
if rules[key] then | if rules[key] then | ||||
rules[key]['score'] = score | rules[key]['score'] = score | ||||
end | end | ||||
end, scores) | end, scores) | ||||
-- Header rules | -- Header rules | ||||
each(function(k, r) | |||||
fun.each(function(k, r) | |||||
local f = function(task) | local f = function(task) | ||||
local raw = false | local raw = false | ||||
end | end | ||||
-- Slow path | -- Slow path | ||||
each(function(h) | |||||
fun.each(function(h) | |||||
local headers = {} | local headers = {} | ||||
local hname = h['header'] | local hname = h['header'] | ||||
end | end | ||||
atoms[k] = f | atoms[k] = f | ||||
end, | end, | ||||
filter(function(k, r) | |||||
fun.filter(function(k, r) | |||||
return r['type'] == 'header' and r['header'] | return r['type'] == 'header' and r['header'] | ||||
end, | end, | ||||
rules)) | rules)) | ||||
-- Custom function rules | -- Custom function rules | ||||
each(function(k, r) | |||||
fun.each(function(k, r) | |||||
local f = function(task) | local f = function(task) | ||||
local res = r['function'](task) | local res = r['function'](task) | ||||
if res and res > 0 then | if res and res > 0 then | ||||
end | end | ||||
atoms[k] = f | atoms[k] = f | ||||
end, | end, | ||||
filter(function(k, r) | |||||
fun.filter(function(k, r) | |||||
return r['type'] == 'function' and r['function'] | return r['type'] == 'function' and r['function'] | ||||
end, | end, | ||||
rules)) | rules)) | ||||
-- Parts rules | -- Parts rules | ||||
each(function(k, r) | |||||
fun.each(function(k, r) | |||||
local f = function(task) | local f = function(task) | ||||
if not r['re'] then | if not r['re'] then | ||||
rspamd_logger.errx(task, 're is missing for rule %1', k) | rspamd_logger.errx(task, 're is missing for rule %1', k) | ||||
end | end | ||||
atoms[k] = f | atoms[k] = f | ||||
end, | end, | ||||
filter(function(k, r) | |||||
fun.filter(function(k, r) | |||||
return r['type'] == 'part' | return r['type'] == 'part' | ||||
end, rules)) | end, rules)) | ||||
-- SA body rules | -- SA body rules | ||||
each(function(k, r) | |||||
fun.each(function(k, r) | |||||
local f = function(task) | local f = function(task) | ||||
if not r['re'] then | if not r['re'] then | ||||
rspamd_logger.errx(task, 're is missing for rule %1', k) | rspamd_logger.errx(task, 're is missing for rule %1', k) | ||||
end | end | ||||
atoms[k] = f | atoms[k] = f | ||||
end, | end, | ||||
filter(function(k, r) | |||||
fun.filter(function(k, r) | |||||
return r['type'] == 'sabody' or r['type'] == 'message' or r['type'] == 'sarawbody' | return r['type'] == 'sabody' or r['type'] == 'message' or r['type'] == 'sarawbody' | ||||
end, rules)) | end, rules)) | ||||
-- URL rules | -- URL rules | ||||
each(function(k, r) | |||||
fun.each(function(k, r) | |||||
local f = function(task) | local f = function(task) | ||||
if not r['re'] then | if not r['re'] then | ||||
rspamd_logger.errx(task, 're is missing for rule %1', k) | rspamd_logger.errx(task, 're is missing for rule %1', k) | ||||
end | end | ||||
atoms[k] = f | atoms[k] = f | ||||
end, | end, | ||||
filter(function(k, r) | |||||
fun.filter(function(k, r) | |||||
return r['type'] == 'uri' | return r['type'] == 'uri' | ||||
end, | end, | ||||
rules)) | rules)) | ||||
-- Meta rules | -- Meta rules | ||||
each(function(k, r) | |||||
fun.each(function(k, r) | |||||
local expression = nil | local expression = nil | ||||
-- Meta function callback | -- Meta function callback | ||||
local meta_cb = function(task) | local meta_cb = function(task) | ||||
end | end | ||||
end | end | ||||
end, | end, | ||||
filter(function(k, r) | |||||
fun.filter(function(k, r) | |||||
return r['type'] == 'meta' | return r['type'] == 'meta' | ||||
end, | end, | ||||
rules)) | rules)) | ||||
-- Check meta rules for foreign symbols and register dependencies | -- Check meta rules for foreign symbols and register dependencies | ||||
-- First direct dependencies: | -- First direct dependencies: | ||||
each(function(k, r) | |||||
fun.each(function(k, r) | |||||
if r['expression'] then | if r['expression'] then | ||||
local expr_atoms = r['expression']:atoms() | local expr_atoms = r['expression']:atoms() | ||||
end | end | ||||
end | end | ||||
end, | end, | ||||
filter(function(k, r) | |||||
fun.filter(function(k, r) | |||||
return r['type'] == 'meta' | return r['type'] == 'meta' | ||||
end, | end, | ||||
rules)) | rules)) | ||||
-- ... And then indirect ones ... | -- ... And then indirect ones ... | ||||
each(function(k, r) | |||||
fun.each(function(k, r) | |||||
if r['expression'] then | if r['expression'] then | ||||
local expr_atoms = r['expression']:atoms() | local expr_atoms = r['expression']:atoms() | ||||
for i,a in ipairs(expr_atoms) do | for i,a in ipairs(expr_atoms) do | ||||
end | end | ||||
end | end | ||||
end, | end, | ||||
filter(function(k, r) | |||||
fun.filter(function(k, r) | |||||
return r['type'] == 'meta' | return r['type'] == 'meta' | ||||
end, | end, | ||||
rules)) | rules)) | ||||
-- Set missing symbols | -- Set missing symbols | ||||
each(function(key, score) | |||||
fun.each(function(key, score) | |||||
if not scores_added[key] then | if not scores_added[key] then | ||||
rspamd_config:set_metric_symbol({ | rspamd_config:set_metric_symbol({ | ||||
name = key, score = score, | name = key, score = score, | ||||
local files = util.glob(elt) | local files = util.glob(elt) | ||||
for i,matched in ipairs(files) do | for i,matched in ipairs(files) do | ||||
f = io.open(matched, "r") | |||||
local f = io.open(matched, "r") | |||||
if f then | if f then | ||||
process_sa_conf(f) | process_sa_conf(f) | ||||
has_rules = true | has_rules = true | ||||
local files = util.glob(fn) | local files = util.glob(fn) | ||||
for i,matched in ipairs(files) do | for i,matched in ipairs(files) do | ||||
f = io.open(matched, "r") | |||||
local f = io.open(matched, "r") | |||||
if f then | if f then | ||||
process_sa_conf(f) | process_sa_conf(f) | ||||
has_rules = true | has_rules = true |
local rspamd_logger = require "rspamd_logger" | local rspamd_logger = require "rspamd_logger" | ||||
local rspamd_util = require "rspamd_util" | local rspamd_util = require "rspamd_util" | ||||
local ucl = require "ucl" | local ucl = require "ucl" | ||||
require "fun" () | |||||
local fun = require "fun" | |||||
local options = { | local options = { | ||||
dmarc_allow_symbol = 'DMARC_POLICY_ALLOW', | dmarc_allow_symbol = 'DMARC_POLICY_ALLOW', | ||||
local dkim_opts = sym[1]['options'] | local dkim_opts = sym[1]['options'] | ||||
if dkim_opts then | if dkim_opts then | ||||
each(function(val) | |||||
fun.each(function(val) | |||||
if not found then | if not found then | ||||
local tld = rspamd_util.get_tld(val) | local tld = rspamd_util.get_tld(val) | ||||
end | end | ||||
if options['rules'] then | if options['rules'] then | ||||
each(function(symbol, rule) | |||||
fun.each(function(symbol, rule) | |||||
if rule['domains'] then | if rule['domains'] then | ||||
if type(rule['domains']) == 'string' then | if type(rule['domains']) == 'string' then | ||||
rule['map'] = rspamd_config:add_kv_map(rule['domains'], | rule['map'] = rspamd_config:add_kv_map(rule['domains'], | ||||
elseif type(rule['domains']) == 'table' then | elseif type(rule['domains']) == 'table' then | ||||
-- Transform ['domain1', 'domain2' ...] to indexes: | -- Transform ['domain1', 'domain2' ...] to indexes: | ||||
-- {'domain1' = 1, 'domain2' = 1 ...] | -- {'domain1' = 1, 'domain2' = 1 ...] | ||||
rule['domains'] = tomap(map(function(d) | |||||
rule['domains'] = fun.tomap(fun.map(function(d) | |||||
local name = d | local name = d | ||||
local value = 1 | local value = 1 | ||||