Mirror von
https://github.com/rspamd/rspamd.git
synchronisiert 2024-08-01 08:18:17 +02:00
545 Zeilen
14 KiB
Lua
545 Zeilen
14 KiB
Lua
--[[
|
|
Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
]]--
|
|
|
|
local logger = require "rspamd_logger"
|
|
local lua_util = require "lua_util"
|
|
local rspamd_util = require "rspamd_util"
|
|
local fun = require "fun"
|
|
|
|
local function is_implicit(t)
|
|
local mt = getmetatable(t)
|
|
|
|
return mt and mt.class and mt.class == 'ucl.type.impl_array'
|
|
end
|
|
|
|
local function metric_pairs(t)
|
|
-- collect the keys
|
|
local keys = {}
|
|
local implicit_array = is_implicit(t)
|
|
|
|
local function gen_keys(tbl)
|
|
if implicit_array then
|
|
for _,v in ipairs(tbl) do
|
|
if v.name then
|
|
table.insert(keys, {v.name, v})
|
|
v.name = nil
|
|
else
|
|
-- Very tricky to distinguish:
|
|
-- group {name = "foo" ... } + group "blah" { ... }
|
|
for gr_name,gr in pairs(v) do
|
|
if type(gr_name) ~= 'number' then
|
|
-- We can also have implicit arrays here
|
|
local gr_implicit = is_implicit(gr)
|
|
|
|
if gr_implicit then
|
|
for _,gr_elt in ipairs(gr) do
|
|
table.insert(keys, {gr_name, gr_elt})
|
|
end
|
|
else
|
|
table.insert(keys, {gr_name, gr})
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
else
|
|
if tbl.name then
|
|
table.insert(keys, {tbl.name, tbl})
|
|
tbl.name = nil
|
|
else
|
|
for k,v in pairs(tbl) do
|
|
if type(k) ~= 'number' then
|
|
-- We can also have implicit arrays here
|
|
local sym_implicit = is_implicit(v)
|
|
|
|
if sym_implicit then
|
|
for _,elt in ipairs(v) do
|
|
table.insert(keys, {k, elt})
|
|
end
|
|
else
|
|
table.insert(keys, {k, v})
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
gen_keys(t)
|
|
|
|
-- return the iterator function
|
|
local i = 0
|
|
return function()
|
|
i = i + 1
|
|
if keys[i] then
|
|
return keys[i][1], keys[i][2]
|
|
end
|
|
end
|
|
end
|
|
|
|
local function group_transform(cfg, k, v)
|
|
if v.name then k = v.name end
|
|
|
|
local new_group = {
|
|
symbols = {}
|
|
}
|
|
|
|
if v.enabled then new_group.enabled = v.enabled end
|
|
if v.disabled then new_group.disabled = v.disabled end
|
|
if v.max_score then new_group.max_score = v.max_score end
|
|
|
|
if v.symbol then
|
|
for sk,sv in metric_pairs(v.symbol) do
|
|
if sv.name then
|
|
sk = sv.name
|
|
sv.name = nil -- Remove field
|
|
end
|
|
|
|
new_group.symbols[sk] = sv
|
|
end
|
|
end
|
|
|
|
if not cfg.group then cfg.group = {} end
|
|
|
|
if cfg.group[k] then
|
|
cfg.group[k] = lua_util.override_defaults(cfg.group[k], new_group)
|
|
else
|
|
cfg.group[k] = new_group
|
|
end
|
|
|
|
logger.infox("overriding group %s from the legacy metric settings", k)
|
|
end
|
|
|
|
local function symbol_transform(cfg, k, v)
|
|
-- first try to find any group where there is a definition of this symbol
|
|
for gr_n, gr in pairs(cfg.group) do
|
|
if gr.symbols and gr.symbols[k] then
|
|
-- We override group symbol with ungrouped symbol
|
|
logger.infox("overriding group symbol %s in the group %s", k, gr_n)
|
|
gr.symbols[k] = lua_util.override_defaults(gr.symbols[k], v)
|
|
return
|
|
end
|
|
end
|
|
-- Now check what Rspamd knows about this symbol
|
|
local sym = rspamd_config:get_metric_symbol(k)
|
|
|
|
if not sym or not sym.group then
|
|
-- Otherwise we just use group 'ungrouped'
|
|
if not cfg.group.ungrouped then
|
|
cfg.group.ungrouped = {
|
|
symbols = {}
|
|
}
|
|
end
|
|
|
|
cfg.group.ungrouped.symbols[k] = v
|
|
logger.debugx("adding symbol %s to the group 'ungrouped'", k)
|
|
end
|
|
end
|
|
|
|
local function test_groups(groups)
|
|
for gr_name, gr in pairs(groups) do
|
|
if not gr.symbols then
|
|
local cnt = 0
|
|
for _,_ in pairs(gr) do cnt = cnt + 1 end
|
|
|
|
if cnt == 0 then
|
|
logger.debugx('group %s is empty', gr_name)
|
|
else
|
|
logger.infox('group %s has no symbols', gr_name)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
local function convert_metric(cfg, metric)
|
|
if metric.actions then
|
|
cfg.actions = lua_util.override_defaults(cfg.actions, metric.actions)
|
|
logger.infox("overriding actions from the legacy metric settings")
|
|
end
|
|
if metric.unknown_weight then
|
|
cfg.actions.unknown_weight = metric.unknown_weight
|
|
end
|
|
|
|
if metric.subject then
|
|
logger.infox("overriding subject from the legacy metric settings")
|
|
cfg.actions.subject = metric.subject
|
|
end
|
|
|
|
if metric.group then
|
|
for k, v in metric_pairs(metric.group) do
|
|
group_transform(cfg, k, v)
|
|
end
|
|
else
|
|
if not cfg.group then
|
|
cfg.group = {
|
|
ungrouped = {
|
|
symbols = {}
|
|
}
|
|
}
|
|
end
|
|
end
|
|
|
|
if metric.symbol then
|
|
for k, v in metric_pairs(metric.symbol) do
|
|
symbol_transform(cfg, k, v)
|
|
end
|
|
end
|
|
|
|
return cfg
|
|
end
|
|
|
|
-- Converts a table of groups indexed by number (implicit array) to a
|
|
-- merged group definition
|
|
local function merge_groups(groups)
|
|
local ret = {}
|
|
for k,gr in pairs(groups) do
|
|
if type(k) == 'number' then
|
|
for key,sec in pairs(gr) do
|
|
ret[key] = sec
|
|
end
|
|
else
|
|
ret[k] = gr
|
|
end
|
|
end
|
|
|
|
return ret
|
|
end
|
|
|
|
-- Checks configuration files for statistics
|
|
local function check_statistics_sanity()
|
|
local local_conf = rspamd_paths['LOCAL_CONFDIR']
|
|
local local_stat = string.format('%s/local.d/%s', local_conf,
|
|
'statistic.conf')
|
|
local local_bayes = string.format('%s/local.d/%s', local_conf,
|
|
'classifier-bayes.conf')
|
|
|
|
if rspamd_util.file_exists(local_stat) and
|
|
rspamd_util.file_exists(local_bayes) then
|
|
logger.warnx(rspamd_config, 'conflicting files %s and %s are found: '..
|
|
'Rspamd classifier configuration might be broken!', local_stat, local_bayes)
|
|
end
|
|
end
|
|
|
|
-- Converts surbl module config to rbl module
|
|
local function surbl_section_convert(cfg, section)
|
|
local rbl_section = cfg.rbl.rbls
|
|
local wl = section.whitelist
|
|
for name,value in pairs(section.rules or {}) do
|
|
if rbl_section[name] then
|
|
logger.warnx(rspamd_config, 'conflicting names in surbl and rbl rules: %s, prefer surbl rule!',
|
|
name)
|
|
end
|
|
local converted = {
|
|
urls = true,
|
|
ignore_defaults = true,
|
|
}
|
|
|
|
if wl then
|
|
converted.whitelist = wl
|
|
end
|
|
|
|
for k,v in pairs(value) do
|
|
local skip = false
|
|
-- Rename
|
|
if k == 'suffix' then k = 'rbl' end
|
|
if k == 'ips' then k = 'returncodes' end
|
|
if k == 'bits' then k = 'returnbits' end
|
|
if k == 'noip' then k = 'no_ip' end
|
|
-- Crappy legacy
|
|
if k == 'options' then
|
|
if v == 'noip' or v == 'no_ip' then
|
|
converted.no_ip = true
|
|
skip = true
|
|
end
|
|
end
|
|
if k:match('check_') then
|
|
local n = k:match('check_(.*)')
|
|
k = n
|
|
end
|
|
|
|
if k == 'dkim' and v then
|
|
converted.dkim_domainonly = false
|
|
converted.dkim_match_from = true
|
|
end
|
|
|
|
if k == 'emails' and v then
|
|
-- To match surbl behaviour
|
|
converted.emails_domainonly = true
|
|
end
|
|
|
|
if not skip then
|
|
converted[k] = lua_util.deepcopy(v)
|
|
end
|
|
end
|
|
rbl_section[name] = lua_util.override_defaults(rbl_section[name], converted)
|
|
end
|
|
end
|
|
|
|
-- Converts surbl module config to rbl module
|
|
local function emails_section_convert(cfg, section)
|
|
local rbl_section = cfg.rbl.rbls
|
|
local wl = section.whitelist
|
|
for name,value in pairs(section.rules or {}) do
|
|
if rbl_section[name] then
|
|
logger.warnx(rspamd_config, 'conflicting names in emails and rbl rules: %s, prefer emails rule!',
|
|
name)
|
|
end
|
|
local converted = {
|
|
emails = true,
|
|
ignore_defaults = true,
|
|
}
|
|
|
|
if wl then
|
|
converted.whitelist = wl
|
|
end
|
|
|
|
for k,v in pairs(value) do
|
|
local skip = false
|
|
-- Rename
|
|
if k == 'dnsbl' then k = 'rbl' end
|
|
if k == 'check_replyto' then k = 'replyto' end
|
|
if k == 'hashlen' then k = 'hash_len' end
|
|
if k == 'encoding' then k = 'hash_format' end
|
|
if k == 'domain_only' then k = 'emails_domainonly' end
|
|
if k == 'delimiter' then k = 'emails_delimiter' end
|
|
if k == 'skip_body' then
|
|
skip = true
|
|
if v then
|
|
-- Hack
|
|
converted.emails = false
|
|
converted.replyto = true
|
|
else
|
|
converted.emails = true
|
|
end
|
|
end
|
|
if k == 'expect_ip' then
|
|
-- Another stupid hack
|
|
if not converted.return_codes then
|
|
converted.returncodes = {}
|
|
end
|
|
local symbol = value.symbol or name
|
|
converted.returncodes[symbol] = { v }
|
|
skip = true
|
|
end
|
|
|
|
if not skip then
|
|
converted[k] = lua_util.deepcopy(v)
|
|
end
|
|
end
|
|
rbl_section[name] = lua_util.override_defaults(rbl_section[name], converted)
|
|
end
|
|
end
|
|
|
|
return function(cfg)
|
|
local ret = false
|
|
|
|
if cfg['metric'] then
|
|
for _, v in metric_pairs(cfg.metric) do
|
|
cfg = convert_metric(cfg, v)
|
|
end
|
|
ret = true
|
|
end
|
|
|
|
if cfg.symbols then
|
|
for k, v in metric_pairs(cfg.symbols) do
|
|
symbol_transform(cfg, k, v)
|
|
end
|
|
end
|
|
|
|
check_statistics_sanity()
|
|
|
|
if not cfg.actions then
|
|
logger.errx('no actions defined')
|
|
else
|
|
-- Perform sanity check for actions
|
|
local actions_defs = {'no action', 'no_action', -- In case if that's added
|
|
'greylist', 'add header', 'add_header',
|
|
'rewrite subject', 'rewrite_subject', 'reject'}
|
|
|
|
if not cfg.actions['no action'] and not cfg.actions['no_action'] and
|
|
not cfg.actions['accept'] then
|
|
for _,d in ipairs(actions_defs) do
|
|
if cfg.actions[d] and type(cfg.actions[d]) == 'number' then
|
|
if cfg.actions[d] < 0 then
|
|
cfg.actions['no action'] = cfg.actions[d] - 0.001
|
|
logger.infox('set no action score to: %s, as action %s has negative score',
|
|
cfg.actions['no action'], d)
|
|
break
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
local actions_set = {}
|
|
for _,d in ipairs(actions_defs) do
|
|
actions_set[d] = true
|
|
end
|
|
|
|
-- Now check actions section for garbadge
|
|
actions_set['unknown_weight'] = true
|
|
actions_set['grow_factor'] = true
|
|
actions_set['subject'] = true
|
|
|
|
for k,_ in pairs(cfg.actions) do
|
|
if not actions_set[k] then
|
|
logger.warnx('unknown element in actions section: %s', k)
|
|
end
|
|
end
|
|
end
|
|
|
|
if not cfg.group then
|
|
logger.errx('no symbol groups defined')
|
|
else
|
|
if cfg.group[1] then
|
|
-- We need to merge groups
|
|
cfg.group = merge_groups(cfg.group)
|
|
ret = true
|
|
end
|
|
test_groups(cfg.group)
|
|
end
|
|
|
|
-- Deal with dkim settings
|
|
if not cfg.dkim then
|
|
cfg.dkim = {}
|
|
else
|
|
if cfg.dkim.sign_condition then
|
|
-- We have an obsoleted sign condition, so we need to either add dkim_signing and move it
|
|
-- there or just move sign condition there...
|
|
if not cfg.dkim_signing then
|
|
logger.warnx('obsoleted DKIM signing method used, converting it to "dkim_signing" module')
|
|
cfg.dkim_signing = {
|
|
sign_condition = cfg.dkim.sign_condition
|
|
}
|
|
else
|
|
if not cfg.dkim_signing.sign_condition then
|
|
logger.warnx('obsoleted DKIM signing method used, move it to "dkim_signing" module')
|
|
cfg.dkim_signing.sign_condition = cfg.dkim.sign_condition
|
|
else
|
|
logger.warnx('obsoleted DKIM signing method used, ignore it as "dkim_signing" also defines condition!')
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
-- Again: legacy stuff :(
|
|
if not cfg.dkim.sign_headers then
|
|
local sec = cfg.dkim_signing
|
|
if sec and sec[1] then sec = cfg.dkim_signing[1] end
|
|
|
|
if sec and sec.sign_headers then
|
|
cfg.dkim.sign_headers = sec.sign_headers
|
|
end
|
|
end
|
|
|
|
if cfg.dkim and cfg.dkim.sign_headers and type(cfg.dkim.sign_headers) == 'table' then
|
|
-- Flatten
|
|
cfg.dkim.sign_headers = table.concat(cfg.dkim.sign_headers, ':')
|
|
end
|
|
|
|
-- Try to find some obvious issues with configuration
|
|
for k,v in pairs(cfg) do
|
|
if type(v) == 'table' and v[k] and type (v[k]) == 'table' then
|
|
logger.errx('nested section: %s { %s { ... } }, it is likely a configuration error',
|
|
k, k)
|
|
end
|
|
end
|
|
|
|
-- If neural network is enabled we MUST have `check_all_filters` flag
|
|
if cfg.neural then
|
|
if not cfg.options then
|
|
cfg.options = {}
|
|
end
|
|
|
|
if not cfg.options.check_all_filters then
|
|
logger.infox(rspamd_config, 'enable `options.check_all_filters` for neural network')
|
|
cfg.options.check_all_filters = true
|
|
end
|
|
end
|
|
|
|
-- Deal with IP_SCORE
|
|
if cfg.ip_score and (cfg.ip_score.servers or cfg.redis.servers) then
|
|
logger.warnx(rspamd_config, 'ip_score module is deprecated in honor of reputation module!')
|
|
|
|
if not cfg.reputation then
|
|
cfg.reputation = {
|
|
rules = {}
|
|
}
|
|
end
|
|
|
|
if not cfg.reputation.rules then cfg.reputation.rules = {} end
|
|
|
|
if not fun.any(function(_, v) return v.selector and v.selector.ip end,
|
|
cfg.reputation.rules) then
|
|
logger.infox(rspamd_config, 'attach ip reputation element to use it')
|
|
|
|
cfg.reputation.rules.ip_score = {
|
|
selector = {
|
|
ip = {},
|
|
},
|
|
backend = {
|
|
redis = {},
|
|
}
|
|
}
|
|
|
|
if cfg.ip_score.servers then
|
|
cfg.reputation.rules.ip_score.backend.redis.servers = cfg.ip_score.servers
|
|
end
|
|
|
|
if cfg.symbols and cfg.symbols['IP_SCORE'] then
|
|
local t = cfg.symbols['IP_SCORE']
|
|
|
|
if not cfg.symbols['SENDER_REP_SPAM'] then
|
|
cfg.symbols['SENDER_REP_SPAM'] = t
|
|
cfg.symbols['SENDER_REP_HAM'] = t
|
|
cfg.symbols['SENDER_REP_HAM'].weight = -(t.weight or 0)
|
|
end
|
|
end
|
|
else
|
|
logger.infox(rspamd_config, 'ip reputation already exists, do not do any IP_SCORE transforms')
|
|
end
|
|
end
|
|
|
|
if cfg.surbl then
|
|
if not cfg.rbl then
|
|
cfg.rbl = {
|
|
rbls = {}
|
|
}
|
|
end
|
|
if not cfg.rbl.rbls then
|
|
cfg.rbl.rbls = {}
|
|
end
|
|
surbl_section_convert(cfg, cfg.surbl)
|
|
logger.infox(rspamd_config, 'converted surbl rules to rbl rules')
|
|
cfg.surbl = {}
|
|
end
|
|
|
|
if cfg.emails then
|
|
if not cfg.rbl then
|
|
cfg.rbl = {
|
|
rbls = {}
|
|
}
|
|
end
|
|
if not cfg.rbl.rbls then
|
|
cfg.rbl.rbls = {}
|
|
end
|
|
emails_section_convert(cfg, cfg.emails)
|
|
logger.infox(rspamd_config, 'converted emails rules to rbl rules')
|
|
cfg.emails = {}
|
|
end
|
|
|
|
return ret, cfg
|
|
end
|