123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544 |
- --[[
- Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ]]--
-
- local logger = require "rspamd_logger"
- local lua_util = require "lua_util"
- local rspamd_util = require "rspamd_util"
- local fun = require "fun"
-
- local function is_implicit(t)
- local mt = getmetatable(t)
-
- return mt and mt.class and mt.class == 'ucl.type.impl_array'
- end
-
- local function metric_pairs(t)
- -- collect the keys
- local keys = {}
- local implicit_array = is_implicit(t)
-
- local function gen_keys(tbl)
- if implicit_array then
- for _,v in ipairs(tbl) do
- if v.name then
- table.insert(keys, {v.name, v})
- v.name = nil
- else
- -- Very tricky to distinguish:
- -- group {name = "foo" ... } + group "blah" { ... }
- for gr_name,gr in pairs(v) do
- if type(gr_name) ~= 'number' then
- -- We can also have implicit arrays here
- local gr_implicit = is_implicit(gr)
-
- if gr_implicit then
- for _,gr_elt in ipairs(gr) do
- table.insert(keys, {gr_name, gr_elt})
- end
- else
- table.insert(keys, {gr_name, gr})
- end
- end
- end
- end
- end
- else
- if tbl.name then
- table.insert(keys, {tbl.name, tbl})
- tbl.name = nil
- else
- for k,v in pairs(tbl) do
- if type(k) ~= 'number' then
- -- We can also have implicit arrays here
- local sym_implicit = is_implicit(v)
-
- if sym_implicit then
- for _,elt in ipairs(v) do
- table.insert(keys, {k, elt})
- end
- else
- table.insert(keys, {k, v})
- end
- end
- end
- end
- end
- end
-
- gen_keys(t)
-
- -- return the iterator function
- local i = 0
- return function()
- i = i + 1
- if keys[i] then
- return keys[i][1], keys[i][2]
- end
- end
- end
-
- local function group_transform(cfg, k, v)
- if v.name then k = v.name end
-
- local new_group = {
- symbols = {}
- }
-
- if v.enabled then new_group.enabled = v.enabled end
- if v.disabled then new_group.disabled = v.disabled end
- if v.max_score then new_group.max_score = v.max_score end
-
- if v.symbol then
- for sk,sv in metric_pairs(v.symbol) do
- if sv.name then
- sk = sv.name
- sv.name = nil -- Remove field
- end
-
- new_group.symbols[sk] = sv
- end
- end
-
- if not cfg.group then cfg.group = {} end
-
- if cfg.group[k] then
- cfg.group[k] = lua_util.override_defaults(cfg.group[k], new_group)
- else
- cfg.group[k] = new_group
- end
-
- logger.infox("overriding group %s from the legacy metric settings", k)
- end
-
- local function symbol_transform(cfg, k, v)
- -- first try to find any group where there is a definition of this symbol
- for gr_n, gr in pairs(cfg.group) do
- if gr.symbols and gr.symbols[k] then
- -- We override group symbol with ungrouped symbol
- logger.infox("overriding group symbol %s in the group %s", k, gr_n)
- gr.symbols[k] = lua_util.override_defaults(gr.symbols[k], v)
- return
- end
- end
- -- Now check what Rspamd knows about this symbol
- local sym = rspamd_config:get_metric_symbol(k)
-
- if not sym or not sym.group then
- -- Otherwise we just use group 'ungrouped'
- if not cfg.group.ungrouped then
- cfg.group.ungrouped = {
- symbols = {}
- }
- end
-
- cfg.group.ungrouped.symbols[k] = v
- logger.debugx("adding symbol %s to the group 'ungrouped'", k)
- end
- end
-
- local function test_groups(groups)
- for gr_name, gr in pairs(groups) do
- if not gr.symbols then
- local cnt = 0
- for _,_ in pairs(gr) do cnt = cnt + 1 end
-
- if cnt == 0 then
- logger.debugx('group %s is empty', gr_name)
- else
- logger.infox('group %s has no symbols', gr_name)
- end
- end
- end
- end
-
- local function convert_metric(cfg, metric)
- if metric.actions then
- cfg.actions = lua_util.override_defaults(cfg.actions, metric.actions)
- logger.infox("overriding actions from the legacy metric settings")
- end
- if metric.unknown_weight then
- cfg.actions.unknown_weight = metric.unknown_weight
- end
-
- if metric.subject then
- logger.infox("overriding subject from the legacy metric settings")
- cfg.actions.subject = metric.subject
- end
-
- if metric.group then
- for k, v in metric_pairs(metric.group) do
- group_transform(cfg, k, v)
- end
- else
- if not cfg.group then
- cfg.group = {
- ungrouped = {
- symbols = {}
- }
- }
- end
- end
-
- if metric.symbol then
- for k, v in metric_pairs(metric.symbol) do
- symbol_transform(cfg, k, v)
- end
- end
-
- return cfg
- end
-
- -- Converts a table of groups indexed by number (implicit array) to a
- -- merged group definition
- local function merge_groups(groups)
- local ret = {}
- for k,gr in pairs(groups) do
- if type(k) == 'number' then
- for key,sec in pairs(gr) do
- ret[key] = sec
- end
- else
- ret[k] = gr
- end
- end
-
- return ret
- end
-
- -- Checks configuration files for statistics
- local function check_statistics_sanity()
- local local_conf = rspamd_paths['LOCAL_CONFDIR']
- local local_stat = string.format('%s/local.d/%s', local_conf,
- 'statistic.conf')
- local local_bayes = string.format('%s/local.d/%s', local_conf,
- 'classifier-bayes.conf')
-
- if rspamd_util.file_exists(local_stat) and
- rspamd_util.file_exists(local_bayes) then
- logger.warnx(rspamd_config, 'conflicting files %s and %s are found: '..
- 'Rspamd classifier configuration might be broken!', local_stat, local_bayes)
- end
- end
-
- -- Converts surbl module config to rbl module
- local function surbl_section_convert(cfg, section)
- local rbl_section = cfg.rbl.rbls
- local wl = section.whitelist
- for name,value in pairs(section.rules or {}) do
- if rbl_section[name] then
- logger.warnx(rspamd_config, 'conflicting names in surbl and rbl rules: %s, prefer surbl rule!',
- name)
- end
- local converted = {
- urls = true,
- ignore_defaults = true,
- }
-
- if wl then
- converted.whitelist = wl
- end
-
- for k,v in pairs(value) do
- local skip = false
- -- Rename
- if k == 'suffix' then k = 'rbl' end
- if k == 'ips' then k = 'returncodes' end
- if k == 'bits' then k = 'returnbits' end
- if k == 'noip' then k = 'no_ip' end
- -- Crappy legacy
- if k == 'options' then
- if v == 'noip' or v == 'no_ip' then
- converted.no_ip = true
- skip = true
- end
- end
- if k:match('check_') then
- local n = k:match('check_(.*)')
- k = n
- end
-
- if k == 'dkim' and v then
- converted.dkim_domainonly = false
- converted.dkim_match_from = true
- end
-
- if k == 'emails' and v then
- -- To match surbl behaviour
- converted.emails_domainonly = true
- end
-
- if not skip then
- converted[k] = lua_util.deepcopy(v)
- end
- end
- rbl_section[name] = lua_util.override_defaults(rbl_section[name], converted)
- end
- end
-
- -- Converts surbl module config to rbl module
- local function emails_section_convert(cfg, section)
- local rbl_section = cfg.rbl.rbls
- local wl = section.whitelist
- for name,value in pairs(section.rules or {}) do
- if rbl_section[name] then
- logger.warnx(rspamd_config, 'conflicting names in emails and rbl rules: %s, prefer emails rule!',
- name)
- end
- local converted = {
- emails = true,
- ignore_defaults = true,
- }
-
- if wl then
- converted.whitelist = wl
- end
-
- for k,v in pairs(value) do
- local skip = false
- -- Rename
- if k == 'dnsbl' then k = 'rbl' end
- if k == 'check_replyto' then k = 'replyto' end
- if k == 'hashlen' then k = 'hash_len' end
- if k == 'encoding' then k = 'hash_format' end
- if k == 'domain_only' then k = 'emails_domainonly' end
- if k == 'delimiter' then k = 'emails_delimiter' end
- if k == 'skip_body' then
- skip = true
- if v then
- -- Hack
- converted.emails = false
- converted.replyto = true
- else
- converted.emails = true
- end
- end
- if k == 'expect_ip' then
- -- Another stupid hack
- if not converted.return_codes then
- converted.returncodes = {}
- end
- local symbol = value.symbol or name
- converted.returncodes[symbol] = { v }
- skip = true
- end
-
- if not skip then
- converted[k] = lua_util.deepcopy(v)
- end
- end
- rbl_section[name] = lua_util.override_defaults(rbl_section[name], converted)
- end
- end
-
- return function(cfg)
- local ret = false
-
- if cfg['metric'] then
- for _, v in metric_pairs(cfg.metric) do
- cfg = convert_metric(cfg, v)
- end
- ret = true
- end
-
- if cfg.symbols then
- for k, v in metric_pairs(cfg.symbols) do
- symbol_transform(cfg, k, v)
- end
- end
-
- check_statistics_sanity()
-
- if not cfg.actions then
- logger.errx('no actions defined')
- else
- -- Perform sanity check for actions
- local actions_defs = {'no action', 'no_action', -- In case if that's added
- 'greylist', 'add header', 'add_header',
- 'rewrite subject', 'rewrite_subject', 'reject'}
-
- if not cfg.actions['no action'] and not cfg.actions['no_action'] and
- not cfg.actions['accept'] then
- for _,d in ipairs(actions_defs) do
- if cfg.actions[d] and type(cfg.actions[d]) == 'number' then
- if cfg.actions[d] < 0 then
- cfg.actions['no action'] = cfg.actions[d] - 0.001
- logger.infox('set no action score to: %s, as action %s has negative score',
- cfg.actions['no action'], d)
- break
- end
- end
- end
- end
-
- local actions_set = {}
- for _,d in ipairs(actions_defs) do
- actions_set[d] = true
- end
-
- -- Now check actions section for garbadge
- actions_set['unknown_weight'] = true
- actions_set['grow_factor'] = true
- actions_set['subject'] = true
-
- for k,_ in pairs(cfg.actions) do
- if not actions_set[k] then
- logger.warnx('unknown element in actions section: %s', k)
- end
- end
- end
-
- if not cfg.group then
- logger.errx('no symbol groups defined')
- else
- if cfg.group[1] then
- -- We need to merge groups
- cfg.group = merge_groups(cfg.group)
- ret = true
- end
- test_groups(cfg.group)
- end
-
- -- Deal with dkim settings
- if not cfg.dkim then
- cfg.dkim = {}
- else
- if cfg.dkim.sign_condition then
- -- We have an obsoleted sign condition, so we need to either add dkim_signing and move it
- -- there or just move sign condition there...
- if not cfg.dkim_signing then
- logger.warnx('obsoleted DKIM signing method used, converting it to "dkim_signing" module')
- cfg.dkim_signing = {
- sign_condition = cfg.dkim.sign_condition
- }
- else
- if not cfg.dkim_signing.sign_condition then
- logger.warnx('obsoleted DKIM signing method used, move it to "dkim_signing" module')
- cfg.dkim_signing.sign_condition = cfg.dkim.sign_condition
- else
- logger.warnx('obsoleted DKIM signing method used, ignore it as "dkim_signing" also defines condition!')
- end
- end
- end
- end
-
- -- Again: legacy stuff :(
- if not cfg.dkim.sign_headers then
- local sec = cfg.dkim_signing
- if sec and sec[1] then sec = cfg.dkim_signing[1] end
-
- if sec and sec.sign_headers then
- cfg.dkim.sign_headers = sec.sign_headers
- end
- end
-
- if cfg.dkim and cfg.dkim.sign_headers and type(cfg.dkim.sign_headers) == 'table' then
- -- Flatten
- cfg.dkim.sign_headers = table.concat(cfg.dkim.sign_headers, ':')
- end
-
- -- Try to find some obvious issues with configuration
- for k,v in pairs(cfg) do
- if type(v) == 'table' and v[k] and type (v[k]) == 'table' then
- logger.errx('nested section: %s { %s { ... } }, it is likely a configuration error',
- k, k)
- end
- end
-
- -- If neural network is enabled we MUST have `check_all_filters` flag
- if cfg.neural then
- if not cfg.options then
- cfg.options = {}
- end
-
- if not cfg.options.check_all_filters then
- logger.infox(rspamd_config, 'enable `options.check_all_filters` for neural network')
- cfg.options.check_all_filters = true
- end
- end
-
- -- Deal with IP_SCORE
- if cfg.ip_score and (cfg.ip_score.servers or cfg.redis.servers) then
- logger.warnx(rspamd_config, 'ip_score module is deprecated in honor of reputation module!')
-
- if not cfg.reputation then
- cfg.reputation = {
- rules = {}
- }
- end
-
- if not cfg.reputation.rules then cfg.reputation.rules = {} end
-
- if not fun.any(function(_, v) return v.selector and v.selector.ip end,
- cfg.reputation.rules) then
- logger.infox(rspamd_config, 'attach ip reputation element to use it')
-
- cfg.reputation.rules.ip_score = {
- selector = {
- ip = {},
- },
- backend = {
- redis = {},
- }
- }
-
- if cfg.ip_score.servers then
- cfg.reputation.rules.ip_score.backend.redis.servers = cfg.ip_score.servers
- end
-
- if cfg.symbols and cfg.symbols['IP_SCORE'] then
- local t = cfg.symbols['IP_SCORE']
-
- if not cfg.symbols['SENDER_REP_SPAM'] then
- cfg.symbols['SENDER_REP_SPAM'] = t
- cfg.symbols['SENDER_REP_HAM'] = t
- cfg.symbols['SENDER_REP_HAM'].weight = -(t.weight or 0)
- end
- end
- else
- logger.infox(rspamd_config, 'ip reputation already exists, do not do any IP_SCORE transforms')
- end
- end
-
- if cfg.surbl then
- if not cfg.rbl then
- cfg.rbl = {
- rbls = {}
- }
- end
- if not cfg.rbl.rbls then
- cfg.rbl.rbls = {}
- end
- surbl_section_convert(cfg, cfg.surbl)
- logger.infox(rspamd_config, 'converted surbl rules to rbl rules')
- cfg.surbl = {}
- end
-
- if cfg.emails then
- if not cfg.rbl then
- cfg.rbl = {
- rbls = {}
- }
- end
- if not cfg.rbl.rbls then
- cfg.rbl.rbls = {}
- end
- emails_section_convert(cfg, cfg.emails)
- logger.infox(rspamd_config, 'converted emails rules to rbl rules')
- cfg.emails = {}
- end
-
- return ret, cfg
- end
|