From eb75c68f30b346b352e69eae90c363c8ef113175 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 23 Feb 2018 13:20:14 +0000 Subject: [Refactor] Rename all lua libraries in the common way --- lualib/auth_results.lua | 203 --------------- lualib/dkim_sign_tools.lua | 206 --------------- lualib/global_functions.lua | 4 +- lualib/lua_auth_results.lua | 203 +++++++++++++++ lualib/lua_cfg_transform.lua | 295 ++++++++++++++++++++++ lualib/lua_dkim_tools.lua | 206 +++++++++++++++ lualib/lua_maps.lua | 188 ++++++++++++++ lualib/lua_meta.lua | 396 +++++++++++++++++++++++++++++ lualib/lua_stat.lua | 502 +++++++++++++++++++++++++++++++++++++ lualib/maps.lua | 188 -------------- lualib/meta_functions.lua | 396 ----------------------------- lualib/rspamadm/configwizard.lua | 2 +- lualib/rspamadm/stat_convert.lua | 2 +- lualib/rspamd_config_transform.lua | 295 ---------------------- lualib/stat_tools.lua | 502 ------------------------------------- src/libserver/cfg_rcl.c | 2 +- src/plugins/lua/arc.lua | 4 +- src/plugins/lua/dkim_signing.lua | 2 +- src/plugins/lua/fann_redis.lua | 2 +- src/plugins/lua/milter_headers.lua | 2 +- src/plugins/lua/ratelimit.lua | 2 +- src/plugins/lua/reputation.lua | 2 +- src/plugins/lua/settings.lua | 2 +- 23 files changed, 1803 insertions(+), 1803 deletions(-) delete mode 100644 lualib/auth_results.lua delete mode 100644 lualib/dkim_sign_tools.lua create mode 100644 lualib/lua_auth_results.lua create mode 100644 lualib/lua_cfg_transform.lua create mode 100644 lualib/lua_dkim_tools.lua create mode 100644 lualib/lua_maps.lua create mode 100644 lualib/lua_meta.lua create mode 100644 lualib/lua_stat.lua delete mode 100644 lualib/maps.lua delete mode 100644 lualib/meta_functions.lua delete mode 100644 lualib/rspamd_config_transform.lua delete mode 100644 lualib/stat_tools.lua diff --git a/lualib/auth_results.lua b/lualib/auth_results.lua deleted file mode 100644 index 92f702950..000000000 --- a/lualib/auth_results.lua +++ /dev/null @@ -1,203 +0,0 @@ ---[[ -Copyright (c) 2016, Andrew Lewis -Copyright (c) 2017, Vsevolod Stakhov - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -local global = require "global_functions" - -local default_settings = { - spf_symbols = { - pass = 'R_SPF_ALLOW', - fail = 'R_SPF_FAIL', - softfail = 'R_SPF_SOFTFAIL', - neutral = 'R_SPF_NEUTRAL', - temperror = 'R_SPF_DNSFAIL', - none = 'R_SPF_NA', - permerror = 'R_SPF_PERMFAIL', - }, - dkim_symbols = { - pass = 'R_DKIM_ALLOW', - fail = 'R_DKIM_REJECT', - temperror = 'R_DKIM_TEMPFAIL', - none = 'R_DKIM_NA', - permerror = 'R_DKIM_PERMFAIL', - }, - dmarc_symbols = { - pass = 'DMARC_POLICY_ALLOW', - permerror = 'DMARC_BAD_POLICY', - temperror = 'DMARC_DNSFAIL', - none = 'DMARC_NA', - reject = 'DMARC_POLICY_REJECT', - softfail = 'DMARC_POLICY_SOFTFAIL', - quarantine = 'DMARC_POLICY_QUARANTINE', - }, - arc_symbols = { - pass = 'ARC_ALLOW', - permerror = 'ARC_INVALID', - temperror = 'ARC_DNSFAIL', - none = 'ARC_NA', - reject = 'ARC_REJECT', - }, - add_smtp_user = true, -} - -local exports = {} - -local function gen_auth_results(task, settings) - local table = table - local pairs = pairs - local ipairs = ipairs - local auth_results, hdr_parts = {}, {} - - if not settings then - settings = default_settings - end - - local auth_types = { - dkim = settings.dkim_symbols, - dmarc = settings.dmarc_symbols, - spf = settings.spf_symbols, - arc = settings.arc_symbols, - } - - local common = { - symbols = {} - } - - local received = task:get_received_headers() or {} - local mxname = (received[1] or {}).by_hostname - if mxname then - table.insert(hdr_parts, mxname) - end - - for auth_type, symbols in pairs(auth_types) do - for key, sym in pairs(symbols) do - if not common.symbols.sym then - local s = task:get_symbol(sym) - if not s then - common.symbols[sym] = false - else - common.symbols[sym] = s - if not auth_results[auth_type] then - auth_results[auth_type] = {key} - else - table.insert(auth_results[auth_type], key) - end - - if auth_type ~= 'dkim' then - break - end - end - end - end - end - - for auth_type, keys in pairs(auth_results) do - for _, key in ipairs(keys) do - local hdr = '' - if auth_type == 'dmarc' and key ~= 'none' then - local opts = common.symbols[auth_types['dmarc'][key]][1]['options'] or {} - hdr = hdr .. 'dmarc=' - if key == 'reject' or key == 'quarantine' or key == 'softfail' then - hdr = hdr .. 'fail' - else - hdr = hdr .. key - end - if key == 'pass' then - hdr = hdr .. ' (policy=' .. opts[2] .. ')' - hdr = hdr .. ' header.from=' .. opts[1] - elseif key ~= 'none' then - local t = global.rspamd_str_split(opts[1], ' : ') - local dom = t[1] - local rsn = t[2] - if rsn then - hdr = hdr .. ' reason="' .. rsn .. '"' - end - hdr = hdr .. ' header.from=' .. dom - if key == 'softfail' then - hdr = hdr .. ' (policy=none)' - else - hdr = hdr .. ' (policy=' .. key .. ')' - end - end - table.insert(hdr_parts, hdr) - elseif auth_type == 'dkim' and key ~= 'none' then - if common.symbols[auth_types['dkim'][key]][1] then - local dkim_parts = {} - local opts = common.symbols[auth_types['dkim'][key]][1]['options'] - for _, v in ipairs(opts) do - table.insert(dkim_parts, auth_type .. '=' .. key .. ' header.d=' .. v) - end - table.insert(hdr_parts, table.concat(dkim_parts, '; ')) - end - elseif auth_type == 'arc' and key ~= 'none' then - if common.symbols[auth_types['arc'][key]][1] then - local opts = common.symbols[auth_types['arc'][key]][1]['options'] or {} - for _, v in ipairs(opts) do - hdr = hdr .. auth_type .. '=' .. key .. ' (' .. v .. ')' - table.insert(hdr_parts, hdr) - end - end - elseif auth_type == 'spf' and key ~= 'none' then - hdr = hdr .. auth_type .. '=' .. key - local smtp_from = task:get_from('smtp') - if smtp_from and smtp_from[1] and smtp_from[1]['addr'] ~= '' and smtp_from[1]['addr'] ~= nil then - hdr = hdr .. ' smtp.mailfrom=' .. smtp_from[1]['addr'] - else - local helo = task:get_helo() - if helo then - hdr = hdr .. ' smtp.helo=' .. task:get_helo() - end - end - table.insert(hdr_parts, hdr) - end - end - end - - local u = task:get_user() - local smtp_from = task:get_from('smtp') - - if u and smtp_from then - local hdr - - if #smtp_from[1]['addr'] > 0 then - if settings['add_smtp_user'] then - hdr = string.format('auth=pass smtp.auth=%s smtp.mailfrom=%s', - u, smtp_from[1]['addr']) - else - hdr = string.format('auth=pass smtp.mailfrom=%s', - smtp_from[1]['addr']) - end - else - if settings['add_smtp_user'] then - hdr = string.format('auth=pass smtp.auth=%s', u) - else - hdr = 'auth=pass' - end - end - - table.insert(hdr_parts, hdr) - end - - if #hdr_parts > 0 then - return table.concat(hdr_parts, '; ') - end - - return nil -end - -exports.gen_auth_results = gen_auth_results - -return exports diff --git a/lualib/dkim_sign_tools.lua b/lualib/dkim_sign_tools.lua deleted file mode 100644 index a0c93b4ff..000000000 --- a/lualib/dkim_sign_tools.lua +++ /dev/null @@ -1,206 +0,0 @@ ---[[ -Copyright (c) 2016, Andrew Lewis -Copyright (c) 2017, Vsevolod Stakhov - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -local exports = {} - -local E = {} -local rspamd_logger = require "rspamd_logger" -local rspamd_util = require "rspamd_util" - -local function prepare_dkim_signing(N, task, settings) - local is_local, is_sign_networks - local auser = task:get_user() - local ip = task:get_from_ip() - - if ip and ip:is_local() then - is_local = true - end - - if settings.auth_only and auser then - rspamd_logger.debugm(N, task, 'user is authenticated') - elseif (settings.sign_networks and settings.sign_networks:get_key(ip)) then - is_sign_networks = true - rspamd_logger.debugm(N, task, 'mail is from address in sign_networks') - elseif settings.sign_local and is_local then - rspamd_logger.debugm(N, task, 'mail is from local address') - elseif settings.sign_inbound and not is_local and not auser then - rspamd_logger.debugm(N, task, 'mail was sent to us') - else - rspamd_logger.debugm(N, task, 'ignoring unauthenticated mail') - return false,{} - end - - local efrom = task:get_from('smtp') - if not settings.allow_envfrom_empty and - #(((efrom or E)[1] or E).addr or '') == 0 then - rspamd_logger.debugm(N, task, 'empty envelope from not allowed') - return false,{} - end - - local hfrom = task:get_from('mime') - if not settings.allow_hdrfrom_multiple and (hfrom or E)[2] then - rspamd_logger.debugm(N, task, 'multiple header from not allowed') - return false,{} - end - - local eto = task:get_recipients(0) - - local dkim_domain - local hdom = ((hfrom or E)[1] or E).domain - local edom = ((efrom or E)[1] or E).domain - local tdom = ((eto or E)[1] or E).domain - local udom = string.match(auser or '', '.*@(.*)') - - local function get_dkim_domain(dtype) - if settings[dtype] == 'header' then - return hdom - elseif settings[dtype] == 'envelope' then - return edom - elseif settings[dtype] == 'auth' then - return udom - elseif settings[dtype] == 'recipient' then - return tdom - end - end - - if hdom then - hdom = hdom:lower() - end - if edom then - edom = edom:lower() - end - if udom then - udom = udom:lower() - end - if tdom then - tdom = tdom:lower() - end - - if settings.use_domain_sign_networks and is_sign_networks then - dkim_domain = get_dkim_domain('use_domain_sign_networks') - rspamd_logger.debugm(N, task, 'sign_networks: use domain(%s) for signature: %s', - settings.use_domain_sign_networks, dkim_domain) - elseif settings.use_domain_local and is_local then - dkim_domain = get_dkim_domain('use_domain_local') - rspamd_logger.debugm(N, task, 'local: use domain(%s) for signature: %s', - settings.use_domain_local, dkim_domain) - else - dkim_domain = get_dkim_domain('use_domain') - rspamd_logger.debugm(N, task, 'use domain(%s) for signature: %s', - settings.use_domain, dkim_domain) - end - - if not dkim_domain then - rspamd_logger.debugm(N, task, 'could not extract dkim domain') - return false,{} - end - - if settings.use_esld then - dkim_domain = rspamd_util.get_tld(dkim_domain) - if hdom then - hdom = rspamd_util.get_tld(hdom) - end - if edom then - edom = rspamd_util.get_tld(edom) - end - end - if edom and hdom and not settings.allow_hdrfrom_mismatch and hdom ~= edom then - if settings.allow_hdrfrom_mismatch_local and is_local then - rspamd_logger.debugm(N, task, 'domain mismatch allowed for local IP: %1 != %2', hdom, edom) - elseif settings.allow_hdrfrom_mismatch_sign_networks and is_sign_networks then - rspamd_logger.debugm(N, task, 'domain mismatch allowed for sign_networks: %1 != %2', hdom, edom) - else - rspamd_logger.debugm(N, task, 'domain mismatch not allowed: %1 != %2', hdom, edom) - return false,{} - end - end - - if auser and not settings.allow_username_mismatch then - if not udom then - rspamd_logger.debugm(N, task, 'couldnt find domain in username') - return false,{} - end - if settings.use_esld then - udom = rspamd_util.get_tld(udom) - end - if udom ~= dkim_domain then - rspamd_logger.debugm(N, task, 'user domain mismatch') - return false,{} - end - end - - local p = {} - - if settings.domain[dkim_domain] then - p.selector = settings.domain[dkim_domain].selector - p.key = settings.domain[dkim_domain].path - end - - if not p.key and p.selector then - local key_var = "dkim_key" - local selector_var = "dkim_selector" - if N == "arc" then - key_var = "arc_key" - selector_var = "arc_selector" - end - - p.key = task:get_mempool():get_variable(key_var) - p.selector = task:get_mempool():get_variable(selector_var) - - if (not p.key or not p.selector) and (not (settings.try_fallback or - settings.use_redis or settings.selector_map - or settings.path_map)) then - rspamd_logger.debugm(N, task, 'dkim unconfigured and fallback disabled') - return false,{} - end - end - - if not p.selector and settings.selector_map then - local data = settings.selector_map:get_key(dkim_domain) - if data then - p.selector = data - elseif not settings.try_fallback then - return false,{} - end - end - - if not p.key and settings.path_map then - local data = settings.path_map:get_key(dkim_domain) - if data then - p.key = data - elseif not settings.try_fallback then - return false,{} - end - end - - if not p.key then - if not settings.use_redis then - p.key = settings.path - end - end - - if not p.selector then - p.selector = settings.selector - end - p.domain = dkim_domain - - return true,p -end - -exports.prepare_dkim_signing = prepare_dkim_signing - -return exports diff --git a/lualib/global_functions.lua b/lualib/global_functions.lua index 956db8bed..05c967534 100644 --- a/lualib/global_functions.lua +++ b/lualib/global_functions.lua @@ -17,8 +17,8 @@ limitations under the License. local logger = require "rspamd_logger" local lua_util = require "lua_util" local lua_redis = require "lua_redis" -local meta_functions = require "meta_functions" -local maps = require "maps" +local meta_functions = require "lua_meta" +local maps = require "lua_maps" local exports = {} diff --git a/lualib/lua_auth_results.lua b/lualib/lua_auth_results.lua new file mode 100644 index 000000000..92f702950 --- /dev/null +++ b/lualib/lua_auth_results.lua @@ -0,0 +1,203 @@ +--[[ +Copyright (c) 2016, Andrew Lewis +Copyright (c) 2017, Vsevolod Stakhov + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local global = require "global_functions" + +local default_settings = { + spf_symbols = { + pass = 'R_SPF_ALLOW', + fail = 'R_SPF_FAIL', + softfail = 'R_SPF_SOFTFAIL', + neutral = 'R_SPF_NEUTRAL', + temperror = 'R_SPF_DNSFAIL', + none = 'R_SPF_NA', + permerror = 'R_SPF_PERMFAIL', + }, + dkim_symbols = { + pass = 'R_DKIM_ALLOW', + fail = 'R_DKIM_REJECT', + temperror = 'R_DKIM_TEMPFAIL', + none = 'R_DKIM_NA', + permerror = 'R_DKIM_PERMFAIL', + }, + dmarc_symbols = { + pass = 'DMARC_POLICY_ALLOW', + permerror = 'DMARC_BAD_POLICY', + temperror = 'DMARC_DNSFAIL', + none = 'DMARC_NA', + reject = 'DMARC_POLICY_REJECT', + softfail = 'DMARC_POLICY_SOFTFAIL', + quarantine = 'DMARC_POLICY_QUARANTINE', + }, + arc_symbols = { + pass = 'ARC_ALLOW', + permerror = 'ARC_INVALID', + temperror = 'ARC_DNSFAIL', + none = 'ARC_NA', + reject = 'ARC_REJECT', + }, + add_smtp_user = true, +} + +local exports = {} + +local function gen_auth_results(task, settings) + local table = table + local pairs = pairs + local ipairs = ipairs + local auth_results, hdr_parts = {}, {} + + if not settings then + settings = default_settings + end + + local auth_types = { + dkim = settings.dkim_symbols, + dmarc = settings.dmarc_symbols, + spf = settings.spf_symbols, + arc = settings.arc_symbols, + } + + local common = { + symbols = {} + } + + local received = task:get_received_headers() or {} + local mxname = (received[1] or {}).by_hostname + if mxname then + table.insert(hdr_parts, mxname) + end + + for auth_type, symbols in pairs(auth_types) do + for key, sym in pairs(symbols) do + if not common.symbols.sym then + local s = task:get_symbol(sym) + if not s then + common.symbols[sym] = false + else + common.symbols[sym] = s + if not auth_results[auth_type] then + auth_results[auth_type] = {key} + else + table.insert(auth_results[auth_type], key) + end + + if auth_type ~= 'dkim' then + break + end + end + end + end + end + + for auth_type, keys in pairs(auth_results) do + for _, key in ipairs(keys) do + local hdr = '' + if auth_type == 'dmarc' and key ~= 'none' then + local opts = common.symbols[auth_types['dmarc'][key]][1]['options'] or {} + hdr = hdr .. 'dmarc=' + if key == 'reject' or key == 'quarantine' or key == 'softfail' then + hdr = hdr .. 'fail' + else + hdr = hdr .. key + end + if key == 'pass' then + hdr = hdr .. ' (policy=' .. opts[2] .. ')' + hdr = hdr .. ' header.from=' .. opts[1] + elseif key ~= 'none' then + local t = global.rspamd_str_split(opts[1], ' : ') + local dom = t[1] + local rsn = t[2] + if rsn then + hdr = hdr .. ' reason="' .. rsn .. '"' + end + hdr = hdr .. ' header.from=' .. dom + if key == 'softfail' then + hdr = hdr .. ' (policy=none)' + else + hdr = hdr .. ' (policy=' .. key .. ')' + end + end + table.insert(hdr_parts, hdr) + elseif auth_type == 'dkim' and key ~= 'none' then + if common.symbols[auth_types['dkim'][key]][1] then + local dkim_parts = {} + local opts = common.symbols[auth_types['dkim'][key]][1]['options'] + for _, v in ipairs(opts) do + table.insert(dkim_parts, auth_type .. '=' .. key .. ' header.d=' .. v) + end + table.insert(hdr_parts, table.concat(dkim_parts, '; ')) + end + elseif auth_type == 'arc' and key ~= 'none' then + if common.symbols[auth_types['arc'][key]][1] then + local opts = common.symbols[auth_types['arc'][key]][1]['options'] or {} + for _, v in ipairs(opts) do + hdr = hdr .. auth_type .. '=' .. key .. ' (' .. v .. ')' + table.insert(hdr_parts, hdr) + end + end + elseif auth_type == 'spf' and key ~= 'none' then + hdr = hdr .. auth_type .. '=' .. key + local smtp_from = task:get_from('smtp') + if smtp_from and smtp_from[1] and smtp_from[1]['addr'] ~= '' and smtp_from[1]['addr'] ~= nil then + hdr = hdr .. ' smtp.mailfrom=' .. smtp_from[1]['addr'] + else + local helo = task:get_helo() + if helo then + hdr = hdr .. ' smtp.helo=' .. task:get_helo() + end + end + table.insert(hdr_parts, hdr) + end + end + end + + local u = task:get_user() + local smtp_from = task:get_from('smtp') + + if u and smtp_from then + local hdr + + if #smtp_from[1]['addr'] > 0 then + if settings['add_smtp_user'] then + hdr = string.format('auth=pass smtp.auth=%s smtp.mailfrom=%s', + u, smtp_from[1]['addr']) + else + hdr = string.format('auth=pass smtp.mailfrom=%s', + smtp_from[1]['addr']) + end + else + if settings['add_smtp_user'] then + hdr = string.format('auth=pass smtp.auth=%s', u) + else + hdr = 'auth=pass' + end + end + + table.insert(hdr_parts, hdr) + end + + if #hdr_parts > 0 then + return table.concat(hdr_parts, '; ') + end + + return nil +end + +exports.gen_auth_results = gen_auth_results + +return exports diff --git a/lualib/lua_cfg_transform.lua b/lualib/lua_cfg_transform.lua new file mode 100644 index 000000000..0823012ba --- /dev/null +++ b/lualib/lua_cfg_transform.lua @@ -0,0 +1,295 @@ +--[[ +Copyright (c) 2017, Vsevolod Stakhov + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local logger = require "rspamd_logger" + +local function override_defaults(def, override) + if not override then + return def + end + if not def then + return override + end + for k,v in pairs(override) do + if def[k] then + if type(v) == 'table' then + def[k] = override_defaults(def[k], v) + else + def[k] = v + end + else + def[k] = v + end + end + + return def +end + +local function is_implicit(t) + local mt = getmetatable(t) + + return mt and mt.class and mt.class == 'ucl.type.impl_array' +end + +local function metric_pairs(t) + -- collect the keys + local keys = {} + local implicit_array = is_implicit(t) + + local function gen_keys(tbl) + if implicit_array then + for _,v in ipairs(tbl) do + if v.name then + table.insert(keys, {v.name, v}) + v.name = nil + else + -- Very tricky to distinguish: + -- group {name = "foo" ... } + group "blah" { ... } + for gr_name,gr in pairs(v) do + if type(gr_name) ~= 'number' then + -- We can also have implicit arrays here + local gr_implicit = is_implicit(gr) + + if gr_implicit then + for _,gr_elt in ipairs(gr) do + table.insert(keys, {gr_name, gr_elt}) + end + else + table.insert(keys, {gr_name, gr}) + end + end + end + end + end + else + if tbl.name then + table.insert(keys, {tbl.name, tbl}) + tbl.name = nil + else + for k,v in pairs(tbl) do + if type(k) ~= 'number' then + -- We can also have implicit arrays here + local sym_implicit = is_implicit(v) + + if sym_implicit then + for _,elt in ipairs(v) do + table.insert(keys, {k, elt}) + end + else + table.insert(keys, {k, v}) + end + end + end + end + end + end + + gen_keys(t) + + -- return the iterator function + local i = 0 + return function() + i = i + 1 + if keys[i] then + return keys[i][1], keys[i][2] + end + end +end + +local function group_transform(cfg, k, v) + if v.name then k = v.name end + + local new_group = { + symbols = {} + } + + if v.enabled then new_group.enabled = v.enabled end + if v.disabled then new_group.disabled = v.disabled end + if v.max_score then new_group.max_score = v.max_score end + + if v.symbol then + for sk,sv in metric_pairs(v.symbol) do + if sv.name then + sk = sv.name + sv.name = nil -- Remove field + end + + new_group.symbols[sk] = sv + end + end + + if not cfg.group then cfg.group = {} end + + if cfg.group[k] then + cfg.group[k] = override_defaults(cfg.group[k], new_group) + else + cfg.group[k] = new_group + end + + logger.infox("overriding group %s from the legacy metric settings", k) +end + +local function symbol_transform(cfg, k, v) + -- first try to find any group where there is a definition of this symbol + for gr_n, gr in pairs(cfg.group) do + if gr.symbols and gr.symbols[k] then + -- We override group symbol with ungrouped symbol + logger.infox("overriding group symbol %s in the group %s", k, gr_n) + gr.symbols[k] = override_defaults(gr.symbols[k], v) + return + end + end + -- Now check what Rspamd knows about this symbol + local sym = rspamd_config:get_metric_symbol(k) + + if not sym or not sym.group then + -- Otherwise we just use group 'ungrouped' + if not cfg.group.ungrouped then + cfg.group.ungrouped = { + symbols = {} + } + end + + cfg.group.ungrouped.symbols[k] = v + logger.debugx("adding symbol %s to the group 'ungrouped'", k) + end +end + +local function test_groups(groups) + local all_symbols = {} + for gr_name, gr in pairs(groups) do + if not gr.symbols then + local cnt = 0 + for _,_ in pairs(gr) do cnt = cnt + 1 end + + if cnt == 0 then + logger.errx('group %s is empty', gr_name) + else + logger.infox('group %s has no symbols', gr_name) + end + + else + for sn,_ in pairs(gr.symbols) do + if all_symbols[sn] then + logger.errx('symbol %s has registered in multiple groups: %s and %s', + sn, all_symbols[sn], gr_name) + else + all_symbols[sn] = gr_name + end + end + end + end +end + +local function convert_metric(cfg, metric) + if metric.actions then + cfg.actions = override_defaults(cfg.actions, metric.actions) + logger.infox("overriding actions from the legacy metric settings") + end + if metric.unknown_weight then + cfg.actions.unknown_weight = metric.unknown_weight + end + + if metric.subject then + logger.infox("overriding subject from the legacy metric settings") + cfg.actions.subject = metric.subject + end + + if metric.group then + for k, v in metric_pairs(metric.group) do + group_transform(cfg, k, v) + end + else + if not cfg.group then + cfg.group = { + ungrouped = { + symbols = {} + } + } + end + end + + if metric.symbol then + for k, v in metric_pairs(metric.symbol) do + symbol_transform(cfg, k, v) + end + end + + return cfg +end + +-- Converts a table of groups indexed by number (implicit array) to a +-- merged group definition +local function merge_groups(groups) + local ret = {} + for k,gr in pairs(groups) do + if type(k) == 'number' then + for key,sec in pairs(gr) do + ret[key] = sec + end + else + ret[k] = gr + end + end + + return ret +end + +return function(cfg) + local ret = false + + if cfg['metric'] then + for _, v in metric_pairs(cfg.metric) do + cfg = convert_metric(cfg, v) + end + ret = true + end + + if not cfg.actions then + logger.errx('no actions defined') + else + -- Perform sanity check for actions + local actions_defs = {'greylist', 'add header', 'add_header', + 'rewrite subject', 'rewrite_subject', 'reject'} + + if not cfg.actions['no action'] and not cfg.actions['no_action'] and + not cfg.actions['accept'] then + for _,d in ipairs(actions_defs) do + if cfg.actions[d] then + if cfg.actions[d] < 0 then + cfg.actions['no action'] = cfg.actions[d] - 0.001 + logger.infox('set no action score to: %s, as action %s has negative score', + cfg.actions['no action'], d) + break + end + end + end + end + end + + if not cfg.group then + logger.errx('no symbol groups defined') + else + if cfg.group[1] then + -- We need to merge groups + cfg.group = merge_groups(cfg.group) + ret = true + end + test_groups(cfg.group) + end + + return ret, cfg +end diff --git a/lualib/lua_dkim_tools.lua b/lualib/lua_dkim_tools.lua new file mode 100644 index 000000000..a0c93b4ff --- /dev/null +++ b/lualib/lua_dkim_tools.lua @@ -0,0 +1,206 @@ +--[[ +Copyright (c) 2016, Andrew Lewis +Copyright (c) 2017, Vsevolod Stakhov + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local exports = {} + +local E = {} +local rspamd_logger = require "rspamd_logger" +local rspamd_util = require "rspamd_util" + +local function prepare_dkim_signing(N, task, settings) + local is_local, is_sign_networks + local auser = task:get_user() + local ip = task:get_from_ip() + + if ip and ip:is_local() then + is_local = true + end + + if settings.auth_only and auser then + rspamd_logger.debugm(N, task, 'user is authenticated') + elseif (settings.sign_networks and settings.sign_networks:get_key(ip)) then + is_sign_networks = true + rspamd_logger.debugm(N, task, 'mail is from address in sign_networks') + elseif settings.sign_local and is_local then + rspamd_logger.debugm(N, task, 'mail is from local address') + elseif settings.sign_inbound and not is_local and not auser then + rspamd_logger.debugm(N, task, 'mail was sent to us') + else + rspamd_logger.debugm(N, task, 'ignoring unauthenticated mail') + return false,{} + end + + local efrom = task:get_from('smtp') + if not settings.allow_envfrom_empty and + #(((efrom or E)[1] or E).addr or '') == 0 then + rspamd_logger.debugm(N, task, 'empty envelope from not allowed') + return false,{} + end + + local hfrom = task:get_from('mime') + if not settings.allow_hdrfrom_multiple and (hfrom or E)[2] then + rspamd_logger.debugm(N, task, 'multiple header from not allowed') + return false,{} + end + + local eto = task:get_recipients(0) + + local dkim_domain + local hdom = ((hfrom or E)[1] or E).domain + local edom = ((efrom or E)[1] or E).domain + local tdom = ((eto or E)[1] or E).domain + local udom = string.match(auser or '', '.*@(.*)') + + local function get_dkim_domain(dtype) + if settings[dtype] == 'header' then + return hdom + elseif settings[dtype] == 'envelope' then + return edom + elseif settings[dtype] == 'auth' then + return udom + elseif settings[dtype] == 'recipient' then + return tdom + end + end + + if hdom then + hdom = hdom:lower() + end + if edom then + edom = edom:lower() + end + if udom then + udom = udom:lower() + end + if tdom then + tdom = tdom:lower() + end + + if settings.use_domain_sign_networks and is_sign_networks then + dkim_domain = get_dkim_domain('use_domain_sign_networks') + rspamd_logger.debugm(N, task, 'sign_networks: use domain(%s) for signature: %s', + settings.use_domain_sign_networks, dkim_domain) + elseif settings.use_domain_local and is_local then + dkim_domain = get_dkim_domain('use_domain_local') + rspamd_logger.debugm(N, task, 'local: use domain(%s) for signature: %s', + settings.use_domain_local, dkim_domain) + else + dkim_domain = get_dkim_domain('use_domain') + rspamd_logger.debugm(N, task, 'use domain(%s) for signature: %s', + settings.use_domain, dkim_domain) + end + + if not dkim_domain then + rspamd_logger.debugm(N, task, 'could not extract dkim domain') + return false,{} + end + + if settings.use_esld then + dkim_domain = rspamd_util.get_tld(dkim_domain) + if hdom then + hdom = rspamd_util.get_tld(hdom) + end + if edom then + edom = rspamd_util.get_tld(edom) + end + end + if edom and hdom and not settings.allow_hdrfrom_mismatch and hdom ~= edom then + if settings.allow_hdrfrom_mismatch_local and is_local then + rspamd_logger.debugm(N, task, 'domain mismatch allowed for local IP: %1 != %2', hdom, edom) + elseif settings.allow_hdrfrom_mismatch_sign_networks and is_sign_networks then + rspamd_logger.debugm(N, task, 'domain mismatch allowed for sign_networks: %1 != %2', hdom, edom) + else + rspamd_logger.debugm(N, task, 'domain mismatch not allowed: %1 != %2', hdom, edom) + return false,{} + end + end + + if auser and not settings.allow_username_mismatch then + if not udom then + rspamd_logger.debugm(N, task, 'couldnt find domain in username') + return false,{} + end + if settings.use_esld then + udom = rspamd_util.get_tld(udom) + end + if udom ~= dkim_domain then + rspamd_logger.debugm(N, task, 'user domain mismatch') + return false,{} + end + end + + local p = {} + + if settings.domain[dkim_domain] then + p.selector = settings.domain[dkim_domain].selector + p.key = settings.domain[dkim_domain].path + end + + if not p.key and p.selector then + local key_var = "dkim_key" + local selector_var = "dkim_selector" + if N == "arc" then + key_var = "arc_key" + selector_var = "arc_selector" + end + + p.key = task:get_mempool():get_variable(key_var) + p.selector = task:get_mempool():get_variable(selector_var) + + if (not p.key or not p.selector) and (not (settings.try_fallback or + settings.use_redis or settings.selector_map + or settings.path_map)) then + rspamd_logger.debugm(N, task, 'dkim unconfigured and fallback disabled') + return false,{} + end + end + + if not p.selector and settings.selector_map then + local data = settings.selector_map:get_key(dkim_domain) + if data then + p.selector = data + elseif not settings.try_fallback then + return false,{} + end + end + + if not p.key and settings.path_map then + local data = settings.path_map:get_key(dkim_domain) + if data then + p.key = data + elseif not settings.try_fallback then + return false,{} + end + end + + if not p.key then + if not settings.use_redis then + p.key = settings.path + end + end + + if not p.selector then + p.selector = settings.selector + end + p.domain = dkim_domain + + return true,p +end + +exports.prepare_dkim_signing = prepare_dkim_signing + +return exports diff --git a/lualib/lua_maps.lua b/lualib/lua_maps.lua new file mode 100644 index 000000000..7b48db034 --- /dev/null +++ b/lualib/lua_maps.lua @@ -0,0 +1,188 @@ +--[[ +Copyright (c) 2017, Vsevolod Stakhov + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local exports = {} + +local function rspamd_map_add_from_ucl(opt, mtype, description) + local ret = { + get_key = function(t, k) + if t.__data then + return t.__data:get_key(k) + end + + return nil + end + } + local ret_mt = { + __index = function(t, k) + if t.__data then + return t.get_key(k) + end + + return nil + end + } + + if not opt then + return nil + end + + if type(opt) == 'string' then + -- We have a single string, so we treat it as a map + local map = rspamd_config:add_map{ + type = mtype, + description = description, + url = opt, + } + + if map then + ret.__data = map + setmetatable(ret, ret_mt) + return ret + end + elseif type(opt) == 'table' then + -- it might be plain map or map of plain elements + if opt[1] then + if mtype == 'radix' then + + if string.find(opt[1], '^%d') then + local map = rspamd_config:radix_from_ucl(opt) + + if map then + ret.__data = map + setmetatable(ret, ret_mt) + return ret + end + else + -- Plain table + local map = rspamd_config:add_map{ + type = mtype, + description = description, + url = opt, + } + if map then + ret.__data = map + setmetatable(ret, ret_mt) + return ret + end + end + elseif mtype == 'regexp' then + -- Plain table + local map = rspamd_config:add_map{ + type = mtype, + description = description, + url = opt, + } + if map then + ret.__data = map + setmetatable(ret, ret_mt) + return ret + end + else + if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then + -- Plain table + local map = rspamd_config:add_map{ + type = mtype, + description = description, + url = opt, + } + if map then + ret.__data = map + setmetatable(ret, ret_mt) + return ret + end + else + local data = {} + local nelts = 0 + for _,elt in ipairs(opt) do + if type(elt) == 'string' then + data[elt] = true + nelts = nelts + 1 + end + end + + if nelts > 0 then + ret.__data = data + ret.get_key = function(t, k) + if k ~= '__data' then + return t.__data[k] + end + + return nil + end + return ret + end + end + end + else + local map = rspamd_config:add_map{ + type = mtype, + description = description, + url = opt, + } + if map then + ret.__data = map + setmetatable(ret, ret_mt) + return ret + end + end + end + + return nil +end + +local function rspamd_map_add(mname, optname, mtype, description) + local opt = rspamd_config:get_module_opt(mname, optname) + + return rspamd_map_add_from_ucl(opt, mtype, description) +end + +exports.rspamd_map_add = rspamd_map_add +exports.rspamd_map_add_from_ucl = rspamd_map_add_from_ucl + +-- Check `what` for being lua_map name, otherwise just compares key with what +local function rspamd_maybe_check_map(key, what) + local fun = require "fun" + + local function starts(where,st) + return string.sub(where,1,string.len(st))==st + end + + if type(what) == "table" then + return fun.any(function(elt) return rspamd_maybe_check_map(key, elt) end, what) + end + if type(rspamd_maps) == "table" then + local mn + if starts(what, "map:") then + mn = string.sub(what, 4) + elseif starts(what, "map://") then + mn = string.sub(what, 6) + end + + if mn and rspamd_maps[mn] then + return rspamd_maps[mn]:get_key(key) + else + return what:lower() == key + end + else + return what:lower() == key + end + +end + +exports.rspamd_maybe_check_map = rspamd_maybe_check_map + +return exports diff --git a/lualib/lua_meta.lua b/lualib/lua_meta.lua new file mode 100644 index 000000000..96404192d --- /dev/null +++ b/lualib/lua_meta.lua @@ -0,0 +1,396 @@ +--[[ +Copyright (c) 2017, Vsevolod Stakhov + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local exports = {} + +local N = "metatokens" + +-- Metafunctions +local function meta_size_function(task) + local sizes = { + 100, + 200, + 500, + 1000, + 2000, + 4000, + 10000, + 20000, + 30000, + 100000, + 200000, + 400000, + 800000, + 1000000, + 2000000, + 8000000, + } + + local size = task:get_size() + for i = 1,#sizes do + if sizes[i] >= size then + return {(1.0 * i) / #sizes} + end + end + + return {0} +end + +local function meta_images_function(task) + local images = task:get_images() + local ntotal = 0 + local njpg = 0 + local npng = 0 + local nlarge = 0 + local nsmall = 0 + + if images then + for _,img in ipairs(images) do + if img:get_type() == 'png' then + npng = npng + 1 + elseif img:get_type() == 'jpeg' then + njpg = njpg + 1 + end + + local w = img:get_width() + local h = img:get_height() + + if w > 0 and h > 0 then + if w + h > 256 then + nlarge = nlarge + 1 + else + nsmall = nsmall + 1 + end + end + + ntotal = ntotal + 1 + end + end + if ntotal > 0 then + njpg = 1.0 * njpg / ntotal + npng = 1.0 * npng / ntotal + nlarge = 1.0 * nlarge / ntotal + nsmall = 1.0 * nsmall / ntotal + end + return {ntotal,njpg,npng,nlarge,nsmall} +end + +local function meta_nparts_function(task) + local nattachments = 0 + local ntextparts = 0 + local totalparts = 1 + + local tp = task:get_text_parts() + if tp then + ntextparts = #tp + end + + local parts = task:get_parts() + + if parts then + for _,p in ipairs(parts) do + if p:get_filename() then + nattachments = nattachments + 1 + end + totalparts = totalparts + 1 + end + end + + return {(1.0 * ntextparts)/totalparts, (1.0 * nattachments)/totalparts} +end + +local function meta_encoding_function(task) + local nutf = 0 + local nother = 0 + + local tp = task:get_text_parts() + if tp and #tp > 0 then + for _,p in ipairs(tp) do + if p:is_utf() then + nutf = nutf + 1 + else + nother = nother + 1 + end + end + + return {nutf / #tp, nother / #tp} + end + + return {0, 0} +end + +local function meta_recipients_function(task) + local nmime = 0 + local nsmtp = 0 + + if task:has_recipients('mime') then + nmime = #(task:get_recipients('mime')) + end + if task:has_recipients('smtp') then + nsmtp = #(task:get_recipients('smtp')) + end + + if nmime > 0 then nmime = 1.0 / nmime end + if nsmtp > 0 then nsmtp = 1.0 / nsmtp end + + return {nmime,nsmtp} +end + +local function meta_received_function(task) + local count_factor = 0 + local invalid_factor = 0 + local rh = task:get_received_headers() + local time_factor = 0 + local secure_factor = 0 + local fun = require "fun" + + if rh and #rh > 0 then + + local ntotal = 0.0 + local init_time = 0 + + fun.each(function(rc) + ntotal = ntotal + 1.0 + + if not rc.by_hostname then + invalid_factor = invalid_factor + 1.0 + end + if init_time == 0 and rc.timestamp then + init_time = rc.timestamp + elseif rc.timestamp then + time_factor = time_factor + math.abs(init_time - rc.timestamp) + init_time = rc.timestamp + end + if rc.flags and (rc.flags['ssl'] or rc.flags['authenticated']) then + secure_factor = secure_factor + 1.0 + end + end, + fun.filter(function(rc) return not rc.flags or not rc.flags['artificial'] end, rh)) + + invalid_factor = invalid_factor / ntotal + secure_factor = secure_factor / ntotal + count_factor = 1.0 / ntotal + + if time_factor ~= 0 then + time_factor = 1.0 / time_factor + end + end + + return {count_factor, invalid_factor, time_factor, secure_factor} +end + +local function meta_urls_function(task) + if task:has_urls() then + return {1.0 / #(task:get_urls())} + end + + return {0} +end + +local function meta_words_function(task) + local avg_len = task:get_mempool():get_variable("avg_words_len", "double") or 0.0 + local short_words = task:get_mempool():get_variable("short_words_cnt", "double") or 0.0 + local ret_len = 0 + + local lens = { + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 15, + 20, + } + + for i = 1,#lens do + if lens[i] >= avg_len then + ret_len = (1.0 * i) / #lens + break + end + end + + local tp = task:get_text_parts() + local wres = { + 0, -- spaces rate + 0, -- double spaces rate + 0, -- non spaces rate + 0, -- ascii characters rate + 0, -- non-ascii characters rate + 0, -- capital characters rate + 0, -- numeric cahracters + } + for _,p in ipairs(tp) do + local stats = p:get_stats() + local len = p:get_length() + + if len > 0 then + wres[1] = wres[1] + stats['spaces'] / len + wres[2] = wres[2] + stats['double_spaces'] / len + wres[3] = wres[3] + stats['non_spaces'] / len + wres[4] = wres[4] + stats['ascii_characters'] / len + wres[5] = wres[5] + stats['non_ascii_characters'] / len + wres[6] = wres[6] + stats['capital_letters'] / len + wres[7] = wres[7] + stats['numeric_characters'] / len + end + end + + local ret = { + short_words, + ret_len, + } + + local divisor = 1.0 + if #tp > 0 then + divisor = #tp + end + + for _,wr in ipairs(wres) do + table.insert(ret, wr / divisor) + end + + return ret +end + +local metafunctions = { + { + cb = meta_size_function, + ninputs = 1, + desc = { + "size" + } + }, + { + cb = meta_images_function, + ninputs = 5, + -- 1 - number of images, + -- 2 - number of png images, + -- 3 - number of jpeg images + -- 4 - number of large images (> 128 x 128) + -- 5 - number of small images (< 128 x 128) + desc = { + 'nimages', + 'npng_images', + 'njpeg_images', + 'nlarge_images', + 'nsmall_images' + } + }, + { + cb = meta_nparts_function, + ninputs = 2, + -- 1 - number of text parts + -- 2 - number of attachments + desc = { + 'ntext_parts', + 'nattachments' + } + }, + { + cb = meta_encoding_function, + ninputs = 2, + -- 1 - number of utf parts + -- 2 - number of non-utf parts + desc = { + 'nutf_parts', + 'nascii_parts' + } + }, + { + cb = meta_recipients_function, + ninputs = 2, + -- 1 - number of mime rcpt + -- 2 - number of smtp rcpt + desc = { + 'nmime_rcpt', + 'nsmtp_rcpt' + } + }, + { + cb = meta_received_function, + ninputs = 4, + desc = { + 'nreceived', + 'nreceived_invalid', + 'nreceived_bad_time', + 'nreceived_secure' + } + }, + { + cb = meta_urls_function, + ninputs = 1, + desc = { + 'nurls' + } + }, + { + cb = meta_words_function, + ninputs = 9, + desc = { + 'avg_words_len', + 'nshort_words', + 'spaces_rate', + 'double_spaces_rate', + 'non_spaces_rate', + 'ascii_characters_rate', + 'non_ascii_characters_rate', + 'capital_characters_rate', + 'numeric_cahracters' + } + }, +} + +local function rspamd_gen_metatokens(task) + local rspamd_logger = require "rspamd_logger" + local ipairs = ipairs + local metatokens = {} + local cached = task:cache_get('metatokens') + + if cached then + return cached + else + for _,mt in ipairs(metafunctions) do + local ct = mt.cb(task) + for i,tok in ipairs(ct) do + rspamd_logger.debugm(N, task, "metatoken: %s = %s", mt.desc[i], tok) + table.insert(metatokens, tok) + end + end + + task:cache_set('metatokens', metatokens) + end + + return metatokens +end + +exports.rspamd_gen_metatokens = rspamd_gen_metatokens + +local function rspamd_count_metatokens() + local ipairs = ipairs + local total = 0 + for _,mt in ipairs(metafunctions) do + total = total + mt.ninputs + end + + return total +end + +exports.rspamd_count_metatokens = rspamd_count_metatokens + +return exports diff --git a/lualib/lua_stat.lua b/lualib/lua_stat.lua new file mode 100644 index 000000000..4f5cafe3c --- /dev/null +++ b/lualib/lua_stat.lua @@ -0,0 +1,502 @@ +--[[ +Copyright (c) 2018, Vsevolod Stakhov + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local logger = require "rspamd_logger" +local sqlite3 = require "rspamd_sqlite3" +local util = require "rspamd_util" +local lua_redis = require "lua_redis" +local exports = {} + +local N = "stat_tools" -- luacheck: ignore (maybe unused) + +-- Performs synchronous conversion of redis schema +local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expire) + + -- Old schema is the following one: + -- Keys are named [] + -- Elements are placed within hash: + -- BAYES_SPAM -> {: , : ...} + -- In new schema it is changed to a more extensible schema: + -- Keys are named RS[]_ -> {'H': , 'S': } + -- So we can expire individual records, measure most popular elements by zranges, + -- add new fields, such as tokens etc + + local res,conn = lua_redis.redis_connect_sync(redis_params, true) + + if not res then + logger.errx("cannot connect to redis server") + return false + end + + -- KEYS[1]: key to check (e.g. 'BAYES_SPAM') + -- KEYS[2]: hash key ('S' or 'H') + -- KEYS[3]: expire + local lua_script = [[ +local keys = redis.call('SMEMBERS', KEYS[1]..'_keys') +local nconverted = 0 + +for _,k in ipairs(keys) do + local elts = redis.call('HGETALL', k) + + for k,v in pairs(elts) do + local neutral_prefix = string.gsub(k, KEYS[1], 'RS') + local nkey = string.format('%s_%s', neutral_prefix, k) + redis.call('HSET', nkey, KEYS[2], v) + if KEYS[4] and tonumber(KEYS[3]) ~= 0 then + redis.call('EXPIRE', nkey, KEYS[3]) + end + nconverted = nconverted + 1 + end +end + +return nconverted +]] + + conn:add_cmd('EVAL', {lua_script, '3', symbol_spam, 'S', tostring(expire)}) + local ret + ret, res = conn:exec() + + if not ret then + logger.errx('error converting symbol %s', symbol_spam) + return false + else + logger.messagex('converted %s elements from symbol %s', res, symbol_spam) + end + + conn:add_cmd('EVAL', {lua_script, '3', symbol_ham, 'H', tostring(expire)}) + ret, res = conn:exec() + + if not ret then + logger.errx('error converting symbol %s', symbol_ham) + return false + else + logger.messagex('converted %s elements from symbol %s', res, symbol_ham) + end + + -- We can now convert metadata: set + learned + version + -- KEYS[1]: key to check (e.g. 'BAYES_SPAM') + -- KEYS[2]: learn key (e.g. 'learns_spam' or 'learns_ham') + lua_script = [[ +local keys = redis.call('SMEMBERS', KEYS[1]..'_keys') + +for _,k in ipairs(keys) do + local learns = redis.call('HGET', k, 'learns') + local neutral_prefix = string.gsub(k, KEYS[1], 'RS') + + redis.call('HSET', neutral_prefix, KEYS[2], learns) + redis.call('SADD', KEYS[1]..'_keys', neutral_prefix) + redis.call('SREM', KEYS[1]..'_keys', k) + redis.call('DEL', k) + redis.call('SET', KEYS[1]..'_version', '2') +end +]] + + conn:add_cmd('EVAL', {lua_script, '2', symbol_spam, 'learns_spam'}) + ret = conn:exec() + + if not ret then + logger.errx('error converting metadata for symbol %s', symbol_spam) + return false + end + + conn:add_cmd('EVAL', {lua_script, '2', symbol_ham, 'learns_ham'}) + ret = conn:exec() + + if not ret then + logger.errx('error converting metadata for symbol %s', symbol_ham) + return false + end + + return true +end + +exports.convert_bayes_schema = convert_bayes_schema + +-- It now accepts both ham and spam databases +-- parameters: +-- redis_params - how do we connect to a redis server +-- sqlite_db_spam - name for sqlite database with spam tokens +-- sqlite_db_ham - name for sqlite database with ham tokens +-- symbol_ham - name for symbol representing spam, e.g. BAYES_SPAM +-- symbol_spam - name for symbol representing ham, e.g. BAYES_HAM +-- learn_cache_spam - name for sqlite database with spam learn cache +-- learn_cache_ham - name for sqlite database with ham learn cache +-- reset_previous - if true, then the old database is flushed (slow) +local function convert_sqlite_to_redis(redis_params, + sqlite_db_spam, sqlite_db_ham, symbol_spam, symbol_ham, + learn_cache_db, expire, reset_previous) + local nusers = 0 + local lim = 1000 -- Update each 1000 tokens + local users_map = {} + local converted = 0 + + local db_spam = sqlite3.open(sqlite_db_spam) + if not db_spam then + logger.errx('Cannot open source db: %s', sqlite_db_spam) + return false + end + local db_ham = sqlite3.open(sqlite_db_ham) + if not db_ham then + logger.errx('Cannot open source db: %s', sqlite_db_ham) + return false + end + + local res,conn = lua_redis.redis_connect_sync(redis_params, true) + + if not res then + logger.errx("cannot connect to redis server") + return false + end + + if reset_previous then + -- Do a more complicated cleanup + -- execute a lua script that cleans up data + local script = [[ +local members = redis.call('SMEMBERS', KEYS[1]..'_keys') + +for _,prefix in ipairs(members) do + local keys = redis.call('KEYS', prefix..'*') + redis.call('DEL', keys) +end +]] + -- Common keys + for _,sym in ipairs({symbol_spam, symbol_ham}) do + logger.messagex('Cleaning up old data for %s', sym) + conn:add_cmd('EVAL', {script, '1', sym}) + conn:exec() + conn:add_cmd('DEL', {sym .. "_version"}) + conn:add_cmd('DEL', {sym .. "_keys"}) + conn:exec() + end + + if learn_cache_db then + -- Cleanup learned_cache + logger.messagex('Cleaning up old data learned cache') + conn:add_cmd('DEL', {"learned_ids"}) + conn:exec() + end + end + + local function convert_db(db, is_spam) + -- Map users and languages + local what = 'ham' + if is_spam then + what = 'spam' + end + + local learns = {} + db:sql('BEGIN;') + -- Fill users mapping + for row in db:rows('SELECT * FROM users;') do + if row.id == '0' then + users_map[row.id] = '' + else + users_map[row.id] = row.name + end + learns[row.id] = row.learns + nusers = nusers + 1 + end + + -- Workaround for old databases + for row in db:rows('SELECT * FROM languages') do + if learns['0'] then + learns['0'] = learns['0'] + row.learns + else + learns['0'] = row.learns + end + end + + local function send_batch(tokens, prefix) + -- We use the new schema: RS[user]_token -> H=ham count + -- S=spam count + local hash_key = 'H' + if is_spam then + hash_key = 'S' + end + for _,tok in ipairs(tokens) do + -- tok schema: + -- tok[1] = token_id (uint64 represented as a string) + -- tok[2] = token value (number) + -- tok[3] = user_map[user_id] or '' + local rkey = string.format('%s%s_%s', prefix, tok[3], tok[1]) + conn:add_cmd('HINCRBYFLOAT', {rkey, hash_key, tostring(tok[2])}) + + if expire and expire ~= 0 then + conn:add_cmd('EXPIRE', {rkey, tostring(expire)}) + end + end + + return conn:exec() + end + -- Fill tokens, sending data to redis each `lim` records + + local ntokens = db:query('SELECT count(*) as c FROM tokens')['c'] + local tokens = {} + local num = 0 + local total = 0 + + for row in db:rows('SELECT token,value,user FROM tokens;') do + local user = '' + if row.user ~= 0 and users_map[row.user] then + user = users_map[row.user] + end + + table.insert(tokens, {row.token, row.value, user}) + num = num + 1 + total = total + 1 + if num > lim then + -- TODO: we use the default 'RS' prefix, it can be false in case of + -- classifiers with labels + local ret,err_str = send_batch(tokens, 'RS') + if not ret then + logger.errx('Cannot send tokens to the redis server: ' .. err_str) + db:sql('COMMIT;') + return false + end + + num = 0 + tokens = {} + end + + io.write(string.format('Processed batch %s: %s/%s\r', what, total, ntokens)) + end + -- Last batch + if #tokens > 0 then + local ret,err_str = send_batch(tokens, 'RS') + if not ret then + logger.errx('Cannot send tokens to the redis server: ' .. err_str) + db:sql('COMMIT;') + return false + end + + io.write(string.format('Processed batch %s: %s/%s\r', what, total, ntokens)) + end + io.write('\n') + + converted = converted + total + + -- Close DB + db:sql('COMMIT;') + local symbol = symbol_ham + local learns_elt = "learns_ham" + + if is_spam then + symbol = symbol_spam + learns_elt = "learns_spam" + end + + for id,learned in pairs(learns) do + local user = users_map[id] + if not conn:add_cmd('HSET', {'RS' .. user, learns_elt, learned}) then + logger.errx('Cannot update learns for user: ' .. user) + return false + end + if user ~= '' then + if not conn:add_cmd('SADD', {symbol .. '_keys', 'RS' .. user}) then + logger.errx('Cannot update learns for user: ' .. user) + return false + end + end + end + -- Set version + conn:add_cmd('SET', {symbol..'_version', '2'}) + return conn:exec() + end + + logger.messagex('Convert spam tokens') + if not convert_db(db_spam, true) then + return false + end + + logger.messagex('Convert ham tokens') + if not convert_db(db_ham, false) then + return false + end + + if learn_cache_db then + logger.messagex('Convert learned ids from %s', learn_cache_db) + local db = sqlite3.open(learn_cache_db) + local ret = true + local total = 0 + + if not db then + logger.errx('Cannot open cache database: ' .. learn_cache_db) + return false + end + + db:sql('BEGIN;') + + for row in db:rows('SELECT * FROM learns;') do + local is_spam + local digest = tostring(util.encode_base32(row.digest)) + + if row.flag == '0' then + is_spam = '-1' + else + is_spam = '1' + end + + if not conn:add_cmd('HSET', {'learned_ids', digest, is_spam}) then + logger.errx('Cannot add hash: ' .. digest) + ret = false + else + total = total + 1 + end + end + db:sql('COMMIT;') + + if ret then + conn:exec() + end + + if ret then + logger.messagex('Converted %s cached items from sqlite3 learned cache to redis', + total) + else + logger.errx('Error occurred during sending data to redis') + end + end + + logger.messagex('Migrated %s tokens for %s users for symbols (%s, %s)', + converted, nusers, symbol_spam, symbol_ham) + return true +end + +exports.convert_sqlite_to_redis = convert_sqlite_to_redis + +-- Loads sqlite3 based classifiers and output data in form of array of objects: +-- [ +-- { +-- symbol_spam = XXX +-- symbol_ham = YYY +-- db_spam = XXX.sqlite +-- db_ham = YYY.sqlite +-- learn_cahe = ZZZ.sqlite +-- per_user = true/false +-- label = str +-- } +-- ] +local function load_sqlite_config(cfg) + local result = {} + + local function parse_classifier(cls) + local tbl = {} + if cls.cache then + local cache = cls.cache + if cache.type == 'sqlite3' and (cache.file or cache.path) then + tbl.learn_cache = (cache.file or cache.path) + end + end + + if cls.per_user then + tbl.per_user = cls.per_user + end + + if cls.label then + tbl.label = cls.label + end + + local statfiles = cls.statfile + for _,stf in ipairs(statfiles) do + local path = (stf.file or stf.path or stf.db or stf.dbname) + local symbol = stf.symbol or 'undefined' + + if not path then + logger.errx('no path defined for statfile %s', symbol) + else + + local spam + if stf.spam then + spam = stf.spam + else + if string.match(symbol:upper(), 'SPAM') then + spam = true + else + spam = false + end + end + + if spam then + tbl.symbol_spam = symbol + tbl.db_spam = path + else + tbl.symbol_ham = symbol + tbl.db_ham = path + end + end + end + + if tbl.symbol_spam and tbl.symbol_ham and tbl.db_ham and tbl.db_spam then + table.insert(result, tbl) + end + end + + local classifier = cfg.classifier + + if classifier then + if classifier[1] then + for _,cls in ipairs(classifier) do + if cls.backend and cls.backend == 'sqlite3' then + parse_classifier(cls) + end + end + else + if classifier.bayes then + classifier = classifier.bayes + if classifier[1] then + for _,cls in ipairs(classifier) do + if cls.backend and cls.backend == 'sqlite3' then + parse_classifier(cls) + end + end + else + if classifier.backend and classifier.backend == 'sqlite3' then + parse_classifier(classifier) + end + end + end + end + end + + return result +end + +exports.load_sqlite_config = load_sqlite_config + +-- A helper method that suggests a user how to configure Redis based +-- classifier based on the existing sqlite classifier +local function redis_classifier_from_sqlite(sqlite_classifier) + local result = { + backend = 'redis', + cache = { + backend = 'redis' + }, + statfile = { + [sqlite_classifier.symbol_spam] = { + spam = true + }, + [sqlite_classifier.symbol_ham] = { + spam = false + } + } + } + + return {classifier = {bayes = result}} +end + +exports.redis_classifier_from_sqlite = redis_classifier_from_sqlite + +return exports diff --git a/lualib/maps.lua b/lualib/maps.lua deleted file mode 100644 index 7b48db034..000000000 --- a/lualib/maps.lua +++ /dev/null @@ -1,188 +0,0 @@ ---[[ -Copyright (c) 2017, Vsevolod Stakhov - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -local exports = {} - -local function rspamd_map_add_from_ucl(opt, mtype, description) - local ret = { - get_key = function(t, k) - if t.__data then - return t.__data:get_key(k) - end - - return nil - end - } - local ret_mt = { - __index = function(t, k) - if t.__data then - return t.get_key(k) - end - - return nil - end - } - - if not opt then - return nil - end - - if type(opt) == 'string' then - -- We have a single string, so we treat it as a map - local map = rspamd_config:add_map{ - type = mtype, - description = description, - url = opt, - } - - if map then - ret.__data = map - setmetatable(ret, ret_mt) - return ret - end - elseif type(opt) == 'table' then - -- it might be plain map or map of plain elements - if opt[1] then - if mtype == 'radix' then - - if string.find(opt[1], '^%d') then - local map = rspamd_config:radix_from_ucl(opt) - - if map then - ret.__data = map - setmetatable(ret, ret_mt) - return ret - end - else - -- Plain table - local map = rspamd_config:add_map{ - type = mtype, - description = description, - url = opt, - } - if map then - ret.__data = map - setmetatable(ret, ret_mt) - return ret - end - end - elseif mtype == 'regexp' then - -- Plain table - local map = rspamd_config:add_map{ - type = mtype, - description = description, - url = opt, - } - if map then - ret.__data = map - setmetatable(ret, ret_mt) - return ret - end - else - if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then - -- Plain table - local map = rspamd_config:add_map{ - type = mtype, - description = description, - url = opt, - } - if map then - ret.__data = map - setmetatable(ret, ret_mt) - return ret - end - else - local data = {} - local nelts = 0 - for _,elt in ipairs(opt) do - if type(elt) == 'string' then - data[elt] = true - nelts = nelts + 1 - end - end - - if nelts > 0 then - ret.__data = data - ret.get_key = function(t, k) - if k ~= '__data' then - return t.__data[k] - end - - return nil - end - return ret - end - end - end - else - local map = rspamd_config:add_map{ - type = mtype, - description = description, - url = opt, - } - if map then - ret.__data = map - setmetatable(ret, ret_mt) - return ret - end - end - end - - return nil -end - -local function rspamd_map_add(mname, optname, mtype, description) - local opt = rspamd_config:get_module_opt(mname, optname) - - return rspamd_map_add_from_ucl(opt, mtype, description) -end - -exports.rspamd_map_add = rspamd_map_add -exports.rspamd_map_add_from_ucl = rspamd_map_add_from_ucl - --- Check `what` for being lua_map name, otherwise just compares key with what -local function rspamd_maybe_check_map(key, what) - local fun = require "fun" - - local function starts(where,st) - return string.sub(where,1,string.len(st))==st - end - - if type(what) == "table" then - return fun.any(function(elt) return rspamd_maybe_check_map(key, elt) end, what) - end - if type(rspamd_maps) == "table" then - local mn - if starts(what, "map:") then - mn = string.sub(what, 4) - elseif starts(what, "map://") then - mn = string.sub(what, 6) - end - - if mn and rspamd_maps[mn] then - return rspamd_maps[mn]:get_key(key) - else - return what:lower() == key - end - else - return what:lower() == key - end - -end - -exports.rspamd_maybe_check_map = rspamd_maybe_check_map - -return exports diff --git a/lualib/meta_functions.lua b/lualib/meta_functions.lua deleted file mode 100644 index 96404192d..000000000 --- a/lualib/meta_functions.lua +++ /dev/null @@ -1,396 +0,0 @@ ---[[ -Copyright (c) 2017, Vsevolod Stakhov - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -local exports = {} - -local N = "metatokens" - --- Metafunctions -local function meta_size_function(task) - local sizes = { - 100, - 200, - 500, - 1000, - 2000, - 4000, - 10000, - 20000, - 30000, - 100000, - 200000, - 400000, - 800000, - 1000000, - 2000000, - 8000000, - } - - local size = task:get_size() - for i = 1,#sizes do - if sizes[i] >= size then - return {(1.0 * i) / #sizes} - end - end - - return {0} -end - -local function meta_images_function(task) - local images = task:get_images() - local ntotal = 0 - local njpg = 0 - local npng = 0 - local nlarge = 0 - local nsmall = 0 - - if images then - for _,img in ipairs(images) do - if img:get_type() == 'png' then - npng = npng + 1 - elseif img:get_type() == 'jpeg' then - njpg = njpg + 1 - end - - local w = img:get_width() - local h = img:get_height() - - if w > 0 and h > 0 then - if w + h > 256 then - nlarge = nlarge + 1 - else - nsmall = nsmall + 1 - end - end - - ntotal = ntotal + 1 - end - end - if ntotal > 0 then - njpg = 1.0 * njpg / ntotal - npng = 1.0 * npng / ntotal - nlarge = 1.0 * nlarge / ntotal - nsmall = 1.0 * nsmall / ntotal - end - return {ntotal,njpg,npng,nlarge,nsmall} -end - -local function meta_nparts_function(task) - local nattachments = 0 - local ntextparts = 0 - local totalparts = 1 - - local tp = task:get_text_parts() - if tp then - ntextparts = #tp - end - - local parts = task:get_parts() - - if parts then - for _,p in ipairs(parts) do - if p:get_filename() then - nattachments = nattachments + 1 - end - totalparts = totalparts + 1 - end - end - - return {(1.0 * ntextparts)/totalparts, (1.0 * nattachments)/totalparts} -end - -local function meta_encoding_function(task) - local nutf = 0 - local nother = 0 - - local tp = task:get_text_parts() - if tp and #tp > 0 then - for _,p in ipairs(tp) do - if p:is_utf() then - nutf = nutf + 1 - else - nother = nother + 1 - end - end - - return {nutf / #tp, nother / #tp} - end - - return {0, 0} -end - -local function meta_recipients_function(task) - local nmime = 0 - local nsmtp = 0 - - if task:has_recipients('mime') then - nmime = #(task:get_recipients('mime')) - end - if task:has_recipients('smtp') then - nsmtp = #(task:get_recipients('smtp')) - end - - if nmime > 0 then nmime = 1.0 / nmime end - if nsmtp > 0 then nsmtp = 1.0 / nsmtp end - - return {nmime,nsmtp} -end - -local function meta_received_function(task) - local count_factor = 0 - local invalid_factor = 0 - local rh = task:get_received_headers() - local time_factor = 0 - local secure_factor = 0 - local fun = require "fun" - - if rh and #rh > 0 then - - local ntotal = 0.0 - local init_time = 0 - - fun.each(function(rc) - ntotal = ntotal + 1.0 - - if not rc.by_hostname then - invalid_factor = invalid_factor + 1.0 - end - if init_time == 0 and rc.timestamp then - init_time = rc.timestamp - elseif rc.timestamp then - time_factor = time_factor + math.abs(init_time - rc.timestamp) - init_time = rc.timestamp - end - if rc.flags and (rc.flags['ssl'] or rc.flags['authenticated']) then - secure_factor = secure_factor + 1.0 - end - end, - fun.filter(function(rc) return not rc.flags or not rc.flags['artificial'] end, rh)) - - invalid_factor = invalid_factor / ntotal - secure_factor = secure_factor / ntotal - count_factor = 1.0 / ntotal - - if time_factor ~= 0 then - time_factor = 1.0 / time_factor - end - end - - return {count_factor, invalid_factor, time_factor, secure_factor} -end - -local function meta_urls_function(task) - if task:has_urls() then - return {1.0 / #(task:get_urls())} - end - - return {0} -end - -local function meta_words_function(task) - local avg_len = task:get_mempool():get_variable("avg_words_len", "double") or 0.0 - local short_words = task:get_mempool():get_variable("short_words_cnt", "double") or 0.0 - local ret_len = 0 - - local lens = { - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 15, - 20, - } - - for i = 1,#lens do - if lens[i] >= avg_len then - ret_len = (1.0 * i) / #lens - break - end - end - - local tp = task:get_text_parts() - local wres = { - 0, -- spaces rate - 0, -- double spaces rate - 0, -- non spaces rate - 0, -- ascii characters rate - 0, -- non-ascii characters rate - 0, -- capital characters rate - 0, -- numeric cahracters - } - for _,p in ipairs(tp) do - local stats = p:get_stats() - local len = p:get_length() - - if len > 0 then - wres[1] = wres[1] + stats['spaces'] / len - wres[2] = wres[2] + stats['double_spaces'] / len - wres[3] = wres[3] + stats['non_spaces'] / len - wres[4] = wres[4] + stats['ascii_characters'] / len - wres[5] = wres[5] + stats['non_ascii_characters'] / len - wres[6] = wres[6] + stats['capital_letters'] / len - wres[7] = wres[7] + stats['numeric_characters'] / len - end - end - - local ret = { - short_words, - ret_len, - } - - local divisor = 1.0 - if #tp > 0 then - divisor = #tp - end - - for _,wr in ipairs(wres) do - table.insert(ret, wr / divisor) - end - - return ret -end - -local metafunctions = { - { - cb = meta_size_function, - ninputs = 1, - desc = { - "size" - } - }, - { - cb = meta_images_function, - ninputs = 5, - -- 1 - number of images, - -- 2 - number of png images, - -- 3 - number of jpeg images - -- 4 - number of large images (> 128 x 128) - -- 5 - number of small images (< 128 x 128) - desc = { - 'nimages', - 'npng_images', - 'njpeg_images', - 'nlarge_images', - 'nsmall_images' - } - }, - { - cb = meta_nparts_function, - ninputs = 2, - -- 1 - number of text parts - -- 2 - number of attachments - desc = { - 'ntext_parts', - 'nattachments' - } - }, - { - cb = meta_encoding_function, - ninputs = 2, - -- 1 - number of utf parts - -- 2 - number of non-utf parts - desc = { - 'nutf_parts', - 'nascii_parts' - } - }, - { - cb = meta_recipients_function, - ninputs = 2, - -- 1 - number of mime rcpt - -- 2 - number of smtp rcpt - desc = { - 'nmime_rcpt', - 'nsmtp_rcpt' - } - }, - { - cb = meta_received_function, - ninputs = 4, - desc = { - 'nreceived', - 'nreceived_invalid', - 'nreceived_bad_time', - 'nreceived_secure' - } - }, - { - cb = meta_urls_function, - ninputs = 1, - desc = { - 'nurls' - } - }, - { - cb = meta_words_function, - ninputs = 9, - desc = { - 'avg_words_len', - 'nshort_words', - 'spaces_rate', - 'double_spaces_rate', - 'non_spaces_rate', - 'ascii_characters_rate', - 'non_ascii_characters_rate', - 'capital_characters_rate', - 'numeric_cahracters' - } - }, -} - -local function rspamd_gen_metatokens(task) - local rspamd_logger = require "rspamd_logger" - local ipairs = ipairs - local metatokens = {} - local cached = task:cache_get('metatokens') - - if cached then - return cached - else - for _,mt in ipairs(metafunctions) do - local ct = mt.cb(task) - for i,tok in ipairs(ct) do - rspamd_logger.debugm(N, task, "metatoken: %s = %s", mt.desc[i], tok) - table.insert(metatokens, tok) - end - end - - task:cache_set('metatokens', metatokens) - end - - return metatokens -end - -exports.rspamd_gen_metatokens = rspamd_gen_metatokens - -local function rspamd_count_metatokens() - local ipairs = ipairs - local total = 0 - for _,mt in ipairs(metafunctions) do - total = total + mt.ninputs - end - - return total -end - -exports.rspamd_count_metatokens = rspamd_count_metatokens - -return exports diff --git a/lualib/rspamadm/configwizard.lua b/lualib/rspamadm/configwizard.lua index a86bb78ec..fc9005ce0 100644 --- a/lualib/rspamadm/configwizard.lua +++ b/lualib/rspamadm/configwizard.lua @@ -19,7 +19,7 @@ local local_conf = rspamd_paths['CONFDIR'] local rspamd_util = require "rspamd_util" local rspamd_logger = require "rspamd_logger" local lua_util = require "lua_util" -local lua_stat_tools = require "stat_tools" +local lua_stat_tools = require "lua_stat" local lua_redis = require "lua_redis" local ucl = require "ucl" diff --git a/lualib/rspamadm/stat_convert.lua b/lualib/rspamadm/stat_convert.lua index f70581fa4..6ad3b0332 100644 --- a/lualib/rspamadm/stat_convert.lua +++ b/lualib/rspamadm/stat_convert.lua @@ -1,5 +1,5 @@ local lua_redis = require "lua_redis" -local stat_tools = require "stat_tools" +local stat_tools = require "lua_stat" local ucl = require "ucl" local logger = require "rspamd_logger" diff --git a/lualib/rspamd_config_transform.lua b/lualib/rspamd_config_transform.lua deleted file mode 100644 index 0823012ba..000000000 --- a/lualib/rspamd_config_transform.lua +++ /dev/null @@ -1,295 +0,0 @@ ---[[ -Copyright (c) 2017, Vsevolod Stakhov - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -local logger = require "rspamd_logger" - -local function override_defaults(def, override) - if not override then - return def - end - if not def then - return override - end - for k,v in pairs(override) do - if def[k] then - if type(v) == 'table' then - def[k] = override_defaults(def[k], v) - else - def[k] = v - end - else - def[k] = v - end - end - - return def -end - -local function is_implicit(t) - local mt = getmetatable(t) - - return mt and mt.class and mt.class == 'ucl.type.impl_array' -end - -local function metric_pairs(t) - -- collect the keys - local keys = {} - local implicit_array = is_implicit(t) - - local function gen_keys(tbl) - if implicit_array then - for _,v in ipairs(tbl) do - if v.name then - table.insert(keys, {v.name, v}) - v.name = nil - else - -- Very tricky to distinguish: - -- group {name = "foo" ... } + group "blah" { ... } - for gr_name,gr in pairs(v) do - if type(gr_name) ~= 'number' then - -- We can also have implicit arrays here - local gr_implicit = is_implicit(gr) - - if gr_implicit then - for _,gr_elt in ipairs(gr) do - table.insert(keys, {gr_name, gr_elt}) - end - else - table.insert(keys, {gr_name, gr}) - end - end - end - end - end - else - if tbl.name then - table.insert(keys, {tbl.name, tbl}) - tbl.name = nil - else - for k,v in pairs(tbl) do - if type(k) ~= 'number' then - -- We can also have implicit arrays here - local sym_implicit = is_implicit(v) - - if sym_implicit then - for _,elt in ipairs(v) do - table.insert(keys, {k, elt}) - end - else - table.insert(keys, {k, v}) - end - end - end - end - end - end - - gen_keys(t) - - -- return the iterator function - local i = 0 - return function() - i = i + 1 - if keys[i] then - return keys[i][1], keys[i][2] - end - end -end - -local function group_transform(cfg, k, v) - if v.name then k = v.name end - - local new_group = { - symbols = {} - } - - if v.enabled then new_group.enabled = v.enabled end - if v.disabled then new_group.disabled = v.disabled end - if v.max_score then new_group.max_score = v.max_score end - - if v.symbol then - for sk,sv in metric_pairs(v.symbol) do - if sv.name then - sk = sv.name - sv.name = nil -- Remove field - end - - new_group.symbols[sk] = sv - end - end - - if not cfg.group then cfg.group = {} end - - if cfg.group[k] then - cfg.group[k] = override_defaults(cfg.group[k], new_group) - else - cfg.group[k] = new_group - end - - logger.infox("overriding group %s from the legacy metric settings", k) -end - -local function symbol_transform(cfg, k, v) - -- first try to find any group where there is a definition of this symbol - for gr_n, gr in pairs(cfg.group) do - if gr.symbols and gr.symbols[k] then - -- We override group symbol with ungrouped symbol - logger.infox("overriding group symbol %s in the group %s", k, gr_n) - gr.symbols[k] = override_defaults(gr.symbols[k], v) - return - end - end - -- Now check what Rspamd knows about this symbol - local sym = rspamd_config:get_metric_symbol(k) - - if not sym or not sym.group then - -- Otherwise we just use group 'ungrouped' - if not cfg.group.ungrouped then - cfg.group.ungrouped = { - symbols = {} - } - end - - cfg.group.ungrouped.symbols[k] = v - logger.debugx("adding symbol %s to the group 'ungrouped'", k) - end -end - -local function test_groups(groups) - local all_symbols = {} - for gr_name, gr in pairs(groups) do - if not gr.symbols then - local cnt = 0 - for _,_ in pairs(gr) do cnt = cnt + 1 end - - if cnt == 0 then - logger.errx('group %s is empty', gr_name) - else - logger.infox('group %s has no symbols', gr_name) - end - - else - for sn,_ in pairs(gr.symbols) do - if all_symbols[sn] then - logger.errx('symbol %s has registered in multiple groups: %s and %s', - sn, all_symbols[sn], gr_name) - else - all_symbols[sn] = gr_name - end - end - end - end -end - -local function convert_metric(cfg, metric) - if metric.actions then - cfg.actions = override_defaults(cfg.actions, metric.actions) - logger.infox("overriding actions from the legacy metric settings") - end - if metric.unknown_weight then - cfg.actions.unknown_weight = metric.unknown_weight - end - - if metric.subject then - logger.infox("overriding subject from the legacy metric settings") - cfg.actions.subject = metric.subject - end - - if metric.group then - for k, v in metric_pairs(metric.group) do - group_transform(cfg, k, v) - end - else - if not cfg.group then - cfg.group = { - ungrouped = { - symbols = {} - } - } - end - end - - if metric.symbol then - for k, v in metric_pairs(metric.symbol) do - symbol_transform(cfg, k, v) - end - end - - return cfg -end - --- Converts a table of groups indexed by number (implicit array) to a --- merged group definition -local function merge_groups(groups) - local ret = {} - for k,gr in pairs(groups) do - if type(k) == 'number' then - for key,sec in pairs(gr) do - ret[key] = sec - end - else - ret[k] = gr - end - end - - return ret -end - -return function(cfg) - local ret = false - - if cfg['metric'] then - for _, v in metric_pairs(cfg.metric) do - cfg = convert_metric(cfg, v) - end - ret = true - end - - if not cfg.actions then - logger.errx('no actions defined') - else - -- Perform sanity check for actions - local actions_defs = {'greylist', 'add header', 'add_header', - 'rewrite subject', 'rewrite_subject', 'reject'} - - if not cfg.actions['no action'] and not cfg.actions['no_action'] and - not cfg.actions['accept'] then - for _,d in ipairs(actions_defs) do - if cfg.actions[d] then - if cfg.actions[d] < 0 then - cfg.actions['no action'] = cfg.actions[d] - 0.001 - logger.infox('set no action score to: %s, as action %s has negative score', - cfg.actions['no action'], d) - break - end - end - end - end - end - - if not cfg.group then - logger.errx('no symbol groups defined') - else - if cfg.group[1] then - -- We need to merge groups - cfg.group = merge_groups(cfg.group) - ret = true - end - test_groups(cfg.group) - end - - return ret, cfg -end diff --git a/lualib/stat_tools.lua b/lualib/stat_tools.lua deleted file mode 100644 index 4f5cafe3c..000000000 --- a/lualib/stat_tools.lua +++ /dev/null @@ -1,502 +0,0 @@ ---[[ -Copyright (c) 2018, Vsevolod Stakhov - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -local logger = require "rspamd_logger" -local sqlite3 = require "rspamd_sqlite3" -local util = require "rspamd_util" -local lua_redis = require "lua_redis" -local exports = {} - -local N = "stat_tools" -- luacheck: ignore (maybe unused) - --- Performs synchronous conversion of redis schema -local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expire) - - -- Old schema is the following one: - -- Keys are named [] - -- Elements are placed within hash: - -- BAYES_SPAM -> {: , : ...} - -- In new schema it is changed to a more extensible schema: - -- Keys are named RS[]_ -> {'H': , 'S': } - -- So we can expire individual records, measure most popular elements by zranges, - -- add new fields, such as tokens etc - - local res,conn = lua_redis.redis_connect_sync(redis_params, true) - - if not res then - logger.errx("cannot connect to redis server") - return false - end - - -- KEYS[1]: key to check (e.g. 'BAYES_SPAM') - -- KEYS[2]: hash key ('S' or 'H') - -- KEYS[3]: expire - local lua_script = [[ -local keys = redis.call('SMEMBERS', KEYS[1]..'_keys') -local nconverted = 0 - -for _,k in ipairs(keys) do - local elts = redis.call('HGETALL', k) - - for k,v in pairs(elts) do - local neutral_prefix = string.gsub(k, KEYS[1], 'RS') - local nkey = string.format('%s_%s', neutral_prefix, k) - redis.call('HSET', nkey, KEYS[2], v) - if KEYS[4] and tonumber(KEYS[3]) ~= 0 then - redis.call('EXPIRE', nkey, KEYS[3]) - end - nconverted = nconverted + 1 - end -end - -return nconverted -]] - - conn:add_cmd('EVAL', {lua_script, '3', symbol_spam, 'S', tostring(expire)}) - local ret - ret, res = conn:exec() - - if not ret then - logger.errx('error converting symbol %s', symbol_spam) - return false - else - logger.messagex('converted %s elements from symbol %s', res, symbol_spam) - end - - conn:add_cmd('EVAL', {lua_script, '3', symbol_ham, 'H', tostring(expire)}) - ret, res = conn:exec() - - if not ret then - logger.errx('error converting symbol %s', symbol_ham) - return false - else - logger.messagex('converted %s elements from symbol %s', res, symbol_ham) - end - - -- We can now convert metadata: set + learned + version - -- KEYS[1]: key to check (e.g. 'BAYES_SPAM') - -- KEYS[2]: learn key (e.g. 'learns_spam' or 'learns_ham') - lua_script = [[ -local keys = redis.call('SMEMBERS', KEYS[1]..'_keys') - -for _,k in ipairs(keys) do - local learns = redis.call('HGET', k, 'learns') - local neutral_prefix = string.gsub(k, KEYS[1], 'RS') - - redis.call('HSET', neutral_prefix, KEYS[2], learns) - redis.call('SADD', KEYS[1]..'_keys', neutral_prefix) - redis.call('SREM', KEYS[1]..'_keys', k) - redis.call('DEL', k) - redis.call('SET', KEYS[1]..'_version', '2') -end -]] - - conn:add_cmd('EVAL', {lua_script, '2', symbol_spam, 'learns_spam'}) - ret = conn:exec() - - if not ret then - logger.errx('error converting metadata for symbol %s', symbol_spam) - return false - end - - conn:add_cmd('EVAL', {lua_script, '2', symbol_ham, 'learns_ham'}) - ret = conn:exec() - - if not ret then - logger.errx('error converting metadata for symbol %s', symbol_ham) - return false - end - - return true -end - -exports.convert_bayes_schema = convert_bayes_schema - --- It now accepts both ham and spam databases --- parameters: --- redis_params - how do we connect to a redis server --- sqlite_db_spam - name for sqlite database with spam tokens --- sqlite_db_ham - name for sqlite database with ham tokens --- symbol_ham - name for symbol representing spam, e.g. BAYES_SPAM --- symbol_spam - name for symbol representing ham, e.g. BAYES_HAM --- learn_cache_spam - name for sqlite database with spam learn cache --- learn_cache_ham - name for sqlite database with ham learn cache --- reset_previous - if true, then the old database is flushed (slow) -local function convert_sqlite_to_redis(redis_params, - sqlite_db_spam, sqlite_db_ham, symbol_spam, symbol_ham, - learn_cache_db, expire, reset_previous) - local nusers = 0 - local lim = 1000 -- Update each 1000 tokens - local users_map = {} - local converted = 0 - - local db_spam = sqlite3.open(sqlite_db_spam) - if not db_spam then - logger.errx('Cannot open source db: %s', sqlite_db_spam) - return false - end - local db_ham = sqlite3.open(sqlite_db_ham) - if not db_ham then - logger.errx('Cannot open source db: %s', sqlite_db_ham) - return false - end - - local res,conn = lua_redis.redis_connect_sync(redis_params, true) - - if not res then - logger.errx("cannot connect to redis server") - return false - end - - if reset_previous then - -- Do a more complicated cleanup - -- execute a lua script that cleans up data - local script = [[ -local members = redis.call('SMEMBERS', KEYS[1]..'_keys') - -for _,prefix in ipairs(members) do - local keys = redis.call('KEYS', prefix..'*') - redis.call('DEL', keys) -end -]] - -- Common keys - for _,sym in ipairs({symbol_spam, symbol_ham}) do - logger.messagex('Cleaning up old data for %s', sym) - conn:add_cmd('EVAL', {script, '1', sym}) - conn:exec() - conn:add_cmd('DEL', {sym .. "_version"}) - conn:add_cmd('DEL', {sym .. "_keys"}) - conn:exec() - end - - if learn_cache_db then - -- Cleanup learned_cache - logger.messagex('Cleaning up old data learned cache') - conn:add_cmd('DEL', {"learned_ids"}) - conn:exec() - end - end - - local function convert_db(db, is_spam) - -- Map users and languages - local what = 'ham' - if is_spam then - what = 'spam' - end - - local learns = {} - db:sql('BEGIN;') - -- Fill users mapping - for row in db:rows('SELECT * FROM users;') do - if row.id == '0' then - users_map[row.id] = '' - else - users_map[row.id] = row.name - end - learns[row.id] = row.learns - nusers = nusers + 1 - end - - -- Workaround for old databases - for row in db:rows('SELECT * FROM languages') do - if learns['0'] then - learns['0'] = learns['0'] + row.learns - else - learns['0'] = row.learns - end - end - - local function send_batch(tokens, prefix) - -- We use the new schema: RS[user]_token -> H=ham count - -- S=spam count - local hash_key = 'H' - if is_spam then - hash_key = 'S' - end - for _,tok in ipairs(tokens) do - -- tok schema: - -- tok[1] = token_id (uint64 represented as a string) - -- tok[2] = token value (number) - -- tok[3] = user_map[user_id] or '' - local rkey = string.format('%s%s_%s', prefix, tok[3], tok[1]) - conn:add_cmd('HINCRBYFLOAT', {rkey, hash_key, tostring(tok[2])}) - - if expire and expire ~= 0 then - conn:add_cmd('EXPIRE', {rkey, tostring(expire)}) - end - end - - return conn:exec() - end - -- Fill tokens, sending data to redis each `lim` records - - local ntokens = db:query('SELECT count(*) as c FROM tokens')['c'] - local tokens = {} - local num = 0 - local total = 0 - - for row in db:rows('SELECT token,value,user FROM tokens;') do - local user = '' - if row.user ~= 0 and users_map[row.user] then - user = users_map[row.user] - end - - table.insert(tokens, {row.token, row.value, user}) - num = num + 1 - total = total + 1 - if num > lim then - -- TODO: we use the default 'RS' prefix, it can be false in case of - -- classifiers with labels - local ret,err_str = send_batch(tokens, 'RS') - if not ret then - logger.errx('Cannot send tokens to the redis server: ' .. err_str) - db:sql('COMMIT;') - return false - end - - num = 0 - tokens = {} - end - - io.write(string.format('Processed batch %s: %s/%s\r', what, total, ntokens)) - end - -- Last batch - if #tokens > 0 then - local ret,err_str = send_batch(tokens, 'RS') - if not ret then - logger.errx('Cannot send tokens to the redis server: ' .. err_str) - db:sql('COMMIT;') - return false - end - - io.write(string.format('Processed batch %s: %s/%s\r', what, total, ntokens)) - end - io.write('\n') - - converted = converted + total - - -- Close DB - db:sql('COMMIT;') - local symbol = symbol_ham - local learns_elt = "learns_ham" - - if is_spam then - symbol = symbol_spam - learns_elt = "learns_spam" - end - - for id,learned in pairs(learns) do - local user = users_map[id] - if not conn:add_cmd('HSET', {'RS' .. user, learns_elt, learned}) then - logger.errx('Cannot update learns for user: ' .. user) - return false - end - if user ~= '' then - if not conn:add_cmd('SADD', {symbol .. '_keys', 'RS' .. user}) then - logger.errx('Cannot update learns for user: ' .. user) - return false - end - end - end - -- Set version - conn:add_cmd('SET', {symbol..'_version', '2'}) - return conn:exec() - end - - logger.messagex('Convert spam tokens') - if not convert_db(db_spam, true) then - return false - end - - logger.messagex('Convert ham tokens') - if not convert_db(db_ham, false) then - return false - end - - if learn_cache_db then - logger.messagex('Convert learned ids from %s', learn_cache_db) - local db = sqlite3.open(learn_cache_db) - local ret = true - local total = 0 - - if not db then - logger.errx('Cannot open cache database: ' .. learn_cache_db) - return false - end - - db:sql('BEGIN;') - - for row in db:rows('SELECT * FROM learns;') do - local is_spam - local digest = tostring(util.encode_base32(row.digest)) - - if row.flag == '0' then - is_spam = '-1' - else - is_spam = '1' - end - - if not conn:add_cmd('HSET', {'learned_ids', digest, is_spam}) then - logger.errx('Cannot add hash: ' .. digest) - ret = false - else - total = total + 1 - end - end - db:sql('COMMIT;') - - if ret then - conn:exec() - end - - if ret then - logger.messagex('Converted %s cached items from sqlite3 learned cache to redis', - total) - else - logger.errx('Error occurred during sending data to redis') - end - end - - logger.messagex('Migrated %s tokens for %s users for symbols (%s, %s)', - converted, nusers, symbol_spam, symbol_ham) - return true -end - -exports.convert_sqlite_to_redis = convert_sqlite_to_redis - --- Loads sqlite3 based classifiers and output data in form of array of objects: --- [ --- { --- symbol_spam = XXX --- symbol_ham = YYY --- db_spam = XXX.sqlite --- db_ham = YYY.sqlite --- learn_cahe = ZZZ.sqlite --- per_user = true/false --- label = str --- } --- ] -local function load_sqlite_config(cfg) - local result = {} - - local function parse_classifier(cls) - local tbl = {} - if cls.cache then - local cache = cls.cache - if cache.type == 'sqlite3' and (cache.file or cache.path) then - tbl.learn_cache = (cache.file or cache.path) - end - end - - if cls.per_user then - tbl.per_user = cls.per_user - end - - if cls.label then - tbl.label = cls.label - end - - local statfiles = cls.statfile - for _,stf in ipairs(statfiles) do - local path = (stf.file or stf.path or stf.db or stf.dbname) - local symbol = stf.symbol or 'undefined' - - if not path then - logger.errx('no path defined for statfile %s', symbol) - else - - local spam - if stf.spam then - spam = stf.spam - else - if string.match(symbol:upper(), 'SPAM') then - spam = true - else - spam = false - end - end - - if spam then - tbl.symbol_spam = symbol - tbl.db_spam = path - else - tbl.symbol_ham = symbol - tbl.db_ham = path - end - end - end - - if tbl.symbol_spam and tbl.symbol_ham and tbl.db_ham and tbl.db_spam then - table.insert(result, tbl) - end - end - - local classifier = cfg.classifier - - if classifier then - if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.backend and cls.backend == 'sqlite3' then - parse_classifier(cls) - end - end - else - if classifier.bayes then - classifier = classifier.bayes - if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.backend and cls.backend == 'sqlite3' then - parse_classifier(cls) - end - end - else - if classifier.backend and classifier.backend == 'sqlite3' then - parse_classifier(classifier) - end - end - end - end - end - - return result -end - -exports.load_sqlite_config = load_sqlite_config - --- A helper method that suggests a user how to configure Redis based --- classifier based on the existing sqlite classifier -local function redis_classifier_from_sqlite(sqlite_classifier) - local result = { - backend = 'redis', - cache = { - backend = 'redis' - }, - statfile = { - [sqlite_classifier.symbol_spam] = { - spam = true - }, - [sqlite_classifier.symbol_ham] = { - spam = false - } - } - } - - return {classifier = {bayes = result}} -end - -exports.redis_classifier_from_sqlite = redis_classifier_from_sqlite - -return exports diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index 889bc7072..97148d22b 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -3549,7 +3549,7 @@ rspamd_rcl_maybe_apply_lua_transform (struct rspamd_config *cfg) gint err_idx, ret; GString *tb; gchar str[PATH_MAX]; - static const char *transform_script = "rspamd_config_transform"; + static const char *transform_script = "lua_cfg_transform"; g_assert (L != NULL); diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua index c87946aff..29faa1258 100644 --- a/src/plugins/lua/arc.lua +++ b/src/plugins/lua/arc.lua @@ -16,12 +16,12 @@ limitations under the License. local rspamd_logger = require "rspamd_logger" local lua_util = require "lua_util" -local dkim_sign_tools = require "dkim_sign_tools" +local dkim_sign_tools = require "lua_dkim_tools" local rspamd_util = require "rspamd_util" local rspamd_rsa_privkey = require "rspamd_rsa_privkey" local rspamd_rsa = require "rspamd_rsa" local fun = require "fun" -local auth_results = require "auth_results" +local auth_results = require "lua_auth_results" local hash = require "rspamd_cryptobox_hash" if confighelp then diff --git a/src/plugins/lua/dkim_signing.lua b/src/plugins/lua/dkim_signing.lua index 75720df9d..de9a286ff 100644 --- a/src/plugins/lua/dkim_signing.lua +++ b/src/plugins/lua/dkim_signing.lua @@ -17,7 +17,7 @@ limitations under the License. local lutil = require "lua_util" local rspamd_logger = require "rspamd_logger" -local dkim_sign_tools = require "dkim_sign_tools" +local dkim_sign_tools = require "lua_dkim_tools" local rspamd_util = require "rspamd_util" if confighelp then diff --git a/src/plugins/lua/fann_redis.lua b/src/plugins/lua/fann_redis.lua index 3120d8b18..828060240 100644 --- a/src/plugins/lua/fann_redis.lua +++ b/src/plugins/lua/fann_redis.lua @@ -27,7 +27,7 @@ local rspamd_util = require "rspamd_util" local rspamd_redis = require "lua_redis" local lua_util = require "lua_util" local fun = require "fun" -local meta_functions = require "meta_functions" +local meta_functions = require "lua_meta" local use_torch = false local torch local nn diff --git a/src/plugins/lua/milter_headers.lua b/src/plugins/lua/milter_headers.lua index a0cca43d3..176597b60 100644 --- a/src/plugins/lua/milter_headers.lua +++ b/src/plugins/lua/milter_headers.lua @@ -350,7 +350,7 @@ local function milter_headers(task) routines['authentication-results'] = function() if skip_wanted('authentication-results') then return end - local ar = require "auth_results" + local ar = require "lua_auth_results" if settings.routines['authentication-results'].remove then remove[settings.routines['authentication-results'].header] = diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua index f77e2ae76..d18b79bfe 100644 --- a/src/plugins/lua/ratelimit.lua +++ b/src/plugins/lua/ratelimit.lua @@ -50,7 +50,7 @@ local rspamd_util = require "rspamd_util" local rspamd_lua_utils = require "lua_util" local lua_redis = require "lua_redis" local fun = require "fun" -local lua_maps = require "maps" +local lua_maps = require "lua_maps" local lua_util = require "lua_util" local user_keywords = {'user'} diff --git a/src/plugins/lua/reputation.lua b/src/plugins/lua/reputation.lua index 0d870eee8..9c5f2aaad 100644 --- a/src/plugins/lua/reputation.lua +++ b/src/plugins/lua/reputation.lua @@ -26,7 +26,7 @@ local N = 'reputation' local rspamd_logger = require "rspamd_logger" local rspamd_util = require "rspamd_util" local lua_util = require "lua_util" -local lua_maps = require "maps" +local lua_maps = require "lua_maps" local hash = require 'rspamd_cryptobox_hash' local fun = require "fun" local redis_params = nil diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua index 1963f9d70..d35c91adf 100644 --- a/src/plugins/lua/settings.lua +++ b/src/plugins/lua/settings.lua @@ -23,7 +23,7 @@ end -- https://rspamd.com/doc/configuration/settings.html local rspamd_logger = require "rspamd_logger" -local rspamd_maps = require "maps" +local rspamd_maps = require "lua_maps" local redis_params local settings = {} -- cgit v1.2.3