diff options
| -rw-r--r-- | conf/options.inc | 2 | ||||
| -rw-r--r-- | lualib/lua_util.lua | 58 | ||||
| -rw-r--r-- | src/CMakeLists.txt | 3 | ||||
| -rw-r--r-- | src/libserver/spf.c | 7 | ||||
| -rw-r--r-- | src/lua/lua_spf.c | 106 | ||||
| -rw-r--r-- | src/lua/lua_task.c | 21 | ||||
| -rw-r--r-- | src/plugins/dkim_check.c | 3 | ||||
| -rw-r--r-- | src/plugins/lua/dmarc.lua | 31 | ||||
| -rw-r--r-- | src/plugins/lua/spf.lua | 230 | ||||
| -rw-r--r-- | src/plugins/spf.c | 670 | ||||
| -rw-r--r-- | test/functional/configs/dmarc.conf | 1 |
11 files changed, 392 insertions, 740 deletions
diff --git a/conf/options.inc b/conf/options.inc index e635262e3..78fd3bee6 100644 --- a/conf/options.inc +++ b/conf/options.inc @@ -13,7 +13,7 @@ # # Relevant documentation: https://rspamd.com/doc/configuration/options.html -filters = "chartable,dkim,spf,regexp,fuzzy_check"; +filters = "chartable,dkim,regexp,fuzzy_check"; raw_mode = false; one_shot = false; cache_file = "$DBDIR/symbols.cache"; diff --git a/lualib/lua_util.lua b/lualib/lua_util.lua index 207bdc0dc..89a4016b2 100644 --- a/lualib/lua_util.lua +++ b/lualib/lua_util.lua @@ -1293,4 +1293,62 @@ exports.toboolean = function(v) end end +---[[[ +-- @function lua_util.config_check_local_or_authed(config, modname) +-- Reads check_local and check_authed from the config as this is used in many modules +-- @param {rspamd_config} config `rspamd_config` global +-- @param {name} module name +-- @return {boolean} v converted to boolean +--]]] +exports.config_check_local_or_authed = function(rspamd_config, modname, def_local, def_authed) + local check_local = def_local or false + local check_authed = def_authed or false + + local function try_section(where) + local ret = false + local opts = rspamd_config:get_all_opt(where) + if type(opts) == 'table' then + if type(opts['check_local']) == 'boolean' then + check_local = opts['check_local'] + ret = true + end + if type(opts['check_authed']) == 'boolean' then + check_authed = opts['check_authed'] + ret = true + end + end + + return ret + end + + if not try_section(modname) then + try_section('options') + end + + return {check_local, check_authed} +end + +---[[[ +-- @function lua_util.is_skip_local_or_authed(task, conf[, ip]) +-- Returns `true` if local or authenticated task should be skipped for this module +-- @param {rspamd_task} task +-- @param {table} conf table returned from `config_check_local_or_authed` +-- @param {rspamd_ip} ip optional ip address (can be obtained from a task) +-- @return {boolean} true if check should be skipped +--]]] +exports.is_skip_local_or_authed = function(task, conf, ip) + if not ip then + ip = task:get_from_ip() + end + if not conf then + conf = {false, false} + end + if ((not conf[2] and task:get_user()) or + (not conf[1] and type(ip) == 'userdata' and ip:is_local())) then + return true + end + + return false +end + return exports diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 480578831..9a34d2ac4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -96,11 +96,10 @@ SET(RSPAMDSRC controller.c SET(PLUGINSSRC plugins/regexp.c plugins/chartable.c plugins/fuzzy_check.c - plugins/spf.c plugins/dkim_check.c libserver/rspamd_control.c) -SET(MODULES_LIST regexp chartable fuzzy_check spf dkim) +SET(MODULES_LIST regexp chartable fuzzy_check dkim) SET(WORKERS_LIST normal controller fuzzy rspamd_proxy) IF (ENABLE_HYPERSCAN MATCHES "ON") LIST(APPEND WORKERS_LIST "hs_helper") diff --git a/src/libserver/spf.c b/src/libserver/spf.c index 5fac43aa1..ec8decb50 100644 --- a/src/libserver/spf.c +++ b/src/libserver/spf.c @@ -2436,11 +2436,10 @@ rspamd_spf_resolve (struct rspamd_task *task, spf_cb_t callback, if (cached) { cached->flags |= RSPAMD_SPF_FLAG_CACHED; - } - - callback (cached, task, cbdata); + callback (cached, task, cbdata); - return TRUE; + return TRUE; + } } diff --git a/src/lua/lua_spf.c b/src/lua/lua_spf.c index cf88bc838..478a7bbc2 100644 --- a/src/lua/lua_spf.c +++ b/src/lua/lua_spf.c @@ -54,6 +54,7 @@ static luaL_reg rspamd_spf_record_m[] = { struct rspamd_lua_spf_cbdata { struct rspamd_task *task; lua_State *L; + struct rspamd_symcache_item *item; gint cbref; ref_entry_t ref; }; @@ -153,6 +154,10 @@ lua_spf_dtor (struct rspamd_lua_spf_cbdata *cbd) { if (cbd) { luaL_unref (cbd->L, LUA_REGISTRYINDEX, cbd->cbref); + if (cbd->item) { + rspamd_symcache_item_async_dec_check (cbd->task, cbd->item, + "lua_spf"); + } } } @@ -162,26 +167,36 @@ spf_lua_lib_callback (struct spf_resolved *record, struct rspamd_task *task, { struct rspamd_lua_spf_cbdata *cbd = (struct rspamd_lua_spf_cbdata *)ud; - if (record && (record->flags & RSPAMD_SPF_RESOLVED_NA)) { - lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_NA, record, - "no record found"); - } - else if (record && record->elts->len == 0 && (record->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED)) { - lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_TEMP_FAILED, record, - "temporary resolution error"); - } - else if (record && record->elts->len == 0 && (record->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED)) { - lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, record, - "permanent resolution error"); - } - else if (record && record->elts->len == 0) { - lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, record, - "record is empty"); + if (record) { + if ((record->flags & RSPAMD_SPF_RESOLVED_NA)) { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_NA, NULL, + "no record found"); + } + else if (record->elts->len == 0 && (record->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED)) { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_TEMP_FAILED, NULL, + "temporary resolution error"); + } + else if (record->elts->len == 0 && (record->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED)) { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL, + "permanent resolution error"); + } + else if (record->elts->len == 0) { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL, + "record is empty"); + } + else if (record->domain) { + spf_record_ref (record); + lua_spf_push_result (cbd, record->flags, record, NULL); + spf_record_unref (record); + } + else { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL, + "internal error: non empty record for no domain"); + } } - else if (record && record->domain) { - spf_record_ref (record); - lua_spf_push_result (cbd, record->flags, record, NULL); - spf_record_unref (record); + else { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL, + "internal error: no record"); } REF_RELEASE (cbd); @@ -209,12 +224,21 @@ lua_spf_resolve (lua_State * L) cbd->cbref = luaL_ref (L, LUA_REGISTRYINDEX); /* TODO: make it as an optional parameter */ spf_cred = rspamd_spf_get_cred (task); + cbd->item = rspamd_symcache_get_cur_item (task); + rspamd_symcache_item_async_inc (task, cbd->item, "lua_spf"); REF_INIT_RETAIN (cbd, lua_spf_dtor); if (!rspamd_spf_resolve (task, spf_lua_lib_callback, cbd, spf_cred)) { - msg_info_task ("cannot make spf request for %s", spf_cred->domain); - lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_TEMP_FAILED, - NULL, "DNS failed"); + msg_info_task ("cannot make spf request for %s", + spf_cred ? spf_cred->domain : "empty domain"); + if (spf_cred) { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_TEMP_FAILED, + NULL, "DNS failed"); + } + else { + lua_spf_push_result (cbd, RSPAMD_SPF_RESOLVED_NA, + NULL, "No domain"); + } REF_RELEASE (cbd); } } @@ -276,11 +300,8 @@ spf_check_element (lua_State *L, struct spf_resolved *rec, struct spf_addr *addr if (addr->flags & RSPAMD_SPF_FLAG_TEMPFAIL) { /* Ignore failed addresses */ - lua_pushboolean (L, false); - lua_pushinteger (L, RSPAMD_SPF_FLAG_TEMPFAIL); - lua_pushstring (L, "temp failed"); - return 3; + return -1; } af = rspamd_inet_address_get_af (ip->addr); @@ -333,12 +354,12 @@ spf_check_element (lua_State *L, struct spf_resolved *rec, struct spf_addr *addr if (rec->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) { lua_pushboolean (L, false); lua_pushinteger (L, RSPAMD_SPF_RESOLVED_PERM_FAILED); - lua_spf_push_spf_addr (L, addr); + lua_pushstring (L, "any perm fail"); } else if (rec->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED) { lua_pushboolean (L, false); lua_pushinteger (L, RSPAMD_SPF_RESOLVED_TEMP_FAILED); - lua_spf_push_spf_addr (L, addr); + lua_pushfstring (L, "any temp fail"); } else { lua_pushboolean (L, true); @@ -365,7 +386,7 @@ spf_check_element (lua_State *L, struct spf_resolved *rec, struct spf_addr *addr * 1. Boolean check result * 2. If result is `false` then the second value is the error flag (e.g. rspamd_spf.flags.temp_fail), otherwise it will be an SPF method * 3. If result is `false` then this will be an error string, otherwise - an SPF string (e.g. `mx` or `ip4:x.y.z.1`) - * @param {rspamd_ip} ip address + * @param {rspamd_ip|string} ip address * @return {result,flag_or_policy,error_or_addr} - triplet */ static gint @@ -374,8 +395,29 @@ lua_spf_record_check_ip (lua_State *L) struct spf_resolved *record = * (struct spf_resolved **)rspamd_lua_check_udata (L, 1, SPF_RECORD_CLASS); - struct rspamd_lua_ip *ip = lua_check_ip (L, 2); + struct rspamd_lua_ip *ip = NULL; gint nres = 0; + gboolean need_free_ip = FALSE; + + if (lua_type (L, 2) == LUA_TUSERDATA) { + ip = lua_check_ip (L, 2); + } + else if (lua_type (L, 2) == LUA_TSTRING) { + const gchar *ip_str; + gsize iplen; + + ip = g_malloc0 (sizeof (struct rspamd_lua_ip)); + ip_str = lua_tolstring (L, 2, &iplen); + + if (!rspamd_parse_inet_address (&ip->addr, + ip_str, iplen, RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) { + g_free (ip); + ip = NULL; + } + else { + need_free_ip = TRUE; + } + } if (record && ip && ip->addr) { for (guint i = 0; i < record->elts->len; i ++) { @@ -389,6 +431,10 @@ lua_spf_record_check_ip (lua_State *L) return luaL_error (L, "invalid arguments"); } + if (need_free_ip) { + g_free (ip); + } + lua_pushboolean (L, false); lua_pushinteger (L, RSPAMD_SPF_RESOLVED_NA); lua_pushstring (L, "no result"); diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index aff6d8499..26ad15dec 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -4889,14 +4889,19 @@ lua_task_get_timeval (lua_State *L) struct timeval tv; if (task != NULL) { - double_to_tv (task->task_timestamp, &tv); - lua_createtable (L, 0, 2); - lua_pushstring (L, "tv_sec"); - lua_pushinteger (L, (lua_Integer)tv.tv_sec); - lua_settable (L, -3); - lua_pushstring (L, "tv_usec"); - lua_pushinteger (L, (lua_Integer)tv.tv_usec); - lua_settable (L, -3); + if (lua_isboolean (L, 2) && !!lua_toboolean (L, 2)) { + lua_pushnumber (L, task->task_timestamp); + } + else { + double_to_tv (task->task_timestamp, &tv); + lua_createtable (L, 0, 2); + lua_pushstring (L, "tv_sec"); + lua_pushinteger (L, (lua_Integer) tv.tv_sec); + lua_settable (L, -3); + lua_pushstring (L, "tv_usec"); + lua_pushinteger (L, (lua_Integer) tv.tv_usec); + lua_settable (L, -3); + } } else { return luaL_error (L, "invalid arguments"); diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index 3a88d9bb6..133feef2f 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -1121,7 +1121,8 @@ dkim_symbol_callback (struct rspamd_task *task, GError *err = NULL; struct rspamd_mime_header *rh, *rh_cur; struct dkim_check_result *res = NULL, *cur; - guint checked = 0, *dmarc_checks; + guint checked = 0; + gdouble *dmarc_checks; struct dkim_ctx *dkim_module_ctx = dkim_get_context (task->cfg); /* Allow dmarc */ diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua index c2fc4d9bb..e6a520e8e 100644 --- a/src/plugins/lua/dmarc.lua +++ b/src/plugins/lua/dmarc.lua @@ -23,8 +23,7 @@ local rspamd_url = require "rspamd_url" local rspamd_util = require "rspamd_util" local rspamd_redis = require "lua_redis" local lua_util = require "lua_util" -local check_local = false -local check_authed = false +local auth_and_local_conf if confighelp then return @@ -559,7 +558,7 @@ local function dmarc_callback(task) local hfromdom = ((from or E)[1] or E).domain local dmarc_domain local ip_addr = task:get_ip() - local dmarc_checks = task:get_mempool():get_variable('dmarc_checks', 'int') or 0 + local dmarc_checks = task:get_mempool():get_variable('dmarc_checks', 'double') or 0 local seen_invalid = false if dmarc_checks ~= 2 then @@ -567,8 +566,7 @@ local function dmarc_callback(task) return end - if ((not check_authed and task:get_user()) or - (not check_local and ip_addr and ip_addr:is_local())) then + if lua_util.is_skip_local_or_authed(task, auth_and_local_conf, ip_addr) then rspamd_logger.infox(task, "skip DMARC checks for local networks and authorized users") return end @@ -709,29 +707,14 @@ local function dmarc_callback(task) end -local function try_opts(where) - local ret = false - local opts = rspamd_config:get_all_opt(where) - if type(opts) == 'table' then - if type(opts['check_local']) == 'boolean' then - check_local = opts['check_local'] - ret = true - end - if type(opts['check_authed']) == 'boolean' then - check_authed = opts['check_authed'] - ret = true - end - end - - return ret -end - -if not try_opts(N) then try_opts('options') end - local opts = rspamd_config:get_all_opt('dmarc') if not opts or type(opts) ~= 'table' then return end + +auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N, + false, false) + no_sampling_domains = rspamd_map_add(N, 'no_sampling_domains', 'map', 'Domains not to apply DMARC sampling to') no_reporting_domains = rspamd_map_add(N, 'no_reporting_domains', 'map', 'Domains not to apply DMARC reporting to') diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua new file mode 100644 index 000000000..f664661f9 --- /dev/null +++ b/src/plugins/lua/spf.lua @@ -0,0 +1,230 @@ +--[[ +Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local N = "spf" +local lua_util = require "lua_util" +local rspamd_spf = require "rspamd_spf" +local bit = require "bit" +local rspamd_logger = require "rspamd_logger" + +if confighelp then + rspamd_config:add_example(nil, N, + 'Performs SPF checks', + [[ +spf { + # Enable module + enabled = true + # Number of elements in the cache of parsed SPF records + spf_cache_size = 2048; + # Default max expire for an element in this cache + spf_cache_expire = 1d; + # Whitelist IPs from checks + whitelist = "/path/to/some/file"; + # Maximum number of recursive DNS subrequests (e.g. includes chanin length) + max_dns_nesting = 10; + # Maximum count of DNS requests per record + max_dns_requests = 30; + # Minimum TTL enforced for all elements in SPF records + min_cache_ttl = 5m; + # Disable all IPv6 lookups + disable_ipv6 = false; + # Use IP address from a received header produced by this relay (using by attribute) + external_relay = "192.168.1.1"; +} + ]]) + return +end + +local symbols = { + fail = "R_SPF_FAIL", + softfail = "R_SPF_SOFTFAIL", + neutral = "R_SPF_NEUTRAL", + allow = "R_SPF_ALLOW", + dnsfail = "R_SPF_DNSFAIL", + permfail = "R_SPF_PERMFAIL", + na = "R_SPF_NA", +} + +local default_config = { + spf_cache_size = 2048, + max_dns_nesting = 10, + max_dns_requests = 30, + whitelist = nil, + min_cache_ttl = 60 * 5, + disable_ipv6 = false, + symbols = symbols, + external_relay = nil, +} + +local local_config = rspamd_config:get_all_opt('spf') +local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N, + false, false) + +if local_config then + local_config = lua_util.override_defaults(default_config, local_config) +else + local_config = default_config +end + +local function spf_check_callback(task) + + local ip + + if local_config.external_relay then + -- Search received headers to get header produced by an external relay + local rh = task:get_received_headers() or {} + local found = false + + for i,hdr in ipairs(rh) do + if hdr.real_ip and hdr.real_ip == local_config.external_relay then + -- We can use the next header as a source of IP address + if rh[i + 1] then + local nhdr = rh[i + 1] + lua_util.debugm(N, task, 'found external relay %s at received header %s -> %s', + local_config.external_relay, hdr, nhdr.real_ip) + + if nhdr.real_ip then + ip = nhdr.real_ip + found = true + end + end + + break + end + end + if not found then + rspamd_logger.warnx(task, "cannot find external relay with IP %s", + local_config.external_relay) + ip = task:get_from_ip() + end + else + ip = task:get_from_ip() + end + + local function flag_to_symbol(fl) + if bit.band(fl, rspamd_spf.flags.temp_fail) ~= 0 then + return local_config.symbols.dnsfail + elseif bit.band(fl, rspamd_spf.flags.perm_fail) ~= 0 then + return local_config.symbols.permfail + elseif bit.band(fl, rspamd_spf.flags.na) ~= 0 then + return local_config.symbols.na + end + + return 'SPF_UNKNOWN' + end + + local function policy_decode(res) + if res == rspamd_spf.policy.fail then + return local_config.symbols.fail,'-' + elseif res == rspamd_spf.policy.pass then + return local_config.symbols.allow,'+' + elseif res == rspamd_spf.policy.soft_fail then + return local_config.symbols.softfail,'~' + elseif res == rspamd_spf.policy.neutral then + return local_config.symbols.neutral,'?' + end + + return 'SPF_UNKNOWN','?' + end + + local function spf_resolved_cb(record, flags, err) + lua_util.debugm(N, task, 'got spf results: %s flags, %s err', + flags, err) + + if record then + local result, flag_or_policy, error_or_addr = record:check_ip(ip) + + lua_util.debugm(N, task, + 'checked ip %s: result=%s, flag_or_policy=%s, error_or_addr=%s', + ip, flags, err, error_or_addr) + + if result then + local sym,code = policy_decode(flag_or_policy) + local opt = string.format('%s%s', code, error_or_addr.str or '???') + if bit.band(flags, rspamd_spf.flags.cached) ~= 0 then + opt = opt .. ':c' + rspamd_logger.infox(task, + "use cached record for %s (0x%s) in LRU cache for %s seconds", + record:get_domain(), + record:get_digest(), + record:get_ttl() - math.floor(task:get_timeval(true) - + record:get_timestamp())); + end + task:insert_result(sym, 1.0, opt) + else + local sym = flag_to_symbol(flag_or_policy) + task:insert_result(sym, 1.0, error_or_addr) + end + else + local sym = flag_to_symbol(flags) + task:insert_result(sym, 1.0, err) + end + end + + if ip then + if local_config.whitelist and ip and local_config.whitelist:get_key(ip) then + rspamd_logger.infox(task, 'whitelisted SPF checks from %s', + tostring(ip)) + return + end + + if lua_util.is_skip_local_or_authed(task, auth_and_local_conf, ip) then + rspamd_logger.infox(task, 'skip SPF checks for local networks and authorized users') + return + end + + rspamd_spf.resolve(task, spf_resolved_cb) + else + lua_util.debugm(N, task, "spf checks are not possible as no source IP address is defined") + end + + -- FIXME: we actually need to set this variable when we really checked SPF + -- However, the old C module has set it all the times + -- Hence, we follow the same rule for now. It should be better designed at some day + local mpool = task:get_mempool() + local dmarc_checks = mpool:get_variable('dmarc_checks', 'double') or 0 + dmarc_checks = dmarc_checks + 1 + mpool:set_variable('dmarc_checks', dmarc_checks) +end + +-- Register all symbols and init rspamd_spf library +rspamd_spf.config(local_config) +local sym_id = rspamd_config:register_symbol{ + name = 'SPF_CHECK', + type = 'callback', + flags = 'fine,empty', + groups = {'policies','spf'}, + score = 0.0, + callback = spf_check_callback +} + +if local_config.whitelist then + local lua_maps = require "lua_maps" + + local_config.whitelist = lua_maps.map_add_from_ucl(local_config.whitelist, + "radix", "SPF whitelist map") +end + +for _,sym in pairs(local_config.symbols) do + rspamd_config:register_symbol{ + name = sym, + type = 'virtual', + parent = sym_id, + groups = {'policies', 'spf'}, + } +end + + diff --git a/src/plugins/spf.c b/src/plugins/spf.c deleted file mode 100644 index f24bea004..000000000 --- a/src/plugins/spf.c +++ /dev/null @@ -1,670 +0,0 @@ -/*- - * Copyright 2016 Vsevolod Stakhov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/***MODULE:spf - * rspamd module that checks spf records of incoming email - * - * Allowed options: - * - symbol_allow (string): symbol to insert (default: 'R_SPF_ALLOW') - * - symbol_fail (string): symbol to insert (default: 'R_SPF_FAIL') - * - symbol_softfail (string): symbol to insert (default: 'R_SPF_SOFTFAIL') - * - symbol_na (string): symbol to insert (default: 'R_SPF_NA') - * - symbol_dnsfail (string): symbol to insert (default: 'R_SPF_DNSFAIL') - * - symbol_permfail (string): symbol to insert (default: 'R_SPF_PERMFAIL') - * - whitelist (map): map of whitelisted networks - */ - - -#include "config.h" -#include "libmime/message.h" -#include "libserver/spf.h" -#include "libutil/hash.h" -#include "libutil/map.h" -#include "libutil/map_helpers.h" -#include "rspamd.h" -#include "libserver/mempool_vars_internal.h" - -#define DEFAULT_SYMBOL_FAIL "R_SPF_FAIL" -#define DEFAULT_SYMBOL_SOFTFAIL "R_SPF_SOFTFAIL" -#define DEFAULT_SYMBOL_NEUTRAL "R_SPF_NEUTRAL" -#define DEFAULT_SYMBOL_ALLOW "R_SPF_ALLOW" -#define DEFAULT_SYMBOL_DNSFAIL "R_SPF_DNSFAIL" -#define DEFAULT_SYMBOL_PERMFAIL "R_SPF_PERMFAIL" -#define DEFAULT_SYMBOL_NA "R_SPF_NA" -#define DEFAULT_CACHE_SIZE 2048 - -static const gchar *M = "rspamd spf plugin"; - -struct spf_ctx { - struct module_ctx ctx; - const gchar *symbol_fail; - const gchar *symbol_softfail; - const gchar *symbol_neutral; - const gchar *symbol_allow; - const gchar *symbol_dnsfail; - const gchar *symbol_na; - const gchar *symbol_permfail; - - struct rspamd_radix_map_helper *whitelist_ip; - - gboolean check_local; - gboolean check_authed; -}; - -static void spf_symbol_callback (struct rspamd_task *task, - struct rspamd_symcache_item *item, - void *unused); - -/* Initialization */ -gint spf_module_init (struct rspamd_config *cfg, struct module_ctx **ctx); -gint spf_module_config (struct rspamd_config *cfg); -gint spf_module_reconfig (struct rspamd_config *cfg); - -module_t spf_module = { - "spf", - spf_module_init, - spf_module_config, - spf_module_reconfig, - NULL, - RSPAMD_MODULE_VER, - (guint)-1, -}; - -static inline struct spf_ctx * -spf_get_context (struct rspamd_config *cfg) -{ - return (struct spf_ctx *)g_ptr_array_index (cfg->c_modules, - spf_module.ctx_offset); -} - - -gint -spf_module_init (struct rspamd_config *cfg, struct module_ctx **ctx) -{ - struct spf_ctx *spf_module_ctx; - - spf_module_ctx = rspamd_mempool_alloc0 (cfg->cfg_pool, - sizeof (*spf_module_ctx)); - *ctx = (struct module_ctx *)spf_module_ctx; - - rspamd_rcl_add_doc_by_path (cfg, - NULL, - "SPF check plugin", - "spf", - UCL_OBJECT, - NULL, - 0, - NULL, - 0); - - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Map of IP addresses that should be excluded from SPF checks (in addition to `local_networks`)", - "whitelist", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF check is successful", - "symbol_allow", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF policy is set to 'deny'", - "symbol_fail", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF policy is set to 'undefined'", - "symbol_softfail", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF policy is set to 'neutral'", - "symbol_neutral", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF policy is failed due to DNS failure", - "symbol_dnsfail", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if no SPF policy is found", - "symbol_na", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Symbol that is added if SPF policy is invalid", - "symbol_permfail", - UCL_STRING, - NULL, - 0, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Size of SPF parsed records cache", - "spf_cache_size", - UCL_INT, - NULL, - 0, - NULL, - 0); - - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Minimum cached records TTL, 0 to disable (default: 5min)", - "min_cache_ttl", - UCL_INT, - NULL, - RSPAMD_CL_FLAG_UINT, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Maximum number of nested requests (default: " G_STRINGIFY(SPF_MAX_NESTING) ")", - "max_dns_nesting", - UCL_INT, - NULL, - RSPAMD_CL_FLAG_UINT, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Maximum number of dns requests to resolve SPF (default: " G_STRINGIFY(SPF_MAX_DNS_REQUESTS) ")", - "max_dns_requests", - UCL_INT, - NULL, - RSPAMD_CL_FLAG_UINT, - NULL, - 0); - rspamd_rcl_add_doc_by_path (cfg, - "spf", - "Disable ipv6 resolving when doing SPF resolution", - "disable_ipv6", - UCL_BOOLEAN, - NULL, - 0, - NULL, - 0); - - return 0; -} - - -gint -spf_module_config (struct rspamd_config *cfg) -{ - const ucl_object_t *value; - gint res = TRUE, cb_id; - struct spf_ctx *spf_module_ctx = spf_get_context (cfg); - - if (!rspamd_config_is_module_enabled (cfg, "spf")) { - return TRUE; - } - - spf_module_ctx->whitelist_ip = NULL; - - value = rspamd_config_get_module_opt (cfg, "spf", "check_local"); - - if (value == NULL) { - rspamd_config_get_module_opt (cfg, "options", "check_local"); - } - - if (value != NULL) { - spf_module_ctx->check_local = ucl_obj_toboolean (value); - } - else { - spf_module_ctx->check_local = FALSE; - } - - value = rspamd_config_get_module_opt (cfg, "spf", "check_authed"); - - if (value == NULL) { - rspamd_config_get_module_opt (cfg, "options", "check_authed"); - } - - if (value != NULL) { - spf_module_ctx->check_authed = ucl_obj_toboolean (value); - } - else { - spf_module_ctx->check_authed = FALSE; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_fail")) != NULL) { - spf_module_ctx->symbol_fail = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_fail = DEFAULT_SYMBOL_FAIL; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_softfail")) != NULL) { - spf_module_ctx->symbol_softfail = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_softfail = DEFAULT_SYMBOL_SOFTFAIL; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_neutral")) != NULL) { - spf_module_ctx->symbol_neutral = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_neutral = DEFAULT_SYMBOL_NEUTRAL; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_allow")) != NULL) { - spf_module_ctx->symbol_allow = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_allow = DEFAULT_SYMBOL_ALLOW; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_dnsfail")) != NULL) { - spf_module_ctx->symbol_dnsfail = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_dnsfail = DEFAULT_SYMBOL_DNSFAIL; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_na")) != NULL) { - spf_module_ctx->symbol_na = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_na = DEFAULT_SYMBOL_NA; - } - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "symbol_permfail")) != NULL) { - spf_module_ctx->symbol_permfail = ucl_obj_tostring (value); - } - else { - spf_module_ctx->symbol_permfail = DEFAULT_SYMBOL_PERMFAIL; - } - - spf_library_config (ucl_obj_get_key (cfg->rcl_obj, "spf")); - - if ((value = - rspamd_config_get_module_opt (cfg, "spf", "whitelist")) != NULL) { - - rspamd_config_radix_from_ucl (cfg, value, "SPF whitelist", - &spf_module_ctx->whitelist_ip, NULL, NULL); - } - - cb_id = rspamd_symcache_add_symbol (cfg->cache, - "SPF_CHECK", - 0, - spf_symbol_callback, - NULL, - SYMBOL_TYPE_CALLBACK | SYMBOL_TYPE_FINE | SYMBOL_TYPE_EMPTY, -1); - rspamd_config_add_symbol (cfg, - "SPF_CHECK", - 0.0, - "SPF check callback", - "policies", - RSPAMD_SYMBOL_FLAG_IGNORE_METRIC, - 1, - 1); - rspamd_config_add_symbol_group (cfg, "SPF_CHECK", "spf"); - - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_fail, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_softfail, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_permfail, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_na, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_neutral, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_allow, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - rspamd_symcache_add_symbol (cfg->cache, - spf_module_ctx->symbol_dnsfail, 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL, - cb_id); - - - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)rspamd_map_helper_destroy_radix, - spf_module_ctx->whitelist_ip); - - msg_info_config ("init internal spf module"); - - return res; -} - -gint -spf_module_reconfig (struct rspamd_config *cfg) -{ - return spf_module_config (cfg); -} - -static gboolean -spf_check_element (struct spf_resolved *rec, struct spf_addr *addr, - struct rspamd_task *task, gboolean cached) -{ - gboolean res = FALSE; - const guint8 *s, *d; - gchar *spf_result; - guint af, mask, bmask, addrlen; - const gchar *spf_message, *spf_symbol; - struct spf_ctx *spf_module_ctx = spf_get_context (task->cfg); - - if (task->from_addr == NULL) { - return FALSE; - } - - if (addr->flags & RSPAMD_SPF_FLAG_TEMPFAIL) { - /* Ignore failed addresses */ - return FALSE; - } - - af = rspamd_inet_address_get_af (task->from_addr); - /* Basic comparing algorithm */ - if (((addr->flags & RSPAMD_SPF_FLAG_IPV6) && af == AF_INET6) || - ((addr->flags & RSPAMD_SPF_FLAG_IPV4) && af == AF_INET)) { - d = rspamd_inet_address_get_hash_key (task->from_addr, &addrlen); - - if (af == AF_INET6) { - s = (const guint8 *)addr->addr6; - mask = addr->m.dual.mask_v6; - } - else { - s = (const guint8 *)addr->addr4; - mask = addr->m.dual.mask_v4; - } - - /* Compare the first bytes */ - bmask = mask / CHAR_BIT; - if (mask > addrlen * CHAR_BIT) { - msg_info_task ("bad mask length: %d", mask); - } - else if (memcmp (s, d, bmask) == 0) { - if (bmask * CHAR_BIT < mask) { - /* Compare the remaining bits */ - s += bmask; - d += bmask; - mask = (0xff << (CHAR_BIT - (mask - bmask * 8))) & 0xff; - - if ((*s & mask) == (*d & mask)) { - res = TRUE; - } - } - else { - res = TRUE; - } - } - } - else { - if (addr->flags & RSPAMD_SPF_FLAG_ANY) { - res = TRUE; - } - else { - res = FALSE; - } - } - - if (res) { - spf_result = rspamd_mempool_alloc (task->task_pool, - strlen (addr->spf_string) + 5); - - switch (addr->mech) { - case SPF_FAIL: - spf_symbol = spf_module_ctx->symbol_fail; - spf_result[0] = '-'; - spf_message = "(SPF): spf fail"; - if (addr->flags & RSPAMD_SPF_FLAG_ANY) { - if (rec->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) { - msg_info_task ("do not apply SPF failed policy, as we have " - "some addresses unresolved"); - spf_symbol = spf_module_ctx->symbol_permfail; - } - else if (rec->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED) { - msg_info_task ("do not apply SPF failed policy, as we have " - "some addresses unresolved"); - spf_symbol = spf_module_ctx->symbol_dnsfail; - spf_message = "(SPF): spf DNS fail"; - } - } - break; - case SPF_SOFT_FAIL: - spf_symbol = spf_module_ctx->symbol_softfail; - spf_message = "(SPF): spf softfail"; - spf_result[0] = '~'; - - if (addr->flags & RSPAMD_SPF_FLAG_ANY) { - if (rec->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) { - msg_info_task ("do not apply SPF failed policy, as we have " - "some addresses unresolved"); - spf_symbol = spf_module_ctx->symbol_permfail; - } - else if (rec->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED) { - msg_info_task ("do not apply SPF failed policy, as we have " - "some addresses unresolved"); - spf_symbol = spf_module_ctx->symbol_dnsfail; - spf_message = "(SPF): spf DNS fail"; - } - } - break; - case SPF_NEUTRAL: - spf_symbol = spf_module_ctx->symbol_neutral; - spf_message = "(SPF): spf neutral"; - spf_result[0] = '?'; - break; - default: - spf_symbol = spf_module_ctx->symbol_allow; - spf_message = "(SPF): spf allow"; - spf_result[0] = '+'; - break; - } - - gint r = rspamd_strlcpy (spf_result + 1, addr->spf_string, - strlen (addr->spf_string) + 1); - - if (cached) { - rspamd_strlcpy (spf_result + r + 1, ":c", 3); - } - - rspamd_task_insert_result (task, - spf_symbol, - 1, - spf_result); - ucl_object_insert_key (task->messages, - ucl_object_fromstring (spf_message), "spf", 0, - false); - - return TRUE; - } - - return FALSE; -} - -static void -spf_check_list (struct spf_resolved *rec, struct rspamd_task *task, gboolean cached) -{ - guint i; - struct spf_addr *addr; - - if (cached) { - msg_info_task ("use cached record for %s (0x%xuL) in LRU cache for %d seconds", - rec->domain, - rec->digest, - rec->ttl - (guint)(task->task_timestamp - rec->timestamp)); - } - - for (i = 0; i < rec->elts->len; i ++) { - addr = &g_array_index (rec->elts, struct spf_addr, i); - if (spf_check_element (rec, addr, task, cached)) { - break; - } - } -} - -static void -spf_plugin_callback (struct spf_resolved *record, struct rspamd_task *task, - gpointer ud) -{ - struct rspamd_symcache_item *item = (struct rspamd_symcache_item *)ud; - struct spf_ctx *spf_module_ctx = spf_get_context (task->cfg); - - if (record && (record->flags & RSPAMD_SPF_RESOLVED_NA)) { - rspamd_task_insert_result (task, - spf_module_ctx->symbol_na, - 1, - NULL); - } - else if (record && record->elts->len == 0 && (record->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED)) { - rspamd_task_insert_result (task, - spf_module_ctx->symbol_dnsfail, - 1, - NULL); - } - else if (record && record->elts->len == 0 && (record->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED)) { - rspamd_task_insert_result (task, - spf_module_ctx->symbol_permfail, - 1, - NULL); - } - else if (record && record->elts->len == 0) { - rspamd_task_insert_result (task, - spf_module_ctx->symbol_permfail, - 1, - NULL); - } - else if (record && record->domain) { - spf_record_ref (record); - spf_check_list (record, task, record->flags & RSPAMD_SPF_FLAG_CACHED); - spf_record_unref (record); - } - - rspamd_symcache_item_async_dec_check (task, item, M); -} - - -static void -spf_symbol_callback (struct rspamd_task *task, - struct rspamd_symcache_item *item, - void *unused) -{ - struct rspamd_spf_cred *spf_cred; - gint *dmarc_checks; - struct spf_ctx *spf_module_ctx = spf_get_context (task->cfg); - - /* Allow dmarc */ - dmarc_checks = rspamd_mempool_get_variable (task->task_pool, - RSPAMD_MEMPOOL_DMARC_CHECKS); - - if (dmarc_checks) { - (*dmarc_checks) ++; - } - else { - dmarc_checks = rspamd_mempool_alloc (task->task_pool, - sizeof (*dmarc_checks)); - *dmarc_checks = 1; - rspamd_mempool_set_variable (task->task_pool, - RSPAMD_MEMPOOL_DMARC_CHECKS, - dmarc_checks, NULL); - } - - if (rspamd_match_radix_map_addr (spf_module_ctx->whitelist_ip, - task->from_addr) != NULL) { - rspamd_symcache_finalize_item (task, item); - return; - } - - if ((!spf_module_ctx->check_authed && task->user != NULL) - || (!spf_module_ctx->check_local && - rspamd_inet_address_is_local (task->from_addr, TRUE))) { - msg_info_task ("skip SPF checks for local networks and authorized users"); - rspamd_symcache_finalize_item (task, item); - - return; - } - - spf_cred = rspamd_spf_get_cred (task); - /* Refcount = 1 */ - rspamd_symcache_item_async_inc (task, item, M); - - if (spf_cred && spf_cred->domain) { - /* Refcount = 2 */ - rspamd_symcache_item_async_inc (task, item, M); - - /* spf_plugin_callback can be called immediately */ - if (!rspamd_spf_resolve (task, spf_plugin_callback, item, spf_cred)) { - msg_info_task ("cannot make spf request for %s", spf_cred->domain); - rspamd_task_insert_result (task, - spf_module_ctx->symbol_dnsfail, - 1, - "(SPF): spf DNS fail"); - } - else { - /* Refcount is either 2 or 1, so it'll be 3 or 2 upon increase */ - rspamd_symcache_item_async_inc (task, item, M); - } - - /* Refcount 3 or 2 */ - rspamd_symcache_item_async_dec_check (task, item, M); - /* Refcount 2 or 1 */ - } - - /* Refcount 1 or 0 */ - rspamd_symcache_item_async_dec_check (task, item, M); -} diff --git a/test/functional/configs/dmarc.conf b/test/functional/configs/dmarc.conf index ddfc1ac61..0d931b7d3 100644 --- a/test/functional/configs/dmarc.conf +++ b/test/functional/configs/dmarc.conf @@ -1 +1,2 @@ dmarc { } +spf { } |