--[[ Copyright (c) 2011-2016, Vsevolod Stakhov Copyright (c) 2015-2016, Andrew Lewis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ]]-- -- Dmarc policy filter local rspamd_logger = require "rspamd_logger" local mempool = require "rspamd_mempool" local rspamd_url = require "rspamd_url" local rspamd_util = require "rspamd_util" local rspamd_redis = require "lua_redis" local lua_util = require "lua_util" local check_local = false local check_authed = false if confighelp then return end local N = 'dmarc' local no_sampling_domains local no_reporting_domains local statefile = string.format('%s/%s', rspamd_paths['DBDIR'], 'dmarc_reports_last_sent') local VAR_NAME = 'dmarc_reports_last_sent' local INTERVAL = 86400 local pool local report_settings = { helo = 'rspamd', hscan_count = 1000, smtp = '127.0.0.1', smtp_port = 25, retries = 2, from_name = 'Rspamd', } local report_template = [[From: "{% from_name %}" <{% from_addr %}> To: {% rcpt %} Subject: Report Domain: {% reporting_domain %} Submitter: {% submitter %} Report-ID: {% report_id %} Date: {% report_date %} MIME-Version: 1.0 Message-ID: <{% message_id %}> Content-Type: multipart/mixed; boundary="----=_NextPart_000_024E_01CC9B0A.AFE54C00" This is a multipart message in MIME format. ------=_NextPart_000_024E_01CC9B0A.AFE54C00 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit This is an aggregate report from {% submitter %}. Report domain: {% reporting_domain %} Submitter: {% submitter %} Report ID: {% report_id %} ------=_NextPart_000_024E_01CC9B0A.AFE54C00 Content-Type: application/gzip Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="{% submitter %}!{% reporting_domain %}!{% report_start %}!{% report_end %}.xml.gz" ]] local report_footer = [[ ------=_NextPart_000_024E_01CC9B0A.AFE54C00--]] local symbols = { spf_allow_symbol = 'R_SPF_ALLOW', spf_deny_symbol = 'R_SPF_FAIL', spf_softfail_symbol = 'R_SPF_SOFTFAIL', spf_neutral_symbol = 'R_SPF_NEUTRAL', spf_tempfail_symbol = 'R_SPF_DNSFAIL', spf_permfail_symbol = 'R_SPF_PERMFAIL', spf_na_symbol = 'R_SPF_NA', dkim_allow_symbol = 'R_DKIM_ALLOW', dkim_deny_symbol = 'R_DKIM_REJECT', dkim_tempfail_symbol = 'R_DKIM_TEMPFAIL', dkim_na_symbol = 'R_DKIM_NA', dkim_permfail_symbol = 'R_DKIM_PERMFAIL', } local dmarc_symbols = { allow = 'DMARC_POLICY_ALLOW', badpolicy = 'DMARC_BAD_POLICY', dnsfail = 'DMARC_DNSFAIL', na = 'DMARC_NA', reject = 'DMARC_POLICY_REJECT', softfail = 'DMARC_POLICY_SOFTFAIL', quarantine = 'DMARC_POLICY_QUARANTINE', } local redis_keys = { index_prefix = 'dmarc_idx', report_prefix = 'dmarc', join_char = ';', } local function gen_xml_grammar() local lpeg = require 'lpeg' local lt = lpeg.P('<') / '<' local gt = lpeg.P('>') / '>' local amp = lpeg.P('&') / '&' local quot = lpeg.P('"') / '"' local apos = lpeg.P("'") / ''' local special = lt + gt + amp + quot + apos local grammar = lpeg.Cs((special + 1)^0) return grammar end local xml_grammar = gen_xml_grammar() local function escape_xml(goo) return xml_grammar:match(goo) end -- Default port for redis upstreams local redis_params = nil -- 2 days local dmarc_reporting = false local dmarc_actions = {} local E = {} local take_report_id local take_report_script = [[ local index_key = KEYS[1] local report_key = KEYS[2] local dmarc_domain = ARGV[1] local report = ARGV[2] redis.call('SADD', index_key, report_key) redis.call('EXPIRE', index_key, 172800) redis.call('HINCRBY', report_key, report, 1) redis.call('EXPIRE', report_key, 172800) ]] -- return the timezone offset in seconds, as it was on the time given by ts -- Eric Feliksik local function get_timezone_offset(ts) local utcdate = os.date("!*t", ts) local localdate = os.date("*t", ts) localdate.isdst = false -- this is the trick return os.difftime(os.time(localdate), os.time(utcdate)) end local tz_offset = get_timezone_offset(os.time()) local function gen_dmarc_grammar() local lpeg = require "lpeg" lpeg.locale(lpeg) local space = lpeg.space^0 local name = lpeg.C(lpeg.alpha^1) * space local sep = lpeg.S("\\;") * space local value = lpeg.C(lpeg.P(lpeg.graph - sep)^1) local pair = lpeg.Cg(name * "=" * space * value) * sep^-1 local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset) local version = lpeg.P("v") * space * lpeg.P("=") * space * lpeg.P("DMARC1") local record = version * space * sep * list return record end local dmarc_grammar = gen_dmarc_grammar() local function dmarc_report(task, spf_ok, dkim_ok, disposition, sampled_out, hfromdom, spfdom, dres, spf_result) local ip = task:get_from_ip() if not ip:is_valid() then return nil end local rspamd_lua_utils = require "lua_util" if rspamd_lua_utils.is_rspamc_or_controller(task) then return end local dkim_pass = table.concat(dres.pass or E, '|') local dkim_fail = table.concat(dres.fail or E, '|') local dkim_temperror = table.concat(dres.temperror or E, '|') local dkim_permerror = table.concat(dres.permerror or E, '|') local res = table.concat({ ip:to_string(), spf_ok, dkim_ok, disposition, (sampled_out and 'sampled_out' or ''), hfromdom, dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, spfdom, spf_result}, ',') return res end local function maybe_force_action(task, disposition) if disposition then local force_action = dmarc_actions[disposition] if force_action then -- Don't do anything if pre-result has been already set if task:has_pre_result() then return end task:set_pre_result(force_action, 'Action set by DMARC', N, nil, nil, 'least') end end end --[[ -- Used to check dmarc record, check elements and produce dmarc policy processed -- result. -- Returns: -- false,false - record is garbadge -- false,error_message - record is invalid -- true,policy_table - record is valid and parsed ]] local function dmarc_check_record(task, record, is_tld) local failed_policy local result = { dmarc_policy = 'none' } local elts = dmarc_grammar:match(record) lua_util.debugm(N, task, "got DMARC record: %s, tld_flag=%s, processed=%s", record, is_tld, elts) if elts then local dkim_pol = elts['adkim'] if dkim_pol then if dkim_pol == 's' then result.strict_dkim = true elseif dkim_pol ~= 'r' then failed_policy = 'adkim tag has invalid value: ' .. dkim_pol return false,failed_policy end end local spf_pol = elts['aspf'] if spf_pol then if spf_pol == 's' then result.strict_spf = true elseif spf_pol ~= 'r' then failed_policy = 'aspf tag has invalid value: ' .. spf_pol return false,failed_policy end end local policy = elts['p'] if policy then if (policy == 'reject') then result.dmarc_policy = 'reject' elseif (policy == 'quarantine') then result.dmarc_policy = 'quarantine' elseif (policy ~= 'none') then failed_policy = 'p tag has invalid value: ' .. policy return false,failed_policy end end -- Adjust policy if we are in tld mode local subdomain_policy = elts['sp'] if elts['sp'] and is_tld then result.subdomain_policy = elts['sp'] if (subdomain_policy == 'reject') then result.dmarc_policy = 'reject' elseif (subdomain_policy == 'quarantine') then result.dmarc_policy = 'quarantine' elseif (subdomain_policy == 'none') then result.dmarc_policy = 'none' elseif (subdomain_policy ~= 'none') then failed_policy = 'sp tag has invalid value: ' .. subdomain_policy return false,failed_policy end end result.pct = elts['pct'] if result.pct then result.pct = tonumber(result.pct) end if elts.rua then result.rua = elts['rua'] end else return false,false -- Ignore garbadge end return true, result end local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) local reason = {} -- Check dkim and spf symbols local spf_ok = false local dkim_ok = false local spf_tmpfail = false local dkim_tmpfail = false local spf_domain = ((task:get_from(1) or E)[1] or E).domain if not spf_domain or spf_domain == '' then spf_domain = task:get_helo() or '' end if task:has_symbol(symbols['spf_allow_symbol']) then if policy.strict_spf then if rspamd_util.strequal_caseless(spf_domain, hdrfromdom) then spf_ok = true else table.insert(reason, "SPF not aligned (strict)") end else local spf_tld = rspamd_util.get_tld(spf_domain) if rspamd_util.strequal_caseless(spf_tld, dmarc_esld) then spf_ok = true else table.insert(reason, "SPF not aligned (relaxed)") end end else if task:has_symbol(symbols['spf_tempfail_symbol']) then if policy.strict_spf then if rspamd_util.strequal_caseless(spf_domain, hdrfromdom) then spf_tmpfail = true end else local spf_tld = rspamd_util.get_tld(spf_domain) if rspamd_util.strequal_caseless(spf_tld, dmarc_esld) then spf_tmpfail = true end end end table.insert(reason, "No valid SPF") end local opts = ((task:get_symbol('DKIM_TRACE') or E)[1] or E).options local dkim_results = { pass = {}, temperror = {}, permerror = {}, fail = {}, } if opts then dkim_results.pass = {} local dkim_violated for _,opt in ipairs(opts) do local check_res = string.sub(opt, -1) local domain = string.sub(opt, 1, -3) if check_res == '+' then table.insert(dkim_results.pass, domain) if policy.strict_dkim then if rspamd_util.strequal_caseless(hdrfromdom, domain) then dkim_ok = true else dkim_violated = "DKIM not aligned (strict)" end else local dkim_tld = rspamd_util.get_tld(domain) if rspamd_util.strequal_caseless(dkim_tld, dmarc_esld) then dkim_ok = true else dkim_violated = "DKIM not aligned (relaxed)" end end elseif check_res == '?' then -- Check for dkim tempfail if not dkim_ok then if policy.strict_dkim then if rspamd_util.strequal_caseless(hdrfromdom, domain) then dkim_tmpfail = true end else local dkim_tld = rspamd_util.get_tld(domain) if rspamd_util.strequal_caseless(dkim_tld, dmarc_esld) then dkim_tmpfail = true end end end table.insert(dkim_results.temperror, domain) elseif check_res == '-' then table.insert(dkim_results.fail, domain) else table.insert(dkim_results.permerror, domain) end end if not dkim_ok and dkim_violated then table.insert(reason, dkim_violated) end else table.insert(reason, "No valid DKIM") end lua_util.debugm(N, task, "validated dmarc policy for %s: %s; dkim_ok=%s, dkim_tempfail=%s, spf_ok=%s, spf_tempfail=%s", policy.domain, policy.dmarc_policy, dkim_ok, dkim_tmpfail, spf_ok, spf_tmpfail) local disposition = 'none' local sampled_out = false local function handle_dmarc_failure(what, reason_str) if not policy.pct or policy.pct == 100 then task:insert_result(dmarc_symbols[what], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy) disposition = what else local coin = math.random(100) if (coin > policy.pct) then if (not no_sampling_domains or not no_sampling_domains:get_key(policy.domain)) then if what == 'reject' then disposition = 'quarantine' else disposition = 'softfail' end task:insert_result(dmarc_symbols[disposition], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy, "sampled_out") sampled_out = true lua_util.debugm(N, task, 'changed dmarc policy from %s to %s, sampled out: %s < %s', what, disposition, coin, policy.pct) else task:insert_result(dmarc_symbols[what], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy, "local_policy") disposition = what end else task:insert_result(dmarc_symbols[what], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy) disposition = what end end maybe_force_action(task, disposition) end if spf_ok or dkim_ok then --[[ https://tools.ietf.org/html/rfc7489#section-6.6.2 DMARC evaluation can only yield a "pass" result after one of the underlying authentication mechanisms passes for an aligned identifier. ]]-- task:insert_result(dmarc_symbols['allow'], 1.0, policy.domain, policy.dmarc_policy) else --[[ https://tools.ietf.org/html/rfc7489#section-6.6.2 If neither passes and one or both of them fail due to a temporary error, the Receiver evaluating the message is unable to conclude that the DMARC mechanism had a permanent failure; they therefore cannot apply the advertised DMARC policy. ]]-- if spf_tmpfail or dkim_tmpfail then task:insert_result(dmarc_symbols['dnsfail'], 1.0, policy.domain.. ' : ' .. 'SPF/DKIM temp error', policy.dmarc_policy) else -- We can now check the failed policy and maybe send report data elt local reason_str = table.concat(reason, ', ') if policy.dmarc_policy == 'quarantine' then handle_dmarc_failure('quarantine', reason_str) elseif policy.dmarc_policy == 'reject' then handle_dmarc_failure('reject', reason_str) else task:insert_result(dmarc_symbols['softfail'], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy) end end end if policy.rua and redis_params and dmarc_reporting then if no_reporting_domains then if no_reporting_domains:get_key(policy.domain) or no_reporting_domains:get_key(rspamd_util.get_tld(policy.domain)) then rspamd_logger.infox(task, 'DMARC reporting suppressed for %1', policy.domain) return end end local function dmarc_report_cb(err) if not err then rspamd_logger.infox(task, '<%1> dmarc report saved for %2', task:get_message_id(), hdrfromdom) else rspamd_logger.errx(task, '<%1> dmarc report is not saved for %2: %3', task:get_message_id(), hdrfromdom, err) end end local spf_result if spf_ok then spf_result = 'pass' elseif spf_tmpfail then spf_result = 'temperror' else if task:has_symbol(symbols.spf_deny_symbol) then spf_result = 'fail' elseif task:has_symbol(symbols.spf_softfail_symbol) then spf_result = 'softfail' elseif task:has_symbol(symbols.spf_neutral_symbol) then spf_result = 'neutral' elseif task:has_symbol(symbols.spf_permfail_symbol) then spf_result = 'permerror' else spf_result = 'none' end end -- Prepare and send redis report element local period = os.date('!%Y%m%d', task:get_date({format = 'connect', gmt = true})) local dmarc_domain_key = table.concat( {redis_keys.report_prefix, hdrfromdom, period}, redis_keys.join_char) local report_data = dmarc_report(task, spf_ok and 'pass' or 'fail', dkim_ok and 'pass' or 'fail', disposition, sampled_out, hdrfromdom, spf_domain, dkim_results, spf_result) local idx_key = table.concat({redis_keys.index_prefix, period}, redis_keys.join_char) if report_data then rspamd_redis.exec_redis_script(take_report_id, {task = task, is_write = true}, dmarc_report_cb, {idx_key, dmarc_domain_key}, {hdrfromdom, report_data}) end end end local function dmarc_callback(task) local from = task:get_from(2) local hfromdom = ((from or E)[1] or E).domain local dmarc_domain local ip_addr = task:get_ip() local dmarc_checks = task:get_mempool():get_variable('dmarc_checks', 'int') or 0 local seen_invalid = false if dmarc_checks ~= 2 then rspamd_logger.infox(task, "skip DMARC checks as either SPF or DKIM were not checked") return end if ((not check_authed and task:get_user()) or (not check_local and ip_addr and ip_addr:is_local())) then rspamd_logger.infox(task, "skip DMARC checks for local networks and authorized users") return end -- Do some initial sanity checks, detect tld domain if different if hfromdom and hfromdom ~= '' and not (from or E)[2] then dmarc_domain = rspamd_util.get_tld(hfromdom) elseif (from or E)[2] then task:insert_result(dmarc_symbols['na'], 1.0, 'Duplicate From header') return maybe_force_action(task, 'na') elseif (from or E)[1] then task:insert_result(dmarc_symbols['na'], 1.0, 'No domain in From header') return maybe_force_action(task,'na') else task:insert_result(dmarc_symbols['na'], 1.0, 'No From header') return maybe_force_action(task,'na') end local dns_checks_inflight = 0 local dmarc_domain_policy = {} local dmarc_tld_policy = {} local function process_dmarc_policy(policy, final) lua_util.debugm(N, task, "validate DMARC policy (final=%s): %s", true, policy) if policy.err and policy.symbol then -- In case of fatal errors or final check for tld, we give up and -- insert result if final or policy.fatal then task:insert_result(policy.symbol, 1.0, policy.err) maybe_force_action(task, policy.disposition) return true end elseif policy.dmarc_policy then dmarc_validate_policy(task, policy, hfromdom, dmarc_domain) return true -- We have a more specific version, use it end return false -- Missing record end local function gen_dmarc_cb(lookup_domain, is_tld) local policy_target = dmarc_domain_policy if is_tld then policy_target = dmarc_tld_policy end return function (_, _, results, err) dns_checks_inflight = dns_checks_inflight - 1 if not seen_invalid then policy_target.domain = lookup_domain if err then if (err ~= 'requested record is not found' and err ~= 'no records with this name') then policy_target.err = lookup_domain .. ' : ' .. err policy_target.symbol = dmarc_symbols['dnsfail'] else policy_target.err = lookup_domain policy_target.symbol = dmarc_symbols['na'] end else local has_valid_policy = false for _,rec in ipairs(results) do local ret,results_or_err = dmarc_check_record(task, rec, is_tld) if not ret then if results_or_err then -- We have a fatal parsing error, give up policy_target.err = lookup_domain .. ' : ' .. results_or_err policy_target.symbol = dmarc_symbols['badpolicy'] policy_target.fatal = true seen_invalid = true end else if has_valid_policy then policy_target.err = lookup_domain .. ' : ' .. 'Multiple policies defined in DNS' policy_target.symbol = dmarc_symbols['badpolicy'] policy_target.fatal = true seen_invalid = true end has_valid_policy = true for k,v in pairs(results_or_err) do policy_target[k] = v end end end if not has_valid_policy and not seen_invalid then policy_target.err = lookup_domain .. ':' .. ' no valid DMARC record' policy_target.symbol = dmarc_symbols['na'] end end end if dns_checks_inflight == 0 then lua_util.debugm(N, task, "finished DNS queries, validate policies") -- We have checked both tld and real domain (if different) if not process_dmarc_policy(dmarc_domain_policy, false) then -- Try tld policy as well if not process_dmarc_policy(dmarc_tld_policy, true) then process_dmarc_policy(dmarc_domain_policy, true) end end end end end local resolve_name = '_dmarc.' .. hfromdom task:get_resolver():resolve_txt({ task=task, name = resolve_name, callback = gen_dmarc_cb(hfromdom, false), forced = true }) dns_checks_inflight = dns_checks_inflight + 1 if dmarc_domain ~= hfromdom then resolve_name = '_dmarc.' .. dmarc_domain task:get_resolver():resolve_txt({ task=task, name = resolve_name, callback = gen_dmarc_cb(dmarc_domain, true), forced = true }) dns_checks_inflight = dns_checks_inflight + 1 end end local function try_opts(where) local ret = false local opts = rspamd_config:get_all_opt(where) if type(opts) == 'table' then if type(opts['check_local']) == 'boolean' then check_local = opts['check_local'] ret = true end if type(opts['check_authed']) == 'boolean' then check_authed = opts['check_authed'] ret = true end end return ret end if not try_opts(N) then try_opts('options') end local opts = rspamd_config:get_all_opt('dmarc') if not opts or type(opts) ~= 'table' then return end no_sampling_domains = rspamd_map_add(N, 'no_sampling_domains', 'map', 'Domains not to apply DMARC sampling to') no_reporting_domains = rspamd_map_add(N, 'no_reporting_domains', 'map', 'Domains not to apply DMARC reporting to') if opts['symbols'] then for k,_ in pairs(dmarc_symbols) do if opts['symbols'][k] then dmarc_symbols[k] = opts['symbols'][k] end end end -- XXX: rework this shitty code some day please if opts['reporting'] == true then redis_params = rspamd_parse_redis_server('dmarc') if not redis_params then rspamd_logger.errx(rspamd_config, 'cannot parse servers parameter') elseif not opts['send_reports'] then dmarc_reporting = true take_report_id = rspamd_redis.add_redis_script(take_report_script, redis_params) else dmarc_reporting = true if type(opts['report_settings']) == 'table' then for k, v in pairs(opts['report_settings']) do report_settings[k] = v end end for _, e in ipairs({'email', 'domain', 'org_name'}) do if not report_settings[e] then rspamd_logger.errx(rspamd_config, 'Missing required setting: report_settings.%s', e) return end end take_report_id = rspamd_redis.add_redis_script(take_report_script, redis_params) rspamd_config:add_on_load(function(cfg, ev_base, worker) if not worker:is_primary_controller() then return end pool = mempool.create() rspamd_config:register_finish_script(function () local stamp = pool:get_variable(VAR_NAME, 'double') if not stamp then rspamd_logger.warnx(rspamd_config, 'No last DMARC report information to persist to disk') return end local f, err = io.open(statefile, 'w') if err then rspamd_logger.errx(rspamd_config, 'Unable to write statefile to disk: %s', err) return end assert(f:write(pool:get_variable(VAR_NAME, 'double'))) assert(f:close()) pool:destroy() end) local get_reporting_domain, reporting_domain, report_start, report_end, report_id, want_period, report_key local reporting_addr = {} local domain_policy = {} local to_verify = {} local cursor = 0 local function entry_to_xml(data) local buf = { table.concat({ '', data.ip, '', data.count, '', data.disposition, '', data.dkim_disposition, '', data.spf_disposition, '' }), } if data.override ~= '' then table.insert(buf, string.format('%s', data.override)) end table.insert(buf, table.concat({ '', data.header_from, '', })) table.insert(buf, '') if data.dkim_results[1] then for _, d in ipairs(data.dkim_results) do table.insert(buf, table.concat({ '', d.domain, '', d.result, '', })) end end table.insert(buf, table.concat({ '', data.spf_domain, '', data.spf_result, '', })) return table.concat(buf) end local function dmarc_report_xml() local entries = {} report_id = string.format('%s.%d.%d', reporting_domain, report_start, report_end) lua_util.debugm(N, rspamd_config, 'new report: %s', report_id) local actions = { push = function(t) local data = t[1] local split = rspamd_str_split(data, ',') local row = { ip = split[1], spf_disposition = split[2], dkim_disposition = split[3], disposition = split[4], override = split[5], header_from = split[6], dkim_results = {}, spf_domain = split[11], spf_result = split[12], count = t[2], } if split[7] and split[7] ~= '' then local tmp = rspamd_str_split(split[7], '|') for _, d in ipairs(tmp) do table.insert(row.dkim_results, {domain = d, result = 'pass'}) end end if split[8] and split[8] ~= '' then local tmp = rspamd_str_split(split[8], '|') for _, d in ipairs(tmp) do table.insert(row.dkim_results, {domain = d, result = 'fail'}) end end if split[9] and split[9] ~= '' then local tmp = rspamd_str_split(split[9], '|') for _, d in ipairs(tmp) do table.insert(row.dkim_results, {domain = d, result = 'temperror'}) end end if split[10] and split[10] ~= '' then local tmp = lua_util.str_split(split[10], '|') for _, d in ipairs(tmp) do table.insert(row.dkim_results, {domain = d, result = 'permerror'}) end end table.insert(entries, row) end, -- TODO: please rework this shit header = function() return table.concat({ '', escape_xml(report_settings.org_name), '', escape_xml(report_settings.email), '', report_id, '', report_start, '', report_end, '', reporting_domain, '', escape_xml(domain_policy.adkim), '', escape_xml(domain_policy.aspf), '

', escape_xml(domain_policy.p), '

', escape_xml(domain_policy.sp), '', escape_xml(domain_policy.pct), '' }) end, footer = function() return [[]] end, entries = function() local buf = {} for _, e in pairs(entries) do table.insert(buf, entry_to_xml(e)) end return table.concat(buf, '') end, } return function(action, p) local f = actions[action] if not f then error('invalid action: ' .. action) end return f(p) end end local function send_report_via_email(xmlf, retry) if not retry then retry = 0 end local function sendmail_cb(ret, err) if not ret then rspamd_logger.errx(rspamd_config, "Couldn't send mail for %s: %s", err) if retry >= report_settings.retries then rspamd_logger.errx(rspamd_config, "Couldn't send mail for %s: retries exceeded", reporting_domain) return get_reporting_domain() else send_report_via_email(xmlf, retry + 1) end end end -- Format message local tmp_addr = {} for k in pairs(reporting_addr) do table.insert(tmp_addr, k) end local encoded = rspamd_util.encode_base64(rspamd_util.gzip_compress( table.concat( {xmlf('header'), xmlf('entries'), xmlf('footer')})), 73) local atmp = {} for k in pairs(reporting_addr) do table.insert(atmp, k) end local addr_string = table.concat(atmp, ', ') local rhead = lua_util.jinja_template(report_template, { from_name = report_settings.from_name, from_addr = report_settings.email, rcpt = addr_string, reporting_domain = reporting_domain, submitter = report_settings.domain, report_id = report_id, report_date = rspamd_util.time_to_string(rspamd_util.get_time()), message_id = rspamd_util.random_hex(12) .. '@rspamd', report_start = report_start, report_end = report_end }, true) local message = { rhead:gsub("\n", "\r\n"), encoded, report_footer:gsub("\n", "\r\n") } local lua_smtp = require "lua_smtp" lua_smtp.sendmail({ ev_base = ev_base, config = rspamd_config, host = report_settings.smtp, port = report_settings.smtp_port, resolver = rspamd_config:get_resolver(), from = report_settings.email, recipients = tmp_addr, helo = report_settings.helo, }, message, sendmail_cb) end local function make_report() if type(report_settings.override_address) == 'string' then reporting_addr = {[report_settings.override_address] = true} end if type(report_settings.additional_address) == 'string' then reporting_addr[report_settings.additional_address] = true end rspamd_logger.infox(ev_base, 'sending report for %s <%s>', reporting_domain, table.concat(reporting_addr, ',')) local dmarc_xml = dmarc_report_xml() local dmarc_push_cb dmarc_push_cb = function(err, data) if err then rspamd_logger.errx(ev_base, 'Redis request failed: %s', err) -- XXX: data is orphaned; replace key or delete data get_reporting_domain() elseif type(data) == 'table' then cursor = tonumber(data[1]) for i = 1, #data[2], 2 do dmarc_xml('push', {data[2][i], data[2][i+1]}) end if cursor ~= 0 then local ret = rspamd_redis.redis_make_request_taskless(ev_base, rspamd_config, redis_params, nil, false, -- is write dmarc_push_cb, --callback 'HSCAN', -- command {report_key, cursor, 'COUNT', report_settings.hscan_count} ) if not ret then rspamd_logger.errx(ev_base, 'Failed to schedule redis request') get_reporting_domain() end else send_report_via_email(dmarc_xml) end end end local ret = rspamd_redis.redis_make_request_taskless(ev_base, rspamd_config, redis_params, nil, false, -- is write dmarc_push_cb, --callback 'HSCAN', -- command {report_key, cursor, 'COUNT', report_settings.hscan_count} ) if not ret then rspamd_logger.errx(rspamd_config, 'Failed to schedule redis request') -- XXX: data is orphaned; replace key or delete data get_reporting_domain() end end local function delete_reports() local function delete_reports_cb(err) if err then rspamd_logger.errx(rspamd_config, 'Error deleting reports: %s', err) end rspamd_logger.infox(rspamd_config, 'Deleted reports for %s', reporting_domain) get_reporting_domain() end local ret = rspamd_redis.redis_make_request_taskless(ev_base, rspamd_config, redis_params, nil, true, -- is write delete_reports_cb, --callback 'DEL', -- command {report_key} ) if not ret then rspamd_logger.errx(rspamd_config, 'Failed to schedule redis request') get_reporting_domain() end end local function verify_reporting_address() local function verifier(test_addr, vdom) local retry = 0 local function verify_cb(resolver, to_resolve, results, err, _, authenticated) if err then if err == 'no records with this name' or err == 'requested record is not found' then rspamd_logger.infox(rspamd_config, 'Reports to %s for %s not authorised', test_addr, reporting_domain) to_verify[test_addr] = nil else rspamd_logger.errx(rspamd_config, 'Lookup error [%s]: %s', to_resolve, err) if retry < report_settings.retries then retry = retry + 1 rspamd_config:get_resolver():resolve('txt', { ev_base = ev_base, name = string.format('%s._report._dmarc.%s', reporting_domain, vdom), callback = verify_cb, }) else delete_reports() end end else local is_authed = false -- XXX: reporting address could be overridden for _, r in ipairs(results) do if string.match(r, 'v=DMARC1') then is_authed = true break end end if not is_authed then to_verify[test_addr] = nil rspamd_logger.infox(rspamd_config, 'Reports to %s for %s not authorised', test_addr, reporting_domain) else to_verify[test_addr] = nil reporting_addr[test_addr] = true end end local t, nvdom = next(to_verify) if not t then if next(reporting_addr) then make_report() else rspamd_logger.infox(rspamd_config, 'No valid reporting addresses for %s', reporting_domain) delete_reports() end else verifier(t, nvdom) end end rspamd_config:get_resolver():resolve('txt', { ev_base = ev_base, name = string.format('%s._report._dmarc.%s', reporting_domain, vdom), callback = verify_cb, }) end local t, vdom = next(to_verify) verifier(t, vdom) end local function get_reporting_address() local retry = 0 local esld = rspamd_util.get_tld(reporting_domain) local function check_addr_cb(resolver, to_resolve, results, err, _, authenticated) if err then if err == 'no records with this name' or err == 'requested record is not found' then if reporting_domain ~= esld then rspamd_config:get_resolver():resolve('txt', { ev_base = ev_base, name = string.format('_dmarc.%s', esld), callback = check_addr_cb, }) else rspamd_logger.errx(rspamd_config, 'No DMARC record found for %s', reporting_domain) delete_reports() end else rspamd_logger.errx(rspamd_config, 'Lookup error [%s]: %s', to_resolve, err) if retry < report_settings.retries then retry = retry + 1 rspamd_config:get_resolver():resolve('txt', { ev_base = ev_base, name = to_resolve, callback = check_addr_cb, }) else rspamd_logger.errx(rspamd_config, "Couldn't get reporting address for %s: retries exceeded", reporting_domain) delete_reports() end end else local policy local found_policy, failed_policy = false, false for _, r in ipairs(results) do local elts = dmarc_grammar:match(r) if elts and found_policy then failed_policy = true elseif elts then found_policy = true policy = elts end end if not found_policy then rspamd_logger.errx(rspamd_config, 'No policy: %s', to_resolve) if reporting_domain ~= esld then rspamd_config:get_resolver():resolve('txt', { ev_base = ev_base, name = string.format('_dmarc.%s', esld), callback = check_addr_cb, }) else delete_reports() end elseif failed_policy then rspamd_logger.errx(rspamd_config, 'Duplicate policies: %s', to_resolve) delete_reports() elseif not policy['rua'] then rspamd_logger.errx(rspamd_config, 'No reporting address: %s', to_resolve) delete_reports() else local upool = mempool.create() local split = rspamd_str_split(policy['rua'], ',') for _, m in ipairs(split) do local url = rspamd_url.create(upool, m) if not url then rspamd_logger.errx(rspamd_config, 'Couldnt extract reporting address: %s', policy['rua']) else local urlt = url:to_table() if urlt['protocol'] ~= 'mailto' then rspamd_logger.errx(rspamd_config, 'Invalid URL: %s', url) else if urlt['tld'] == rspamd_util.get_tld(reporting_domain) then reporting_addr[string.format('%s@%s', urlt['user'], urlt['host'])] = true else to_verify[string.format('%s@%s', urlt['user'], urlt['host'])] = urlt['host'] end end end end upool:destroy() domain_policy['pct'] = policy['pct'] or 100 domain_policy['adkim'] = policy['adkim'] or 'r' domain_policy['aspf'] = policy['aspf'] or 'r' domain_policy['p'] = policy['p'] or 'none' domain_policy['sp'] = policy['sp'] or 'none' if next(to_verify) then verify_reporting_address() elseif next(reporting_addr) then make_report() else rspamd_logger.errx(rspamd_config, 'No reporting address for %s', reporting_domain) delete_reports() end end end end rspamd_config:get_resolver():resolve('txt', { ev_base = ev_base, name = string.format('_dmarc.%s', reporting_domain), callback = check_addr_cb, }) end get_reporting_domain = function() reporting_domain = nil reporting_addr = {} domain_policy = {} cursor = 0 local function get_reporting_domain_cb(err, data) if err then rspamd_logger.errx(cfg, 'Unable to get DMARC domain: %s', err) else if type(data) == 'userdata' then reporting_domain = nil else report_key = data local tmp = rspamd_str_split(data, redis_keys.join_char) reporting_domain = tmp[2] end if not reporting_domain then rspamd_logger.infox(cfg, 'No more domains to generate reports for') else get_reporting_address() end end end local idx_key = table.concat({redis_keys.index_prefix, want_period}, redis_keys.join_char) local ret = rspamd_redis.redis_make_request_taskless(ev_base, rspamd_config, redis_params, nil, true, -- is write get_reporting_domain_cb, --callback 'SPOP', -- command {idx_key} ) if not ret then rspamd_logger.errx(cfg, 'Unable to get DMARC domain') end end local function send_reports(time) rspamd_logger.infox(ev_base, 'sending reports ostensibly %1', time) pool:set_variable(VAR_NAME, time) local yesterday = os.date('!*t', rspamd_util.get_time() - INTERVAL) local today = os.date('!*t', rspamd_util.get_time()) report_start = os.time({ year = yesterday.year, month = yesterday.month, day = yesterday.day, hour = 0}) + tz_offset report_end = os.time({ year = today.year, month = today.month, day = today.day, hour = 0}) + tz_offset want_period = table.concat({ yesterday.year, string.format('%02d', yesterday.month), string.format('%02d', yesterday.day) }) get_reporting_domain() end -- Push reports at regular intervals local function schedule_regular_send() rspamd_config:add_periodic(ev_base, INTERVAL, function () send_reports() return true end) end -- Push reports to backend and reschedule check local function schedule_intermediate_send(when) rspamd_config:add_periodic(ev_base, when, function () schedule_regular_send() send_reports(rspamd_util.get_time()) return false end) end -- Try read statefile on startup local stamp local f, err = io.open(statefile, 'r') if err then rspamd_logger.errx('Failed to open statefile: %s', err) end if f then io.input(f) stamp = tonumber(io.read()) pool:set_variable(VAR_NAME, stamp) end local time = rspamd_util.get_time() if not stamp then lua_util.debugm(N, rspamd_config, 'No state found - sending reports immediately') schedule_regular_send() send_reports(time) return end local delta = stamp - time + INTERVAL if delta <= 0 then lua_util.debugm(N, rspamd_config, 'Last send is too old - sending reports immediately') schedule_regular_send() send_reports(time) return end lua_util.debugm(N, rspamd_config, 'Scheduling next send in %s seconds', delta) schedule_intermediate_send(delta) end) end end if type(opts['actions']) == 'table' then dmarc_actions = opts['actions'] end if type(opts['report_settings']) == 'table' then for k, v in pairs(opts['report_settings']) do report_settings[k] = v end end if opts['send_reports'] then for _, e in ipairs({'email', 'domain', 'org_name'}) do if not report_settings[e] then rspamd_logger.errx(rspamd_config, 'Missing required setting: report_settings.%s', e) return end end end -- Check spf and dkim sections for changed symbols local function check_mopt(var, m_opts, name) if m_opts[name] then symbols[var] = tostring(m_opts[name]) end end local spf_opts = rspamd_config:get_all_opt('spf') if spf_opts then check_mopt('spf_deny_symbol', spf_opts, 'symbol_fail') check_mopt('spf_allow_symbol', spf_opts, 'symbol_allow') check_mopt('spf_softfail_symbol', spf_opts, 'symbol_softfail') check_mopt('spf_neutral_symbol', spf_opts, 'symbol_neutral') check_mopt('spf_tempfail_symbol', spf_opts, 'symbol_dnsfail') check_mopt('spf_na_symbol', spf_opts, 'symbol_na') end local dkim_opts = rspamd_config:get_all_opt('dkim') if dkim_opts then check_mopt('dkim_deny_symbol', dkim_opts, 'symbol_reject') check_mopt('dkim_allow_symbol', dkim_opts, 'symbol_allow') check_mopt('dkim_tempfail_symbol', dkim_opts, 'symbol_tempfail') check_mopt('dkim_na_symbol', dkim_opts, 'symbol_na') end local id = rspamd_config:register_symbol({ name = 'DMARC_CALLBACK', type = 'callback', group = 'policies', groups = {'dmarc'}, callback = dmarc_callback }) rspamd_config:register_symbol({ name = dmarc_symbols['allow'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ name = dmarc_symbols['reject'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ name = dmarc_symbols['quarantine'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ name = dmarc_symbols['softfail'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ name = dmarc_symbols['dnsfail'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ name = dmarc_symbols['na'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_dependency('DMARC_CALLBACK', symbols['spf_allow_symbol']) rspamd_config:register_dependency('DMARC_CALLBACK', symbols['dkim_allow_symbol'])