From b75b02a67bad7cba5635ec93123814d91baf1f53 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Thu, 29 Jul 2021 15:54:25 +0100 Subject: [PATCH] [Rework] Reorganize dmarc plugin and remove unsupported reporting code --- src/plugins/lua/dmarc.lua | 894 +++++--------------------------------- 1 file changed, 113 insertions(+), 781 deletions(-) diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua index ef2c89881..e38ab822b 100644 --- a/src/plugins/lua/dmarc.lua +++ b/src/plugins/lua/dmarc.lua @@ -18,135 +18,58 @@ limitations under the License. -- Dmarc policy filter local rspamd_logger = require "rspamd_logger" -local mempool = require "rspamd_mempool" -local rspamd_url = require "rspamd_url" local rspamd_util = require "rspamd_util" local rspamd_redis = require "lua_redis" local lua_util = require "lua_util" -local auth_and_local_conf if confighelp then return end local N = 'dmarc' -local no_sampling_domains -local no_reporting_domains -local statefile = string.format('%s/%s', rspamd_paths['DBDIR'], 'dmarc_reports_last_sent') -local VAR_NAME = 'dmarc_reports_last_sent' -local INTERVAL = 86400 -local pool - -local report_settings = { - helo = 'rspamd', - hscan_count = 1000, - smtp = '127.0.0.1', - smtp_port = 25, - retries = 2, - from_name = 'Rspamd', - msgid_from = 'rspamd', -} - -local report_template = [[From: "{= from_name =}" <{= from_addr =}> -To: {= rcpt =} -{%+ if is_string(bcc) %}Bcc: {= bcc =}{%- endif %} -Subject: Report Domain: {= reporting_domain =} - Submitter: {= submitter =} - Report-ID: {= report_id =} -Date: {= report_date =} -MIME-Version: 1.0 -Message-ID: <{= message_id =}> -Content-Type: multipart/mixed; - boundary="----=_NextPart_000_024E_01CC9B0A.AFE54C00" - -This is a multipart message in MIME format. - -------=_NextPart_000_024E_01CC9B0A.AFE54C00 -Content-Type: text/plain; charset="us-ascii" -Content-Transfer-Encoding: 7bit - -This is an aggregate report from {= submitter =}. - -Report domain: {= reporting_domain =} -Submitter: {= submitter =} -Report ID: {= report_id =} - -------=_NextPart_000_024E_01CC9B0A.AFE54C00 -Content-Type: application/gzip -Content-Transfer-Encoding: base64 -Content-Disposition: attachment; - filename="{= submitter =}!{= reporting_domain =}!{= report_start =}!{= report_end =}.xml.gz" - -]] -local report_footer = [[ - -------=_NextPart_000_024E_01CC9B0A.AFE54C00--]] - -local symbols = { - spf_allow_symbol = 'R_SPF_ALLOW', - spf_deny_symbol = 'R_SPF_FAIL', - spf_softfail_symbol = 'R_SPF_SOFTFAIL', - spf_neutral_symbol = 'R_SPF_NEUTRAL', - spf_tempfail_symbol = 'R_SPF_DNSFAIL', - spf_permfail_symbol = 'R_SPF_PERMFAIL', - spf_na_symbol = 'R_SPF_NA', - - dkim_allow_symbol = 'R_DKIM_ALLOW', - dkim_deny_symbol = 'R_DKIM_REJECT', - dkim_tempfail_symbol = 'R_DKIM_TEMPFAIL', - dkim_na_symbol = 'R_DKIM_NA', - dkim_permfail_symbol = 'R_DKIM_PERMFAIL', -} -local dmarc_symbols = { - allow = 'DMARC_POLICY_ALLOW', - badpolicy = 'DMARC_BAD_POLICY', - dnsfail = 'DMARC_DNSFAIL', - na = 'DMARC_NA', - reject = 'DMARC_POLICY_REJECT', - softfail = 'DMARC_POLICY_SOFTFAIL', - quarantine = 'DMARC_POLICY_QUARANTINE', +local settings = { + auth_and_local_conf = false, + symbols = { + spf_allow_symbol = 'R_SPF_ALLOW', + spf_deny_symbol = 'R_SPF_FAIL', + spf_softfail_symbol = 'R_SPF_SOFTFAIL', + spf_neutral_symbol = 'R_SPF_NEUTRAL', + spf_tempfail_symbol = 'R_SPF_DNSFAIL', + spf_permfail_symbol = 'R_SPF_PERMFAIL', + spf_na_symbol = 'R_SPF_NA', + + dkim_allow_symbol = 'R_DKIM_ALLOW', + dkim_deny_symbol = 'R_DKIM_REJECT', + dkim_tempfail_symbol = 'R_DKIM_TEMPFAIL', + dkim_na_symbol = 'R_DKIM_NA', + dkim_permfail_symbol = 'R_DKIM_PERMFAIL', + + -- DMARC symbols + allow = 'DMARC_POLICY_ALLOW', + badpolicy = 'DMARC_BAD_POLICY', + dnsfail = 'DMARC_DNSFAIL', + na = 'DMARC_NA', + reject = 'DMARC_POLICY_REJECT', + softfail = 'DMARC_POLICY_SOFTFAIL', + quarantine = 'DMARC_POLICY_QUARANTINE', + }, + no_sampling_domains = nil, + no_reporting_domains = nil, + reporting = { + redis_keys = { + index_prefix = 'dmarc_idx', + report_prefix = 'dmarc', + join_char = ';', + }, + enabled = false, + max_entries = 1000, + only_domains = nil, + }, + actions = {}, } -local redis_keys = { - index_prefix = 'dmarc_idx', - report_prefix = 'dmarc', - join_char = ';', -} - -local function gen_xml_grammar() - local lpeg = require 'lpeg' - local lt = lpeg.P('<') / '<' - local gt = lpeg.P('>') / '>' - local amp = lpeg.P('&') / '&' - local quot = lpeg.P('"') / '"' - local apos = lpeg.P("'") / ''' - local special = lt + gt + amp + quot + apos - local grammar = lpeg.Cs((special + 1)^0) - return grammar -end - -local xml_grammar = gen_xml_grammar() - -local function escape_xml(input) - if type(input) == 'string' or type(input) == 'userdata' then - return xml_grammar:match(input) - else - input = tostring(input) - - if input then - return xml_grammar:match(input) - end - end - - return '' -end - --- Default port for redis upstreams local redis_params = nil --- 2 days -local dmarc_reporting = false -local dmarc_actions = {} local E = {} @@ -162,17 +85,6 @@ redis.call('HINCRBY', report_key, report, 1) redis.call('EXPIRE', report_key, 172800) ]] --- return the timezone offset in seconds, as it was on the time given by ts --- Eric Feliksik -local function get_timezone_offset(ts) - local utcdate = os.date("!*t", ts) - local localdate = os.date("*t", ts) - localdate.isdst = false -- this is the trick - return os.difftime(os.time(localdate), os.time(utcdate)) -end - -local tz_offset = get_timezone_offset(os.time()) - local function gen_dmarc_grammar() local lpeg = require "lpeg" lpeg.locale(lpeg) @@ -207,6 +119,8 @@ local function dmarc_key_value_case(elts) return result end + +-- Returns a key used to be inserted into dmarc report sample local function dmarc_report(task, spf_ok, dkim_ok, disposition, sampled_out, hfromdom, spfdom, dres, spf_result) local ip = task:get_from_ip() @@ -230,7 +144,7 @@ end local function maybe_force_action(task, disposition) if disposition then - local force_action = dmarc_actions[disposition] + local force_action = settings.actions[disposition] if force_action then -- Set least action task:set_pre_result(force_action, 'Action set by DMARC', N, nil, nil, 'least') @@ -337,7 +251,7 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) spf_domain = task:get_helo() or '' end - if task:has_symbol(symbols['spf_allow_symbol']) then + if task:has_symbol(settings.symbols['spf_allow_symbol']) then if policy.strict_spf then if rspamd_util.strequal_caseless(spf_domain, hdrfromdom) then spf_ok = true @@ -353,7 +267,7 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) end end else - if task:has_symbol(symbols['spf_tempfail_symbol']) then + if task:has_symbol(settings.symbols['spf_tempfail_symbol']) then if policy.strict_spf then if rspamd_util.strequal_caseless(spf_domain, hdrfromdom) then spf_tmpfail = true @@ -446,14 +360,14 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) local function handle_dmarc_failure(what, reason_str) if not policy.pct or policy.pct == 100 then - task:insert_result(dmarc_symbols[what], 1.0, + task:insert_result(settings.symbols[what], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy) disposition = what else local coin = math.random(100) if (coin > policy.pct) then - if (not no_sampling_domains or - not no_sampling_domains:get_key(policy.domain)) then + if (not settings.no_sampling_domains or + not settings.no_sampling_domains:get_key(policy.domain)) then if what == 'reject' then disposition = 'quarantine' @@ -461,19 +375,19 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) disposition = 'softfail' end - task:insert_result(dmarc_symbols[disposition], 1.0, + task:insert_result(settings.symbols[disposition], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy, "sampled_out") sampled_out = true lua_util.debugm(N, task, 'changed dmarc policy from %s to %s, sampled out: %s < %s', what, disposition, coin, policy.pct) else - task:insert_result(dmarc_symbols[what], 1.0, + task:insert_result(settings.symbols[what], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy, "local_policy") disposition = what end else - task:insert_result(dmarc_symbols[what], 1.0, + task:insert_result(settings.symbols[what], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy) disposition = what end @@ -489,7 +403,7 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) underlying authentication mechanisms passes for an aligned identifier. ]]-- - task:insert_result(dmarc_symbols['allow'], 1.0, policy.domain, + task:insert_result(settings.symbols['allow'], 1.0, policy.domain, policy.dmarc_policy) else --[[ @@ -501,7 +415,7 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) therefore cannot apply the advertised DMARC policy. ]]-- if spf_tmpfail or dkim_tmpfail then - task:insert_result(dmarc_symbols['dnsfail'], 1.0, policy.domain.. + task:insert_result(settings.symbols['dnsfail'], 1.0, policy.domain.. ' : ' .. 'SPF/DKIM temp error', policy.dmarc_policy) else -- We can now check the failed policy and maybe send report data elt @@ -512,17 +426,17 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) elseif policy.dmarc_policy == 'reject' then handle_dmarc_failure('reject', reason_str) else - task:insert_result(dmarc_symbols['softfail'], 1.0, + task:insert_result(settings.symbols['softfail'], 1.0, policy.domain .. ' : ' .. reason_str, policy.dmarc_policy) end end end - if policy.rua and redis_params and dmarc_reporting then - if no_reporting_domains then - if no_reporting_domains:get_key(policy.domain) or - no_reporting_domains:get_key(rspamd_util.get_tld(policy.domain)) then + if policy.rua and redis_params and settings.reporting.enabled then + if settings.no_reporting_domains then + if settings.no_reporting_domains:get_key(policy.domain) or + settings.no_reporting_domains:get_key(rspamd_util.get_tld(policy.domain)) then rspamd_logger.infox(task, 'DMARC reporting suppressed for %1', policy.domain) return end @@ -544,13 +458,13 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) elseif spf_tmpfail then spf_result = 'temperror' else - if task:has_symbol(symbols.spf_deny_symbol) then + if task:has_symbol(settings.symbols.spf_deny_symbol) then spf_result = 'fail' - elseif task:has_symbol(symbols.spf_softfail_symbol) then + elseif task:has_symbol(settings.symbols.spf_softfail_symbol) then spf_result = 'softfail' - elseif task:has_symbol(symbols.spf_neutral_symbol) then + elseif task:has_symbol(settings.symbols.spf_neutral_symbol) then spf_result = 'neutral' - elseif task:has_symbol(symbols.spf_permfail_symbol) then + elseif task:has_symbol(settings.symbols.spf_permfail_symbol) then spf_result = 'permerror' else spf_result = 'none' @@ -561,7 +475,8 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) local period = os.date('!%Y%m%d', task:get_date({format = 'connect', gmt = true})) local dmarc_domain_key = table.concat( - {redis_keys.report_prefix, hdrfromdom, period}, redis_keys.join_char) + {settings.reporting.redis_keys.report_prefix, hdrfromdom, period}, + settings.reporting.redis_keys.join_char) local report_data = dmarc_report(task, spf_ok and 'pass' or 'fail', dkim_ok and 'pass' or 'fail', @@ -572,8 +487,8 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) dkim_results, spf_result) - local idx_key = table.concat({redis_keys.index_prefix, period}, - redis_keys.join_char) + local idx_key = table.concat({settings.redis_keys.index_prefix, period}, + settings.redis_keys.join_char) if report_data then rspamd_redis.exec_redis_script(take_report_id, @@ -598,7 +513,7 @@ local function dmarc_callback(task) return end - if lua_util.is_skip_local_or_authed(task, auth_and_local_conf, ip_addr) then + if lua_util.is_skip_local_or_authed(task, settings.auth_and_local_conf, ip_addr) then rspamd_logger.infox(task, "skip DMARC checks for local networks and authorized users") return end @@ -607,13 +522,13 @@ local function dmarc_callback(task) if hfromdom and hfromdom ~= '' and not (from or E)[2] then dmarc_domain = rspamd_util.get_tld(hfromdom) elseif (from or E)[2] then - task:insert_result(dmarc_symbols['na'], 1.0, 'Duplicate From header') + task:insert_result(settings.symbols['na'], 1.0, 'Duplicate From header') return maybe_force_action(task, 'na') elseif (from or E)[1] then - task:insert_result(dmarc_symbols['na'], 1.0, 'No domain in From header') + task:insert_result(settings.symbols['na'], 1.0, 'No domain in From header') return maybe_force_action(task,'na') else - task:insert_result(dmarc_symbols['na'], 1.0, 'No From header') + task:insert_result(settings.symbols['na'], 1.0, 'No From header') return maybe_force_action(task,'na') end @@ -659,10 +574,10 @@ local function dmarc_callback(task) if (err ~= 'requested record is not found' and err ~= 'no records with this name') then policy_target.err = lookup_domain .. ' : ' .. err - policy_target.symbol = dmarc_symbols['dnsfail'] + policy_target.symbol = settings.symbols['dnsfail'] else policy_target.err = lookup_domain - policy_target.symbol = dmarc_symbols['na'] + policy_target.symbol = settings.symbols['na'] end else local has_valid_policy = false @@ -674,7 +589,7 @@ local function dmarc_callback(task) if results_or_err then -- We have a fatal parsing error, give up policy_target.err = lookup_domain .. ' : ' .. results_or_err - policy_target.symbol = dmarc_symbols['badpolicy'] + policy_target.symbol = settings.symbols['badpolicy'] policy_target.fatal = true seen_invalid = true end @@ -682,7 +597,7 @@ local function dmarc_callback(task) if has_valid_policy then policy_target.err = lookup_domain .. ' : ' .. 'Multiple policies defined in DNS' - policy_target.symbol = dmarc_symbols['badpolicy'] + policy_target.symbol = settings.symbols['badpolicy'] policy_target.fatal = true seen_invalid = true end @@ -696,7 +611,7 @@ local function dmarc_callback(task) if not has_valid_policy and not seen_invalid then policy_target.err = lookup_domain .. ':' .. ' no valid DMARC record' - policy_target.symbol = dmarc_symbols['na'] + policy_target.symbol = settings.symbols['na'] end end end @@ -740,622 +655,41 @@ end local opts = rspamd_config:get_all_opt('dmarc') -if not opts or type(opts) ~= 'table' then - return -end +settings = lua_util.override_defaults(settings, opts) -auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N, +settings.auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N, false, false) -no_sampling_domains = rspamd_map_add(N, 'no_sampling_domains', 'map', 'Domains not to apply DMARC sampling to') -no_reporting_domains = rspamd_map_add(N, 'no_reporting_domains', 'map', 'Domains not to apply DMARC reporting to') +local lua_maps = require "lua_maps" +lua_maps.fill_config_maps(N, settings, { + no_sampling_domains = { + optional = true, + type = 'map', + description = 'Domains not to apply DMARC sampling to' + }, + no_reporting_domains = { + optional = true, + type = 'map', + description = 'Domains not to apply DMARC reporting to' + }, +}) -if opts['symbols'] then - for k,_ in pairs(dmarc_symbols) do - if opts['symbols'][k] then - dmarc_symbols[k] = opts['symbols'][k] - end - end -end --- Reporting related code -- - ---- ---- Converts a reporting entry to an XML format ---- @param data data table ---- @return string with an XML representation -local function entry_to_xml(data) - local buf = { - table.concat({ - '', data.ip, '', - data.count, '', - data.disposition, '', data.dkim_disposition, - '', data.spf_disposition, '' - }), - } - if data.override ~= '' then - table.insert(buf, string.format('%s', data.override)) - end - table.insert(buf, table.concat({ - '', data.header_from, - '', - })) - table.insert(buf, '') - if data.dkim_results[1] then - for _, d in ipairs(data.dkim_results) do - table.insert(buf, table.concat({ - '', d.domain, '', - d.result, '', - })) - end - end - table.insert(buf, table.concat({ - '', data.spf_domain, '', - data.spf_result, '', - })) - return table.concat(buf) -end - -if opts['reporting'] == true then +if settings.reporting == true then + rspamd_logger.errx(rspamd_config, 'old style dmarc reporting is NO LONGER supported, please read the documentation') +elseif settings.reporting.enabled then redis_params = rspamd_parse_redis_server('dmarc') if not redis_params then rspamd_logger.errx(rspamd_config, 'cannot parse servers parameter') - elseif not opts['send_reports'] then - dmarc_reporting = true - take_report_id = rspamd_redis.add_redis_script(take_report_script, redis_params) else - dmarc_reporting = true - if type(opts['report_settings']) == 'table' then - for k, v in pairs(opts['report_settings']) do - report_settings[k] = v - end - end - for _, e in ipairs({'email', 'domain', 'org_name'}) do - if not report_settings[e] then - rspamd_logger.errx(rspamd_config, 'Missing required setting: report_settings.%s', e) - return - end - end take_report_id = rspamd_redis.add_redis_script(take_report_script, redis_params) - rspamd_config:add_on_load(function(cfg, ev_base, worker) - if not worker:is_primary_controller() then return end - - pool = mempool.create() - - rspamd_config:register_finish_script(function () - local stamp = pool:get_variable(VAR_NAME, 'double') - if not stamp then - rspamd_logger.warnx(rspamd_config, 'No last DMARC report information to persist to disk') - return - end - local f, err = io.open(statefile, 'w') - if err then - rspamd_logger.errx(rspamd_config, 'Unable to write statefile to disk: %s', err) - return - end - assert(f:write(pool:get_variable(VAR_NAME, 'double'))) - assert(f:close()) - pool:destroy() - end) - - local get_reporting_domain, reporting_domain, report_start, - report_end, report_id, want_period, report_key - local reporting_addrs = {} - local bcc_addrs = {} - local domain_policy = {} - local to_verify = {} - local cursor = 0 - local function dmarc_report_xml() - local entries = {} - report_id = string.format('%s.%d.%d', - reporting_domain, report_start, report_end) - lua_util.debugm(N, rspamd_config, 'new report: %s', report_id) - local actions = { - push = function(t) - local data = t[1] - local split = rspamd_str_split(data, ',') - local row = { - ip = split[1], - spf_disposition = split[2], - dkim_disposition = split[3], - disposition = split[4], - override = split[5], - header_from = split[6], - dkim_results = {}, - spf_domain = split[11], - spf_result = split[12], - count = t[2], - } - if split[7] and split[7] ~= '' then - local tmp = rspamd_str_split(split[7], '|') - for _, d in ipairs(tmp) do - table.insert(row.dkim_results, {domain = d, result = 'pass'}) - end - end - if split[8] and split[8] ~= '' then - local tmp = rspamd_str_split(split[8], '|') - for _, d in ipairs(tmp) do - table.insert(row.dkim_results, {domain = d, result = 'fail'}) - end - end - if split[9] and split[9] ~= '' then - local tmp = rspamd_str_split(split[9], '|') - for _, d in ipairs(tmp) do - table.insert(row.dkim_results, {domain = d, result = 'temperror'}) - end - end - if split[10] and split[10] ~= '' then - local tmp = lua_util.str_split(split[10], '|') - for _, d in ipairs(tmp) do - table.insert(row.dkim_results, - {domain = d, result = 'permerror'}) - end - end - table.insert(entries, row) - end, - -- TODO: please rework this shit - header = function() - return table.concat({ - '', - escape_xml(report_settings.org_name), '', - escape_xml(report_settings.email), '', - report_id, '', report_start, - '', report_end, '', - reporting_domain, '', escape_xml(domain_policy.adkim), '', - escape_xml(domain_policy.aspf), '

', escape_xml(domain_policy.p), - '

', escape_xml(domain_policy.sp), '', - escape_xml(domain_policy.pct), - '' - }) - end, - footer = function() - return [[]] - end, - entries = function() - local buf = {} - for _, e in pairs(entries) do - table.insert(buf, entry_to_xml(e)) - end - return table.concat(buf, '') - end, - } - return function(action, p) - local f = actions[action] - if not f then error('invalid action: ' .. action) end - return f(p) - end - end - - - local function send_report_via_email(xmlf, retry) - if not retry then retry = 0 end - - local function sendmail_cb(ret, err) - if not ret then - rspamd_logger.errx(rspamd_config, "Couldn't send mail for %s: %s", err) - if retry >= report_settings.retries then - rspamd_logger.errx(rspamd_config, "Couldn't send mail for %s: retries exceeded", reporting_domain) - return get_reporting_domain() - else - send_report_via_email(xmlf, retry + 1) - end - else - get_reporting_domain() - end - end - - -- Format message - local list_rcpt = lua_util.keys(reporting_addrs) - - local encoded = rspamd_util.encode_base64(rspamd_util.gzip_compress( - table.concat( - {xmlf('header'), - xmlf('entries'), - xmlf('footer')})), 73) - local addr_string = table.concat(list_rcpt, ', ') - - local bcc_addrs_keys = lua_util.keys(bcc_addrs) - local bcc_string - if #bcc_addrs_keys > 0 then - bcc_string = table.concat(bcc_addrs_keys, ', ') - end - - local rhead = lua_util.jinja_template(report_template, - { - from_name = report_settings.from_name, - from_addr = report_settings.email, - rcpt = addr_string, - bcc = bcc_string, - reporting_domain = reporting_domain, - submitter = report_settings.domain, - report_id = report_id, - report_date = rspamd_util.time_to_string(rspamd_util.get_time()), - message_id = rspamd_util.random_hex(16) .. '@' .. report_settings.msgid_from, - report_start = report_start, - report_end = report_end - }, true) - local message = { - (rhead:gsub("\n", "\r\n")), - encoded, - (report_footer:gsub("\n", "\r\n")) - } - - local lua_smtp = require "lua_smtp" - lua_smtp.sendmail({ - ev_base = ev_base, - config = rspamd_config, - host = report_settings.smtp, - port = report_settings.smtp_port, - resolver = rspamd_config:get_resolver(), - from = report_settings.email, - recipients = list_rcpt, - helo = report_settings.helo, - }, message, sendmail_cb) - end - - - local function make_report() - if type(report_settings.override_address) == 'string' then - reporting_addrs = { [report_settings.override_address] = true} - end - if type(report_settings.additional_address) == 'string' then - if report_settings.additional_address_bcc then - bcc_addrs[report_settings.additional_address] = true - else - reporting_addrs[report_settings.additional_address] = true - end - end - rspamd_logger.infox(rspamd_config, 'sending report for %s <%s> (<%s> bcc)', - reporting_domain, reporting_addrs, bcc_addrs) - local dmarc_xml = dmarc_report_xml() - local dmarc_push_cb - dmarc_push_cb = function(err, data) - if err then - rspamd_logger.errx(rspamd_config, 'redis request failed: %s', err) - -- XXX: data is orphaned; replace key or delete data - get_reporting_domain() - elseif type(data) == 'table' then - cursor = tonumber(data[1]) - for i = 1, #data[2], 2 do - dmarc_xml('push', {data[2][i], data[2][i+1]}) - end - if cursor ~= 0 then - local ret = rspamd_redis.redis_make_request_taskless(ev_base, - rspamd_config, - redis_params, - nil, - false, -- is write - dmarc_push_cb, --callback - 'HSCAN', -- command - {report_key, cursor, 'COUNT', report_settings.hscan_count} - ) - if not ret then - rspamd_logger.errx(rspamd_config, 'failed to schedule redis request') - get_reporting_domain() - end - else - send_report_via_email(dmarc_xml) - end - end - end - - local ret = rspamd_redis.redis_make_request_taskless(ev_base, - rspamd_config, - redis_params, - nil, - false, -- is write - dmarc_push_cb, --callback - 'HSCAN', -- command - {report_key, cursor, 'COUNT', report_settings.hscan_count} - ) - if not ret then - rspamd_logger.errx(rspamd_config, 'failed to schedule redis request') - -- XXX: data is orphaned; replace key or delete data - get_reporting_domain() - end - end - local function delete_reports() - local function delete_reports_cb(err) - if err then - rspamd_logger.errx(rspamd_config, 'error deleting reports: %s', err) - end - rspamd_logger.infox(rspamd_config, 'deleted reports for %s', reporting_domain) - get_reporting_domain() - end - local ret = rspamd_redis.redis_make_request_taskless(ev_base, - rspamd_config, - redis_params, - nil, - true, -- is write - delete_reports_cb, --callback - 'DEL', -- command - {report_key} - ) - if not ret then - rspamd_logger.errx(rspamd_config, 'failed to schedule redis request') - get_reporting_domain() - end - end - local function verify_reporting_address() - local function verifier(test_addr, vdom) - local function verify_cb(resolver, to_resolve, results, err, _, authenticated) - if err then - rspamd_logger.errx(rspamd_config, 'lookup error [%s]: %s', - to_resolve, err) - rspamd_logger.infox(rspamd_config, 'reports to %s for %s not authorised', - test_addr, reporting_domain) - to_verify[test_addr] = nil - else - local is_authed = false - -- XXX: reporting address could be overridden - for _, r in ipairs(results) do - -- Oh wow - if string.match(r, 'v=DMARC1') then - is_authed = true - break - end - end - if not is_authed then - to_verify[test_addr] = nil - rspamd_logger.infox(rspamd_config, 'Reports to %s for %s not authorised', test_addr, reporting_domain) - else - to_verify[test_addr] = nil - reporting_addrs[test_addr] = true - end - end - -- TODO: wtf this code does??? - local t, nvdom = next(to_verify) - if not t then - if next(reporting_addrs) then - make_report() - else - rspamd_logger.infox(rspamd_config, 'No valid reporting addresses for %s', reporting_domain) - delete_reports() - end - else - verifier(t, nvdom) - end - end - rspamd_config:get_resolver():resolve('txt', { - ev_base = ev_base, - name = string.format('%s._report._dmarc.%s', - reporting_domain, vdom), - callback = verify_cb, - }) - end - -- TODO: recursion and wtf again - local t, vdom = next(to_verify) - verifier(t, vdom) - end - local function get_reporting_address() - local retry = 0 - local esld = rspamd_util.get_tld(reporting_domain) - local function check_addr_cb(resolver, to_resolve, results, err, _, authenticated) - if err then - if err == 'no records with this name' or err == 'requested record is not found' then - if reporting_domain ~= esld then - rspamd_config:get_resolver():resolve('txt', { - ev_base = ev_base, - name = string.format('_dmarc.%s', esld), - callback = check_addr_cb, - }) - else - rspamd_logger.errx(rspamd_config, 'no DMARC record found for %s', reporting_domain) - delete_reports() - end - else - rspamd_logger.errx(rspamd_config, 'lookup error [%s]: %s', to_resolve, err) - if retry < report_settings.retries then - retry = retry + 1 - rspamd_config:get_resolver():resolve('txt', { - ev_base = ev_base, - name = to_resolve, - callback = check_addr_cb, - }) - else - rspamd_logger.errx(rspamd_config, "couldn't get reporting address for %s: retries exceeded", - reporting_domain) - delete_reports() - end - end - else - local policy - local found_policy, failed_policy = false, false - for _, r in ipairs(results) do - local elts = dmarc_grammar:match(r) - if elts and found_policy then - failed_policy = true - elseif elts then - found_policy = true - policy = dmarc_key_value_case(elts) - end - end - if not found_policy then - rspamd_logger.errx(rspamd_config, 'no policy: %s', to_resolve) - if reporting_domain ~= esld then - rspamd_config:get_resolver():resolve('txt', { - ev_base = ev_base, - name = string.format('_dmarc.%s', esld), - callback = check_addr_cb, - }) - else - delete_reports() - end - elseif failed_policy then - rspamd_logger.errx(rspamd_config, 'duplicate policies: %s', to_resolve) - delete_reports() - elseif not policy['rua'] then - rspamd_logger.errx(rspamd_config, 'no reporting address: %s', to_resolve) - delete_reports() - else - local upool = mempool.create() - local split = rspamd_str_split(policy['rua']:gsub('%s+', ''), ',') - for _, m in ipairs(split) do - local url = rspamd_url.create(upool, m) - if not url then - rspamd_logger.errx(rspamd_config, "couldn't extract reporting address: %s", policy['rua']) - else - local urlt = url:to_table() - if urlt['protocol'] ~= 'mailto' then - rspamd_logger.errx(rspamd_config, 'invalid URL: %s', url) - else - if string.lower(urlt['tld']) == string.lower(rspamd_util.get_tld(reporting_domain)) then - reporting_addrs[string.format('%s@%s', urlt['user'], urlt['host'])] = true - else - to_verify[string.format('%s@%s', urlt['user'], urlt['host'])] = urlt['host'] - end - end - end - end - upool:destroy() - domain_policy['pct'] = policy['pct'] or 100 - domain_policy['adkim'] = policy['adkim'] or 'r' - domain_policy['aspf'] = policy['aspf'] or 'r' - domain_policy['p'] = policy['p'] or 'none' - domain_policy['sp'] = policy['sp'] or 'none' - if next(to_verify) then - verify_reporting_address() - elseif next(reporting_addrs) then - make_report() - else - rspamd_logger.errx(rspamd_config, 'no reporting address for %s', reporting_domain) - delete_reports() - end - end - end - end - - rspamd_config:get_resolver():resolve('txt', { - ev_base = ev_base, - name = string.format('_dmarc.%s', reporting_domain), - callback = check_addr_cb, - }) - end - get_reporting_domain = function() - reporting_domain = nil - reporting_addrs = {} - domain_policy = {} - cursor = 0 - local function get_reporting_domain_cb(err, data) - if err then - rspamd_logger.errx(cfg, 'unable to get DMARC domain: %s', err) - else - if type(data) == 'userdata' then - reporting_domain = nil - else - report_key = data - local tmp = rspamd_str_split(data, redis_keys.join_char) - reporting_domain = tmp[2] - end - if not reporting_domain then - rspamd_logger.infox(cfg, 'no more domains to generate reports for') - else - get_reporting_address() - end - end - end - local idx_key = table.concat({redis_keys.index_prefix, want_period}, redis_keys.join_char) - local ret = rspamd_redis.redis_make_request_taskless(ev_base, - rspamd_config, - redis_params, - nil, - true, -- is write - get_reporting_domain_cb, --callback - 'SPOP', -- command - {idx_key} - ) - if not ret then - rspamd_logger.errx(cfg, 'unable to get DMARC domain') - end - end - local function send_reports(time) - rspamd_logger.infox(rspamd_config, 'sending reports ostensibly %1', time) - pool:set_variable(VAR_NAME, time) - local yesterday = os.date('!*t', rspamd_util.get_time() - INTERVAL) - local today = os.date('!*t', rspamd_util.get_time()) - report_start = os.time({ - year = yesterday.year, - month = yesterday.month, - day = yesterday.day, - hour = 0}) + tz_offset - report_end = os.time({ - year = today.year, - month = today.month, - day = today.day, - hour = 0}) + tz_offset - want_period = table.concat({ - yesterday.year, - string.format('%02d', yesterday.month), - string.format('%02d', yesterday.day) - }) - get_reporting_domain() - end - -- Push reports at regular intervals - local function schedule_regular_send() - rspamd_config:add_periodic(ev_base, INTERVAL, function () - send_reports() - return true - end) - end - -- Push reports to backend and reschedule check - local function schedule_intermediate_send(when) - rspamd_config:add_periodic(ev_base, when, function () - schedule_regular_send() - send_reports(rspamd_util.get_time()) - return false - end) - end - -- Try read statefile on startup - local stamp - local f, err = io.open(statefile, 'r') - if err then - rspamd_logger.errx(rspamd_config, 'failed to open statefile: %s', err) - end - if f then - io.input(f) - stamp = tonumber(io.read()) - pool:set_variable(VAR_NAME, stamp) - end - local time = rspamd_util.get_time() - if not stamp then - lua_util.debugm(N, rspamd_config, 'no state found - sending reports immediately') - schedule_regular_send() - send_reports(time) - return - end - local delta = stamp - time + INTERVAL - if delta <= 0 then - lua_util.debugm(N, rspamd_config, 'last send is too old - sending reports immediately') - schedule_regular_send() - send_reports(time) - return - end - lua_util.debugm(N, rspamd_config, 'scheduling next send in %s seconds', delta) - schedule_intermediate_send(delta) - end) - end -end -if type(opts['actions']) == 'table' then - dmarc_actions = opts['actions'] -end -if type(opts['report_settings']) == 'table' then - for k, v in pairs(opts['report_settings']) do - report_settings[k] = v - end -end -if opts['send_reports'] then - for _, e in ipairs({'email', 'domain', 'org_name'}) do - if not report_settings[e] then - rspamd_logger.errx(rspamd_config, 'missing required setting: report_settings.%s', e) - return - end end end -- Check spf and dkim sections for changed symbols local function check_mopt(var, m_opts, name) if m_opts[name] then - symbols[var] = tostring(m_opts[name]) + settings.symbols[var] = tostring(m_opts[name]) end end @@ -1388,62 +722,60 @@ rspamd_config:register_symbol({ parent = id, }) rspamd_config:register_symbol({ - name = dmarc_symbols['allow'], + name = settings.symbols['allow'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ - name = dmarc_symbols['reject'], + name = settings.symbols['reject'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ - name = dmarc_symbols['quarantine'], + name = settings.symbols['quarantine'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ - name = dmarc_symbols['softfail'], + name = settings.symbols['softfail'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ - name = dmarc_symbols['dnsfail'], + name = settings.symbols['dnsfail'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ - name = dmarc_symbols['badpolicy'], + name = settings.symbols['badpolicy'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ - name = dmarc_symbols['na'], + name = settings.symbols['na'], parent = id, group = 'policies', groups = {'dmarc'}, type = 'virtual' }) -rspamd_config:register_dependency('DMARC_CHECK', symbols['spf_allow_symbol']) -rspamd_config:register_dependency('DMARC_CHECK', symbols['dkim_allow_symbol']) +rspamd_config:register_dependency('DMARC_CHECK', settings.symbols['spf_allow_symbol']) +rspamd_config:register_dependency('DMARC_CHECK', settings.symbols['dkim_allow_symbol']) -- DMARC munging support - -if opts.munging then - local lua_maps = require "lua_maps" +if settings.munging then local lua_maps_expressions = require "lua_maps_expressions" local lua_mime = require "lua_mime" @@ -1456,7 +788,7 @@ if opts.munging then munge_map_condition = nil, -- maps expression to enable munging } - local munging_opts = lua_util.override_defaults(munging_defaults, opts.munging) + local munging_opts = lua_util.override_defaults(munging_defaults, settings.munging) if not munging_opts.list_map then rspamd_logger.errx(rspamd_config, 'cannot enable DMARC munging with no list_map parameter') @@ -1480,27 +812,27 @@ if opts.munging then local function dmarc_munge_callback(task) if munging_opts.mitigate_allow_only then - if not task:has_symbol(dmarc_symbols.allow) then + if not task:has_symbol(settings.symbols.allow) then lua_util.debugm(N, task, 'skip munging, no %s symbol', - dmarc_symbols.allow) + settings.symbols.allow) -- Excepted return end else - local has_dmarc = task:has_symbol(dmarc_symbols.allow) or - task:has_symbol(dmarc_symbols.quarantine) or - task:has_symbol(dmarc_symbols.reject) or - task:has_symbol(dmarc_symbols.softfail) + local has_dmarc = task:has_symbol(settings.symbols.allow) or + task:has_symbol(settings.symbols.quarantine) or + task:has_symbol(settings.symbols.reject) or + task:has_symbol(settings.symbols.softfail) if not has_dmarc then lua_util.debugm(N, task, 'skip munging, no %s symbol', - dmarc_symbols.allow) + settings.symbols.allow) -- Excepted return end end if munging_opts.mitigate_strict_only then - local s = task:get_symbol(dmarc_symbols.allow) or {[1] = {}} + local s = task:get_symbol(settings.symbols.allow) or {[1] = {}} local sopts = s[1].options or {} local seen_strict @@ -1513,7 +845,7 @@ if opts.munging then if not seen_strict then lua_util.debugm(N, task, 'skip munging, no strict policy found in %s', - dmarc_symbols.allow) + settings.symbols.allow) -- Excepted return end -- 2.39.5