[Rework] Move common and rarely used dmarc code to the library

2 files changed, 246 insertions, 212 deletions

diff --git a/lualib/plugins/dmarc.lua b/lualib/plugins/dmarc.lua
new file mode 100644
index 000000000..9477f3c0d
--- /dev/null
+++ b/lualib/plugins/dmarc.lua
@@ -0,0 +1,233 @@
+--[[
+Copyright (c) 2011-2016, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2015-2016, Andrew Lewis <nerf@judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Common dmarc stuff
+local rspamd_logger = require "rspamd_logger"
+local N = "dmarc"
+
+local exports = {}
+
+exports.default_settings = {
+  auth_and_local_conf = false,
+  symbols = {
+    spf_allow_symbol = 'R_SPF_ALLOW',
+    spf_deny_symbol = 'R_SPF_FAIL',
+    spf_softfail_symbol = 'R_SPF_SOFTFAIL',
+    spf_neutral_symbol = 'R_SPF_NEUTRAL',
+    spf_tempfail_symbol = 'R_SPF_DNSFAIL',
+    spf_permfail_symbol = 'R_SPF_PERMFAIL',
+    spf_na_symbol = 'R_SPF_NA',
+
+    dkim_allow_symbol = 'R_DKIM_ALLOW',
+    dkim_deny_symbol = 'R_DKIM_REJECT',
+    dkim_tempfail_symbol = 'R_DKIM_TEMPFAIL',
+    dkim_na_symbol = 'R_DKIM_NA',
+    dkim_permfail_symbol = 'R_DKIM_PERMFAIL',
+
+    -- DMARC symbols
+    allow = 'DMARC_POLICY_ALLOW',
+    badpolicy = 'DMARC_BAD_POLICY',
+    dnsfail = 'DMARC_DNSFAIL',
+    na = 'DMARC_NA',
+    reject = 'DMARC_POLICY_REJECT',
+    softfail = 'DMARC_POLICY_SOFTFAIL',
+    quarantine = 'DMARC_POLICY_QUARANTINE',
+  },
+  no_sampling_domains = nil,
+  no_reporting_domains = nil,
+  reporting = {
+    report_local_controller = false, -- Store reports for local/controller scans (for testing only)
+    redis_keys = {
+      index_prefix = 'dmarc_idx',
+      report_prefix = 'dmarc_rpt',
+      join_char = ';',
+    },
+    enabled = false,
+    max_entries = 1000,
+    keys_expire = 172800,
+    only_domains = nil,
+  },
+  actions = {},
+}
+
+
+-- Returns a key used to be inserted into dmarc report sample
+exports.dmarc_report = function (task, settings, data)
+  local rspamd_lua_utils = require "lua_util"
+  local E = {}
+
+  local ip = task:get_from_ip()
+  if ip and not ip:is_valid() then
+    rspamd_logger.infox(task, 'cannot store dmarc report for %s: no valid source IP',
+        data.domain)
+    return nil
+  end
+
+  ip = ip:to_string()
+
+  if rspamd_lua_utils.is_rspamc_or_controller(task) and not settings.reporting.report_local_controller then
+    rspamd_logger.infox(task, 'cannot store dmarc report for %s from IP %s: has come from controller/rspamc',
+        data.domain, ip)
+    return
+  end
+
+  local dkim_pass = table.concat(data.dkim_results.pass or E, '|')
+  local dkim_fail = table.concat(data.dkim_results.fail or E, '|')
+  local dkim_temperror = table.concat(data.dkim_results.temperror or E, '|')
+  local dkim_permerror = table.concat(data.dkim_results.permerror or E, '|')
+  local disposition_to_return = data.disposition
+  local res = table.concat({
+    ip, data.spf_ok, data.dkim_ok,
+    disposition_to_return, (data.sampled_out and 'sampled_out' or ''), data.domain,
+    dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, data.spf_domain, data.spf_result}, ',')
+
+  return res
+end
+
+
+exports.gen_munging_callback = function(munging_opts, settings)
+  local lua_util = require "lua_util"
+  local rspamd_util = require "rspamd_util"
+  local lua_mime = require "lua_mime"
+  return function (task)
+    if munging_opts.mitigate_allow_only then
+      if not task:has_symbol(settings.symbols.allow) then
+        lua_util.debugm(N, task, 'skip munging, no %s symbol',
+            settings.symbols.allow)
+        -- Excepted
+        return
+      end
+    else
+      local has_dmarc = task:has_symbol(settings.symbols.allow) or
+          task:has_symbol(settings.symbols.quarantine) or
+          task:has_symbol(settings.symbols.reject) or
+          task:has_symbol(settings.symbols.softfail)
+
+      if not has_dmarc then
+        lua_util.debugm(N, task, 'skip munging, no %s symbol',
+            settings.symbols.allow)
+        -- Excepted
+        return
+      end
+    end
+    if munging_opts.mitigate_strict_only then
+      local s = task:get_symbol(settings.symbols.allow) or {[1] = {}}
+      local sopts = s[1].options or {}
+
+      local seen_strict
+      for _,o in ipairs(sopts) do
+        if o == 'reject' or o == 'quarantine' then
+          seen_strict = true
+          break
+        end
+      end
+
+      if not seen_strict then
+        lua_util.debugm(N, task, 'skip munging, no strict policy found in %s',
+            settings.symbols.allow)
+        -- Excepted
+        return
+      end
+    end
+    if munging_opts.munge_map_condition then
+      local accepted,trace = munging_opts.munge_map_condition:process(task)
+      if not accepted then
+        lua_util.debugm(task, 'skip munging, maps condition not satisified: (%s)',
+            trace)
+        -- Excepted
+        return
+      end
+    end
+    -- Now, look for domain for munging
+    local mr = task:get_recipients({ 'mime', 'orig'})
+    local rcpt_found
+    if mr then
+      for _,r in ipairs(mr) do
+        if r.domain and munging_opts.list_map:get_key(r.addr) then
+          rcpt_found = r
+          break
+        end
+      end
+    end
+
+    if not rcpt_found then
+      lua_util.debugm(task, 'skip munging, recipients are not in list_map')
+      -- Excepted
+      return
+    end
+
+    local from = task:get_from({ 'mime', 'orig'})
+
+    if not from or not from[1] then
+      lua_util.debugm(task, 'skip munging, from is bad')
+      -- Excepted
+      return
+    end
+
+    from = from[1]
+    local via_user = rcpt_found.user
+    local via_addr = rcpt_found.addr
+    local via_name
+
+    if from.name then
+      via_name = string.format('%s via %s', from.name, via_user)
+    else
+      via_name = string.format('%s via %s', from.user or 'unknown', via_user)
+    end
+
+    local hdr_encoded = rspamd_util.fold_header('From',
+        rspamd_util.mime_header_encode(string.format('%s <%s>',
+            via_name, via_addr)))
+    local orig_from_encoded = rspamd_util.fold_header('X-Original-From',
+        rspamd_util.mime_header_encode(string.format('%s <%s>',
+            from.name or '', from.addr)))
+    local add_hdrs = {
+      ['From'] = { order = 1, value = hdr_encoded },
+    }
+    local remove_hdrs = {['From'] = 0}
+
+    local nreply = from.addr
+    if munging_opts.reply_goes_to_list then
+      -- Reply-to goes to the list
+      nreply = via_addr
+    end
+
+    if task:has_header('Reply-To') then
+      -- If we have reply-to header, then we need to insert an additional
+      -- address there
+      local orig_reply = task:get_header_full('Reply-To')[1]
+      if orig_reply.value then
+        nreply = string.format('%s, %s', orig_reply.value, nreply)
+      end
+      remove_hdrs['Reply-To'] = 1
+    end
+
+    add_hdrs['Reply-To'] = {order = 0, value = nreply}
+
+    add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded}
+    lua_mime.modify_headers(task, {
+      remove = remove_hdrs,
+      add = add_hdrs
+    })
+    lua_util.debugm(N, task, 'munged DMARC header for %s: %s -> %s',
+        from.domain, hdr_encoded, from.addr)
+    rspamd_logger.infox(task, 'munged DMARC header for %s', from.addr)
+    task:insert_result('DMARC_MUNGED', 1.0, from.addr)
+  end
+end
+
+return exports
\ No newline at end of file
diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua
index 417bd89eb..0209dedb8 100644
--- a/src/plugins/lua/dmarc.lua
+++ b/src/plugins/lua/dmarc.lua
@@ -21,6 +21,7 @@ local rspamd_logger = require "rspamd_logger"
 local rspamd_util = require "rspamd_util"
 local lua_redis = require "lua_redis"
 local lua_util = require "lua_util"
+local dmarc_common = require "plugins/dmarc"
 
 if confighelp then
   return
@@ -28,48 +29,7 @@ end
 
 local N = 'dmarc'
 
-local settings = {
-  auth_and_local_conf = false,
-  symbols = {
-    spf_allow_symbol = 'R_SPF_ALLOW',
-    spf_deny_symbol = 'R_SPF_FAIL',
-    spf_softfail_symbol = 'R_SPF_SOFTFAIL',
-    spf_neutral_symbol = 'R_SPF_NEUTRAL',
-    spf_tempfail_symbol = 'R_SPF_DNSFAIL',
-    spf_permfail_symbol = 'R_SPF_PERMFAIL',
-    spf_na_symbol = 'R_SPF_NA',
-
-    dkim_allow_symbol = 'R_DKIM_ALLOW',
-    dkim_deny_symbol = 'R_DKIM_REJECT',
-    dkim_tempfail_symbol = 'R_DKIM_TEMPFAIL',
-    dkim_na_symbol = 'R_DKIM_NA',
-    dkim_permfail_symbol = 'R_DKIM_PERMFAIL',
-
-    -- DMARC symbols
-    allow = 'DMARC_POLICY_ALLOW',
-    badpolicy = 'DMARC_BAD_POLICY',
-    dnsfail = 'DMARC_DNSFAIL',
-    na = 'DMARC_NA',
-    reject = 'DMARC_POLICY_REJECT',
-    softfail = 'DMARC_POLICY_SOFTFAIL',
-    quarantine = 'DMARC_POLICY_QUARANTINE',
-  },
-  no_sampling_domains = nil,
-  no_reporting_domains = nil,
-  reporting = {
-    report_local_controller = false, -- Store reports for local/controller scans (for testing only)
-    redis_keys = {
-      index_prefix = 'dmarc_idx',
-      report_prefix = 'dmarc_rpt',
-      join_char = ';',
-    },
-    enabled = false,
-    max_entries = 1000,
-    keys_expire = 172800,
-    only_domains = nil,
-  },
-  actions = {},
-}
+local settings = dmarc_common.default_settings
 
 local redis_params = nil
 
@@ -132,40 +92,6 @@ local function dmarc_key_value_case(elts)
   return result
 end
 
-
--- Returns a key used to be inserted into dmarc report sample
-local function dmarc_report(task, spf_ok, dkim_ok, disposition,
-    sampled_out, hfromdom, spfdom, dres, spf_result)
-  local rspamd_lua_utils = require "lua_util"
-
-  local ip = task:get_from_ip()
-  if ip and not ip:is_valid() then
-    rspamd_logger.infox(task, 'cannot store dmarc report for %s: no valid source IP',
-        hfromdom)
-    return nil
-  end
-
-  ip = ip:to_string()
-
-  if rspamd_lua_utils.is_rspamc_or_controller(task) and not settings.reporting.report_local_controller then
-    rspamd_logger.infox(task, 'cannot store dmarc report for %s from IP %s: has come from controller/rspamc',
-        hfromdom, ip)
-    return
-  end
-
-  local dkim_pass = table.concat(dres.pass or E, '|')
-  local dkim_fail = table.concat(dres.fail or E, '|')
-  local dkim_temperror = table.concat(dres.temperror or E, '|')
-  local dkim_permerror = table.concat(dres.permerror or E, '|')
-  local disposition_to_return = (disposition == "softfail") and "none" or disposition
-  local res = table.concat({
-    ip, spf_ok, dkim_ok,
-    disposition_to_return, (sampled_out and 'sampled_out' or ''), hfromdom,
-    dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, spfdom, spf_result}, ',')
-
-  return res
-end
-
 local function maybe_force_action(task, disposition)
   if disposition then
     local force_action = settings.actions[disposition]
@@ -503,15 +429,16 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld)
     local dmarc_domain_key = table.concat(
         {settings.reporting.redis_keys.report_prefix, hdrfromdom, policy.rua, period},
         settings.reporting.redis_keys.join_char)
-    local report_data = dmarc_report(task,
-        spf_ok and 'pass' or 'fail',
-        dkim_ok and 'pass' or 'fail',
-        disposition,
-        sampled_out,
-        hdrfromdom,
-        spf_domain,
-        dkim_results,
-        spf_result)
+    local report_data = dmarc_common.dmarc_report(task, settings, {
+        spf_ok = spf_ok and 'pass' or 'fail',
+        dkim_ok = dkim_ok and 'pass' or 'fail',
+        disposition = (disposition == "softfail") and "none" or disposition,
+        sampled_out = sampled_out,
+        domain = hdrfromdom,
+        spf_domain = spf_domain,
+        dkim_results = dkim_results,
+        spf_result = spf_result
+    })
 
 
     local idx_key = table.concat({settings.reporting.redis_keys.index_prefix, period},
@@ -806,7 +733,6 @@ rspamd_config:register_dependency('DMARC_CHECK', settings.symbols['dkim_allow_sy
 -- DMARC munging support
 if settings.munging then
   local lua_maps_expressions = require "lua_maps_expressions"
-  local lua_mime = require "lua_mime"
 
   local munging_defaults = {
     reply_goes_to_list = false,
@@ -839,131 +765,6 @@ if settings.munging then
             munging_opts.munge_map_condition, N)
   end
 
-  local function dmarc_munge_callback(task)
-    if munging_opts.mitigate_allow_only then
-      if not task:has_symbol(settings.symbols.allow) then
-        lua_util.debugm(N, task, 'skip munging, no %s symbol',
-                settings.symbols.allow)
-        -- Excepted
-        return
-      end
-    else
-      local has_dmarc = task:has_symbol(settings.symbols.allow) or
-              task:has_symbol(settings.symbols.quarantine) or
-              task:has_symbol(settings.symbols.reject) or
-              task:has_symbol(settings.symbols.softfail)
-
-      if not has_dmarc then
-        lua_util.debugm(N, task, 'skip munging, no %s symbol',
-                settings.symbols.allow)
-        -- Excepted
-        return
-      end
-    end
-    if munging_opts.mitigate_strict_only then
-      local s = task:get_symbol(settings.symbols.allow) or {[1] = {}}
-      local sopts = s[1].options or {}
-
-      local seen_strict
-      for _,o in ipairs(sopts) do
-        if o == 'reject' or o == 'quarantine' then
-          seen_strict = true
-          break
-        end
-      end
-
-      if not seen_strict then
-        lua_util.debugm(N, task, 'skip munging, no strict policy found in %s',
-                settings.symbols.allow)
-        -- Excepted
-        return
-      end
-    end
-    if munging_opts.munge_map_condition then
-      local accepted,trace = munging_opts.munge_map_condition:process(task)
-      if not accepted then
-        lua_util.debugm(task, 'skip munging, maps condition not satisified: (%s)',
-                trace)
-        -- Excepted
-        return
-      end
-    end
-    -- Now, look for domain for munging
-    local mr = task:get_recipients({ 'mime', 'orig'})
-    local rcpt_found
-    if mr then
-      for _,r in ipairs(mr) do
-        if r.domain and munging_opts.list_map:get_key(r.addr) then
-          rcpt_found = r
-          break
-        end
-      end
-    end
-
-    if not rcpt_found then
-      lua_util.debugm(task, 'skip munging, recipients are not in list_map')
-      -- Excepted
-      return
-    end
-
-    local from = task:get_from({ 'mime', 'orig'})
-
-    if not from or not from[1] then
-      lua_util.debugm(task, 'skip munging, from is bad')
-      -- Excepted
-      return
-    end
-
-    from = from[1]
-    local via_user = rcpt_found.user
-    local via_addr = rcpt_found.addr
-    local via_name
-
-    if from.name then
-      via_name = string.format('%s via %s', from.name, via_user)
-    else
-      via_name = string.format('%s via %s', from.user or 'unknown', via_user)
-    end
-
-    local hdr_encoded = rspamd_util.fold_header('From',
-            rspamd_util.mime_header_encode(string.format('%s <%s>',
-                    via_name, via_addr)))
-    local orig_from_encoded = rspamd_util.fold_header('X-Original-From',
-            rspamd_util.mime_header_encode(string.format('%s <%s>',
-                    from.name or '', from.addr)))
-    local add_hdrs = {
-      ['From'] = { order = 1, value = hdr_encoded },
-    }
-    local remove_hdrs = {['From'] = 0}
-
-    local nreply = from.addr
-    if munging_opts.reply_goes_to_list then
-      -- Reply-to goes to the list
-      nreply = via_addr
-    end
-
-    if task:has_header('Reply-To') then
-      -- If we have reply-to header, then we need to insert an additional
-      -- address there
-      local orig_reply = task:get_header_full('Reply-To')[1]
-      if orig_reply.value then
-        nreply = string.format('%s, %s', orig_reply.value, nreply)
-      end
-      remove_hdrs['Reply-To'] = 1
-    end
-
-    add_hdrs['Reply-To'] = {order = 0, value = nreply}
-
-    add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded}
-    lua_mime.modify_headers(task, {
-      remove = remove_hdrs,
-      add = add_hdrs
-    })
-    lua_util.debugm(N, task, 'munged DMARC header for %s: %s -> %s',
-            from.domain, hdr_encoded, from.addr)
-    rspamd_logger.infox(task, 'munged DMARC header for %s', from.addr)
-    task:insert_result('DMARC_MUNGED', 1.0, from.addr)
-  end
 
   rspamd_config:register_symbol({
     name = 'DMARC_MUNGED',
@@ -972,7 +773,7 @@ if settings.munging then
     score = 0,
     group = 'policies',
     groups = {'dmarc'},
-    callback = dmarc_munge_callback
+    callback = dmarc_common.gen_munging_callback(munging_opts, settings)
   })
 
   rspamd_config:register_dependency('DMARC_MUNGED', 'DMARC_CHECK')