From: Andrew Lewis <nerf@judo.za.org>
Date: Mon, 7 Jun 2021 16:36:58 +0000 (+0200)
Subject: [Feature] external_relay plugin
X-Git-Tag: 3.0~327^2
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=37d7bee0fe739c3262e40036309573a41e6f4b0a;p=rspamd.git

[Feature] external_relay plugin
---

diff --git a/conf/modules.d/external_relay.conf b/conf/modules.d/external_relay.conf
new file mode 100644
index 000000000..7d52ced0d
--- /dev/null
+++ b/conf/modules.d/external_relay.conf
@@ -0,0 +1,22 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify 'local.d/external_relay.conf' to add and merge
+# parameters defined inside this section
+#
+# You can modify 'override.d/external_relay.conf' to strictly override all
+# parameters defined inside this section
+#
+# See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+# for details
+#
+# Module documentation can be found at  https://rspamd.com/doc/modules/external_relay.html
+
+external_relay {
+  # This module is default-disabled
+  enabled = false;
+
+  .include(try=true,priority=5) "${DBDIR}/dynamic/external_relay.conf"
+  .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/external_relay.conf"
+  .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/external_relay.conf"
+}
diff --git a/src/plugins/lua/external_relay.lua b/src/plugins/lua/external_relay.lua
new file mode 100644
index 000000000..dc973b9a6
--- /dev/null
+++ b/src/plugins/lua/external_relay.lua
@@ -0,0 +1,250 @@
+--[[
+Copyright (c) 2021, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[
+external_relay plugin - sets IP/hostname from Received headers
+]]--
+
+if confighelp then
+  return
+end
+
+local lua_maps = require "lua_maps"
+local lua_util = require "lua_util"
+local rspamd_ip = require "rspamd_ip"
+local rspamd_logger = require "rspamd_logger"
+local ts = require("tableshape").types
+
+local E = {}
+local N = "external_relay"
+
+local settings = {
+  rules = {},
+}
+
+local config_schema = ts.shape{
+  enabled = ts.boolean:is_optional(),
+  rules = ts.map_of(
+    ts.string, ts.one_of{
+      ts.shape{
+        priority = ts.number:is_optional(),
+        strategy = 'authenticated',
+        symbol = ts.string:is_optional(),
+        user_map = lua_maps.map_schema:is_optional(),
+      },
+      ts.shape{
+        count = ts.number,
+        priority = ts.number:is_optional(),
+        strategy = 'count',
+        symbol = ts.string:is_optional(),
+      },
+      ts.shape{
+        priority = ts.number:is_optional(),
+        strategy = 'local',
+        symbol = ts.string:is_optional(),
+      },
+      ts.shape{
+        hostname_map = lua_maps.map_schema,
+        priority = ts.number:is_optional(),
+        strategy = 'hostname_map',
+        symbol = ts.string:is_optional(),
+      },
+    }
+  ),
+}
+
+local function set_from_rcvd(task, rcvd, remote_rcvd_ip)
+  if not remote_rcvd_ip then
+    if not rcvd.from_ip then
+      rspamd_logger.errx(task, 'no IP in header: %s', rcvd)
+      return
+    end
+    remote_rcvd_ip = rspamd_ip.from_string(rcvd.from_ip)
+    if not remote_rcvd_ip and remote_rcvd_ip:is_valid() then
+      rspamd_logger.errx(task, 'invalid remote IP: %s', rcvd.from_ip)
+      return
+    end
+  end
+  task:set_from_ip(remote_rcvd_ip)
+  if rcvd.from_hostname then
+    task:set_hostname(rcvd.from_hostname)
+    task:set_helo(rcvd.from_hostname) -- use fake value for HELO
+  else
+    rspamd_logger.warnx(task, "couldn't get hostname from headers")
+    local ipstr = string.format('[%s]', rcvd.from_ip)
+    task:set_hostname(ipstr) -- returns nil from task:get_hostname()
+    task:set_helo(ipstr)
+  end
+  return true
+end
+
+local strategies = {}
+
+strategies.authenticated = function(rule)
+  local user_map
+  if rule.user_map then
+    user_map = lua_maps.map_add_from_ucl(rule.user_map, 'set', 'external relay usernames')
+    if not user_map then
+      rspamd_logger.errx(rspamd_config, "couldn't add map %s; won't register symbol %s",
+          rule.user_map, rule.symbol)
+      return
+    end
+  end
+
+  return function(task)
+    local user = task:get_user()
+    if not user then
+      lua_util.debugm(N, task, 'sender is unauthenticated')
+      return
+    end
+    if user_map then
+      if not user_map:get_key(user) then
+        lua_util.debugm(N, task, 'sender (%s) is not in user_map', user)
+        return
+      end
+    end
+
+    local rcvd_hdrs = task:get_received_headers()
+    -- Try find end of authentication chain
+    for _, rcvd in ipairs(rcvd_hdrs) do
+      if not rcvd.flags.authenticated then
+        -- Found unauthenticated hop, use this header
+        return set_from_rcvd(task, rcvd)
+      end
+    end
+
+    rspamd_logger.errx(task, 'found nothing useful in Received headers')
+  end
+end
+
+strategies.count = function(rule)
+  return function(task)
+    local rcvd_hdrs = task:get_received_headers()
+    -- Reduce count by 1 if artificial header is present
+    local hdr_count
+    if ((rcvd_hdrs[1] or E).flags or E).artificial then
+      hdr_count = rule.count - 1
+    else
+      hdr_count = rule.count
+    end
+
+    local rcvd = rcvd_hdrs[hdr_count]
+    if not rcvd then
+      rspamd_logger.errx(task, 'found no received header #%s', hdr_count)
+      return
+    end
+
+    return set_from_rcvd(task, rcvd)
+  end
+end
+
+strategies.hostname_map = function(rule)
+  local hostname_map = lua_maps.map_add_from_ucl(rule.hostname_map, 'map', 'external relay hostnames')
+  if not hostname_map then
+    rspamd_logger.errx(rspamd_config, "couldn't add map %s; won't register symbol %s",
+        rule.hostname_map, rule.symbol)
+    return
+  end
+
+  return function(task)
+    local from_hn = task:get_hostname()
+    if not from_hn then
+      lua_util.debugm(N, task, 'sending hostname is missing')
+      return
+    end
+
+    if hostname_map:get_key(from_hn) ~= 'direct' then
+      lua_util.debugm(N, task, 'sending hostname (%s) is not a direct relay', from_hn)
+      return
+    end
+
+    local rcvd_hdrs = task:get_received_headers()
+    -- Try find sending hostname in Received headers
+    for _, rcvd in ipairs(rcvd_hdrs) do
+      if rcvd.by_hostname == from_hn and rcvd.from_ip then
+        if not hostname_map:get_key(rcvd.from_hostname) then
+          -- Remote hostname is not another relay, use this header
+          return set_from_rcvd(task, rcvd)
+        else
+          -- Keep checking with new hostname
+          from_hn = rcvd.from_hostname
+        end
+      end
+    end
+
+    rspamd_logger.errx(task, 'found nothing useful in Received headers')
+  end
+end
+
+strategies['local'] = function(rule)
+  return function(task)
+    local from_ip = task:get_from_ip()
+    if not from_ip then
+      lua_util.debugm(N, task, 'sending IP is missing')
+      return
+    end
+
+    if not from_ip:is_local() then
+      lua_util.debugm(N, task, 'sending IP (%s) is non-local', from_ip)
+      return
+    end
+
+    local rcvd_hdrs = task:get_received_headers()
+    local num_rcvd = #rcvd_hdrs
+    -- Try find first non-local IP in Received headers
+    for i, rcvd in ipairs(rcvd_hdrs) do
+      if rcvd.from_ip then
+        local remote_rcvd_ip = rspamd_ip.from_string(rcvd.from_ip)
+        if remote_rcvd_ip and remote_rcvd_ip:is_valid() and (not remote_rcvd_ip:is_local() or i == num_rcvd) then
+          return set_from_rcvd(task, rcvd, remote_rcvd_ip)
+        end
+      end
+    end
+
+    rspamd_logger.errx(task, 'found nothing useful in Received headers')
+  end
+end
+
+local opts = rspamd_config:get_all_opt(N)
+if opts then
+  settings = lua_util.override_defaults(settings, opts)
+
+  local ok, schema_err = config_schema:transform(settings)
+  if not ok then
+    rspamd_logger.errx(rspamd_config, 'config schema error: %s', schema_err)
+    lua_util.disable_module(N, "config")
+    return
+  end
+
+  for k, rule in pairs(settings.rules) do
+
+    if not rule.symbol then
+      rule.symbol = k
+    end
+
+    local cb = strategies[rule.strategy](rule)
+
+    if cb then
+      rspamd_config:register_symbol({
+        name = rule.symbol,
+        type = 'prefilter',
+        priority = rule.priority or 20,
+        group = N,
+        callback = cb,
+      })
+    end
+  end
+end
diff --git a/test/functional/cases/001_merged/__init__.robot b/test/functional/cases/001_merged/__init__.robot
index 07f877c46..7f2c7bb43 100644
--- a/test/functional/cases/001_merged/__init__.robot
+++ b/test/functional/cases/001_merged/__init__.robot
@@ -6,9 +6,10 @@ Resource        ${RSPAMD_TESTDIR}/lib/rspamd.robot
 Variables       ${RSPAMD_TESTDIR}/lib/vars.py
 
 *** Variables ***
-${CONFIG}             ${RSPAMD_TESTDIR}/configs/merged.conf
-${REDIS_SCOPE}        Suite
-${RSPAMD_MAP_MAP}     ${RSPAMD_TESTDIR}/configs/maps/map.list
-${RSPAMD_RADIX_MAP}   ${RSPAMD_TESTDIR}/configs/maps/ip2.list
-${RSPAMD_REGEXP_MAP}  ${RSPAMD_TESTDIR}/configs/maps/regexp.list
-${RSPAMD_SCOPE}       Suite
+${CONFIG}                         ${RSPAMD_TESTDIR}/configs/merged.conf
+${REDIS_SCOPE}                    Suite
+${RSPAMD_EXTERNAL_RELAY_ENABLED}  false
+${RSPAMD_MAP_MAP}                 ${RSPAMD_TESTDIR}/configs/maps/map.list
+${RSPAMD_RADIX_MAP}               ${RSPAMD_TESTDIR}/configs/maps/ip2.list
+${RSPAMD_REGEXP_MAP}              ${RSPAMD_TESTDIR}/configs/maps/regexp.list
+${RSPAMD_SCOPE}                   Suite
diff --git a/test/functional/cases/380_external_relay.robot b/test/functional/cases/380_external_relay.robot
new file mode 100644
index 000000000..ff301626e
--- /dev/null
+++ b/test/functional/cases/380_external_relay.robot
@@ -0,0 +1,41 @@
+*** Settings ***
+Suite Setup     Rspamd Setup
+Suite Teardown  Rspamd Teardown
+Library         ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource        ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables       ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG}                         ${RSPAMD_TESTDIR}/configs/merged.conf
+${RSPAMD_EXTERNAL_RELAY_ENABLED}  true
+${RSPAMD_SCOPE}                   Suite
+
+*** Test Cases ***
+EXTERNAL RELAY AUTHENTICATED
+  Scan File  ${RSPAMD_TESTDIR}/messages/received5.eml
+  ...  Settings={symbols_enabled [EXTERNAL_RELAY_TEST, EXTERNAL_RELAY_AUTHENTICATED]}
+  ...  IP=8.8.8.8  User=user@example.net
+  Expect Symbol With Exact Options  EXTERNAL_RELAY_TEST
+  ...  IP=192.0.2.1  HOSTNAME=mail.example.org  HELO=mail.example.org
+
+EXTERNAL RELAY COUNT
+  Scan File  ${RSPAMD_TESTDIR}/messages/received4.eml
+  ...  Settings={symbols_enabled [EXTERNAL_RELAY_TEST, EXTERNAL_RELAY_COUNT]}
+  ...  IP=8.8.8.8
+  Expect Symbol With Exact Options  EXTERNAL_RELAY_TEST
+  ...  IP=151.18.193.131  HOSTNAME=ca-18-193-131.service.infuturo.it
+  ...  HELO=ca-18-193-131.service.infuturo.it
+
+EXTERNAL RELAY HOSTNAME MAP
+  Scan File  ${RSPAMD_TESTDIR}/messages/received6.eml
+  ...  Settings={symbols_enabled [EXTERNAL_RELAY_TEST, EXTERNAL_RELAY_HOSTNAME_MAP]}
+  ...  Hostname=lame.example.net  IP=192.0.2.10
+  Expect Symbol With Exact Options  EXTERNAL_RELAY_TEST
+  ...  IP=192.0.2.1  HOSTNAME=mail.example.org  HELO=mail.example.org
+
+EXTERNAL RELAY LOCAL
+  Scan File  ${RSPAMD_TESTDIR}/messages/ham.eml
+  ...  Settings={symbols_enabled [EXTERNAL_RELAY_TEST, EXTERNAL_RELAY_LOCAL]}
+  ...  IP=127.0.0.1
+  Expect Symbol With Exact Options  EXTERNAL_RELAY_TEST
+  ...  IP=4.31.198.44  HOSTNAME=mail.ietf.org  HELO=mail.ietf.org
diff --git a/test/functional/configs/maps/external_relay.hostname_map b/test/functional/configs/maps/external_relay.hostname_map
new file mode 100644
index 000000000..fdb4fc07a
--- /dev/null
+++ b/test/functional/configs/maps/external_relay.hostname_map
@@ -0,0 +1,3 @@
+cool.example.org direct
+lame.example.net
+
diff --git a/test/functional/configs/maps/external_relay.user_map b/test/functional/configs/maps/external_relay.user_map
new file mode 100644
index 000000000..bd04568da
--- /dev/null
+++ b/test/functional/configs/maps/external_relay.user_map
@@ -0,0 +1,2 @@
+user@example.net
+
diff --git a/test/functional/configs/merged-local.conf b/test/functional/configs/merged-local.conf
index 84bffe8dd..dd93a7ba3 100644
--- a/test/functional/configs/merged-local.conf
+++ b/test/functional/configs/merged-local.conf
@@ -31,6 +31,28 @@ emails {
   }
 }
 
+external_relay {
+  enabled = {= env.EXTERNAL_RELAY_ENABLED =};
+
+  rules {
+    EXTERNAL_RELAY_AUTHENTICATED {
+      strategy = "authenticated";
+      user_map = "{= env.TESTDIR =}/configs/maps/external_relay.user_map";
+    }
+    EXTERNAL_RELAY_COUNT {
+      count = 4;
+      strategy = "count";
+    }
+    EXTERNAL_RELAY_HOSTNAME_MAP {
+      hostname_map = "{= env.TESTDIR =}/configs/maps/external_relay.hostname_map";
+      strategy = "hostname_map";
+    }
+    EXTERNAL_RELAY_LOCAL {
+      strategy = "local";
+    }
+  }
+}
+
 greylist {
   check_local = true;
   timeout = 4;
diff --git a/test/functional/configs/merged.conf b/test/functional/configs/merged.conf
index 8bec67a41..bda7044e1 100644
--- a/test/functional/configs/merged.conf
+++ b/test/functional/configs/merged.conf
@@ -31,5 +31,8 @@ lua = "{= env.TESTDIR =}/lua/udp.lua"
 # 350_magic
 lua = "{= env.TESTDIR =}/lua/magic.lua"
 
+# 380_external_relay
+lua = "{= env.TESTDIR =}/lua/external_relay.lua"
+
 .include(priority=1,duplicate=merge) "{= env.TESTDIR =}/configs/merged-local.conf"
 .include(priority=2,duplicate=replace) "{= env.TESTDIR =}/configs/merged-override.conf"
diff --git a/test/functional/lua/external_relay.lua b/test/functional/lua/external_relay.lua
new file mode 100644
index 000000000..6aa3a292c
--- /dev/null
+++ b/test/functional/lua/external_relay.lua
@@ -0,0 +1,10 @@
+rspamd_config:register_symbol({
+  name = 'EXTERNAL_RELAY_TEST',
+  score = 0.0,
+  callback = function(task)
+    local from_ip = string.format('IP=%s', task:get_from_ip() or 'NIL')
+    local hostname = string.format('HOSTNAME=%s', task:get_hostname() or 'NIL')
+    local helo = string.format('HELO=%s', task:get_helo() or 'NIL')
+    return true, from_ip, hostname, helo
+  end
+})
diff --git a/test/functional/messages/received5.eml b/test/functional/messages/received5.eml
new file mode 100644
index 000000000..84d89e968
--- /dev/null
+++ b/test/functional/messages/received5.eml
@@ -0,0 +1,13 @@
+Received: from localhost (localhost [127.0.0.1])
+ by ietfa.amsl.com (Postfix) with ESMTPA id 00E7712024B
+ for <cfrg@ietfa.amsl.com>; Tue,  7 May 2019 14:01:07 -0700 (PDT)
+Received: from mail.ietf.org ([4.31.198.44])
+ by localhost (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTPA id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue,  7 May 2019 14:01:04 -0700 (PDT)
+Received: from mail.example.org ([192.0.2.1])
+ by localhost (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTP id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue,  7 May 2019 14:01:04 -0700 (PDT)
+
+aa
diff --git a/test/functional/messages/received6.eml b/test/functional/messages/received6.eml
new file mode 100644
index 000000000..38dd801c6
--- /dev/null
+++ b/test/functional/messages/received6.eml
@@ -0,0 +1,17 @@
+Received: from localhost (localhost [127.0.0.1])
+ by ietfa.amsl.com (Postfix) with ESMTPA id 00E7712024B
+ for <cfrg@ietfa.amsl.com>; Tue,  7 May 2019 14:01:07 -0700 (PDT)
+Received: from cool.example.org ([4.31.198.44])
+ by lame.example.net (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTPA id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue,  7 May 2019 14:01:04 -0700 (PDT)
+Received: from mail.example.org ([192.0.3.1])
+ by localhost (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTP id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue,  7 May 2019 14:01:04 -0700 (PDT)
+Received: from mail.example.org ([192.0.2.1])
+ by cool.example.org (ietfa.amsl.com [127.0.0.1]) (amavisd-new, port 10024)
+ with ESMTP id k8UsBTUjeiTe for <cfrg@ietfa.amsl.com>;
+ Tue,  7 May 2019 14:01:04 -0700 (PDT)
+
+aa