From fea6bf4c35ae7fd40c9d61edc2283d335347d998 Mon Sep 17 00:00:00 2001
From: Andrew Lewis <nerf@judo.za.org>
Date: Tue, 24 Oct 2023 13:23:13 +0200
Subject: [Minor] rbl: support use of different matchers for return codes

---
 src/plugins/lua/rbl.lua | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua
index 7f97506bc..b6467c103 100644
--- a/src/plugins/lua/rbl.lua
+++ b/src/plugins/lua/rbl.lua
@@ -215,7 +215,17 @@ local function gen_check_rcvd_conditions(rbl, received_total)
   end
 end
 
-local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_table_elt)
+local matchers = {}
+
+matchers.equality = function(to_match, pattern)
+  return to_match == pattern
+end
+
+matchers.luapattern = function(to_match, pattern)
+  return string.find(to_match, '^' .. pattern .. '$') and true or false
+end
+
+local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_table_elt, match)
   local function make_option(ip, label)
     if ip then
       return string.format('%s:%s:%s',
@@ -293,7 +303,7 @@ local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_tabl
     elseif rbl.returncodes then
       for s, codes in pairs(rbl.returncodes) do
         for _, v in ipairs(codes) do
-          if string.find(ipstr, '^' .. v .. '$') then
+          if match(ipstr, v) then
             foundrc = true
             insert_results(s)
             break
@@ -858,6 +868,11 @@ local function gen_rbl_callback(rule)
     description[#description + 1] = 'selector'
   end
 
+  if not rule.matcher then
+    rule.matcher = 'equality'
+  end
+  local match = matchers[rule.matcher]
+
   local callback_f = function(task)
     -- DNS requests to issue (might be hashed afterwards)
     local dns_req = {}
@@ -865,7 +880,7 @@ local function gen_rbl_callback(rule)
 
     local function gen_rbl_dns_callback(resolve_table_elt)
       return function(_, to_resolve, results, err)
-        rbl_dns_process(task, rule, to_resolve, results, err, resolve_table_elt)
+        rbl_dns_process(task, rule, to_resolve, results, err, resolve_table_elt, match)
       end
     end
 
-- 
cgit v1.2.3


From 06618f8a52242c8678eed659b56cebb23a34ed38 Mon Sep 17 00:00:00 2001
From: Andrew Lewis <nerf@judo.za.org>
Date: Tue, 24 Oct 2023 13:53:03 +0200
Subject: [Feature] rbl: support checking returncodes by regex

---
 lualib/plugins/rbl.lua                       |  2 +-
 src/plugins/lua/rbl.lua                      | 13 +++++++++++++
 test/functional/configs/merged-override.conf | 10 ++++++----
 3 files changed, 20 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/lualib/plugins/rbl.lua b/lualib/plugins/rbl.lua
index 1a25c1ed3..bc5fbe278 100644
--- a/lualib/plugins/rbl.lua
+++ b/lualib/plugins/rbl.lua
@@ -107,7 +107,7 @@ local rule_schema_tbl = {
   ipv6 = ts.boolean:is_optional(),
   is_whitelist = ts.boolean:is_optional(),
   local_exclude_ip_map = ts.string:is_optional(),
-  matcher = ts.one_of { "equality", "luapattern" }:is_optional(),
+  matcher = ts.one_of { "equality", "luapattern", "regexp" }:is_optional(),
   monitored_address = ts.string:is_optional(),
   no_ip = ts.boolean:is_optional(),
   process_script = ts.string:is_optional(),
diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua
index b6467c103..a36131b1b 100644
--- a/src/plugins/lua/rbl.lua
+++ b/src/plugins/lua/rbl.lua
@@ -21,6 +21,7 @@ end
 
 local hash = require 'rspamd_cryptobox_hash'
 local rspamd_logger = require 'rspamd_logger'
+local rspamd_regexp = require 'rspamd_regexp'
 local rspamd_util = require 'rspamd_util'
 local rspamd_ip = require "rspamd_ip"
 local fun = require 'fun'
@@ -225,6 +226,18 @@ matchers.luapattern = function(to_match, pattern)
   return string.find(to_match, '^' .. pattern .. '$') and true or false
 end
 
+matchers.regexp = function(to_match, pattern)
+  local re = rspamd_regexp.get_cached(pattern)
+  if not re then
+    re = rspamd_regexp.create_cached(pattern)
+    if not re then
+      rspamd_logger.errx('regexp did not compile: %s', pattern)
+      return false
+    end
+  end
+  return re:match(to_match)
+end
+
 local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_table_elt, match)
   local function make_option(ip, label)
     if ip then
diff --git a/test/functional/configs/merged-override.conf b/test/functional/configs/merged-override.conf
index 28d554b83..0c9b7a3e2 100644
--- a/test/functional/configs/merged-override.conf
+++ b/test/functional/configs/merged-override.conf
@@ -269,9 +269,10 @@ rbl {
         from = 'FAKE_RBL',
       }
       unknown = true;
+      matcher = "regexp";
       returncodes = {
-        "CODE_2" = "127.0.0.2";
-        "CODE_3" = "127.0.0.3";
+        "CODE_2" = '^127\.0\.0\.2$';
+        "CODE_3" = '^127\.0\.0\.3$';
       }
     }
     fake_whitelist {
@@ -283,9 +284,10 @@ rbl {
       rbl = "fake.wl";
       symbol = "FAKE_WL_RBL_UNKNOWN";
       unknown = true;
+      #matcher = "luapattern";
       returncodes = {
-        "FAKE_WL_RBL_CODE_2" = "127.0.0.2";
-        "FAKE_WL_RBL_CODE_3" = "127.0.0.3";
+        "FAKE_WL_RBL_CODE_2" = "127%.0%.0%.2";
+        "FAKE_WL_RBL_CODE_3" = "127%.0%.0%.3";
       }
     }
     RSPAMD_EMAILBL {
-- 
cgit v1.2.3


From 3d17d2d857bb2d719b6e32e1b85e81dd1431d5bd Mon Sep 17 00:00:00 2001
From: Andrew Lewis <nerf@judo.za.org>
Date: Wed, 25 Oct 2023 14:34:47 +0200
Subject: [Feature] rbl: support checking returncodes by CIDR

---
 lualib/plugins/rbl.lua                       |  2 +-
 src/plugins/lua/rbl.lua                      | 60 +++++++++++++++++++---------
 test/functional/configs/merged-override.conf |  3 +-
 3 files changed, 44 insertions(+), 21 deletions(-)

(limited to 'src')

diff --git a/lualib/plugins/rbl.lua b/lualib/plugins/rbl.lua
index bc5fbe278..4eb7b987d 100644
--- a/lualib/plugins/rbl.lua
+++ b/lualib/plugins/rbl.lua
@@ -107,7 +107,7 @@ local rule_schema_tbl = {
   ipv6 = ts.boolean:is_optional(),
   is_whitelist = ts.boolean:is_optional(),
   local_exclude_ip_map = ts.string:is_optional(),
-  matcher = ts.one_of { "equality", "luapattern", "regexp" }:is_optional(),
+  matcher = ts.one_of { "equality", "luapattern", "radix", "regexp" }:is_optional(),
   monitored_address = ts.string:is_optional(),
   no_ip = ts.boolean:is_optional(),
   process_script = ts.string:is_optional(),
diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua
index a36131b1b..4365d245f 100644
--- a/src/plugins/lua/rbl.lua
+++ b/src/plugins/lua/rbl.lua
@@ -21,7 +21,6 @@ end
 
 local hash = require 'rspamd_cryptobox_hash'
 local rspamd_logger = require 'rspamd_logger'
-local rspamd_regexp = require 'rspamd_regexp'
 local rspamd_util = require 'rspamd_util'
 local rspamd_ip = require "rspamd_ip"
 local fun = require 'fun'
@@ -218,24 +217,34 @@ end
 
 local matchers = {}
 
-matchers.equality = function(to_match, pattern)
-  return to_match == pattern
+matchers.radix = function(_, _, real_ip, map)
+  return map and map:get_key(real_ip) or false
 end
 
-matchers.luapattern = function(to_match, pattern)
-  return string.find(to_match, '^' .. pattern .. '$') and true or false
+matchers.equality = function(codes, to_match)
+  if type(codes) ~= 'table' then return codes == to_match end
+  for _, ip in ipairs(codes) do
+    if to_match == ip then
+      return true
+    end
+  end
+  return false
 end
 
-matchers.regexp = function(to_match, pattern)
-  local re = rspamd_regexp.get_cached(pattern)
-  if not re then
-    re = rspamd_regexp.create_cached(pattern)
-    if not re then
-      rspamd_logger.errx('regexp did not compile: %s', pattern)
-      return false
+matchers.luapattern = function(codes, to_match)
+  if type(codes) ~= 'table' then
+    return string.find(to_match, '^' .. codes .. '$') and true or false
+  end
+  for _, pattern in ipairs(codes) do
+    if string.find(to_match, '^' .. pattern .. '$') then
+      return true
     end
   end
-  return re:match(to_match)
+  return false
+end
+
+matchers.regexp = function(_, to_match, _, map)
+  return map and map:get_key(to_match) or false
 end
 
 local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_table_elt, match)
@@ -297,6 +306,8 @@ local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_tabl
     return
   end
 
+  local returncodes_maps = rbl.returncodes_maps or {}
+
   for _, result in ipairs(results) do
     local ipstr = result:to_string()
     lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr)
@@ -315,12 +326,10 @@ local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_tabl
       end
     elseif rbl.returncodes then
       for s, codes in pairs(rbl.returncodes) do
-        for _, v in ipairs(codes) do
-          if match(ipstr, v) then
-            foundrc = true
-            insert_results(s)
-            break
-          end
+        local res = match(codes, ipstr, result, returncodes_maps[s])
+        if res then
+          foundrc = true
+          insert_results(s)
         end
       end
     end
@@ -1077,6 +1086,19 @@ local function add_rbl(key, rbl, global_opts)
         def_type, rbl.symbol)
   end
 
+  local match_type = rbl.matcher
+  if match_type and rbl.returncodes and (match_type == 'radix' or match_type == 'regexp') then
+    if not rbl.returncodes_maps then
+      rbl.returncodes_maps = {}
+    end
+    for label, v in pairs(rbl.returncodes) do
+      if type(v) ~= 'table' then
+        v = {v}
+      end
+      rbl.returncodes_maps[label] = lua_maps.map_add_from_ucl(v, match_type, string.format('%s_%s RBL returncodes', label, rbl.symbol))
+    end
+  end
+
   if rbl.url_compose_map then
     local lua_urls_compose = require "lua_urls_compose"
     rbl.url_compose_map = lua_urls_compose.add_composition_map(rspamd_config, rbl.url_compose_map)
diff --git a/test/functional/configs/merged-override.conf b/test/functional/configs/merged-override.conf
index 0c9b7a3e2..9ba31b7a1 100644
--- a/test/functional/configs/merged-override.conf
+++ b/test/functional/configs/merged-override.conf
@@ -296,8 +296,9 @@ rbl {
       ignore_defaults = true;
       emails = true;
       emails_domainonly = true
+      matcher = "radix";
       returncodes = {
-        RSPAMD_EMAILBL = "127.0.0.2";
+        RSPAMD_EMAILBL = "127.0.0.2/32";
       }
     }
     URIBL_NUMERIC {
-- 
cgit v1.2.3


From 5ef8dfff5dcff5e8922e5a2c8e41d7e381dbcd37 Mon Sep 17 00:00:00 2001
From: Andrew Lewis <nerf@judo.za.org>
Date: Wed, 25 Oct 2023 15:00:08 +0200
Subject: [Feature] rbl: support globbed return codes

---
 lualib/plugins/rbl.lua                       |  2 +-
 src/plugins/lua/rbl.lua                      | 12 +++++++++++-
 test/functional/configs/merged-override.conf |  6 +++++-
 3 files changed, 17 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/lualib/plugins/rbl.lua b/lualib/plugins/rbl.lua
index 4eb7b987d..89304baa0 100644
--- a/lualib/plugins/rbl.lua
+++ b/lualib/plugins/rbl.lua
@@ -107,7 +107,7 @@ local rule_schema_tbl = {
   ipv6 = ts.boolean:is_optional(),
   is_whitelist = ts.boolean:is_optional(),
   local_exclude_ip_map = ts.string:is_optional(),
-  matcher = ts.one_of { "equality", "luapattern", "radix", "regexp" }:is_optional(),
+  matcher = ts.one_of { "equality", "glob", "luapattern", "radix", "regexp" }:is_optional(),
   monitored_address = ts.string:is_optional(),
   no_ip = ts.boolean:is_optional(),
   process_script = ts.string:is_optional(),
diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua
index 4365d245f..d1f2df69c 100644
--- a/src/plugins/lua/rbl.lua
+++ b/src/plugins/lua/rbl.lua
@@ -247,6 +247,10 @@ matchers.regexp = function(_, to_match, _, map)
   return map and map:get_key(to_match) or false
 end
 
+matchers.glob = function(_, to_match, _, map)
+  return map and map:get_key(to_match) or false
+end
+
 local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_table_elt, match)
   local function make_option(ip, label)
     if ip then
@@ -1009,6 +1013,12 @@ local function gen_rbl_callback(rule)
   return callback_f, string.format('checks: %s', table.concat(description, ','))
 end
 
+local map_match_types = {
+  glob = true,
+  radix = true,
+  regexp = true,
+}
+
 local function add_rbl(key, rbl, global_opts)
   if not rbl.symbol then
     rbl.symbol = key:upper()
@@ -1087,7 +1097,7 @@ local function add_rbl(key, rbl, global_opts)
   end
 
   local match_type = rbl.matcher
-  if match_type and rbl.returncodes and (match_type == 'radix' or match_type == 'regexp') then
+  if match_type and rbl.returncodes and map_match_types[match_type] then
     if not rbl.returncodes_maps then
       rbl.returncodes_maps = {}
     end
diff --git a/test/functional/configs/merged-override.conf b/test/functional/configs/merged-override.conf
index 9ba31b7a1..752711369 100644
--- a/test/functional/configs/merged-override.conf
+++ b/test/functional/configs/merged-override.conf
@@ -310,10 +310,14 @@ rbl {
       images = true;
       rbl = "test9.uribl";
     }
-    URIBL_NUMERIC_CONTENT {
+    UNKNOWN_URIBL_NUMERIC_CONTENT {
       checks = ["numeric_urls"];
       content_urls = true;
       rbl = "test9.uribl";
+      matcher = "glob";
+      returncodes = {
+        URIBL_NUMERIC_CONTENT = "*.*.*.*";
+      }
     }
     URIBL_NUMERIC_EVERYTHING {
       checks = ["numeric_urls"];
-- 
cgit v1.2.3


From 15e3f277fad7c692b641423b38d6b508b764a3f8 Mon Sep 17 00:00:00 2001
From: Andrew Lewis <nerf@judo.za.org>
Date: Thu, 26 Oct 2023 08:40:00 +0200
Subject: [Minor] Reiterate on the previous changes

 - Demote message to info level
 - Name it returncodes_matcher for better specificity
---
 conf/modules.d/rbl.conf                      |  6 +++---
 lualib/plugins/rbl.lua                       | 10 +++++-----
 src/plugins/lua/rbl.lua                      |  8 ++++----
 test/functional/configs/merged-override.conf |  8 ++++----
 4 files changed, 16 insertions(+), 16 deletions(-)

(limited to 'src')

diff --git a/conf/modules.d/rbl.conf b/conf/modules.d/rbl.conf
index cb90d5d31..dcb5661c0 100644
--- a/conf/modules.d/rbl.conf
+++ b/conf/modules.d/rbl.conf
@@ -105,7 +105,7 @@ rbl {
       ipv6 = true;
       checks = ['from', 'received'];
       is_whitelist = true;
-      matcher = "luapattern";
+      returncodes_matcher = "luapattern";
       whitelist_exception = "RCVD_IN_DNSWL";
       whitelist_exception = "RCVD_IN_DNSWL_NONE";
       whitelist_exception = "RCVD_IN_DNSWL_LOW";
@@ -153,7 +153,7 @@ rbl {
       rbl = "dwl.dnswl.org";
       checks = ['dkim'];
       ignore_whitelist = true;
-      matcher = "luapattern";
+      returncodes_matcher = "luapattern";
       unknown = false;
 
       returncodes {
@@ -224,7 +224,7 @@ rbl {
       selector = "specific_urls_filter_map('surbl_hashbl_map', {limit = 10}).apply_methods('get_host', 'get_path').join_tables('/')",
       hash = 'md5';
       hash_len = 32;
-      matcher = "luapattern";
+      returncodes_matcher = "luapattern";
       returncodes = {
         SURBL_HASHBL_PHISH = "127.0.0.8";
         SURBL_HASHBL_MALWARE = "127.0.0.16";
diff --git a/lualib/plugins/rbl.lua b/lualib/plugins/rbl.lua
index 89304baa0..320fd7725 100644
--- a/lualib/plugins/rbl.lua
+++ b/lualib/plugins/rbl.lua
@@ -107,7 +107,6 @@ local rule_schema_tbl = {
   ipv6 = ts.boolean:is_optional(),
   is_whitelist = ts.boolean:is_optional(),
   local_exclude_ip_map = ts.string:is_optional(),
-  matcher = ts.one_of { "equality", "glob", "luapattern", "radix", "regexp" }:is_optional(),
   monitored_address = ts.string:is_optional(),
   no_ip = ts.boolean:is_optional(),
   process_script = ts.string:is_optional(),
@@ -131,6 +130,7 @@ local rule_schema_tbl = {
   return_codes = return_codes_schema:is_optional(),
   returnbits = return_bits_schema:is_optional(),
   returncodes = return_codes_schema:is_optional(),
+  returncodes_matcher = ts.one_of { "equality", "glob", "luapattern", "radix", "regexp" }:is_optional(),
   selector = ts.one_of { ts.string, ts.table }:is_optional(),
   selector_flatten = ts.boolean:is_optional(),
   symbol = ts.string:is_optional(),
@@ -200,16 +200,16 @@ local function convert_checks(rule)
     rule.from = true
   end
 
-  if rule.returncodes and not rule.matcher then
+  if rule.returncodes and not rule.returncodes_matcher then
     for _, v in pairs(rule.returncodes) do
       for _, e in ipairs(v) do
         if e:find('%', 1, true) then
-          rspamd_logger.warnx(rspamd_config, 'implicitly enabling luapattern matcher for rule %s', rule.symbol)
-          rule.matcher = 'luapattern'
+          rspamd_logger.info(rspamd_config, 'implicitly enabling luapattern returncodes_matcher for rule %s', rule.symbol)
+          rule.returncodes_matcher = 'luapattern'
           break
         end
       end
-      if rule.matcher then
+      if rule.returncodes_matcher then
         break
       end
     end
diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua
index d1f2df69c..f2233a3e5 100644
--- a/src/plugins/lua/rbl.lua
+++ b/src/plugins/lua/rbl.lua
@@ -894,10 +894,10 @@ local function gen_rbl_callback(rule)
     description[#description + 1] = 'selector'
   end
 
-  if not rule.matcher then
-    rule.matcher = 'equality'
+  if not rule.returncodes_matcher then
+    rule.returncodes_matcher = 'equality'
   end
-  local match = matchers[rule.matcher]
+  local match = matchers[rule.returncodes_matcher]
 
   local callback_f = function(task)
     -- DNS requests to issue (might be hashed afterwards)
@@ -1096,7 +1096,7 @@ local function add_rbl(key, rbl, global_opts)
         def_type, rbl.symbol)
   end
 
-  local match_type = rbl.matcher
+  local match_type = rbl.returncodes_matcher
   if match_type and rbl.returncodes and map_match_types[match_type] then
     if not rbl.returncodes_maps then
       rbl.returncodes_maps = {}
diff --git a/test/functional/configs/merged-override.conf b/test/functional/configs/merged-override.conf
index 752711369..c94d8df00 100644
--- a/test/functional/configs/merged-override.conf
+++ b/test/functional/configs/merged-override.conf
@@ -269,7 +269,7 @@ rbl {
         from = 'FAKE_RBL',
       }
       unknown = true;
-      matcher = "regexp";
+      returncodes_matcher = "regexp";
       returncodes = {
         "CODE_2" = '^127\.0\.0\.2$';
         "CODE_3" = '^127\.0\.0\.3$';
@@ -284,7 +284,7 @@ rbl {
       rbl = "fake.wl";
       symbol = "FAKE_WL_RBL_UNKNOWN";
       unknown = true;
-      #matcher = "luapattern";
+      #returncodes_matcher = "luapattern";
       returncodes = {
         "FAKE_WL_RBL_CODE_2" = "127%.0%.0%.2";
         "FAKE_WL_RBL_CODE_3" = "127%.0%.0%.3";
@@ -296,7 +296,7 @@ rbl {
       ignore_defaults = true;
       emails = true;
       emails_domainonly = true
-      matcher = "radix";
+      returncodes_matcher = "radix";
       returncodes = {
         RSPAMD_EMAILBL = "127.0.0.2/32";
       }
@@ -314,7 +314,7 @@ rbl {
       checks = ["numeric_urls"];
       content_urls = true;
       rbl = "test9.uribl";
-      matcher = "glob";
+      returncodes_matcher = "glob";
       returncodes = {
         URIBL_NUMERIC_CONTENT = "*.*.*.*";
       }
-- 
cgit v1.2.3