From 05eda3bb5a60014614a5eea3111bd5af994d716c Mon Sep 17 00:00:00 2001
From: Jason Stephenson <jstephenson@mimecast.com>
Date: Mon, 13 May 2024 11:55:47 +0100
Subject: Fix regex to no longer match on dmarc=

---
 src/plugins/lua/arc.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua
index ff19aef4c..1533655d5 100644
--- a/src/plugins/lua/arc.lua
+++ b/src/plugins/lua/arc.lua
@@ -639,7 +639,7 @@ local function prepare_arc_selector(task, sel)
       local ar_header = task:get_header('Authentication-Results')
 
       if ar_header then
-        local arc_match = string.match(ar_header, 'arc=(%w+)')
+        local arc_match = string.match(ar_header, '[; ]arc=(%w+)')
 
         if arc_match then
           if arc_match == 'none' or arc_match == 'pass' then
-- 
cgit v1.2.3


From 470fe5ae7ddba65078d9d456dc0068ccb8b9905b Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Mon, 8 Jul 2024 14:42:03 +0100
Subject: [Minor] Push dynamic symbol weight

---
 src/lua/lua_task.c | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
index dc41d4ab7..61aac63f2 100644
--- a/src/lua/lua_task.c
+++ b/src/lua/lua_task.c
@@ -2642,7 +2642,7 @@ struct rspamd_url_query_to_inject_cbd {
 
 static gboolean
 inject_url_query_callback(struct rspamd_url *url, gsize start_offset,
-						gsize end_offset, gpointer ud)
+						  gsize end_offset, gpointer ud)
 {
 	struct rspamd_url_query_to_inject_cbd *cbd =
 		(struct rspamd_url_query_to_inject_cbd *) ud;
@@ -2661,7 +2661,7 @@ inject_url_query_callback(struct rspamd_url *url, gsize start_offset,
 
 static void
 inject_url_query(struct rspamd_task *task, struct rspamd_url *url,
-					   GPtrArray *part_urls)
+				 GPtrArray *part_urls)
 {
 	if (url->querylen > 0) {
 		struct rspamd_url_query_to_inject_cbd cbd;
@@ -2692,11 +2692,11 @@ lua_task_inject_url(lua_State *L)
 	if (lua_isuserdata(L, 3)) {
 		/* We also have a mime part there */
 		mpart = *((struct rspamd_mime_part **)
-							   rspamd_lua_check_udata_maybe(L, 3, rspamd_mimepart_classname));
+					  rspamd_lua_check_udata_maybe(L, 3, rspamd_mimepart_classname));
 	}
 	if (task && task->message && url && url->url) {
 		if (rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls), url->url, false)) {
-			if(mpart && mpart->urls) {
+			if (mpart && mpart->urls) {
 				inject_url_query(task, url->url, mpart->urls);
 			}
 		}
@@ -4682,7 +4682,7 @@ lua_push_symbol_result(lua_State *L,
 	struct rspamd_symbol_option *opt;
 	struct rspamd_symbols_group *sym_group;
 	unsigned int i;
-	int j = 1, table_fields_cnt = 4;
+	int j = 1, table_fields_cnt = 5;
 
 	if (!metric_res) {
 		metric_res = task->result;
@@ -4710,10 +4710,22 @@ lua_push_symbol_result(lua_State *L,
 			lua_pushstring(L, symbol);
 			lua_settable(L, -3);
 		}
+
 		lua_pushstring(L, "score");
 		lua_pushnumber(L, s->score);
 		lua_settable(L, -3);
 
+		/* Dynamic weight of the symbol */
+		if (s->sym->score != 0) {
+			lua_pushstring(L, "weight");
+			lua_pushnumber(L, s->score / s->sym->score);
+		}
+		else {
+			lua_pushstring(L, "weight");
+			lua_pushnumber(L, 0.0);
+		}
+		lua_settable(L, -3);
+
 		if (s->sym && s->sym->gr) {
 			lua_pushstring(L, "group");
 			lua_pushstring(L, s->sym->gr->name);
-- 
cgit v1.2.3


From ca8675103beadca0c6a6a2059b63125d1e3ca1af Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Mon, 8 Jul 2024 14:49:19 +0100
Subject: [Feature] Allow to specify minimum weight in GPT plugin

---
 src/plugins/lua/gpt.lua | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

(limited to 'src')

diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua
index ddd2f0186..6adbce3bf 100644
--- a/src/plugins/lua/gpt.lua
+++ b/src/plugins/lua/gpt.lua
@@ -59,13 +59,13 @@ local fun = require "fun"
 
 -- Exclude checks if one of those is found
 local default_symbols_to_except = {
-  'BAYES_SPAM', -- We already know that it is a spam, so we can safely skip it, but no same logic for HAM!
-  'WHITELIST_SPF',
-  'WHITELIST_DKIM',
-  'WHITELIST_DMARC',
-  'FUZZY_DENIED',
-  'REPLY',
-  'BOUNCE',
+  BAYES_SPAM = 0.9, -- We already know that it is a spam, so we can safely skip it, but no same logic for HAM!
+  WHITELIST_SPF = -1,
+  WHITELIST_DKIM = -1,
+  WHITELIST_DMARC = -1,
+  FUZZY_DENIED = -1,
+  REPLY = -1,
+  BOUNCE = -1,
 }
 
 local settings = {
@@ -105,9 +105,20 @@ local function default_condition(task)
     end
   end
   -- We also exclude some symbols
-  for _, s in ipairs(settings.symbols_to_except) do
+  for s, required_weight in pairs(settings.symbols_to_except) do
     if task:has_symbol(s) then
-      return false, 'skip as "' .. s .. '" is found'
+      if required_weight > 0 then
+        -- Also check score
+        local sym = task:get_symbol(s)
+        -- Must exist as we checked it before with `has_symbol`
+        if math.abs(sym.weight) >= required_weight then
+          return false, 'skip as "' .. s .. '" is found (weight: ' .. sym.weight .. ')'
+        end
+        lua_util.debugm(N, task, 'symbol %s has weight %s, but required %s', s,
+            sym.weight, required_weight)
+      else
+        return false, 'skip as "' .. s .. '" is found'
+      end
     end
   end
 
-- 
cgit v1.2.3


From 2f9eebdf46e4c4bf7c2cb06724c1697431ce73bb Mon Sep 17 00:00:00 2001
From: Andrew Lewis <nerf@judo.za.org>
Date: Mon, 8 Jul 2024 16:05:52 +0200
Subject: [Fix] metric_exporter: avoid sending `nil` in metric values

---
 src/plugins/lua/metric_exporter.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/plugins/lua/metric_exporter.lua b/src/plugins/lua/metric_exporter.lua
index 75885516c..3de87c157 100644
--- a/src/plugins/lua/metric_exporter.lua
+++ b/src/plugins/lua/metric_exporter.lua
@@ -117,7 +117,7 @@ local function graphite_push(kwargs)
     elseif #split == 2 then
       mvalue = kwargs['stats'][split[1]][split[2]]
     end
-    table.insert(metrics_str, string.format('%s %s %s', mname, mvalue, stamp))
+    table.insert(metrics_str, string.format('%s %s %s', mname, mvalue or 'null', stamp))
   end
 
   metrics_str = table.concat(metrics_str, '\n')
-- 
cgit v1.2.3


From 69621218bc47eb54a0dffda7ce819c9309e42c67 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Mon, 8 Jul 2024 19:22:25 +0100
Subject: [Rework] Introduce dynamic_rate_limit for ratelimits

---
 src/plugins/lua/ratelimit.lua | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

(limited to 'src')

diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua
index add5741e8..470ac5f07 100644
--- a/src/plugins/lua/ratelimit.lua
+++ b/src/plugins/lua/ratelimit.lua
@@ -42,6 +42,8 @@ local settings = {
   -- Do not check ratelimits for these recipients
   whitelisted_rcpts = { 'postmaster', 'mailer-daemon' },
   prefix = 'RL',
+  -- If enabled, we apply dynamic rate limiting based on the verdict
+  dynamic_rate_limit = false,
   ham_factor_rate = 1.01,
   spam_factor_rate = 0.99,
   ham_factor_burst = 1.02,
@@ -361,17 +363,19 @@ local function make_prefix(redis_key, name, bucket)
   local hash = settings.prefix ..
       string.sub(rspamd_hash.create(redis_key):base32(), 1, hash_len)
   -- Fill defaults
+  -- If settings.dynamic_rate_limit is false, then the default dynamic rate limits are 1.0
+  -- We always allow per-bucket overrides of the dyn rate limits
   if not bucket.spam_factor_rate then
-    bucket.spam_factor_rate = settings.spam_factor_rate
+    bucket.spam_factor_rate = settings.dynamic_rate_limit and settings.spam_factor_rate or 1.0
   end
   if not bucket.ham_factor_rate then
-    bucket.ham_factor_rate = settings.ham_factor_rate
+    bucket.ham_factor_rate = settings.dynamic_rate_limit and settings.ham_factor_rate or 1.0
   end
   if not bucket.spam_factor_burst then
-    bucket.spam_factor_burst = settings.spam_factor_burst
+    bucket.spam_factor_burst = settings.dynamic_rate_limit and settings.spam_factor_burst or 1.0
   end
   if not bucket.ham_factor_burst then
-    bucket.ham_factor_burst = settings.ham_factor_burst
+    bucket.ham_factor_burst = settings.dynamic_rate_limit and settings.ham_factor_burst or 1.0
   end
 
   return {
-- 
cgit v1.2.3


From 87702745e128977d238284b2a69415bcf94974e6 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Tue, 9 Jul 2024 14:22:31 +0100
Subject: [Project] Enable compatibility with the existing buckets

---
 lualib/redis_scripts/ratelimit_check.lua  | 22 ++++++---
 lualib/redis_scripts/ratelimit_update.lua | 74 +++++++++++++++++--------------
 src/plugins/lua/ratelimit.lua             | 24 +++++++++-
 3 files changed, 78 insertions(+), 42 deletions(-)

(limited to 'src')

diff --git a/lualib/redis_scripts/ratelimit_check.lua b/lualib/redis_scripts/ratelimit_check.lua
index d39cdf148..f24e0daf0 100644
--- a/lualib/redis_scripts/ratelimit_check.lua
+++ b/lualib/redis_scripts/ratelimit_check.lua
@@ -7,6 +7,7 @@
 -- KEYS[4]: The maximum allowed burst
 -- KEYS[5]: The expiration time for a bucket
 -- KEYS[6]: The number of recipients for the message
+-- KEYS[7]: Enable dynamic ratelimits
 
 -- Redis keys used:
 -- l: Last hit (time in milliseconds)
@@ -29,6 +30,7 @@ local nrcpt = tonumber(KEYS[6])
 local leak_rate = tonumber(KEYS[3])
 local max_burst = tonumber(KEYS[4])
 local prefix = KEYS[1]
+local enable_dynamic = KEYS[7] == 'true'
 local dynr, dynb, leaked = 0, 0, 0
 if not last then
   -- New bucket
@@ -52,9 +54,13 @@ pending = pending + nrcpt -- this message
 if burst + pending > 0 then
   -- If we have any time passed
   if burst > 0 and last < now then
-    dynr = tonumber(redis.call('HGET', prefix, 'dr')) / 10000.0
-    if dynr == 0 then
-      dynr = 0.0001
+    if enable_dynamic then
+      dynr = tonumber(redis.call('HGET', prefix, 'dr')) / 10000.0
+      if dynr == 0 then
+        dynr = 0.0001
+      end
+    else
+      dynr = 1.0
     end
     leak_rate = leak_rate * dynr
     leaked = ((now - last) * leak_rate)
@@ -66,9 +72,13 @@ if burst + pending > 0 then
     redis.call('HSET', prefix, 'l', tostring(now))
   end
 
-  dynb = tonumber(redis.call('HGET', prefix, 'db')) / 10000.0
-  if dynb == 0 then
-    dynb = 0.0001
+  if enable_dynamic then
+    dynb = tonumber(redis.call('HGET', prefix, 'db')) / 10000.0
+    if dynb == 0 then
+      dynb = 0.0001
+    end
+  else
+    dynb = 1.0
   end
 
   burst = burst + pending
diff --git a/lualib/redis_scripts/ratelimit_update.lua b/lualib/redis_scripts/ratelimit_update.lua
index caee8fb31..8b7a934dc 100644
--- a/lualib/redis_scripts/ratelimit_update.lua
+++ b/lualib/redis_scripts/ratelimit_update.lua
@@ -9,12 +9,14 @@
 -- KEYS[6] - max_burst_rate: The maximum allowed value for the dynamic burst multiplier.
 -- KEYS[7] - expire: The expiration time for the Redis key storing the bucket information, in seconds.
 -- KEYS[8] - number_of_recipients: The number of requests to be allowed (or the increase rate).
+-- KEYS[9] - Enable dynamic ratelimits
 
 -- 1. Retrieve the last hit time and initialize variables
 local prefix = KEYS[1]
 local last = redis.call('HGET', prefix, 'l')
 local now = tonumber(KEYS[2])
 local nrcpt = tonumber(KEYS[8])
+local enable_dynamic = KEYS[9] == 'true'
 if not last then
   -- 2. Initialize a new bucket if the last hit time is not found (must not happen)
   redis.call('HMSET', prefix, 'l', tostring(now), 'b', tostring(nrcpt), 'dr', '10000', 'db', '10000', 'p', '0')
@@ -25,48 +27,52 @@ end
 -- 3. Update the dynamic rate multiplier based on input parameters
 local dr, db = 1.0, 1.0
 
-local max_dr = tonumber(KEYS[5])
+if enable_dynamic then
+  local max_dr = tonumber(KEYS[5])
 
-if max_dr > 1 then
-  local rate_mult = tonumber(KEYS[3])
-  dr = tonumber(redis.call('HGET', prefix, 'dr')) / 10000
+  if max_dr > 1 then
+    local rate_mult = tonumber(KEYS[3])
+    dr = tonumber(redis.call('HGET', prefix, 'dr')) / 10000
 
-  if rate_mult > 1.0 and dr < max_dr then
-    dr = dr * rate_mult
-    if dr > 0.0001 then
-      redis.call('HSET', prefix, 'dr', tostring(math.floor(dr * 10000)))
-    else
-      redis.call('HSET', prefix, 'dr', '1')
-    end
-  elseif rate_mult < 1.0 and dr > (1.0 / max_dr) then
-    dr = dr * rate_mult
-    if dr > 0.0001 then
-      redis.call('HSET', prefix, 'dr', tostring(math.floor(dr * 10000)))
-    else
-      redis.call('HSET', prefix, 'dr', '1')
+    if rate_mult > 1.0 and dr < max_dr then
+      dr = dr * rate_mult
+      if dr > 0.0001 then
+        redis.call('HSET', prefix, 'dr', tostring(math.floor(dr * 10000)))
+      else
+        redis.call('HSET', prefix, 'dr', '1')
+      end
+    elseif rate_mult < 1.0 and dr > (1.0 / max_dr) then
+      dr = dr * rate_mult
+      if dr > 0.0001 then
+        redis.call('HSET', prefix, 'dr', tostring(math.floor(dr * 10000)))
+      else
+        redis.call('HSET', prefix, 'dr', '1')
+      end
     end
   end
 end
 
 -- 4. Update the dynamic burst multiplier based on input parameters
-local max_db = tonumber(KEYS[6])
-if max_db > 1 then
-  local rate_mult = tonumber(KEYS[4])
-  db = tonumber(redis.call('HGET', prefix, 'db')) / 10000
+if enable_dynamic then
+  local max_db = tonumber(KEYS[6])
+  if max_db > 1 then
+    local rate_mult = tonumber(KEYS[4])
+    db = tonumber(redis.call('HGET', prefix, 'db')) / 10000
 
-  if rate_mult > 1.0 and db < max_db then
-    db = db * rate_mult
-    if db > 0.0001 then
-      redis.call('HSET', prefix, 'db', tostring(math.floor(db * 10000)))
-    else
-      redis.call('HSET', prefix, 'db', '1')
-    end
-  elseif rate_mult < 1.0 and db > (1.0 / max_db) then
-    db = db * rate_mult
-    if db > 0.0001 then
-      redis.call('HSET', prefix, 'db', tostring(math.floor(db * 10000)))
-    else
-      redis.call('HSET', prefix, 'db', '1')
+    if rate_mult > 1.0 and db < max_db then
+      db = db * rate_mult
+      if db > 0.0001 then
+        redis.call('HSET', prefix, 'db', tostring(math.floor(db * 10000)))
+      else
+        redis.call('HSET', prefix, 'db', '1')
+      end
+    elseif rate_mult < 1.0 and db > (1.0 / max_db) then
+      db = db * rate_mult
+      if db > 0.0001 then
+        redis.call('HSET', prefix, 'db', tostring(math.floor(db * 10000)))
+      else
+        redis.call('HSET', prefix, 'db', '1')
+      end
     end
   end
 end
diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua
index 470ac5f07..c3602f915 100644
--- a/src/plugins/lua/ratelimit.lua
+++ b/src/plugins/lua/ratelimit.lua
@@ -365,17 +365,33 @@ local function make_prefix(redis_key, name, bucket)
   -- Fill defaults
   -- If settings.dynamic_rate_limit is false, then the default dynamic rate limits are 1.0
   -- We always allow per-bucket overrides of the dyn rate limits
+
+  local seen_specific_dyn_rate = false
+
   if not bucket.spam_factor_rate then
     bucket.spam_factor_rate = settings.dynamic_rate_limit and settings.spam_factor_rate or 1.0
+  else
+    seen_specific_dyn_rate = true
   end
   if not bucket.ham_factor_rate then
     bucket.ham_factor_rate = settings.dynamic_rate_limit and settings.ham_factor_rate or 1.0
+  else
+    seen_specific_dyn_rate = true
   end
   if not bucket.spam_factor_burst then
     bucket.spam_factor_burst = settings.dynamic_rate_limit and settings.spam_factor_burst or 1.0
+  else
+    seen_specific_dyn_rate = true
   end
   if not bucket.ham_factor_burst then
     bucket.ham_factor_burst = settings.dynamic_rate_limit and settings.ham_factor_burst or 1.0
+  else
+    seen_specific_dyn_rate = true
+  end
+
+  if seen_specific_dyn_rate then
+    -- Use if afterwards in case we don't use global dyn rates
+    bucket.specific_dyn_rate = true
   end
 
   return {
@@ -555,13 +571,15 @@ local function ratelimit_cb(task)
         bincr = 1
       end
 
+      local dyn_rate_enabled = settings.dynamic_rate_limit or bucket.specific_dyn_rate
+
       lua_util.debugm(N, task, "check limit %s:%s -> %s (%s/%s)",
           value.name, pr, value.hash, bucket.burst, bucket.rate)
       lua_redis.exec_redis_script(bucket_check_id,
           { key = value.hash, task = task, is_write = true },
           gen_check_cb(pr, bucket, value.name, value.hash),
           { value.hash, tostring(now), tostring(rate), tostring(bucket.burst),
-            tostring(settings.expire), tostring(bincr) })
+            tostring(settings.expire), tostring(bincr), tostring(dyn_rate_enabled) })
     end
   end
 end
@@ -661,12 +679,14 @@ local function ratelimit_update_cb(task)
         bincr = 1
       end
 
+      local dyn_rate_enabled = settings.dynamic_rate_limit or bucket.specific_dyn_rate
+
       lua_redis.exec_redis_script(bucket_update_id,
           { key = v.hash, task = task, is_write = true },
           update_bucket_cb,
           { v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst),
             tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult),
-            tostring(settings.expire), tostring(bincr) })
+            tostring(settings.expire), tostring(bincr), tostring(dyn_rate_enabled) })
     end
   end
 end
-- 
cgit v1.2.3


From 7f19699410f7d85ee9468b7ab6afab2fd1a4cedd Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Tue, 9 Jul 2024 16:53:44 +0100
Subject: [Minor] Mention mode of dynamic limits in the log

---
 src/plugins/lua/ratelimit.lua | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua
index c3602f915..f3331e850 100644
--- a/src/plugins/lua/ratelimit.lua
+++ b/src/plugins/lua/ratelimit.lua
@@ -527,11 +527,20 @@ local function ratelimit_cb(task)
                   string.format('%s(%s)', lim_name, lim_key))
             end
           end
-          rspamd_logger.infox(task,
-              'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn); redis key: %s',
-              lim_name, prefix,
-              bucket.burst, bucket.rate,
-              data[2], data[3], data[4], lim_key)
+
+          if bucket.dyn_rate_enabled then
+            rspamd_logger.infox(task,
+                'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn); redis key: %s',
+                lim_name, prefix,
+                bucket.burst, bucket.rate,
+                data[2], data[3], data[4], lim_key)
+          else
+            rspamd_logger.infox(task,
+                'ratelimit "%s(%s)" exceeded, (%s / %s): %s (dynamic ratelimits disabled); redis key: %s',
+                lim_name, prefix,
+                bucket.burst, bucket.rate,
+                data[2], lim_key)
+          end
 
           if not (bucket.symbol or settings.symbol) and not bucket.skip_soft_reject then
             if not bucket.message then
-- 
cgit v1.2.3


From 1cc29aa2341dd04ce71912c824c53cc096e9bdba Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Thu, 11 Jul 2024 13:47:05 +0100
Subject: [Minor] Fix logic

---
 src/libserver/cfg_utils.cxx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/libserver/cfg_utils.cxx b/src/libserver/cfg_utils.cxx
index dd85eddfb..bcd26d59c 100644
--- a/src/libserver/cfg_utils.cxx
+++ b/src/libserver/cfg_utils.cxx
@@ -928,9 +928,9 @@ rspamd_config_post_load(struct rspamd_config *cfg,
 
 	if (opts & RSPAMD_CONFIG_INIT_LIBS) {
 		/* Config other libraries */
-		ret = rspamd_config_libs(cfg->libs_ctx, cfg) && ret;
+		auto libs_ret = rspamd_config_libs(cfg->libs_ctx, cfg);
 
-		if (!ret) {
+		if (!libs_ret) {
 			msg_err_config("cannot configure libraries, fatal error");
 			return FALSE;
 		}
-- 
cgit v1.2.3


From 77d78f6ab7cf5590930c46d6d71c7345f6422fad Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Thu, 11 Jul 2024 14:03:40 +0100
Subject: [Minor] Specify failure reason clearly

---
 src/libserver/cfg_utils.cxx | 34 +++++++++++++++++++++++++---------
 src/rspamd.c                |  4 ++++
 2 files changed, 29 insertions(+), 9 deletions(-)

(limited to 'src')

diff --git a/src/libserver/cfg_utils.cxx b/src/libserver/cfg_utils.cxx
index bcd26d59c..afa4009cc 100644
--- a/src/libserver/cfg_utils.cxx
+++ b/src/libserver/cfg_utils.cxx
@@ -57,7 +57,7 @@
 #ifdef HAVE_SYS_RESOURCE_H
 #include <sys/resource.h>
 #endif
-#include <math.h>
+#include <cmath>
 #include "libserver/composites/composites.h"
 
 #include "blas-config.h"
@@ -69,6 +69,7 @@
 #include "cxx/util.hxx"
 #include "frozen/unordered_map.h"
 #include "frozen/string.h"
+#include "contrib/expected/expected.hpp"
 #include "contrib/ankerl/unordered_dense.h"
 
 #define DEFAULT_SCORE 10.0
@@ -826,8 +827,7 @@ gboolean
 rspamd_config_post_load(struct rspamd_config *cfg,
 						enum rspamd_post_load_options opts)
 {
-
-	auto ret = TRUE;
+	auto ret = tl::expected<void, std::string>{};
 
 	rspamd_adjust_clocks_resolution(cfg);
 	rspamd_logger_configure_modules(cfg->debug_modules);
@@ -867,14 +867,15 @@ rspamd_config_post_load(struct rspamd_config *cfg,
 			else {
 				if (opts & RSPAMD_CONFIG_INIT_VALIDATE) {
 					msg_err_config("no url_tld option has been specified");
-					ret = FALSE;
+					ret = tl::make_unexpected(std::string{"no url_tld option has been specified"});
 				}
 			}
 		}
 		else {
 			if (access(cfg->tld_file, R_OK) == -1) {
 				if (opts & RSPAMD_CONFIG_INIT_VALIDATE) {
-					ret = FALSE;
+					ret = tl::make_unexpected(fmt::format("cannot access tld file {}: {}",
+														  cfg->tld_file, strerror(errno)));
 					msg_err_config("cannot access tld file %s: %s", cfg->tld_file,
 								   strerror(errno));
 				}
@@ -905,13 +906,17 @@ rspamd_config_post_load(struct rspamd_config *cfg,
 	if (!rspamd_config_parse_log_format(cfg)) {
 		msg_err_config("cannot parse log format, task logging will not be available");
 		if (opts & RSPAMD_CONFIG_INIT_VALIDATE) {
-			ret = FALSE;
+			ret = tl::make_unexpected(std::string{"cannot parse log format"});
 		}
 	}
 
 	if (opts & RSPAMD_CONFIG_INIT_SYMCACHE) {
 		/* Init config cache */
-		ret = rspamd_symcache_init(cfg->cache) && ret;
+		auto symcache_ret = rspamd_symcache_init(cfg->cache);
+
+		if (!symcache_ret) {
+			ret = tl::make_unexpected(std::string{"symcache init failed"});
+		}
 
 		/* Init re cache */
 		rspamd_re_cache_init(cfg->re_cache, cfg);
@@ -959,7 +964,11 @@ rspamd_config_post_load(struct rspamd_config *cfg,
 							" Rspamd features will be broken");
 		}
 
-		ret = rspamd_symcache_validate(cfg->cache, cfg, FALSE) && ret;
+		auto val_ret = rspamd_symcache_validate(cfg->cache, cfg, FALSE);
+
+		if (!val_ret) {
+			ret = tl::make_unexpected(std::string{"symcache validation failed"});
+		}
 	}
 
 	if (opts & RSPAMD_CONFIG_INIT_POST_LOAD_LUA) {
@@ -970,7 +979,14 @@ rspamd_config_post_load(struct rspamd_config *cfg,
 		rspamd_map_preload(cfg);
 	}
 
-	return ret;
+	if (ret) {
+		return true;
+	}
+	else {
+		msg_err_config("error on post-init stage: %s", ret.error().c_str());
+
+		return false;
+	}
 }
 
 struct rspamd_classifier_config *
diff --git a/src/rspamd.c b/src/rspamd.c
index 117f3b995..b6c361cb2 100644
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -979,12 +979,16 @@ load_rspamd_config(struct rspamd_main *rspamd_main,
 
 		if (init_modules) {
 			if (!rspamd_init_filters(cfg, reload, false)) {
+				msg_err_main("init filters failed");
+
 				return FALSE;
 			}
 		}
 
 		/* Do post-load actions */
 		if (!rspamd_config_post_load(cfg, opts)) {
+			msg_err_main("post load failed");
+
 			return FALSE;
 		}
 	}
-- 
cgit v1.2.3


From 47c7cf97134d9c828c2dc859e55ce5594a48e319 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Thu, 11 Jul 2024 14:13:26 +0100
Subject: [Minor] cmath is a bit more strict

---
 src/libserver/cfg_utils.cxx | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'src')

diff --git a/src/libserver/cfg_utils.cxx b/src/libserver/cfg_utils.cxx
index afa4009cc..1344bc4f9 100644
--- a/src/libserver/cfg_utils.cxx
+++ b/src/libserver/cfg_utils.cxx
@@ -136,14 +136,14 @@ struct rspamd_actions_list {
 	void sort()
 	{
 		std::sort(actions.begin(), actions.end(), [](const action_ptr &a1, const action_ptr &a2) -> bool {
-			if (!isnan(a1->threshold) && !isnan(a2->threshold)) {
+			if (!std::isnan(a1->threshold) && !std::isnan(a2->threshold)) {
 				return a1->threshold < a2->threshold;
 			}
 
-			if (isnan(a1->threshold) && isnan(a2->threshold)) {
+			if (std::isnan(a1->threshold) && std::isnan(a2->threshold)) {
 				return false;
 			}
-			else if (isnan(a1->threshold)) {
+			else if (std::isnan(a1->threshold)) {
 				return true;
 			}
 
@@ -1541,7 +1541,7 @@ rspamd_config_new_symbol(struct rspamd_config *cfg, const char *symbol,
 		rspamd_mempool_alloc0_type(cfg->cfg_pool, struct rspamd_symbol);
 	score_ptr = rspamd_mempool_alloc_type(cfg->cfg_pool, double);
 
-	if (isnan(score)) {
+	if (std::isnan(score)) {
 		/* In fact, it could be defined later */
 		msg_debug_config("score is not defined for symbol %s, set it to zero",
 						 symbol);
@@ -1652,7 +1652,7 @@ rspamd_config_add_symbol(struct rspamd_config *cfg,
 		}
 
 		if (sym_def->priority > priority &&
-			(isnan(score) || !(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNSCORED))) {
+			(std::isnan(score) || !(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNSCORED))) {
 			msg_debug_config("symbol %s has been already registered with "
 							 "priority %ud, do not override (new priority: %ud)",
 							 symbol,
@@ -1673,7 +1673,7 @@ rspamd_config_add_symbol(struct rspamd_config *cfg,
 		}
 		else {
 
-			if (!isnan(score)) {
+			if (!std::isnan(score)) {
 				msg_debug_config("symbol %s has been already registered with "
 								 "priority %ud, override it with new priority: %ud, "
 								 "old score: %.2f, new score: %.2f",
@@ -2013,7 +2013,7 @@ rspamd_config_action_from_ucl(struct rspamd_config *cfg,
 
 	/* TODO: add lua references support */
 
-	if (isnan(threshold) && !(flags & RSPAMD_ACTION_NO_THRESHOLD)) {
+	if (std::isnan(threshold) && !(flags & RSPAMD_ACTION_NO_THRESHOLD)) {
 		msg_err_config("action %s has no threshold being set and it is not"
 					   " a no threshold action",
 					   act->name);
-- 
cgit v1.2.3


From 3cb946c77d0c06750ccb3a4cbb709244dfd69344 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Thu, 11 Jul 2024 16:08:00 +0100
Subject: [Fix] Do not apply tableshape for known senders

---
 src/plugins/lua/known_senders.lua | 53 ++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 23 deletions(-)

(limited to 'src')

diff --git a/src/plugins/lua/known_senders.lua b/src/plugins/lua/known_senders.lua
index d4e966908..6d57acea3 100644
--- a/src/plugins/lua/known_senders.lua
+++ b/src/plugins/lua/known_senders.lua
@@ -17,7 +17,6 @@ limitations under the License.
 -- This plugin implements known senders logic for Rspamd
 
 local rspamd_logger = require "rspamd_logger"
-local ts = (require "tableshape").types
 local N = 'known_senders'
 local lua_util = require "lua_util"
 local lua_redis = require "lua_redis"
@@ -65,8 +64,10 @@ local settings = {
   reply_sender_privacy_length = 16,
 }
 
+--[[
+XXX: please fix tableshape one day
 local settings_schema = lua_redis.enrich_schema({
-  domains = lua_maps.map_schema,
+  domains = lua_maps.map_schema:is_optional(),
   enabled = ts.boolean:is_optional(),
   max_senders = (ts.integer + ts.string / tonumber):is_optional(),
   max_ttl = (ts.integer + ts.string / tonumber):is_optional(),
@@ -74,7 +75,16 @@ local settings_schema = lua_redis.enrich_schema({
   redis_key = ts.string:is_optional(),
   symbol = ts.string:is_optional(),
   symbol_unknown = ts.string:is_optional(),
+  max_recipients = ts.integer:is_optional(),
+  sender_prefix = ts.string:is_optional(),
+  sender_key_global = ts.string:is_optional(),
+  sender_key_size = ts.integer:is_optional(),
+  reply_sender_privacy = ts.boolean:is_optional(),
+  reply_sender_privacy_alg = ts.string:is_optional(),
+  reply_sender_privacy_prefix = ts.string:is_optional(),
+  reply_sender_privacy_length = ts.integer:is_optional(),
 })
+]]--
 
 local function make_key(input)
   local hash = rspamd_cryptobox_hash.create_specific('md5')
@@ -250,7 +260,8 @@ local function verify_local_replies_set(task)
 
   local replies_recipients = task:get_recipients('mime')
 
-  local replies_sender_string = lua_util.maybe_obfuscate_string(tostring(replies_sender), settings, settings.sender_prefix)
+  local replies_sender_string = lua_util.maybe_obfuscate_string(tostring(replies_sender), settings,
+      settings.sender_prefix)
   local replies_sender_key = make_key_replies(replies_sender_string:lower(), 8)
 
   local function redis_zscore_script_cb(err, data)
@@ -274,10 +285,10 @@ local function verify_local_replies_set(task)
 
   lua_util.debugm(N, task, 'Making redis request to local replies set')
   lua_redis.exec_redis_script(zscore_script_id,
-          {task = task, is_write = true},
-          redis_zscore_script_cb,
-          { replies_sender_key },
-          replies_recipients_addrs  )
+      { task = task, is_write = true },
+      redis_zscore_script_cb,
+      { replies_sender_key },
+      replies_recipients_addrs)
 end
 
 local function check_known_incoming_mail_callback(task)
@@ -289,14 +300,16 @@ local function check_known_incoming_mail_callback(task)
 
   -- making sender key
   lua_util.debugm(N, task, 'Sender: %s', replies_sender)
-  local replies_sender_string = lua_util.maybe_obfuscate_string(tostring(replies_sender), settings, settings.sender_prefix)
+  local replies_sender_string = lua_util.maybe_obfuscate_string(tostring(replies_sender), settings,
+      settings.sender_prefix)
   local replies_sender_key = make_key_replies(replies_sender_string:lower(), 8)
 
   lua_util.debugm(N, task, 'Sender key: %s', replies_sender_key)
 
   local function redis_zscore_global_cb(err, data)
     if err ~= nil then
-      rspamd_logger.errx(task, 'Couldn\'t find sender %s in global replies set. Ended with error: %s', replies_sender, err)
+      rspamd_logger.errx(task, 'Couldn\'t find sender %s in global replies set. Ended with error: %s', replies_sender,
+          err)
       return
     end
 
@@ -311,34 +324,28 @@ local function check_known_incoming_mail_callback(task)
 
   -- key for global replies set
   local replies_global_key = make_key_replies(settings.sender_key_global,
-          settings.sender_key_size, settings.sender_prefix)
+      settings.sender_key_size, settings.sender_prefix)
   -- using zscore to find sender in global set
   lua_util.debugm(N, task, 'Making redis request to global replies set')
   lua_redis.redis_make_request(task,
-          redis_params, -- connect params
-          replies_sender_key, -- hash key
-          false, -- is write
-          redis_zscore_global_cb, --callback
-          'ZSCORE', -- command
-          { replies_global_key, replies_sender } -- arguments
+      redis_params, -- connect params
+      replies_sender_key, -- hash key
+      false, -- is write
+      redis_zscore_global_cb, --callback
+      'ZSCORE', -- command
+      { replies_global_key, replies_sender } -- arguments
   )
 end
 
 local opts = rspamd_config:get_all_opt('known_senders')
 if opts then
   settings = lua_util.override_defaults(settings, opts)
-  local res, err = settings_schema:transform(settings)
-  if not res then
-    rspamd_logger.errx(rspamd_config, 'cannot parse known_senders options: %1', err)
-  else
-    settings = res
-  end
   redis_params = lua_redis.parse_redis_server(N, opts)
 
   if redis_params then
     local map_conf = settings.domains
     settings.domains = lua_maps.map_add_from_ucl(settings.domains,
-            'set', 'domains to track senders from')
+        'set', 'domains to track senders from')
     if not settings.domains then
       rspamd_logger.errx(rspamd_config, "couldn't add map %s, disable module",
           map_conf)
-- 
cgit v1.2.3


From ea14905c994fa7e0df53f8f6989e8c65c37606d5 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Thu, 11 Jul 2024 16:23:33 +0100
Subject: [Fix] Do not crash if symbol is missing in the metric

---
 src/lua/lua_task.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
index 61aac63f2..21e24fc45 100644
--- a/src/lua/lua_task.c
+++ b/src/lua/lua_task.c
@@ -4716,7 +4716,7 @@ lua_push_symbol_result(lua_State *L,
 		lua_settable(L, -3);
 
 		/* Dynamic weight of the symbol */
-		if (s->sym->score != 0) {
+		if (s->sym != NULL && s->sym->score != 0) {
 			lua_pushstring(L, "weight");
 			lua_pushnumber(L, s->score / s->sym->score);
 		}
-- 
cgit v1.2.3


From f8b0fd41a3b6335d86e9fa3ee25e1e5b12df1c7b Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Wed, 17 Jul 2024 15:35:18 +0100
Subject: [Feature] Treat SPF +all in a special way

Issue: #4996
---
 src/libserver/spf.c     | 2 +-
 src/libserver/spf.h     | 1 +
 src/lua/lua_spf.c       | 2 ++
 src/plugins/lua/spf.lua | 1 +
 4 files changed, 5 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/libserver/spf.c b/src/libserver/spf.c
index 32c020bf3..afd77294b 100644
--- a/src/libserver/spf.c
+++ b/src/libserver/spf.c
@@ -1418,7 +1418,7 @@ parse_spf_all(struct spf_record *rec, struct spf_addr *addr)
 
 	/* Disallow +all */
 	if (addr->mech == SPF_PASS) {
-		addr->flags |= RSPAMD_SPF_FLAG_INVALID;
+		addr->flags |= RSPAMD_SPF_FLAG_INVALID | RSPAMD_SPF_FLAG_PLUSALL;
 		msg_notice_spf("domain %s allows any SPF (+all), ignore SPF record completely",
 					   rec->sender_domain);
 	}
diff --git a/src/libserver/spf.h b/src/libserver/spf.h
index cc0ee4c05..2487b6d57 100644
--- a/src/libserver/spf.h
+++ b/src/libserver/spf.h
@@ -77,6 +77,7 @@ typedef enum spf_action_e {
 #define RSPAMD_SPF_FLAG_PERMFAIL (1u << 10u)
 #define RSPAMD_SPF_FLAG_RESOLVED (1u << 11u)
 #define RSPAMD_SPF_FLAG_CACHED (1u << 12u)
+#define RSPAMD_SPF_FLAG_PLUSALL (1u << 13u)
 
 /** Default SPF limits for avoiding abuse **/
 #define SPF_MAX_NESTING 10
diff --git a/src/lua/lua_spf.c b/src/lua/lua_spf.c
index 46e72202f..a9bcfc80c 100644
--- a/src/lua/lua_spf.c
+++ b/src/lua/lua_spf.c
@@ -89,6 +89,8 @@ lua_load_spf(lua_State *L)
 	lua_setfield(L, -2, "perm_fail");
 	lua_pushinteger(L, RSPAMD_SPF_FLAG_CACHED);
 	lua_setfield(L, -2, "cached");
+	lua_pushinteger(L, RSPAMD_SPF_FLAG_PLUSALL);
+	lua_setfield(L, -2, "plusall");
 
 	lua_setfield(L, -2, "flags");
 
diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua
index 48f3c17be..b9add61a6 100644
--- a/src/plugins/lua/spf.lua
+++ b/src/plugins/lua/spf.lua
@@ -56,6 +56,7 @@ local symbols = {
   dnsfail = "R_SPF_DNSFAIL",
   permfail = "R_SPF_PERMFAIL",
   na = "R_SPF_NA",
+  plusall = "R_SPF_PLUSALL",
 }
 
 local default_config = {
-- 
cgit v1.2.3


From 02e433ff1f9fa327387a7d0453e3e851776fd160 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Thu, 18 Jul 2024 13:26:26 +0100
Subject: [Minor] Fix several issues with flag propagation

---
 src/libserver/spf.c     | 5 ++++-
 src/libserver/spf.h     | 1 +
 src/lua/lua_spf.c       | 7 ++++++-
 src/plugins/lua/spf.lua | 2 ++
 4 files changed, 13 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/libserver/spf.c b/src/libserver/spf.c
index afd77294b..562222042 100644
--- a/src/libserver/spf.c
+++ b/src/libserver/spf.c
@@ -451,6 +451,9 @@ rspamd_spf_process_reference(struct spf_resolved *target,
 			target->flags |= RSPAMD_SPF_RESOLVED_NA;
 			continue;
 		}
+		if (cur->flags & RSPAMD_SPF_FLAG_PLUSALL) {
+			target->flags |= RSPAMD_SPF_RESOLVED_PLUSALL;
+		}
 		if (cur->flags & RSPAMD_SPF_FLAG_INVALID) {
 			/* Ignore invalid elements */
 			continue;
@@ -1418,7 +1421,7 @@ parse_spf_all(struct spf_record *rec, struct spf_addr *addr)
 
 	/* Disallow +all */
 	if (addr->mech == SPF_PASS) {
-		addr->flags |= RSPAMD_SPF_FLAG_INVALID | RSPAMD_SPF_FLAG_PLUSALL;
+		addr->flags |= RSPAMD_SPF_FLAG_PLUSALL;
 		msg_notice_spf("domain %s allows any SPF (+all), ignore SPF record completely",
 					   rec->sender_domain);
 	}
diff --git a/src/libserver/spf.h b/src/libserver/spf.h
index 2487b6d57..b89dc4d0e 100644
--- a/src/libserver/spf.h
+++ b/src/libserver/spf.h
@@ -105,6 +105,7 @@ enum rspamd_spf_resolved_flags {
 	RSPAMD_SPF_RESOLVED_TEMP_FAILED = (1u << 0u),
 	RSPAMD_SPF_RESOLVED_PERM_FAILED = (1u << 1u),
 	RSPAMD_SPF_RESOLVED_NA = (1u << 2u),
+	RSPAMD_SPF_RESOLVED_PLUSALL = (1u << 3u),
 };
 
 struct spf_resolved {
diff --git a/src/lua/lua_spf.c b/src/lua/lua_spf.c
index a9bcfc80c..850ce2120 100644
--- a/src/lua/lua_spf.c
+++ b/src/lua/lua_spf.c
@@ -89,7 +89,7 @@ lua_load_spf(lua_State *L)
 	lua_setfield(L, -2, "perm_fail");
 	lua_pushinteger(L, RSPAMD_SPF_FLAG_CACHED);
 	lua_setfield(L, -2, "cached");
-	lua_pushinteger(L, RSPAMD_SPF_FLAG_PLUSALL);
+	lua_pushinteger(L, RSPAMD_SPF_RESOLVED_PLUSALL);
 	lua_setfield(L, -2, "plusall");
 
 	lua_setfield(L, -2, "flags");
@@ -370,6 +370,11 @@ spf_check_element(lua_State *L, struct spf_resolved *rec, struct spf_addr *addr,
 				lua_pushinteger(L, RSPAMD_SPF_RESOLVED_TEMP_FAILED);
 				lua_pushfstring(L, "%cany", spf_mech_char(addr->mech));
 			}
+			else if (rec->flags & RSPAMD_SPF_RESOLVED_PLUSALL) {
+				lua_pushboolean(L, false);
+				lua_pushinteger(L, RSPAMD_SPF_RESOLVED_PLUSALL);
+				lua_pushfstring(L, "%cany", spf_mech_char(addr->mech));
+			}
 			else {
 				lua_pushboolean(L, true);
 				lua_pushinteger(L, addr->mech);
diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua
index b9add61a6..356507250 100644
--- a/src/plugins/lua/spf.lua
+++ b/src/plugins/lua/spf.lua
@@ -119,6 +119,8 @@ local function spf_check_callback(task)
   local function flag_to_symbol(fl)
     if bit.band(fl, rspamd_spf.flags.temp_fail) ~= 0 then
       return local_config.symbols.dnsfail
+    elseif bit.band(fl, rspamd_spf.flags.plusall) ~= 0 then
+      return local_config.symbols.plusall
     elseif bit.band(fl, rspamd_spf.flags.perm_fail) ~= 0 then
       return local_config.symbols.permfail
     elseif bit.band(fl, rspamd_spf.flags.na) ~= 0 then
-- 
cgit v1.2.3


From d019b0dfc11ca01c19a2889db0605ff1a776562b Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Mon, 22 Jul 2024 13:06:51 +0100
Subject: [Minor] Remove top_p reduce temperature to 0

---
 conf/modules.d/gpt.conf | 4 +---
 src/plugins/lua/gpt.lua | 8 ++------
 2 files changed, 3 insertions(+), 9 deletions(-)

(limited to 'src')

diff --git a/conf/modules.d/gpt.conf b/conf/modules.d/gpt.conf
index 7a2e11d40..eac9952e5 100644
--- a/conf/modules.d/gpt.conf
+++ b/conf/modules.d/gpt.conf
@@ -22,9 +22,7 @@ gpt {
   # Maximum tokens to generate
   max_tokens = 1000;
   # Temperature for sampling
-  temperature = 0.7;
-  # Top p for sampling
-  top_p = 0.9;
+  temperature = 0.0;
   # Timeout for requests
   timeout = 10s;
   # Prompt for the model (use default if not set)
diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua
index 6adbce3bf..046fe5dd7 100644
--- a/src/plugins/lua/gpt.lua
+++ b/src/plugins/lua/gpt.lua
@@ -31,9 +31,7 @@ gpt {
   # Maximum tokens to generate
   max_tokens = 1000;
   # Temperature for sampling
-  temperature = 0.7;
-  # Top p for sampling
-  top_p = 0.9;
+  temperature = 0.0;
   # Timeout for requests
   timeout = 10s;
   # Prompt for the model (use default if not set)
@@ -73,8 +71,7 @@ local settings = {
   api_key = nil,
   model = 'gpt-3.5-turbo',
   max_tokens = 1000,
-  temperature = 0.7,
-  top_p = 0.9,
+  temperature = 0.0,
   timeout = 10,
   prompt = nil,
   condition = nil,
@@ -276,7 +273,6 @@ local function openai_gpt_check(task)
     model = settings.model,
     max_tokens = settings.max_tokens,
     temperature = settings.temperature,
-    top_p = settings.top_p,
     messages = {
       {
         role = 'system',
-- 
cgit v1.2.3


From 272c3aa423eb74eff38bd8836acff744b9ea143b Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Mon, 22 Jul 2024 13:10:02 +0100
Subject: [Minor] Set response_format

---
 src/plugins/lua/gpt.lua | 1 +
 1 file changed, 1 insertion(+)

(limited to 'src')

diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua
index 046fe5dd7..7b582e55d 100644
--- a/src/plugins/lua/gpt.lua
+++ b/src/plugins/lua/gpt.lua
@@ -273,6 +273,7 @@ local function openai_gpt_check(task)
     model = settings.model,
     max_tokens = settings.max_tokens,
     temperature = settings.temperature,
+    response_format = { type = "json_object" },
     messages = {
       {
         role = 'system',
-- 
cgit v1.2.3


From 58d13149875a44329c54567387241b642cc96471 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Mon, 22 Jul 2024 13:25:39 +0100
Subject: [Minor] Ensure some safety when checking weights

Issue: #5065
---
 src/plugins/lua/gpt.lua | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua
index 7b582e55d..65b5758a8 100644
--- a/src/plugins/lua/gpt.lua
+++ b/src/plugins/lua/gpt.lua
@@ -106,10 +106,12 @@ local function default_condition(task)
     if task:has_symbol(s) then
       if required_weight > 0 then
         -- Also check score
-        local sym = task:get_symbol(s)
+        local sym = task:get_symbol(s) or E
         -- Must exist as we checked it before with `has_symbol`
-        if math.abs(sym.weight) >= required_weight then
-          return false, 'skip as "' .. s .. '" is found (weight: ' .. sym.weight .. ')'
+        if sym.weight then
+          if math.abs(sym.weight) >= required_weight then
+            return false, 'skip as "' .. s .. '" is found (weight: ' .. sym.weight .. ')'
+          end
         end
         lua_util.debugm(N, task, 'symbol %s has weight %s, but required %s', s,
             sym.weight, required_weight)
-- 
cgit v1.2.3


From cb3faa71cade474d449cebf2896e81c083cde295 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Mon, 22 Jul 2024 13:33:37 +0100
Subject: [Minor] Improve prompt and add some conversion heursitics

---
 src/plugins/lua/gpt.lua | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua
index 65b5758a8..5e19d2998 100644
--- a/src/plugins/lua/gpt.lua
+++ b/src/plugins/lua/gpt.lua
@@ -194,6 +194,18 @@ local function default_conversion(task, input)
 
   if type(reply) == 'table' and reply.probability then
     local spam_score = tonumber(reply.probability)
+
+    if not spam_score then
+      -- Maybe we need GPT to convert GPT reply here?
+      if reply.probability == "high" then
+        spam_score = 0.9
+      elseif reply.probability == "low" then
+        spam_score = 0.1
+      else
+        rspamd_logger.infox("cannot convert to spam probability: %s", reply.probability)
+      end
+    end
+
     if type(reply.usage) == 'table' then
       rspamd_logger.infox(task, 'usage: %s tokens', reply.usage.total_tokens)
     end
@@ -347,8 +359,8 @@ if opts then
   end
 
   if not settings.prompt then
-    settings.prompt = "You will be provided with the email message, " ..
-        "and your task is to classify its probability to be spam, " ..
+    settings.prompt = "You will be provided with the email message, subject, from and url domains, " ..
+        "and your task is to evaluate the probability to be spam as number from 0 to 1, " ..
         "output result as JSON with 'probability' field."
   end
 
-- 
cgit v1.2.3


From 10b3cefd4e8dd99c73cf11c838d80ad6238a0b57 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Mon, 22 Jul 2024 13:34:37 +0100
Subject: [Minor] Use gpt-4o-mini by default

---
 conf/modules.d/gpt.conf | 2 +-
 src/plugins/lua/gpt.lua | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/conf/modules.d/gpt.conf b/conf/modules.d/gpt.conf
index eac9952e5..c76a08c92 100644
--- a/conf/modules.d/gpt.conf
+++ b/conf/modules.d/gpt.conf
@@ -18,7 +18,7 @@ gpt {
   # Your key to access the API (add this to enable this plugin)
   #api_key = "xxx";
   # Model name
-  model = "gpt-3.5-turbo";
+  model = "gpt-4o-mini";
   # Maximum tokens to generate
   max_tokens = 1000;
   # Temperature for sampling
diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua
index 5e19d2998..61217cbf8 100644
--- a/src/plugins/lua/gpt.lua
+++ b/src/plugins/lua/gpt.lua
@@ -27,7 +27,7 @@ gpt {
   # Your key to access the API
   api_key = "xxx";
   # Model name
-  model = "gpt-3.5-turbo";
+  model = "gpt-4o-mini";
   # Maximum tokens to generate
   max_tokens = 1000;
   # Temperature for sampling
@@ -69,7 +69,7 @@ local default_symbols_to_except = {
 local settings = {
   type = 'openai',
   api_key = nil,
-  model = 'gpt-3.5-turbo',
+  model = 'gpt-4o-mini',
   max_tokens = 1000,
   temperature = 0.0,
   timeout = 10,
-- 
cgit v1.2.3


From 42c4aa280f20aa66a7be5528da119a63fa0073d4 Mon Sep 17 00:00:00 2001
From: Jason Stephenson <jstephenson@mimecast.com>
Date: Tue, 23 Jul 2024 16:12:28 +0100
Subject: Use pattern to get all methodspecs + reasonspecs, and return the
 value of the arc one if present

---
 src/plugins/lua/arc.lua | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua
index 1533655d5..98f9303fa 100644
--- a/src/plugins/lua/arc.lua
+++ b/src/plugins/lua/arc.lua
@@ -635,11 +635,22 @@ local function prepare_arc_selector(task, sel)
       end
     end
 
+    local function arc_result_from_ar(ar_header)
+      ar_header = ar_header or ""
+      for k, v in string.gmatch(ar_header, "(%w+)=(%w+)") do
+        if k == 'arc' then
+         return v
+        end
+      end
+      
+      return nil
+    end
+
     if settings.reuse_auth_results then
       local ar_header = task:get_header('Authentication-Results')
 
       if ar_header then
-        local arc_match = string.match(ar_header, '[; ]arc=(%w+)')
+        local arc_match = arc_result_from_ar(ar_header)
 
         if arc_match then
           if arc_match == 'none' or arc_match == 'pass' then
-- 
cgit v1.2.3


From 5ccf9bc7fb353c2bf20f7eb44feb283d4720bbdd Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Thu, 25 Jul 2024 13:15:12 +0100
Subject: [Fix] GPT: Fix bug in condition check

---
 src/plugins/lua/gpt.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua
index 61217cbf8..823dbd045 100644
--- a/src/plugins/lua/gpt.lua
+++ b/src/plugins/lua/gpt.lua
@@ -94,11 +94,11 @@ local function default_condition(task)
     local action = result.action
 
     if action == 'reject' and result.npositive > 1 then
-      return true, 'already decided as spam'
+      return false, 'already decided as spam'
     end
 
     if action == 'no action' and score < 0 then
-      return true, 'negative score, already decided as ham'
+      return false, 'negative score, already decided as ham'
     end
   end
   -- We also exclude some symbols
-- 
cgit v1.2.3


From a6a7695e71fad5e29fa499817b9109d4ed27304d Mon Sep 17 00:00:00 2001
From: Jason Stephenson <jstephenson@mimecast.com>
Date: Thu, 25 Jul 2024 13:52:02 +0100
Subject: linter fix

---
 src/plugins/lua/arc.lua | 1 -
 1 file changed, 1 deletion(-)

(limited to 'src')

diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua
index 98f9303fa..90e254e78 100644
--- a/src/plugins/lua/arc.lua
+++ b/src/plugins/lua/arc.lua
@@ -642,7 +642,6 @@ local function prepare_arc_selector(task, sel)
          return v
         end
       end
-      
       return nil
     end
 
-- 
cgit v1.2.3


From 08aad032fade28a4647d2f63e2f7196d5b402582 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Fri, 26 Jul 2024 18:04:27 +0100
Subject: [Minor] Fix known senders to work with empty mime recipients

Issue: #5070
---
 src/plugins/lua/known_senders.lua | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/plugins/lua/known_senders.lua b/src/plugins/lua/known_senders.lua
index 6d57acea3..5cb2ddcf5 100644
--- a/src/plugins/lua/known_senders.lua
+++ b/src/plugins/lua/known_senders.lua
@@ -18,6 +18,7 @@ limitations under the License.
 
 local rspamd_logger = require "rspamd_logger"
 local N = 'known_senders'
+local E = {}
 local lua_util = require "lua_util"
 local lua_redis = require "lua_redis"
 local lua_maps = require "lua_maps"
@@ -258,7 +259,7 @@ local function verify_local_replies_set(task)
     return nil
   end
 
-  local replies_recipients = task:get_recipients('mime')
+  local replies_recipients = task:get_recipients('mime') or E
 
   local replies_sender_string = lua_util.maybe_obfuscate_string(tostring(replies_sender), settings,
       settings.sender_prefix)
-- 
cgit v1.2.3


From 5a2dedd00eb68a5d4c7acda93fd3cc4f3bb2d358 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Sun, 28 Jul 2024 08:45:26 +0100
Subject: [Minor] Try to fix ambigious rvalue

---
 src/libserver/cfg_rcl.cxx | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/libserver/cfg_rcl.cxx b/src/libserver/cfg_rcl.cxx
index ce3df4010..8a479fa6d 100644
--- a/src/libserver/cfg_rcl.cxx
+++ b/src/libserver/cfg_rcl.cxx
@@ -3623,7 +3623,8 @@ rspamd_config_parse_ucl(struct rspamd_config *cfg,
 	auto &cfg_file = cfg_file_maybe.value();
 
 	/* Try to load keyfile if available */
-	rspamd::util::raii_file::open(fmt::format("{}.key", filename), O_RDONLY).map([&](const auto &keyfile) {
+	auto keyfile_name = fmt::format("{}.key", filename);
+	rspamd::util::raii_file::open(keyfile_name, O_RDONLY).map([&](const auto &keyfile) {
 		auto *kp_parser = ucl_parser_new(0);
 		if (ucl_parser_add_fd(kp_parser, keyfile.get_fd())) {
 			auto *kp_obj = ucl_parser_get_object(kp_parser);
@@ -3632,8 +3633,8 @@ rspamd_config_parse_ucl(struct rspamd_config *cfg,
 			decrypt_keypair = rspamd_keypair_from_ucl(kp_obj);
 
 			if (decrypt_keypair == nullptr) {
-				msg_err_config_forced("cannot load keypair from %s.key: invalid keypair",
-									  filename);
+				msg_err_config_forced("cannot load keypair from %s: invalid keypair",
+									  keyfile_name.c_str());
 			}
 			else {
 				/* Add decryption support to UCL */
@@ -3645,8 +3646,8 @@ rspamd_config_parse_ucl(struct rspamd_config *cfg,
 			ucl_object_unref(kp_obj);
 		}
 		else {
-			msg_err_config_forced("cannot load keypair from %s.key: %s",
-								  filename, ucl_parser_get_error(kp_parser));
+			msg_err_config_forced("cannot load keypair from %s: %s",
+								  keyfile_name.c_str(), ucl_parser_get_error(kp_parser));
 		}
 		ucl_parser_free(kp_parser);
 	});
-- 
cgit v1.2.3