[Fix] Fix DL lists initialisations

2 files changed, 93 insertions, 75 deletions

diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c
index 5313e157a..7c5e14616 100644
--- a/src/libserver/re_cache.c
+++ b/src/libserver/re_cache.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2024 Vsevolod Stakhov
+ * Copyright 2025 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -324,7 +324,8 @@ rspamd_re_cache_new(void)
 	cache->nre = 0;
 	cache->re = g_ptr_array_new_full(256, rspamd_re_cache_elt_dtor);
 	cache->selectors = kh_init(lua_selectors_hash);
-	cache->next = cache->prev = NULL;
+	cache->next = NULL;
+	cache->prev = cache;
 	cache->scope = NULL;                        /* Default scope */
 	cache->flags = RSPAMD_RE_CACHE_FLAG_LOADED; /* Default scope is always loaded */
 #ifdef WITH_HYPERSCAN
@@ -659,6 +660,9 @@ rspamd_re_cache_runtime_new_single(struct rspamd_re_cache *cache)
 #ifdef WITH_HYPERSCAN
 	rt->has_hs = cache->hyperscan_loaded;
 #endif
+	/* Initialize the doubly-linked list pointers */
+	rt->next = NULL;
+	rt->prev = NULL;
 
 	return rt;
 }
@@ -689,6 +693,9 @@ rspamd_re_cache_runtime_new(struct rspamd_re_cache *cache)
 			}
 			else {
 				rt_head = rt;
+				/* For doubly-linked list, first element's prev should point to itself */
+				rt_head->prev = rt_head;
+				rt_head->next = NULL;
 			}
 		}
 	}
@@ -2657,7 +2664,7 @@ rspamd_re_cache_is_valid_hyperscan_file(struct rspamd_re_cache *cache,
 
 	len = strlen(path);
 
-	if (len < sizeof(rspamd_cryptobox_HASHBYTES + 3)) {
+	if (len < (rspamd_cryptobox_HASHBYTES + 3)) {
 		if (!silent) {
 			msg_err_re_cache("cannot open hyperscan cache file %s: too short filename",
 							 path);
diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua
index 97d68f36b..336a65d0e 100644
--- a/src/plugins/lua/multimap.lua
+++ b/src/plugins/lua/multimap.lua
@@ -62,8 +62,8 @@ local function parse_multimap_value(parse_rule, p_ret)
           (digit ^ 1)
 
       -- Matches: 55.97, -90.8, .9
-      number.decimal = (number.integer *     -- Integer
-            (number.fractional ^ -1)) +      -- Fractional
+      number.decimal = (number.integer * -- Integer
+          (number.fractional ^ -1)) + -- Fractional
           (lpeg.S("+-") * number.fractional) -- Completely fractional number
 
       local sym_start = lpeg.R("az", "AZ") + lpeg.S("_")
@@ -98,7 +98,7 @@ local function parse_multimap_value(parse_rule, p_ret)
     else
       if p_ret ~= '' then
         rspamd_logger.infox(rspamd_config, '%s: cannot parse string "%s"',
-          parse_rule.symbol, p_ret)
+            parse_rule.symbol, p_ret)
       end
 
       return true, nil, 1.0, {}
@@ -113,7 +113,9 @@ end
 -- SpamAssassin-like line processing functions
 local function split_sa_line(str)
   local result = {}
-  if not str then return result end
+  if not str then
+    return result
+  end
 
   for token in string.gmatch(str, '%S+') do
     table.insert(result, token)
@@ -193,7 +195,7 @@ local function create_sa_atom_function(atom_name, re, rule_type, rule_data)
   end
 end
 
-local function process_sa_line(rule, line, map)
+local function process_sa_line(rule, line)
   line = lua_util.str_trim(line)
 
   if string.len(line) == 0 or string.sub(line, 1, 1) == '#' then
@@ -249,7 +251,7 @@ local function process_sa_line(rule, line, map)
         }
 
         lua_util.debugm(N, rspamd_config, 'added SA header atom: %s for header %s (scope: %s)',
-          atom_name, header_name, scope_name)
+            atom_name, header_name, scope_name)
       end
     end
   elseif words[1] == 'body' then
@@ -501,8 +503,8 @@ create_sa_meta_callback = function(meta_rule)
 
     if not (already_processed and already_processed[result_name or 'default']) then
       local expression = rspamd_expression.create(meta_rule.expression,
-        parse_sa_atom,
-        rspamd_config:get_mempool())
+          parse_sa_atom,
+          rspamd_config:get_mempool())
       if not expression then
         rspamd_logger.errx(rspamd_config, 'Cannot parse SA meta expression: %s', meta_rule.expression)
         return
@@ -515,7 +517,7 @@ create_sa_meta_callback = function(meta_rule)
         if res > 0 then
           local score = sa_scores[meta_rule.symbol] or 1.0
           task:insert_result_named(cur_res, meta_rule.symbol, score,
-            fun.totable(fun.filter(exclude_sym_filter, trace)))
+              fun.totable(fun.filter(exclude_sym_filter, trace)))
         end
 
         if not cached[meta_rule.symbol] then
@@ -584,7 +586,7 @@ local function finalize_sa_rules()
     end
 
     lua_util.debugm(N, rspamd_config, 'registered SA meta symbol: %s (score: %s)',
-      meta_name, score)
+        meta_name, score)
   end
 
   -- Mark orphaned symbols - symbols that were previously available but no longer in current rules
@@ -1058,19 +1060,19 @@ local function multimap_query_redis(key, task, value, callback)
 
   local function redis_map_cb(err, data)
     lua_util.debugm(N, task, 'got reply from Redis when trying to get key %s: err=%s, data=%s',
-      key, err, data)
+        key, err, data)
     if not err and type(data) ~= 'userdata' then
       callback(data)
     end
   end
 
   return rspamd_redis_make_request(task,
-    redis_params, -- connect params
-    key,          -- hash key
-    false,        -- is write
-    redis_map_cb, --callback
-    cmd,          -- command
-    srch          -- arguments
+      redis_params, -- connect params
+      key, -- hash key
+      false, -- is write
+      redis_map_cb, --callback
+      cmd, -- command
+      srch          -- arguments
   )
 end
 
@@ -1082,9 +1084,9 @@ local function multimap_callback(task, rule)
 
     local function get_key_callback(ret, err_or_data, err_code)
       lua_util.debugm(N, task, 'got return "%s" (err code = %s) for multimap %s',
-        err_or_data,
-        err_code,
-        rule.symbol)
+          err_or_data,
+          err_code,
+          rule.symbol)
 
       if ret then
         if type(err_or_data) == 'table' then
@@ -1096,12 +1098,12 @@ local function multimap_callback(task, rule)
         end
       elseif err_code ~= 404 then
         rspamd_logger.infox(task, "map %s: get key returned error %s: %s",
-          rule.symbol, err_code, err_or_data)
+            rule.symbol, err_code, err_or_data)
       end
     end
 
     lua_util.debugm(N, task, 'check value %s for multimap %s', value,
-      rule.symbol)
+        rule.symbol)
 
     local ret = false
 
@@ -1140,8 +1142,8 @@ local function multimap_callback(task, rule)
       if rule.symbols_set then
         if not rule.symbols_set[symbol] then
           rspamd_logger.infox(task, 'symbol %s is not registered for map %s, ' ..
-            'replace it with just %s',
-            symbol, rule.symbol, rule.symbol)
+              'replace it with just %s',
+              symbol, rule.symbol, rule.symbol)
           symbol = rule.symbol
         end
       elseif rule.disable_multisymbol then
@@ -1211,7 +1213,7 @@ local function multimap_callback(task, rule)
       if fn then
         local filtered_value = fn(task, r.filter, value, r)
         lua_util.debugm(N, task, 'apply filter %s for rule %s: %s -> %s',
-          r.filter, r.symbol, value, filtered_value)
+            r.filter, r.symbol, value, filtered_value)
         value = filtered_value
       end
     end
@@ -1358,12 +1360,12 @@ local function multimap_callback(task, rule)
 
     if not res or res == 0 then
       lua_util.debugm(N, task, 'condition is false for %s',
-        rule.symbol)
+          rule.symbol)
       return
     else
       lua_util.debugm(N, task, 'condition is true for %s: %s',
-        rule.symbol,
-        trace)
+          rule.symbol,
+          trace)
     end
   end
 
@@ -1380,18 +1382,18 @@ local function multimap_callback(task, rule)
         local to_resolve = ip_to_rbl(ip, rule['map'])
         local function dns_cb(_, _, results, err)
           lua_util.debugm(N, rspamd_config,
-            'resolve() finished: results=%1, err=%2, to_resolve=%3',
-            results, err, to_resolve)
+              'resolve() finished: results=%1, err=%2, to_resolve=%3',
+              results, err, to_resolve)
 
           if err and
               (err ~= 'requested record is not found' and
-                err ~= 'no records with this name') then
+                  err ~= 'no records with this name') then
             rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, results)
           elseif results then
             task:insert_result(rule['symbol'], 1, rule['map'])
             if rule.action then
               task:set_pre_result(rule['action'],
-                'Matched map: ' .. rule['symbol'], N)
+                  'Matched map: ' .. rule['symbol'], N)
             end
           end
         end
@@ -1557,7 +1559,7 @@ local function multimap_callback(task, rule)
           if ext then
             local fake_fname = string.format('detected.%s', ext)
             lua_util.debugm(N, task, 'detected filename %s',
-              fake_fname)
+                fake_fname)
             match_filename(rule, fake_fname)
           end
         end
@@ -1630,7 +1632,7 @@ local function multimap_callback(task, rule)
       if ret and ret ~= 0 then
         for n, t in pairs(trace) do
           insert_results(t.value, string.format("%s=%s",
-            n, t.matched))
+              n, t.matched))
         end
       end
     end,
@@ -1666,7 +1668,7 @@ local function multimap_on_load_gen(rule)
 
       if r and symbol and not known_symbols[symbol] then
         lua_util.debugm(N, rspamd_config, "%s: adding new symbol %s (score = %s), triggered by %s",
-          rule.symbol, symbol, score, key)
+            rule.symbol, symbol, score, key)
         rspamd_config:register_symbol {
           name = symbol,
           parent = rule.callback_id,
@@ -1692,22 +1694,22 @@ local function add_multimap_rule(key, newrule)
     if rule['regexp'] then
       if rule['multi'] then
         rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'regexp_multi',
-          rule.description)
+            rule.description)
       else
         rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'regexp',
-          rule.description)
+            rule.description)
       end
     elseif rule['glob'] then
       if rule['multi'] then
         rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'glob_multi',
-          rule.description)
+            rule.description)
       else
         rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'glob',
-          rule.description)
+            rule.description)
       end
     else
       rule.map_obj = lua_maps.map_add_from_ucl(rule.map, 'hash',
-        rule.description)
+          rule.description)
     end
   end
 
@@ -1748,7 +1750,7 @@ local function add_multimap_rule(key, newrule)
   end
   if not newrule['description'] then
     newrule['description'] = string.format('multimap, type %s: %s', newrule['type'],
-      newrule['symbol'])
+        newrule['symbol'])
   end
   if newrule['type'] == 'mempool' and not newrule['variable'] then
     rspamd_logger.errx(rspamd_config, 'mempool map requires variable')
@@ -1760,11 +1762,11 @@ local function add_multimap_rule(key, newrule)
       return nil
     else
       local selector = lua_selectors.create_selector_closure(
-        rspamd_config, newrule['selector'], newrule['delimiter'] or "")
+          rspamd_config, newrule['selector'], newrule['delimiter'] or "")
 
       if not selector then
         rspamd_logger.errx(rspamd_config, 'selector map has invalid selector: "%s", symbol: %s',
-          newrule['selector'], newrule['symbol'])
+            newrule['selector'], newrule['symbol'])
         return nil
       end
 
@@ -1775,7 +1777,7 @@ local function add_multimap_rule(key, newrule)
       string.find(newrule['map'], '^redis://.*$') then
     if not redis_params then
       rspamd_logger.infox(rspamd_config, 'no redis servers are specified, ' ..
-        'cannot add redis map %s: %s', newrule['symbol'], newrule['map'])
+          'cannot add redis map %s: %s', newrule['symbol'], newrule['map'])
       return nil
     end
 
@@ -1788,17 +1790,17 @@ local function add_multimap_rule(key, newrule)
       string.find(newrule['map'], '^redis%+selector://.*$') then
     if not redis_params then
       rspamd_logger.infox(rspamd_config, 'no redis servers are specified, ' ..
-        'cannot add redis map %s: %s', newrule['symbol'], newrule['map'])
+          'cannot add redis map %s: %s', newrule['symbol'], newrule['map'])
       return nil
     end
 
     local selector_str = string.match(newrule['map'], '^redis%+selector://(.*)$')
     local selector = lua_selectors.create_selector_closure(
-      rspamd_config, selector_str, newrule['delimiter'] or "")
+        rspamd_config, selector_str, newrule['delimiter'] or "")
 
     if not selector then
       rspamd_logger.errx(rspamd_config, 'redis selector map has invalid selector: "%s", symbol: %s',
-        selector_str, newrule['symbol'])
+          selector_str, newrule['symbol'])
       return nil
     end
 
@@ -1807,12 +1809,12 @@ local function add_multimap_rule(key, newrule)
   elseif newrule.type == 'combined' then
     local lua_maps_expressions = require "lua_maps_expressions"
     newrule.combined = lua_maps_expressions.create(rspamd_config,
-      {
-        rules = newrule.rules,
-        expression = newrule.expression,
-        description = newrule.description,
-        on_load = newrule.dynamic_symbols and multimap_on_load_gen(newrule) or nil,
-      }, N, 'Combined map for ' .. newrule.symbol)
+        {
+          rules = newrule.rules,
+          expression = newrule.expression,
+          description = newrule.description,
+          on_load = newrule.dynamic_symbols and multimap_on_load_gen(newrule) or nil,
+        }, N, 'Combined map for ' .. newrule.symbol)
     if not newrule.combined then
       rspamd_logger.errx(rspamd_config, 'cannot add combined map for %s', newrule.symbol)
     else
@@ -1851,17 +1853,23 @@ local function add_multimap_rule(key, newrule)
       type = "callback",
       url = map_ucl.url or map_ucl.urls or map_ucl,
       description = newrule.description or 'SA-style multimap: ' .. newrule.symbol,
-      callback = function(line, map)
+      callback = function(pseudo_key, pseudo_value)
+        -- We have values being parsed as kv pairs, but they are not, so we concat them and use as a line
+        local line = pseudo_key .. ' ' .. pseudo_value
+        -- Add debug logging to see if callback is called
+        lua_util.debugm(N, rspamd_config, 'regexp_rules callback called for line: %s', line)
+
         -- Mark scope as unloaded on first line
         if not first_line_processed then
           first_line_processed = true
+          lua_util.debugm(N, rspamd_config, 'processing first line of regexp_rules map %s', newrule.symbol)
 
           -- Mark all existing symbols for this scope as loading
           for symbol, state_info in pairs(regexp_rules_symbol_states) do
             if state_info.rule_name == newrule.symbol then
               state_info.state = 'loading'
               lua_util.debugm(N, rspamd_config, 'marked regexp_rules symbol %s as loading for scope %s reload',
-                symbol, scope_name)
+                  symbol, scope_name)
             end
           end
 
@@ -1877,7 +1885,7 @@ local function add_multimap_rule(key, newrule)
             sa_atoms[symbol] = nil
             sa_meta_rules[symbol] = nil
             lua_util.debugm(N, rspamd_config, 'cleared regexp_rules symbol %s for scope %s reload',
-              symbol, scope_name)
+                symbol, scope_name)
           end
 
           -- The scope will be created by process_sa_line when first regexp is added
@@ -1885,15 +1893,17 @@ local function add_multimap_rule(key, newrule)
           rspamd_config:set_regexp_scope_loaded(scope_name, false)
           lua_util.debugm(N, rspamd_config, 'marked regexp scope %s as unloaded during processing', scope_name)
         end
-        process_sa_line(newrule, line, map)
+        process_sa_line(newrule, line)
       end,
-      by_line = true,      -- Process line by line
+      by_line = true, -- Process line by line
       opaque_data = false, -- Use plain strings
     })
 
     -- Add on_load callback to mark scope as loaded when map processing is complete
     if newrule.map_obj then
       newrule.map_obj:on_load(function()
+        lua_util.debugm(N, rspamd_config, 'regexp_rules map %s loaded successfully', newrule.symbol)
+
         -- Mark all atoms for this scope as available (if they're still loading)
         for symbol, state_info in pairs(regexp_rules_symbol_states) do
           if state_info.rule_name == newrule.symbol then
@@ -1921,7 +1931,8 @@ local function add_multimap_rule(key, newrule)
 
           -- Trigger hyperscan compilation for this updated scope
           newrule.map_obj:trigger_hyperscan_compilation()
-          lua_util.debugm(N, rspamd_config, 'triggered hyperscan compilation for scope %s after map loading', scope_name)
+          lua_util.debugm(N, rspamd_config, 'triggered hyperscan compilation for scope %s after map loading',
+              scope_name)
         else
           lua_util.debugm(N, rspamd_config, 'regexp scope %s not created (empty map)', scope_name)
         end
@@ -1932,21 +1943,21 @@ local function add_multimap_rule(key, newrule)
       -- Mark this rule as using SA functionality
       newrule.uses_sa = true
       lua_util.debugm(N, rspamd_config, 'created regexp_rules map %s with scope: %s',
-        newrule.symbol, scope_name)
+          newrule.symbol, scope_name)
       ret = true
     else
       rspamd_logger.warnx(rspamd_config, 'Cannot add SA-style rule: map doesn\'t exists: %s',
-        newrule['map'])
+          newrule['map'])
     end
   else
     if newrule['type'] == 'ip' then
       newrule.map_obj = lua_maps.map_add_from_ucl(newrule.map, 'radix',
-        newrule.description)
+          newrule.description)
       if newrule.map_obj then
         ret = true
       else
         rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %s',
-          newrule['map'])
+            newrule['map'])
       end
     elseif newrule['type'] == 'received' then
       if type(newrule['flags']) == 'table' and newrule['flags'][1] then
@@ -1962,12 +1973,12 @@ local function add_multimap_rule(key, newrule)
       local filter = newrule['filter'] or 'real_ip'
       if filter == 'real_ip' or filter == 'from_ip' then
         newrule.map_obj = lua_maps.map_add_from_ucl(newrule.map, 'radix',
-          newrule.description)
+            newrule.description)
         if newrule.map_obj then
           ret = true
         else
           rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %s',
-            newrule['map'])
+              newrule['map'])
         end
       else
         multimap_load_kv_map(newrule)
@@ -1976,13 +1987,13 @@ local function add_multimap_rule(key, newrule)
           ret = true
         else
           rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %s',
-            newrule['map'])
+              newrule['map'])
         end
       end
     elseif known_generic_types[newrule.type] then
       if newrule.filter == 'ip_addr' then
         newrule.map_obj = lua_maps.map_add_from_ucl(newrule.map, 'radix',
-          newrule.description)
+            newrule.description)
       elseif not newrule.combined then
         multimap_load_kv_map(newrule)
       end
@@ -1991,13 +2002,13 @@ local function add_multimap_rule(key, newrule)
         ret = true
       else
         rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %s',
-          newrule['map'])
+            newrule['map'])
       end
     elseif newrule['type'] == 'dnsbl' then
       ret = true
     else
       rspamd_logger.errx(rspamd_config, 'cannot add rule %s: invalid type %s',
-        key, newrule['type'])
+          key, newrule['type'])
     end
   end
 
@@ -2034,13 +2045,13 @@ local function add_multimap_rule(key, newrule)
       end
 
       local expression = rspamd_expression.create(newrule['require_symbols'],
-        { parse_atom, process_atom }, rspamd_config:get_mempool())
+          { parse_atom, process_atom }, rspamd_config:get_mempool())
       if expression then
         newrule['expression'] = expression
 
         fun.each(function(v)
           lua_util.debugm(N, rspamd_config, 'add dependency %s -> %s',
-            newrule['symbol'], v)
+              newrule['symbol'], v)
           rspamd_config:register_dependency(newrule['symbol'], v)
         end, atoms)
       end
@@ -2062,7 +2073,7 @@ if opts and type(opts) == 'table' then
         rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"')
       else
         rspamd_logger.infox(rspamd_config, 'added multimap rule: %s (%s)',
-          k, rule.type)
+            k, rule.type)
         table.insert(rules, rule)
       end
     end