[Rework] Track maps origins

src/controller.c

@@ -3565,7 +3565,7 @@ start_controller_worker (struct rspamd_worker *worker)
 				"Allow unauthenticated requests from these addresses",
 				&ctx->secure_map,
 				NULL,
-				worker);
+				worker, "controller secure ip");
 	}
 
 	ctx->lang_det = ctx->cfg->lang_det;

src/fuzzy_storage.c

@@ -2503,7 +2503,7 @@ start_fuzzy (struct rspamd_worker *worker)
 	if (ctx->update_map != NULL) {
 		rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->update_map,
 				"Allow fuzzy updates from specified addresses",
-				&ctx->update_ips, NULL, worker);
+				&ctx->update_ips, NULL, worker, "fuzzy update");
 	}
 
 	if (ctx->skip_map != NULL) {
@@ -2529,7 +2529,7 @@ start_fuzzy (struct rspamd_worker *worker)
 				"Block fuzzy requests from the specific IPs",
 				&ctx->blocked_ips,
 				NULL,
-				worker);
+				worker, "fuzzy blocked");
 	}
 
 	/* Create radix trees */
@@ -2538,13 +2538,14 @@ start_fuzzy (struct rspamd_worker *worker)
 				"Skip ratelimits from specific ip addresses/networks",
 				&ctx->ratelimit_whitelist,
 				NULL,
-				worker);
+				worker, "fuzzy ratelimit whitelist");
 	}
 
 	if (!isnan (ctx->delay) && ctx->delay_whitelist_map != NULL) {
 		rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->delay_whitelist_map,
 				"Skip delay from the following ips",
-				&ctx->delay_whitelist, NULL, worker);
+				&ctx->delay_whitelist, NULL, worker,
+				"fuzzy delayed whitelist");
 	}
 
 	/* Ratelimits */

src/libserver/cfg_file.h

@@ -760,12 +760,9 @@ void rspamd_actions_sort (struct rspamd_config *cfg);
  */
 struct rspamd_radix_map_helper;
 
-gboolean rspamd_config_radix_from_ucl (struct rspamd_config *cfg,
-									   const ucl_object_t *obj,
-									   const gchar *description,
-									   struct rspamd_radix_map_helper **target,
-									   GError **err,
-									   struct rspamd_worker *worker);
+gboolean rspamd_config_radix_from_ucl (struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description,
+									   struct rspamd_radix_map_helper **target, GError **err,
+									   struct rspamd_worker *worker, const gchar *map_name);
 
 /**
  * Adds new settings id to be preprocessed

src/libserver/cfg_utils.c

@@ -2226,12 +2226,9 @@ rspamd_config_get_action_by_type (struct rspamd_config *cfg,
 }
 
 gboolean
-rspamd_config_radix_from_ucl (struct rspamd_config *cfg,
-							  const ucl_object_t *obj,
-							  const gchar *description,
-							  struct rspamd_radix_map_helper **target,
-							  GError **err,
-							  struct rspamd_worker *worker)
+rspamd_config_radix_from_ucl (struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description,
+							  struct rspamd_radix_map_helper **target, GError **err,
+							  struct rspamd_worker *worker, const gchar *map_name)
 {
 	ucl_type_t type;
 	ucl_object_iter_t it = NULL;
@@ -2269,7 +2266,8 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg,
 			else {
 				/* Just a list */
 				if (!*target) {
-					*target = rspamd_map_helper_new_radix (NULL);
+					*target = rspamd_map_helper_new_radix (
+							rspamd_map_add_fake (cfg, description, map_name));
 				}
 
 				rspamd_map_helper_insert_radix_resolve (*target, str, "");
@@ -2300,7 +2298,8 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg,
 				str = ucl_object_tostring (cur);
 
 				if (!*target) {
-					*target = rspamd_map_helper_new_radix (NULL);
+					*target = rspamd_map_helper_new_radix (
+							rspamd_map_add_fake (cfg, description, map_name));
 				}
 
 				rspamd_map_helper_insert_radix_resolve (*target, str, "");
@@ -2803,9 +2802,9 @@ rspamd_config_libs (struct rspamd_external_libs_ctx *ctx,
 		if (cfg->local_addrs) {
 			rspamd_config_radix_from_ucl (cfg, cfg->local_addrs,
 					"Local addresses",
-					(struct rspamd_radix_map_helper **)ctx->local_addrs,
+					(struct rspamd_radix_map_helper **) ctx->local_addrs,
 					NULL,
-					NULL);
+					NULL, "local addresses");
 		}
 
 		rspamd_free_zstd_dictionary (ctx->in_dict);

src/libserver/logger/logger.c

@@ -258,7 +258,7 @@ rspamd_log_open_specific (rspamd_mempool_t *pool,
 					"IP addresses for which debug logs are enabled",
 					&logger->debug_ip,
 					NULL,
-					NULL);
+					NULL, "debug ip");
 		}
 
 		if (cfg->log_encryption_key) {

src/libserver/maps/map.c

@@ -2730,6 +2730,26 @@ rspamd_map_add (struct rspamd_config *cfg,
 	return map;
 }
 
+struct rspamd_map *
+rspamd_map_add_fake (struct rspamd_config *cfg,
+				const gchar *description,
+				const gchar *name)
+{
+	struct rspamd_map *map;
+
+	map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_map));
+	map->cfg = cfg;
+	map->id = rspamd_random_uint64_fast ();
+	map->name = rspamd_mempool_strdup (cfg->cfg_pool, name);
+	map->user_data = (void **)↦ /* to prevent null pointer dereferencing */
+
+	if (description != NULL) {
+		map->description = rspamd_mempool_strdup (cfg->cfg_pool, description);
+	}
+
+	return map;
+}
+
 static inline void
 rspamd_map_add_backend (struct rspamd_map *map, struct rspamd_map_backend *bk)
 {

src/libserver/maps/map.h

@@ -93,6 +93,17 @@ struct rspamd_map *rspamd_map_add_from_ucl (struct rspamd_config *cfg,
 											struct rspamd_worker *worker,
 											int flags);
 
+/**
+ * Adds a fake map structure (for logging purposes mainly)
+ * @param cfg
+ * @param description
+ * @return
+ */
+struct rspamd_map *rspamd_map_add_fake (struct rspamd_config *cfg,
+								   const gchar *description,
+								   const gchar *name);
+
+
 enum rspamd_map_watch_type {
 	RSPAMD_MAP_WATCH_MIN = 9,
 	RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER,

src/libserver/maps/map_helpers.c

@@ -53,17 +53,20 @@ struct rspamd_radix_map_helper {
 	rspamd_mempool_t *pool;
 	khash_t(rspamd_map_hash) *htb;
 	radix_compressed_t *trie;
+	struct rspamd_map *map;
 	rspamd_cryptobox_fast_hash_state_t hst;
 };
 
 struct rspamd_hash_map_helper {
 	rspamd_mempool_t *pool;
 	khash_t(rspamd_map_hash) *htb;
+	struct rspamd_map *map;
 	rspamd_cryptobox_fast_hash_state_t hst;
 };
 
 struct rspamd_cdb_map_helper {
 	GQueue cdbs;
+	struct rspamd_map *map;
 	rspamd_cryptobox_fast_hash_state_t hst;
 	gsize total_size;
 };
@@ -457,7 +460,9 @@ rspamd_map_helper_insert_radix (gpointer st, gconstpointer key, gconstpointer va
 	gconstpointer nk;
 	rspamd_ftok_t tok;
 	gint res;
+	struct rspamd_map *map;
 
+	map = r->map;
 	vlen = strlen (value);
 	val = rspamd_mempool_alloc0 (r->pool, sizeof (*val) +
 			vlen + 1);
@@ -472,11 +477,16 @@ rspamd_map_helper_insert_radix (gpointer st, gconstpointer key, gconstpointer va
 		tok.begin = nk;
 		k = kh_put (rspamd_map_hash, r->htb, tok, &res);
 	}
+	else {
+		msg_warn_map ("duplicate radix entry found for map %s: %s (old value: %s, new: %s)",
+				map->name, key, kh_value (r->htb, k), val);
+	}
 
 	nk = kh_key (r->htb, k).begin;
 	val->key = nk;
 	kh_value (r->htb, k) = val;
-	rspamd_radix_add_iplist (key, ",", r->trie, val, FALSE);
+	rspamd_radix_add_iplist (key, ",", r->trie, val, FALSE,
+			r->map->name);
 	rspamd_cryptobox_fast_hash_update (&r->hst, nk, tok.len);
 }
 
@@ -490,7 +500,9 @@ rspamd_map_helper_insert_radix_resolve (gpointer st, gconstpointer key, gconstpo
 	gconstpointer nk;
 	rspamd_ftok_t tok;
 	gint res;
+	struct rspamd_map *map;
 
+	map = r->map;
 	vlen = strlen (value);
 	val = rspamd_mempool_alloc0 (r->pool, sizeof (*val) +
 			vlen + 1);
@@ -505,11 +517,16 @@ rspamd_map_helper_insert_radix_resolve (gpointer st, gconstpointer key, gconstpo
 		tok.begin = nk;
 		k = kh_put (rspamd_map_hash, r->htb, tok, &res);
 	}
+	else {
+		msg_warn_map ("duplicate radix entry found for map %s: %s (old value: %s, new: %s)",
+				map->name, key, kh_value (r->htb, k), val);
+	}
 
 	nk = kh_key (r->htb, k).begin;
 	val->key = nk;
 	kh_value (r->htb, k) = val;
-	rspamd_radix_add_iplist (key, ",", r->trie, val, TRUE);
+	rspamd_radix_add_iplist (key, ",", r->trie, val, TRUE,
+			r->map->name);
 	rspamd_cryptobox_fast_hash_update (&r->hst, nk, tok.len);
 }
 
@@ -523,9 +540,11 @@ rspamd_map_helper_insert_hash (gpointer st, gconstpointer key, gconstpointer val
 	gsize vlen;
 	gint r;
 	rspamd_ftok_t tok;
+	struct rspamd_map *map;
 
 	tok.begin = key;
 	tok.len = strlen (key);
+	map = ht->map;
 
 	k = kh_get (rspamd_map_hash, ht->htb, tok);
 	vlen = strlen (value);
@@ -542,6 +561,10 @@ rspamd_map_helper_insert_hash (gpointer st, gconstpointer key, gconstpointer val
 			/* Same element, skip */
 			return;
 		}
+		else {
+			msg_warn_map ("duplicate hash entry found for map %s: %s (old value: %s, new: %s)",
+					map->name, key, kh_value (ht->htb, k), val);
+		}
 	}
 
 	/* Null termination due to alloc0 */
@@ -598,7 +621,7 @@ rspamd_map_helper_insert_re (gpointer st, gconstpointer key, gconstpointer value
 	vlen = strlen (value);
 	val = rspamd_mempool_alloc0 (re_map->pool, sizeof (*val) +
 			vlen + 1);
-	memcpy (val->value, value, vlen);
+	memcpy (val->value, value, vlen); /* Null terminated due to alloc0 previously */
 	tok.begin = key;
 	tok.len = strlen (key);
 
@@ -609,6 +632,10 @@ rspamd_map_helper_insert_re (gpointer st, gconstpointer key, gconstpointer value
 		tok.begin = nk;
 		k = kh_put (rspamd_map_hash, re_map->htb, tok, &r);
 	}
+	else {
+		msg_warn_map ("duplicate re found for map %s: %s (old value: %s, new: %s)",
+				map->name, key, kh_value (re_map->htb, k)->value, val);
+	}
 
 	nk = kh_key (re_map->htb, k).begin;
 	val->key = nk;
@@ -670,6 +697,7 @@ rspamd_map_helper_new_hash (struct rspamd_map *map)
 	htb = rspamd_mempool_alloc0 (pool, sizeof (*htb));
 	htb->htb = kh_init (rspamd_map_hash);
 	htb->pool = pool;
+	htb->map = map;
 	rspamd_cryptobox_fast_hash_init (&htb->hst, map_hash_seed);
 
 	return htb;
@@ -714,10 +742,12 @@ rspamd_map_helper_new_radix (struct rspamd_map *map)
 {
 	struct rspamd_radix_map_helper *r;
 	rspamd_mempool_t *pool;
+	const gchar *name = "unnamed";
 
 	if (map) {
 		pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
 				map->tag, 0);
+		name = map->name;
 	}
 	else {
 		pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
@@ -725,9 +755,10 @@ rspamd_map_helper_new_radix (struct rspamd_map *map)
 	}
 
 	r = rspamd_mempool_alloc0 (pool, sizeof (*r));
-	r->trie = radix_create_compressed_with_pool (pool);
+	r->trie = radix_create_compressed_with_pool (pool, name);
 	r->htb = kh_init (rspamd_map_hash);
 	r->pool = pool;
+	r->map = map;
 	rspamd_cryptobox_fast_hash_init (&r->hst, map_hash_seed);
 
 	return r;
@@ -1449,6 +1480,7 @@ rspamd_map_helper_new_cdb (struct rspamd_map *map)
 
 	n = g_malloc0 (sizeof (*n));
 	n->cdbs = (GQueue)G_QUEUE_INIT;
+	n->map = map;
 
 	rspamd_cryptobox_fast_hash_init (&n->hst, map_hash_seed);
 

src/libserver/monitored.c

@@ -265,7 +265,7 @@ rspamd_monitored_dns_conf (struct rspamd_monitored *m,
 		if (elt) {
 			if (ucl_object_type (elt) == UCL_STRING) {
 				radix_add_generic_iplist (ucl_object_tostring (elt),
-						&conf->expected, FALSE);
+						&conf->expected, FALSE, NULL);
 			}
 			else if (ucl_object_type (elt) == UCL_ARRAY) {
 				const ucl_object_t *cur;
@@ -273,7 +273,7 @@ rspamd_monitored_dns_conf (struct rspamd_monitored *m,
 
 				while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) {
 					radix_add_generic_iplist (ucl_object_tostring (elt),
-							&conf->expected, FALSE);
+							&conf->expected, FALSE, NULL);
 				}
 			}
 		}

src/libutil/radix.c

@@ -41,6 +41,7 @@ INIT_LOG_MODULE(radix)
 struct radix_tree_compressed {
 	rspamd_mempool_t *pool;
 	struct btrie *tree;
+	const gchar *name;
 	size_t size;
 	guint duplicates;
 	gboolean own_pool;
@@ -78,8 +79,8 @@ radix_insert_compressed (radix_compressed_t * tree,
 	g_assert (tree != NULL);
 	g_assert (keybits >= masklen);
 
-	msg_debug_radix ("want insert value %p with mask %z, key: %*xs",
-			(gpointer)value, keybits - masklen, (int)keylen, key);
+	msg_debug_radix ("%s: want insert value %p with mask %z, key: %*xs",
+			tree->name, (gpointer)value, keybits - masklen, (int)keylen, key);
 
 	old = radix_find_compressed (tree, key, keylen);
 
@@ -90,25 +91,30 @@ radix_insert_compressed (radix_compressed_t * tree,
 		tree->duplicates++;
 
 		if (tree->duplicates == max_duplicates) {
-			msg_err_radix ("maximum duplicates limit reached: %d, "
-				  "suppress further errors", max_duplicates);
+			msg_err_radix ("%s: maximum duplicates limit reached: %d, "
+				  "suppress further errors", tree->name, max_duplicates);
 		}
 		else if (tree->duplicates < max_duplicates) {
 			memset (ip_str, 0, sizeof (ip_str));
 
 			if (keybits == 32) {
-				msg_err_radix ("cannot insert %p, key: %s/%d, duplicate value",
+				msg_err_radix ("%s: cannot insert %p, key: %s/%d, duplicate value",
+						tree->name,
 						(gpointer) value,
 						inet_ntop (AF_INET, key, ip_str, sizeof (ip_str) - 1),
 						(gint) (keybits - masklen));
 			} else if (keybits == 128) {
-				msg_err_radix ("cannot insert %p, key: [%s]/%d, duplicate value",
+				msg_err_radix ("%s: cannot insert %p, key: [%s]/%d, duplicate value",
+						tree->name,
 						(gpointer) value,
 						inet_ntop (AF_INET6, key, ip_str, sizeof (ip_str) - 1),
 						(gint) (keybits - masklen));
 			} else {
-				msg_err_radix ("cannot insert %p with mask %z, key: %*xs, duplicate value",
-						(gpointer) value, keybits - masklen, (int) keylen, key);
+				msg_err_radix ("%s: cannot insert %p with mask %z, key: %*xs, duplicate value",
+						tree->name,
+						(gpointer) value,
+						keybits - masklen,
+						(int) keylen, key);
 			}
 		}
 	}
@@ -121,7 +127,7 @@ radix_insert_compressed (radix_compressed_t * tree,
 
 
 radix_compressed_t *
-radix_create_compressed (void)
+radix_create_compressed (const gchar *tree_name)
 {
 	radix_compressed_t *tree;
 
@@ -135,12 +141,13 @@ radix_create_compressed (void)
 	tree->duplicates = 0;
 	tree->tree = btrie_init (tree->pool);
 	tree->own_pool = TRUE;
+	tree->name = tree_name;
 
 	return tree;
 }
 
 radix_compressed_t *
-radix_create_compressed_with_pool (rspamd_mempool_t *pool)
+radix_create_compressed_with_pool (rspamd_mempool_t *pool, const gchar *tree_name)
 {
 	radix_compressed_t *tree;
 
@@ -150,6 +157,7 @@ radix_create_compressed_with_pool (rspamd_mempool_t *pool)
 	tree->duplicates = 0;
 	tree->tree = btrie_init (tree->pool);
 	tree->own_pool = FALSE;
+	tree->name = tree_name;
 
 	return tree;
 }
@@ -199,7 +207,8 @@ radix_find_compressed_addr (radix_compressed_t *tree,
 
 gint
 rspamd_radix_add_iplist (const gchar *list, const gchar *separators,
-		radix_compressed_t *tree, gconstpointer value, gboolean resolve)
+						 radix_compressed_t *tree, gconstpointer value,
+						 gboolean resolve, const gchar *tree_name)
 {
 	gchar *token, *ipnet, *err_str, **strv, **cur, *brace;
 	union {
@@ -232,7 +241,8 @@ rspamd_radix_add_iplist (const gchar *list, const gchar *separators,
 			k = strtoul (ipnet, &err_str, 10);
 			if (errno != 0) {
 				msg_warn_radix (
-						"invalid netmask, error detected on symbol: %s, error: %s",
+						"%s: invalid netmask, error detected on symbol: %s, error: %s",
+						tree_name,
 						err_str,
 						strerror (errno));
 				k = G_MAXINT;
@@ -378,16 +388,16 @@ rspamd_radix_add_iplist (const gchar *list, const gchar *separators,
 
 gboolean
 radix_add_generic_iplist (const gchar *ip_list, radix_compressed_t **tree,
-		gboolean resolve)
+						  gboolean resolve, const gchar *tree_name)
 {
 	static const char fill_ptr[] = "1";
 
 	if (*tree == NULL) {
-		*tree = radix_create_compressed ();
+		*tree = radix_create_compressed (tree_name);
 	}
 
 	return (rspamd_radix_add_iplist (ip_list, ",; ", *tree,
-			fill_ptr, resolve) > 0);
+			fill_ptr, resolve, tree_name) > 0);
 }
 
 

src/libutil/radix.h

@@ -72,9 +72,9 @@ void radix_destroy_compressed (radix_compressed_t *tree);
  * Create new radix trie
  * @return
  */
-radix_compressed_t *radix_create_compressed (void);
+radix_compressed_t *radix_create_compressed (const gchar *tree_name);
 
-radix_compressed_t *radix_create_compressed_with_pool (rspamd_mempool_t *pool);
+radix_compressed_t *radix_create_compressed_with_pool (rspamd_mempool_t *pool, const gchar *tree_name);
 
 /**
  * Insert list of ip addresses and masks to the radix tree
@@ -84,14 +84,18 @@ radix_compressed_t *radix_create_compressed_with_pool (rspamd_mempool_t *pool);
  * @return number of elements inserted
  */
 gint rspamd_radix_add_iplist (const gchar *list, const gchar *separators,
-							  radix_compressed_t *tree, gconstpointer value, gboolean resolve);
+							  radix_compressed_t *tree, gconstpointer value,
+							  gboolean resolve, const gchar *tree_name);
 
 /**
  * Generic version of @see rspamd_radix_add_iplist. This function creates tree
  * if `tree` is NULL.
  */
-gboolean radix_add_generic_iplist (const gchar *ip_list,
-								   radix_compressed_t **tree, gboolean resolve);
+gboolean
+radix_add_generic_iplist (const gchar *ip_list,
+						  radix_compressed_t **tree,
+						  gboolean resolve,
+						  const gchar *tree_name);
 
 /**
  * Returns number of elements in the tree

src/plugins/dkim_check.c

@@ -438,7 +438,7 @@ dkim_module_config (struct rspamd_config *cfg)
 		rspamd_config_get_module_opt (cfg, "dkim", "whitelist")) != NULL) {
 
 		rspamd_config_radix_from_ucl (cfg, value, "DKIM whitelist",
-				&dkim_module_ctx->whitelist_ip, NULL, NULL);
+				&dkim_module_ctx->whitelist_ip, NULL, NULL, "dkim whitelist");
 	}
 
 	if ((value =

src/plugins/fuzzy_check.c

@@ -1067,7 +1067,7 @@ fuzzy_check_module_config (struct rspamd_config *cfg)
 		rspamd_config_radix_from_ucl (cfg, value, "Fuzzy whitelist",
 				&fuzzy_module_ctx->whitelist,
 				NULL,
-				NULL);
+				NULL, "fuzzy ip whitelist");
 	}
 	else {
 		fuzzy_module_ctx->whitelist = NULL;

test/rspamd_radix_test.c

@@ -78,7 +78,7 @@ struct _tv {
 static void
 rspamd_radix_test_vec (void)
 {
-	radix_compressed_t *tree = radix_create_compressed ();
+	radix_compressed_t *tree = radix_create_compressed (NULL);
 	struct _tv *t = &test_vec[0];
 	struct in_addr ina;
 	struct in6_addr in6a;
@@ -209,7 +209,7 @@ rspamd_radix_test_func (void)
 {
 	struct btrie *btrie;
 	rspamd_mempool_t *pool;
-	radix_compressed_t *comp_tree = radix_create_compressed ();
+	radix_compressed_t *comp_tree = radix_create_compressed (NULL);
 	struct {
 		guint32 addr;
 		guint32 mask;