From f84b86d901c8ba53147ae89bce894f58e0448be3 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Mon, 26 Oct 2020 12:14:17 +0000 Subject: [PATCH] [Rework] Track maps origins --- src/controller.c | 2 +- src/fuzzy_storage.c | 9 +++---- src/libserver/cfg_file.h | 9 +++---- src/libserver/cfg_utils.c | 19 +++++++-------- src/libserver/logger/logger.c | 2 +- src/libserver/maps/map.c | 20 ++++++++++++++++ src/libserver/maps/map.h | 11 +++++++++ src/libserver/maps/map_helpers.c | 40 ++++++++++++++++++++++++++++---- src/libserver/monitored.c | 4 ++-- src/libutil/radix.c | 40 ++++++++++++++++++++------------ src/libutil/radix.h | 14 +++++++---- src/plugins/dkim_check.c | 2 +- src/plugins/fuzzy_check.c | 2 +- test/rspamd_radix_test.c | 4 ++-- 14 files changed, 126 insertions(+), 52 deletions(-) diff --git a/src/controller.c b/src/controller.c index e3f810d0e..174382879 100644 --- a/src/controller.c +++ b/src/controller.c @@ -3565,7 +3565,7 @@ start_controller_worker (struct rspamd_worker *worker) "Allow unauthenticated requests from these addresses", &ctx->secure_map, NULL, - worker); + worker, "controller secure ip"); } ctx->lang_det = ctx->cfg->lang_det; diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index b1a340d6b..6d046fcdb 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -2503,7 +2503,7 @@ start_fuzzy (struct rspamd_worker *worker) if (ctx->update_map != NULL) { rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->update_map, "Allow fuzzy updates from specified addresses", - &ctx->update_ips, NULL, worker); + &ctx->update_ips, NULL, worker, "fuzzy update"); } if (ctx->skip_map != NULL) { @@ -2529,7 +2529,7 @@ start_fuzzy (struct rspamd_worker *worker) "Block fuzzy requests from the specific IPs", &ctx->blocked_ips, NULL, - worker); + worker, "fuzzy blocked"); } /* Create radix trees */ @@ -2538,13 +2538,14 @@ start_fuzzy (struct rspamd_worker *worker) "Skip ratelimits from specific ip addresses/networks", &ctx->ratelimit_whitelist, NULL, - worker); + worker, "fuzzy ratelimit whitelist"); } if (!isnan (ctx->delay) && ctx->delay_whitelist_map != NULL) { rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->delay_whitelist_map, "Skip delay from the following ips", - &ctx->delay_whitelist, NULL, worker); + &ctx->delay_whitelist, NULL, worker, + "fuzzy delayed whitelist"); } /* Ratelimits */ diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index a0498e402..723f12c16 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -760,12 +760,9 @@ void rspamd_actions_sort (struct rspamd_config *cfg); */ struct rspamd_radix_map_helper; -gboolean rspamd_config_radix_from_ucl (struct rspamd_config *cfg, - const ucl_object_t *obj, - const gchar *description, - struct rspamd_radix_map_helper **target, - GError **err, - struct rspamd_worker *worker); +gboolean rspamd_config_radix_from_ucl (struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description, + struct rspamd_radix_map_helper **target, GError **err, + struct rspamd_worker *worker, const gchar *map_name); /** * Adds new settings id to be preprocessed diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index 78e3fc79f..e2f886aa6 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -2226,12 +2226,9 @@ rspamd_config_get_action_by_type (struct rspamd_config *cfg, } gboolean -rspamd_config_radix_from_ucl (struct rspamd_config *cfg, - const ucl_object_t *obj, - const gchar *description, - struct rspamd_radix_map_helper **target, - GError **err, - struct rspamd_worker *worker) +rspamd_config_radix_from_ucl (struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description, + struct rspamd_radix_map_helper **target, GError **err, + struct rspamd_worker *worker, const gchar *map_name) { ucl_type_t type; ucl_object_iter_t it = NULL; @@ -2269,7 +2266,8 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg, else { /* Just a list */ if (!*target) { - *target = rspamd_map_helper_new_radix (NULL); + *target = rspamd_map_helper_new_radix ( + rspamd_map_add_fake (cfg, description, map_name)); } rspamd_map_helper_insert_radix_resolve (*target, str, ""); @@ -2300,7 +2298,8 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg, str = ucl_object_tostring (cur); if (!*target) { - *target = rspamd_map_helper_new_radix (NULL); + *target = rspamd_map_helper_new_radix ( + rspamd_map_add_fake (cfg, description, map_name)); } rspamd_map_helper_insert_radix_resolve (*target, str, ""); @@ -2803,9 +2802,9 @@ rspamd_config_libs (struct rspamd_external_libs_ctx *ctx, if (cfg->local_addrs) { rspamd_config_radix_from_ucl (cfg, cfg->local_addrs, "Local addresses", - (struct rspamd_radix_map_helper **)ctx->local_addrs, + (struct rspamd_radix_map_helper **) ctx->local_addrs, NULL, - NULL); + NULL, "local addresses"); } rspamd_free_zstd_dictionary (ctx->in_dict); diff --git a/src/libserver/logger/logger.c b/src/libserver/logger/logger.c index 56b0c6d32..44e1ffdf9 100644 --- a/src/libserver/logger/logger.c +++ b/src/libserver/logger/logger.c @@ -258,7 +258,7 @@ rspamd_log_open_specific (rspamd_mempool_t *pool, "IP addresses for which debug logs are enabled", &logger->debug_ip, NULL, - NULL); + NULL, "debug ip"); } if (cfg->log_encryption_key) { diff --git a/src/libserver/maps/map.c b/src/libserver/maps/map.c index 6b708b9a8..86ecc5d95 100644 --- a/src/libserver/maps/map.c +++ b/src/libserver/maps/map.c @@ -2730,6 +2730,26 @@ rspamd_map_add (struct rspamd_config *cfg, return map; } +struct rspamd_map * +rspamd_map_add_fake (struct rspamd_config *cfg, + const gchar *description, + const gchar *name) +{ + struct rspamd_map *map; + + map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_map)); + map->cfg = cfg; + map->id = rspamd_random_uint64_fast (); + map->name = rspamd_mempool_strdup (cfg->cfg_pool, name); + map->user_data = (void **)↦ /* to prevent null pointer dereferencing */ + + if (description != NULL) { + map->description = rspamd_mempool_strdup (cfg->cfg_pool, description); + } + + return map; +} + static inline void rspamd_map_add_backend (struct rspamd_map *map, struct rspamd_map_backend *bk) { diff --git a/src/libserver/maps/map.h b/src/libserver/maps/map.h index 2d3883e11..0812e1d44 100644 --- a/src/libserver/maps/map.h +++ b/src/libserver/maps/map.h @@ -93,6 +93,17 @@ struct rspamd_map *rspamd_map_add_from_ucl (struct rspamd_config *cfg, struct rspamd_worker *worker, int flags); +/** + * Adds a fake map structure (for logging purposes mainly) + * @param cfg + * @param description + * @return + */ +struct rspamd_map *rspamd_map_add_fake (struct rspamd_config *cfg, + const gchar *description, + const gchar *name); + + enum rspamd_map_watch_type { RSPAMD_MAP_WATCH_MIN = 9, RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER, diff --git a/src/libserver/maps/map_helpers.c b/src/libserver/maps/map_helpers.c index 04741390b..0ce191a55 100644 --- a/src/libserver/maps/map_helpers.c +++ b/src/libserver/maps/map_helpers.c @@ -53,17 +53,20 @@ struct rspamd_radix_map_helper { rspamd_mempool_t *pool; khash_t(rspamd_map_hash) *htb; radix_compressed_t *trie; + struct rspamd_map *map; rspamd_cryptobox_fast_hash_state_t hst; }; struct rspamd_hash_map_helper { rspamd_mempool_t *pool; khash_t(rspamd_map_hash) *htb; + struct rspamd_map *map; rspamd_cryptobox_fast_hash_state_t hst; }; struct rspamd_cdb_map_helper { GQueue cdbs; + struct rspamd_map *map; rspamd_cryptobox_fast_hash_state_t hst; gsize total_size; }; @@ -457,7 +460,9 @@ rspamd_map_helper_insert_radix (gpointer st, gconstpointer key, gconstpointer va gconstpointer nk; rspamd_ftok_t tok; gint res; + struct rspamd_map *map; + map = r->map; vlen = strlen (value); val = rspamd_mempool_alloc0 (r->pool, sizeof (*val) + vlen + 1); @@ -472,11 +477,16 @@ rspamd_map_helper_insert_radix (gpointer st, gconstpointer key, gconstpointer va tok.begin = nk; k = kh_put (rspamd_map_hash, r->htb, tok, &res); } + else { + msg_warn_map ("duplicate radix entry found for map %s: %s (old value: %s, new: %s)", + map->name, key, kh_value (r->htb, k), val); + } nk = kh_key (r->htb, k).begin; val->key = nk; kh_value (r->htb, k) = val; - rspamd_radix_add_iplist (key, ",", r->trie, val, FALSE); + rspamd_radix_add_iplist (key, ",", r->trie, val, FALSE, + r->map->name); rspamd_cryptobox_fast_hash_update (&r->hst, nk, tok.len); } @@ -490,7 +500,9 @@ rspamd_map_helper_insert_radix_resolve (gpointer st, gconstpointer key, gconstpo gconstpointer nk; rspamd_ftok_t tok; gint res; + struct rspamd_map *map; + map = r->map; vlen = strlen (value); val = rspamd_mempool_alloc0 (r->pool, sizeof (*val) + vlen + 1); @@ -505,11 +517,16 @@ rspamd_map_helper_insert_radix_resolve (gpointer st, gconstpointer key, gconstpo tok.begin = nk; k = kh_put (rspamd_map_hash, r->htb, tok, &res); } + else { + msg_warn_map ("duplicate radix entry found for map %s: %s (old value: %s, new: %s)", + map->name, key, kh_value (r->htb, k), val); + } nk = kh_key (r->htb, k).begin; val->key = nk; kh_value (r->htb, k) = val; - rspamd_radix_add_iplist (key, ",", r->trie, val, TRUE); + rspamd_radix_add_iplist (key, ",", r->trie, val, TRUE, + r->map->name); rspamd_cryptobox_fast_hash_update (&r->hst, nk, tok.len); } @@ -523,9 +540,11 @@ rspamd_map_helper_insert_hash (gpointer st, gconstpointer key, gconstpointer val gsize vlen; gint r; rspamd_ftok_t tok; + struct rspamd_map *map; tok.begin = key; tok.len = strlen (key); + map = ht->map; k = kh_get (rspamd_map_hash, ht->htb, tok); vlen = strlen (value); @@ -542,6 +561,10 @@ rspamd_map_helper_insert_hash (gpointer st, gconstpointer key, gconstpointer val /* Same element, skip */ return; } + else { + msg_warn_map ("duplicate hash entry found for map %s: %s (old value: %s, new: %s)", + map->name, key, kh_value (ht->htb, k), val); + } } /* Null termination due to alloc0 */ @@ -598,7 +621,7 @@ rspamd_map_helper_insert_re (gpointer st, gconstpointer key, gconstpointer value vlen = strlen (value); val = rspamd_mempool_alloc0 (re_map->pool, sizeof (*val) + vlen + 1); - memcpy (val->value, value, vlen); + memcpy (val->value, value, vlen); /* Null terminated due to alloc0 previously */ tok.begin = key; tok.len = strlen (key); @@ -609,6 +632,10 @@ rspamd_map_helper_insert_re (gpointer st, gconstpointer key, gconstpointer value tok.begin = nk; k = kh_put (rspamd_map_hash, re_map->htb, tok, &r); } + else { + msg_warn_map ("duplicate re found for map %s: %s (old value: %s, new: %s)", + map->name, key, kh_value (re_map->htb, k)->value, val); + } nk = kh_key (re_map->htb, k).begin; val->key = nk; @@ -670,6 +697,7 @@ rspamd_map_helper_new_hash (struct rspamd_map *map) htb = rspamd_mempool_alloc0 (pool, sizeof (*htb)); htb->htb = kh_init (rspamd_map_hash); htb->pool = pool; + htb->map = map; rspamd_cryptobox_fast_hash_init (&htb->hst, map_hash_seed); return htb; @@ -714,10 +742,12 @@ rspamd_map_helper_new_radix (struct rspamd_map *map) { struct rspamd_radix_map_helper *r; rspamd_mempool_t *pool; + const gchar *name = "unnamed"; if (map) { pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), map->tag, 0); + name = map->name; } else { pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), @@ -725,9 +755,10 @@ rspamd_map_helper_new_radix (struct rspamd_map *map) } r = rspamd_mempool_alloc0 (pool, sizeof (*r)); - r->trie = radix_create_compressed_with_pool (pool); + r->trie = radix_create_compressed_with_pool (pool, name); r->htb = kh_init (rspamd_map_hash); r->pool = pool; + r->map = map; rspamd_cryptobox_fast_hash_init (&r->hst, map_hash_seed); return r; @@ -1449,6 +1480,7 @@ rspamd_map_helper_new_cdb (struct rspamd_map *map) n = g_malloc0 (sizeof (*n)); n->cdbs = (GQueue)G_QUEUE_INIT; + n->map = map; rspamd_cryptobox_fast_hash_init (&n->hst, map_hash_seed); diff --git a/src/libserver/monitored.c b/src/libserver/monitored.c index 54bf5c508..a8eb736ec 100644 --- a/src/libserver/monitored.c +++ b/src/libserver/monitored.c @@ -265,7 +265,7 @@ rspamd_monitored_dns_conf (struct rspamd_monitored *m, if (elt) { if (ucl_object_type (elt) == UCL_STRING) { radix_add_generic_iplist (ucl_object_tostring (elt), - &conf->expected, FALSE); + &conf->expected, FALSE, NULL); } else if (ucl_object_type (elt) == UCL_ARRAY) { const ucl_object_t *cur; @@ -273,7 +273,7 @@ rspamd_monitored_dns_conf (struct rspamd_monitored *m, while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) { radix_add_generic_iplist (ucl_object_tostring (elt), - &conf->expected, FALSE); + &conf->expected, FALSE, NULL); } } } diff --git a/src/libutil/radix.c b/src/libutil/radix.c index 8619d3118..3c617df37 100644 --- a/src/libutil/radix.c +++ b/src/libutil/radix.c @@ -41,6 +41,7 @@ INIT_LOG_MODULE(radix) struct radix_tree_compressed { rspamd_mempool_t *pool; struct btrie *tree; + const gchar *name; size_t size; guint duplicates; gboolean own_pool; @@ -78,8 +79,8 @@ radix_insert_compressed (radix_compressed_t * tree, g_assert (tree != NULL); g_assert (keybits >= masklen); - msg_debug_radix ("want insert value %p with mask %z, key: %*xs", - (gpointer)value, keybits - masklen, (int)keylen, key); + msg_debug_radix ("%s: want insert value %p with mask %z, key: %*xs", + tree->name, (gpointer)value, keybits - masklen, (int)keylen, key); old = radix_find_compressed (tree, key, keylen); @@ -90,25 +91,30 @@ radix_insert_compressed (radix_compressed_t * tree, tree->duplicates++; if (tree->duplicates == max_duplicates) { - msg_err_radix ("maximum duplicates limit reached: %d, " - "suppress further errors", max_duplicates); + msg_err_radix ("%s: maximum duplicates limit reached: %d, " + "suppress further errors", tree->name, max_duplicates); } else if (tree->duplicates < max_duplicates) { memset (ip_str, 0, sizeof (ip_str)); if (keybits == 32) { - msg_err_radix ("cannot insert %p, key: %s/%d, duplicate value", + msg_err_radix ("%s: cannot insert %p, key: %s/%d, duplicate value", + tree->name, (gpointer) value, inet_ntop (AF_INET, key, ip_str, sizeof (ip_str) - 1), (gint) (keybits - masklen)); } else if (keybits == 128) { - msg_err_radix ("cannot insert %p, key: [%s]/%d, duplicate value", + msg_err_radix ("%s: cannot insert %p, key: [%s]/%d, duplicate value", + tree->name, (gpointer) value, inet_ntop (AF_INET6, key, ip_str, sizeof (ip_str) - 1), (gint) (keybits - masklen)); } else { - msg_err_radix ("cannot insert %p with mask %z, key: %*xs, duplicate value", - (gpointer) value, keybits - masklen, (int) keylen, key); + msg_err_radix ("%s: cannot insert %p with mask %z, key: %*xs, duplicate value", + tree->name, + (gpointer) value, + keybits - masklen, + (int) keylen, key); } } } @@ -121,7 +127,7 @@ radix_insert_compressed (radix_compressed_t * tree, radix_compressed_t * -radix_create_compressed (void) +radix_create_compressed (const gchar *tree_name) { radix_compressed_t *tree; @@ -135,12 +141,13 @@ radix_create_compressed (void) tree->duplicates = 0; tree->tree = btrie_init (tree->pool); tree->own_pool = TRUE; + tree->name = tree_name; return tree; } radix_compressed_t * -radix_create_compressed_with_pool (rspamd_mempool_t *pool) +radix_create_compressed_with_pool (rspamd_mempool_t *pool, const gchar *tree_name) { radix_compressed_t *tree; @@ -150,6 +157,7 @@ radix_create_compressed_with_pool (rspamd_mempool_t *pool) tree->duplicates = 0; tree->tree = btrie_init (tree->pool); tree->own_pool = FALSE; + tree->name = tree_name; return tree; } @@ -199,7 +207,8 @@ radix_find_compressed_addr (radix_compressed_t *tree, gint rspamd_radix_add_iplist (const gchar *list, const gchar *separators, - radix_compressed_t *tree, gconstpointer value, gboolean resolve) + radix_compressed_t *tree, gconstpointer value, + gboolean resolve, const gchar *tree_name) { gchar *token, *ipnet, *err_str, **strv, **cur, *brace; union { @@ -232,7 +241,8 @@ rspamd_radix_add_iplist (const gchar *list, const gchar *separators, k = strtoul (ipnet, &err_str, 10); if (errno != 0) { msg_warn_radix ( - "invalid netmask, error detected on symbol: %s, error: %s", + "%s: invalid netmask, error detected on symbol: %s, error: %s", + tree_name, err_str, strerror (errno)); k = G_MAXINT; @@ -378,16 +388,16 @@ rspamd_radix_add_iplist (const gchar *list, const gchar *separators, gboolean radix_add_generic_iplist (const gchar *ip_list, radix_compressed_t **tree, - gboolean resolve) + gboolean resolve, const gchar *tree_name) { static const char fill_ptr[] = "1"; if (*tree == NULL) { - *tree = radix_create_compressed (); + *tree = radix_create_compressed (tree_name); } return (rspamd_radix_add_iplist (ip_list, ",; ", *tree, - fill_ptr, resolve) > 0); + fill_ptr, resolve, tree_name) > 0); } diff --git a/src/libutil/radix.h b/src/libutil/radix.h index f08b6cc1b..45e3d5ee5 100644 --- a/src/libutil/radix.h +++ b/src/libutil/radix.h @@ -72,9 +72,9 @@ void radix_destroy_compressed (radix_compressed_t *tree); * Create new radix trie * @return */ -radix_compressed_t *radix_create_compressed (void); +radix_compressed_t *radix_create_compressed (const gchar *tree_name); -radix_compressed_t *radix_create_compressed_with_pool (rspamd_mempool_t *pool); +radix_compressed_t *radix_create_compressed_with_pool (rspamd_mempool_t *pool, const gchar *tree_name); /** * Insert list of ip addresses and masks to the radix tree @@ -84,14 +84,18 @@ radix_compressed_t *radix_create_compressed_with_pool (rspamd_mempool_t *pool); * @return number of elements inserted */ gint rspamd_radix_add_iplist (const gchar *list, const gchar *separators, - radix_compressed_t *tree, gconstpointer value, gboolean resolve); + radix_compressed_t *tree, gconstpointer value, + gboolean resolve, const gchar *tree_name); /** * Generic version of @see rspamd_radix_add_iplist. This function creates tree * if `tree` is NULL. */ -gboolean radix_add_generic_iplist (const gchar *ip_list, - radix_compressed_t **tree, gboolean resolve); +gboolean +radix_add_generic_iplist (const gchar *ip_list, + radix_compressed_t **tree, + gboolean resolve, + const gchar *tree_name); /** * Returns number of elements in the tree diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index e2a8c4b23..fae0acf0b 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -438,7 +438,7 @@ dkim_module_config (struct rspamd_config *cfg) rspamd_config_get_module_opt (cfg, "dkim", "whitelist")) != NULL) { rspamd_config_radix_from_ucl (cfg, value, "DKIM whitelist", - &dkim_module_ctx->whitelist_ip, NULL, NULL); + &dkim_module_ctx->whitelist_ip, NULL, NULL, "dkim whitelist"); } if ((value = diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 4e1a2263a..640afcc32 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -1067,7 +1067,7 @@ fuzzy_check_module_config (struct rspamd_config *cfg) rspamd_config_radix_from_ucl (cfg, value, "Fuzzy whitelist", &fuzzy_module_ctx->whitelist, NULL, - NULL); + NULL, "fuzzy ip whitelist"); } else { fuzzy_module_ctx->whitelist = NULL; diff --git a/test/rspamd_radix_test.c b/test/rspamd_radix_test.c index 5841dd10f..9a637344c 100644 --- a/test/rspamd_radix_test.c +++ b/test/rspamd_radix_test.c @@ -78,7 +78,7 @@ struct _tv { static void rspamd_radix_test_vec (void) { - radix_compressed_t *tree = radix_create_compressed (); + radix_compressed_t *tree = radix_create_compressed (NULL); struct _tv *t = &test_vec[0]; struct in_addr ina; struct in6_addr in6a; @@ -209,7 +209,7 @@ rspamd_radix_test_func (void) { struct btrie *btrie; rspamd_mempool_t *pool; - radix_compressed_t *comp_tree = radix_create_compressed (); + radix_compressed_t *comp_tree = radix_create_compressed (NULL); struct { guint32 addr; guint32 mask; -- 2.39.5