[Feature] Suppress duplicate warning on very large radix tries

diff --git a/contrib/lc-btrie/btrie.c b/contrib/lc-btrie/btrie.c
index fe768fc56a00c13dd1112e863a591e52555272f8..b1c66f1c398ca1993728632e2572ae75a628721d 100644 (file)
--- a/contrib/lc-btrie/btrie.c
+++ b/contrib/lc-btrie/btrie.c
@@ -1485,7 +1485,7 @@ static size_t count_free(const struct btrie *btrie)
 #endif /* not NDEBUG */
 
 const char *
-btrie_stats(const struct btrie *btrie)
+btrie_stats(const struct btrie *btrie, guint duplicates)
 {
        static char buf[128];
        size_t n_nodes = btrie->n_lc_nodes + btrie->n_tbm_nodes;
@@ -1530,8 +1530,10 @@ btrie_stats(const struct btrie *btrie)
                        , average_depth, (long unsigned)stats.max_depth);
 #else
        snprintf(buf, sizeof(buf),
-                       "ents=%lu tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu"
-                       ,(long unsigned)btrie->n_entries, (long unsigned)btrie->n_tbm_nodes,
+                       "ents=%lu dup=%u tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu",
+                       (long unsigned)btrie->n_entries,
+                       duplicates,
+                       (long unsigned)btrie->n_tbm_nodes,
                        (long unsigned)btrie->n_lc_nodes, (double)btrie->alloc_total / 1024,
                        (long unsigned)alloc_free, (long unsigned)btrie->alloc_waste
                        );

diff --git a/contrib/lc-btrie/btrie.h b/contrib/lc-btrie/btrie.h
index 2cc6629611ca4026954e86af2853388d7f97b72b..370a03e51a1354799b37c589c03fee4de50aefc8 100644 (file)
--- a/contrib/lc-btrie/btrie.h
+++ b/contrib/lc-btrie/btrie.h
@@ -70,7 +70,7 @@ enum btrie_result btrie_add_prefix(struct btrie *btrie,
 const void *btrie_lookup(const struct btrie *btrie, const btrie_oct_t *pfx,
                unsigned len);
 
-const char *btrie_stats(const struct btrie *btrie);
+const char *btrie_stats(const struct btrie *btrie, guint duplicates);
 
 #ifndef NO_MASTER_DUMP
 typedef void btrie_walk_cb_t(const btrie_oct_t *prefix, unsigned len,

diff --git a/src/libutil/radix.c b/src/libutil/radix.c
index a42be7032b942b42a1dbad37729f98f6b318807f..3ea6c6b8c2e58bdde0776fa9462aac91ac850dc9 100644 (file)
--- a/src/libutil/radix.c
+++ b/src/libutil/radix.c
@@ -42,6 +42,7 @@ struct radix_tree_compressed {
        rspamd_mempool_t *pool;
        struct btrie *tree;
        size_t size;
+       guint duplicates;
        gboolean own_pool;
 };
 
@@ -68,6 +69,7 @@ radix_insert_compressed (radix_compressed_t * tree,
        gsize masklen,
        uintptr_t value)
 {
+       static const guint max_duplicates = 32;
        guint keybits = keylen * NBBY;
        uintptr_t old;
        gchar ip_str[INET6_ADDRSTRLEN + 1];
@@ -85,23 +87,29 @@ radix_insert_compressed (radix_compressed_t * tree,
                        (gconstpointer)value);
 
        if (ret != BTRIE_OKAY) {
-               memset (ip_str, 0, sizeof (ip_str));
+               tree->duplicates++;
 
-               if (keybits == 32) {
-                       msg_err_radix ("cannot insert %p, key: %s/%d, duplicate value",
-                                       (gpointer)value,
-                                       inet_ntop (AF_INET, key, ip_str, sizeof (ip_str) - 1),
-                                       (gint)(keybits - masklen));
+               if (tree->duplicates == max_duplicates) {
+                       msg_err_radix ("maximum duplicates limit reached: %d, "
+                                 "suppress further errors", max_duplicates);
                }
-               else if (keybits == 128) {
-                       msg_err_radix ("cannot insert %p, key: [%s]/%d, duplicate value",
-                                       (gpointer)value,
-                                       inet_ntop (AF_INET6, key, ip_str, sizeof (ip_str) - 1),
-                                       (gint)(keybits - masklen));
-               }
-               else {
-                       msg_err_radix ("cannot insert %p with mask %z, key: %*xs, duplicate value",
-                               (gpointer)value, keybits - masklen, (int)keylen, key);
+               else if (tree->duplicates < max_duplicates) {
+                       memset (ip_str, 0, sizeof (ip_str));
+
+                       if (keybits == 32) {
+                               msg_err_radix ("cannot insert %p, key: %s/%d, duplicate value",
+                                               (gpointer) value,
+                                               inet_ntop (AF_INET, key, ip_str, sizeof (ip_str) - 1),
+                                               (gint) (keybits - masklen));
+                       } else if (keybits == 128) {
+                               msg_err_radix ("cannot insert %p, key: [%s]/%d, duplicate value",
+                                               (gpointer) value,
+                                               inet_ntop (AF_INET6, key, ip_str, sizeof (ip_str) - 1),
+                                               (gint) (keybits - masklen));
+                       } else {
+                               msg_err_radix ("cannot insert %p with mask %z, key: %*xs, duplicate value",
+                                               (gpointer) value, keybits - masklen, (int) keylen, key);
+                       }
                }
        }
        else {
@@ -124,6 +132,7 @@ radix_create_compressed (void)
 
        tree->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
        tree->size = 0;
+       tree->duplicates = 0;
        tree->tree = btrie_init (tree->pool);
        tree->own_pool = TRUE;
 
@@ -138,6 +147,7 @@ radix_create_compressed_with_pool (rspamd_mempool_t *pool)
        tree = rspamd_mempool_alloc (pool, sizeof (*tree));
        tree->pool = pool;
        tree->size = 0;
+       tree->duplicates = 0;
        tree->tree = btrie_init (tree->pool);
        tree->own_pool = FALSE;
 
@@ -393,5 +403,5 @@ radix_get_info (radix_compressed_t *tree)
                return NULL;
        }
 
-       return btrie_stats (tree->tree);
+       return btrie_stats (tree->tree, tree->duplicates);
 }