From fcaf800bd6cddb56c1cd88044346996d1f8854ee Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Fri, 1 Jun 2018 11:09:37 +0100
Subject: [PATCH] [Feature] Suppress duplicate warning on very large radix
 tries

---
 contrib/lc-btrie/btrie.c |  8 +++++---
 contrib/lc-btrie/btrie.h |  2 +-
 src/libutil/radix.c      | 42 +++++++++++++++++++++++++---------------
 3 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/contrib/lc-btrie/btrie.c b/contrib/lc-btrie/btrie.c
index fe768fc56..b1c66f1c3 100644
--- a/contrib/lc-btrie/btrie.c
+++ b/contrib/lc-btrie/btrie.c
@@ -1485,7 +1485,7 @@ static size_t count_free(const struct btrie *btrie)
 #endif /* not NDEBUG */
 
 const char *
-btrie_stats(const struct btrie *btrie)
+btrie_stats(const struct btrie *btrie, guint duplicates)
 {
 	static char buf[128];
 	size_t n_nodes = btrie->n_lc_nodes + btrie->n_tbm_nodes;
@@ -1530,8 +1530,10 @@ btrie_stats(const struct btrie *btrie)
 			, average_depth, (long unsigned)stats.max_depth);
 #else
 	snprintf(buf, sizeof(buf),
-			"ents=%lu tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu"
-			,(long unsigned)btrie->n_entries, (long unsigned)btrie->n_tbm_nodes,
+			"ents=%lu dup=%u tbm=%lu lc=%lu mem=%.0fk free=%lu waste=%lu",
+			(long unsigned)btrie->n_entries,
+			duplicates,
+			(long unsigned)btrie->n_tbm_nodes,
 			(long unsigned)btrie->n_lc_nodes, (double)btrie->alloc_total / 1024,
 			(long unsigned)alloc_free, (long unsigned)btrie->alloc_waste
 			);
diff --git a/contrib/lc-btrie/btrie.h b/contrib/lc-btrie/btrie.h
index 2cc662961..370a03e51 100644
--- a/contrib/lc-btrie/btrie.h
+++ b/contrib/lc-btrie/btrie.h
@@ -70,7 +70,7 @@ enum btrie_result btrie_add_prefix(struct btrie *btrie,
 const void *btrie_lookup(const struct btrie *btrie, const btrie_oct_t *pfx,
 		unsigned len);
 
-const char *btrie_stats(const struct btrie *btrie);
+const char *btrie_stats(const struct btrie *btrie, guint duplicates);
 
 #ifndef NO_MASTER_DUMP
 typedef void btrie_walk_cb_t(const btrie_oct_t *prefix, unsigned len,
diff --git a/src/libutil/radix.c b/src/libutil/radix.c
index a42be7032..3ea6c6b8c 100644
--- a/src/libutil/radix.c
+++ b/src/libutil/radix.c
@@ -42,6 +42,7 @@ struct radix_tree_compressed {
 	rspamd_mempool_t *pool;
 	struct btrie *tree;
 	size_t size;
+	guint duplicates;
 	gboolean own_pool;
 };
 
@@ -68,6 +69,7 @@ radix_insert_compressed (radix_compressed_t * tree,
 	gsize masklen,
 	uintptr_t value)
 {
+	static const guint max_duplicates = 32;
 	guint keybits = keylen * NBBY;
 	uintptr_t old;
 	gchar ip_str[INET6_ADDRSTRLEN + 1];
@@ -85,23 +87,29 @@ radix_insert_compressed (radix_compressed_t * tree,
 			(gconstpointer)value);
 
 	if (ret != BTRIE_OKAY) {
-		memset (ip_str, 0, sizeof (ip_str));
+		tree->duplicates++;
 
-		if (keybits == 32) {
-			msg_err_radix ("cannot insert %p, key: %s/%d, duplicate value",
-					(gpointer)value,
-					inet_ntop (AF_INET, key, ip_str, sizeof (ip_str) - 1),
-					(gint)(keybits - masklen));
+		if (tree->duplicates == max_duplicates) {
+			msg_err_radix ("maximum duplicates limit reached: %d, "
+				  "suppress further errors", max_duplicates);
 		}
-		else if (keybits == 128) {
-			msg_err_radix ("cannot insert %p, key: [%s]/%d, duplicate value",
-					(gpointer)value,
-					inet_ntop (AF_INET6, key, ip_str, sizeof (ip_str) - 1),
-					(gint)(keybits - masklen));
-		}
-		else {
-			msg_err_radix ("cannot insert %p with mask %z, key: %*xs, duplicate value",
-				(gpointer)value, keybits - masklen, (int)keylen, key);
+		else if (tree->duplicates < max_duplicates) {
+			memset (ip_str, 0, sizeof (ip_str));
+
+			if (keybits == 32) {
+				msg_err_radix ("cannot insert %p, key: %s/%d, duplicate value",
+						(gpointer) value,
+						inet_ntop (AF_INET, key, ip_str, sizeof (ip_str) - 1),
+						(gint) (keybits - masklen));
+			} else if (keybits == 128) {
+				msg_err_radix ("cannot insert %p, key: [%s]/%d, duplicate value",
+						(gpointer) value,
+						inet_ntop (AF_INET6, key, ip_str, sizeof (ip_str) - 1),
+						(gint) (keybits - masklen));
+			} else {
+				msg_err_radix ("cannot insert %p with mask %z, key: %*xs, duplicate value",
+						(gpointer) value, keybits - masklen, (int) keylen, key);
+			}
 		}
 	}
 	else {
@@ -124,6 +132,7 @@ radix_create_compressed (void)
 
 	tree->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
 	tree->size = 0;
+	tree->duplicates = 0;
 	tree->tree = btrie_init (tree->pool);
 	tree->own_pool = TRUE;
 
@@ -138,6 +147,7 @@ radix_create_compressed_with_pool (rspamd_mempool_t *pool)
 	tree = rspamd_mempool_alloc (pool, sizeof (*tree));
 	tree->pool = pool;
 	tree->size = 0;
+	tree->duplicates = 0;
 	tree->tree = btrie_init (tree->pool);
 	tree->own_pool = FALSE;
 
@@ -393,5 +403,5 @@ radix_get_info (radix_compressed_t *tree)
 		return NULL;
 	}
 
-	return btrie_stats (tree->tree);
+	return btrie_stats (tree->tree, tree->duplicates);
 }
-- 
2.39.5