Start new era of regexp cache

6 files changed, 74 insertions, 28 deletions

diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h
index 05946a79e..36e6371d8 100644
--- a/src/libserver/cfg_file.h
+++ b/src/libserver/cfg_file.h
@@ -13,6 +13,7 @@
 #include "cfg_rcl.h"
 #include "ucl.h"
 #include "regexp.h"
+#include "libserver/re_cache.h"
 #include "ref.h"
 
 #define DEFAULT_BIND_PORT 11333
@@ -327,6 +328,8 @@ struct rspamd_config {
 
 	struct rspamd_external_libs_ctx *libs_ctx;		/**< context for external libraries						*/
 
+	struct rspamd_re_cache *re_cache;				/**< static regexp cache								*/
+
 	ref_entry_t ref;								/**< reference counter									*/
 };
 
diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c
index 042658f8a..2d185a59d 100644
--- a/src/libserver/cfg_utils.c
+++ b/src/libserver/cfg_utils.c
@@ -198,6 +198,7 @@ rspamd_config_new (void)
 	cfg->lua_state = rspamd_lua_init (cfg);
 	cfg->cache = rspamd_symbols_cache_new (cfg);
 	cfg->ups_ctx = rspamd_upstreams_library_init ();
+	cfg->re_cache = rspamd_regexp_cache_new ();
 
 	REF_INIT_RETAIN (cfg, rspamd_config_free);
 
@@ -229,6 +230,7 @@ rspamd_config_free (struct rspamd_config *cfg)
 	g_list_free (cfg->metrics_list);
 	rspamd_symbols_cache_destroy (cfg->cache);
 	REF_RELEASE (cfg->libs_ctx);
+	rspamd_re_cache_unref (cfg->re_cache);
 	rspamd_upstreams_library_unref (cfg->ups_ctx);
 	rspamd_mempool_delete (cfg->cfg_pool);
 	lua_close (cfg->lua_state);
diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c
index 18106a7c6..5ec8dd0aa 100644
--- a/src/libserver/re_cache.c
+++ b/src/libserver/re_cache.c
@@ -28,6 +28,8 @@
 #include "cryptobox.h"
 #include "ref.h"
 #include "libserver/url.h"
+#include "libserver/task.h"
+#include "libutil/util.h"
 
 struct rspamd_re_class {
 	guint64 id;
@@ -69,6 +71,36 @@ rspamd_re_cache_class_id (enum rspamd_re_type type,
 	return XXH64_digest (&st);
 }
 
+static void
+rspamd_re_cache_destroy (struct rspamd_re_cache *cache)
+{
+	GHashTableIter it;
+	gpointer k, v;
+	struct rspamd_re_class *re_class;
+	rspamd_regexp_t *re;
+	guint i;
+
+	g_assert (cache != NULL);
+	g_hash_table_iter_init (&it, cache->re_classes);
+
+	while (g_hash_table_iter_next (&it, &k, &v)) {
+		re_class = v;
+		g_hash_table_iter_steal (&it);
+
+		for (i = 0; i < re_class->all_re->len; i++) {
+			re = g_ptr_array_index (re_class->all_re, i);
+
+			rspamd_regexp_set_cache_id (re, RSPAMD_INVALID_ID);
+			rspamd_regexp_unref (re);
+		}
+
+		g_slice_free1 (sizeof (*re_class), re_class);
+	}
+
+	g_hash_table_unref (cache->re_classes);
+	g_slice_free1 (sizeof (*cache), cache);
+}
+
 struct rspamd_re_cache *
 rspamd_re_cache_new (void)
 {
@@ -412,31 +444,32 @@ rspamd_re_cache_runtime_destroy (struct rspamd_re_runtime *rt)
 }
 
 void
-rspamd_re_cache_destroy (struct rspamd_re_cache *cache)
+rspamd_re_cache_unref (struct rspamd_re_cache *cache)
 {
-	GHashTableIter it;
-	gpointer k, v;
-	struct rspamd_re_class *re_class;
-	rspamd_regexp_t *re;
-	guint i;
+	if (cache) {
+		REF_RELEASE (cache);
+	}
+}
 
-	g_assert (cache != NULL);
-	g_hash_table_iter_init (&it, cache->re_classes);
+struct rspamd_re_cache *
+rspamd_re_cache_ref (struct rspamd_re_cache *cache)
+{
+	if (cache) {
+		REF_RETAIN (cache);
+	}
 
-	while (g_hash_table_iter_next (&it, &k, &v)) {
-		re_class = v;
-		g_hash_table_iter_steal (&it);
+	return cache;
+}
 
-		for (i = 0; i < re_class->all_re->len; i++) {
-			re = g_ptr_array_index (re_class->all_re, i);
+guint
+rspamd_re_cache_set_limit (struct rspamd_re_cache *cache, guint limit)
+{
+	guint old;
 
-			rspamd_regexp_set_cache_id (re, RSPAMD_INVALID_ID);
-			rspamd_regexp_unref (re);
-		}
+	g_assert (cache != NULL);
 
-		g_slice_free1 (sizeof (*re_class), re_class);
-	}
+	old = cache->max_re_data;
+	cache->max_re_data = limit;
 
-	g_hash_table_unref (cache->re_classes);
-	g_slice_free1 (sizeof (*cache), cache);
+	return old;
 }
diff --git a/src/libserver/re_cache.h b/src/libserver/re_cache.h
index ade0a1164..ae23fc89a 100644
--- a/src/libserver/re_cache.h
+++ b/src/libserver/re_cache.h
@@ -25,11 +25,11 @@
 #define RSPAMD_RE_CACHE_H
 
 #include "config.h"
-#include "libserver/task.h"
 #include "libutil/regexp.h"
 
 struct rspamd_re_cache;
 struct rspamd_re_runtime;
+struct rspamd_task;
 
 enum rspamd_re_type {
 	RSPAMD_RE_HEADER,
@@ -91,8 +91,17 @@ gint rspamd_re_cache_process (struct rspamd_task *task,
 void rspamd_re_cache_runtime_destroy (struct rspamd_re_runtime *rt);
 
 /**
- * Destroy re cache
+ * Unref re cache
  */
-void rspamd_re_cache_destroy (struct rspamd_re_cache *cache);
+void rspamd_re_cache_unref (struct rspamd_re_cache *cache);
+/**
+ * Retain reference to re cache
+ */
+struct rspamd_re_cache *rspamd_re_cache_ref (struct rspamd_re_cache *cache);
+
+/**
+ * Set limit for all regular expressions in the cache, returns previous limit
+ */
+guint rspamd_re_cache_set_limit (struct rspamd_re_cache *cache, guint limit);
 
 #endif
diff --git a/src/libserver/task.c b/src/libserver/task.c
index eea9057ee..2f4d7ad89 100644
--- a/src/libserver/task.c
+++ b/src/libserver/task.c
@@ -67,10 +67,7 @@ rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg)
 	rspamd_mempool_add_destructor (new_task->task_pool,
 		(rspamd_mempool_destruct_t) g_hash_table_unref,
 		new_task->results);
-	new_task->re_cache = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
-	rspamd_mempool_add_destructor (new_task->task_pool,
-		(rspamd_mempool_destruct_t) g_hash_table_unref,
-		new_task->re_cache);
+	new_task->re_rt = rspamd_re_cache_runtime_new (cfg->re_cache);
 	new_task->raw_headers = g_hash_table_new (rspamd_strcase_hash,
 			rspamd_strcase_equal);
 	new_task->request_headers = g_hash_table_new_full (rspamd_ftok_icase_hash,
@@ -226,6 +223,7 @@ rspamd_task_free (struct rspamd_task *task)
 			event_del (&task->timeout_ev);
 		}
 
+		rspamd_re_cache_runtime_destroy (task->re_rt);
 		REF_RELEASE (task->cfg);
 
 		rspamd_mempool_delete (task->task_pool);
diff --git a/src/libserver/task.h b/src/libserver/task.h
index 49357e00b..21d885b25 100644
--- a/src/libserver/task.h
+++ b/src/libserver/task.h
@@ -29,6 +29,7 @@
 #include "util.h"
 #include "mem_pool.h"
 #include "dns.h"
+#include "re_cache.h"
 
 #include <gmime/gmime.h>
 
@@ -152,7 +153,7 @@ struct rspamd_task {
 	InternetAddressList *from_envelope;
 
 	GList *messages;								/**< list of messages that would be reported		*/
-	GHashTable *re_cache;							/**< cache for matched or not matched regexps		*/
+	struct rspamd_re_runtime *re_rt;				/**< regexp runtime									*/
 	struct rspamd_config *cfg;						/**< pointer to config object						*/
 	GError *err;
 	rspamd_mempool_t *task_pool;					/**< memory pool for task							*/