* Add more logic for dynamic rules

* Add documentation for dynamic rules
* Add ability to make negations in networks in dynamic rules

6 files changed, 196 insertions, 13 deletions

diff --git a/doc/rspamd.texi b/doc/rspamd.texi
index 666adb7b6..91dda2734 100644
--- a/doc/rspamd.texi
+++ b/doc/rspamd.texi
@@ -1650,6 +1650,59 @@ These internal functions can be easily implemented in lua but I've decided to
 make them built-in as they are widely used in our rules. In fact this list may
 be extended in future.
 
+@subsection Dynamic rules.
+Rspamd regexp module can use dynamic rules that can be written in json syntax.
+Dynamic rules are loaded at runtime and can be modified while rspamd is working.
+Also it is possible to turn dynamic rules for specific networks only and add rules
+that does not contain any regexp (this can be usefull for dynamic lists for example).
+Dynamic rules can be obtained like any other dynamic map via file monitoring or via
+http. Here are examples of dynamic rules definitions:
+@example
+<module name="regexp">
+  <option name="dynamic_rules">file:///tmp/rules.json</option>
+</module>
+@end example
+@noindent
+or for http map:
+@example
+<module name="regexp">
+  <option name="dynamic_rules">http://somehost/rules.json</option>
+</module>
+@end example
+@noindent
+Rules are presented as json array (in brackets @emph{'[]'}). Each rule is json object.
+This object can have several properties (properties with @strong{*} are required):
+@multitable @columnfractions 0.3 0.7
+@headitem Property @tab Mean
+@item symbol(*)
+@tab Symbol for rule.
+@item factor(*)
+@tab Factor for rule.
+@item rule
+@tab Rule itself (regexp expression).
+@item enabled
+@tab Boolean flag that define whether this rule is enabled (rule is enabled if 
+this flag is not present by default).
+@item networks
+@tab Json array of networks (in CIDR format, also it is possible to add negation
+by prepending @emph{!} symbol before item.
+@end multitable
+Here is an example of dynamic rule:
+@example
+[
+	{
+		"rule": "/test/rP",
+		"symbol": "R_TMP_1",
+		"factor": 1.1,
+		"networks": ["!192.168.1.0/24", "172.16.0.0/16"],
+		"enabled": false
+	}
+]
+@end example
+Note that dynamic rules are constantly monitored for changes and are reloaded 
+completely when modification is detected. If you change dynamic rules they
+would be reloaded in a minute and would be applied for new messages.
+
 @subsection Conclusion.
 Rspamd regexp module is powerfull tool for matching different patterns in
 messages. You may use logical expressions of regexps and internal rspamd
diff --git a/src/mem_pool.c b/src/mem_pool.c
index 11f74f2f3..363116ce5 100644
--- a/src/mem_pool.c
+++ b/src/mem_pool.c
@@ -429,6 +429,23 @@ memory_pool_add_destructor (memory_pool_t * pool, pool_destruct_func func, void
 }
 
 void
+memory_pool_replace_destructor (memory_pool_t * pool, pool_destruct_func func, void *old_data, void *new_data)
+{
+	struct _pool_destructors       *tmp;
+
+	tmp = pool->destructors;
+	while (tmp) {
+		if (tmp->func == func && tmp->data == old_data) {
+			tmp->func = func;
+			tmp->data = new_data;
+			break;
+		}
+		tmp = tmp->prev;
+	}
+
+}
+
+void
 memory_pool_delete (memory_pool_t * pool)
 {
 	struct _pool_chain             *cur = pool->first_pool, *tmp;
diff --git a/src/mem_pool.h b/src/mem_pool.h
index 81d4a85fa..14f83d9c5 100644
--- a/src/mem_pool.h
+++ b/src/mem_pool.h
@@ -169,6 +169,15 @@ void memory_pool_unlock_shared (memory_pool_t *pool, void *pointer);
 void memory_pool_add_destructor (memory_pool_t *pool, pool_destruct_func func, void *data);
 
 /**
+ * Replace destructor callback to pool for specified pointer
+ * @param pool memory pool object
+ * @param func pointer to function-destructor
+ * @param old_data pointer to old data
+ * @param new_data pointer to data that would be passed to destructor
+ */
+void memory_pool_replace_destructor (memory_pool_t *pool, pool_destruct_func func, void *old_data, void *new_data);
+
+/**
  * Delete pool, free all its chunks and call destructors chain
  * @param pool memory pool object
  */
diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c
index 636b26c03..00fd7ad4c 100644
--- a/src/plugins/regexp.c
+++ b/src/plugins/regexp.c
@@ -102,6 +102,14 @@ parse_regexp_ipmask (const char *begin, struct dynamic_map_item *addr)
 	pos = begin;
 	p = ip_buf;
 
+	if (*pos == '!') {
+		addr->negative = TRUE;
+		pos ++;
+	}
+	else {
+		addr->negative = FALSE;
+	}
+
 	while (*pos) {
 		switch (state) {
 			case 0:
@@ -244,6 +252,7 @@ json_regexp_fin_cb (memory_pool_t * pool, struct map_cb_data *data)
 	json_error_t                    je;
 	char                           *cur_rule, *cur_symbol;
 	double                          score;
+	gboolean                        enabled;
 	struct regexp_module_item      *cur_item;
 	GList                          *cur_networks = NULL;
 	struct dynamic_map_item        *cur_nitem;
@@ -303,6 +312,7 @@ json_regexp_fin_cb (memory_pool_t * pool, struct map_cb_data *data)
 	for (i = 0; i < nelts; i++) {
 		cur_networks = NULL;
 		cur_rule = NULL;
+		enabled = TRUE;
 
 		cur_elt = json_array_get (js, i);
 		if (!cur_elt || !json_is_object (cur_elt)) {
@@ -323,6 +333,14 @@ json_regexp_fin_cb (memory_pool_t * pool, struct map_cb_data *data)
 			continue;
 		}
 		cur_symbol = memory_pool_strdup (new_pool, json_string_value (cur_nm)); 
+		/* Enabled flag */
+		cur_nm = json_object_get (cur_elt, "enabled");
+		if (cur_nm != NULL && json_is_boolean (cur_nm)) {
+			if (json_is_false (cur_nm)) {
+				msg_info ("rule %s is disabled in json", cur_symbol);
+				continue;
+			}
+		}
 		/* Now check other settings */
 		/* Rule */
 		cur_nm = json_object_get (cur_elt, "rule");
@@ -460,10 +478,16 @@ regexp_module_config (struct config_file *cfg)
 	while (cur_opt) {
 		cur = cur_opt->data;
 		if (strcmp (cur->param, "metric") == 0 || strcmp (cur->param, "statfile_prefix") == 0) {
+			cur_opt = g_list_next (cur_opt);
 			continue;
 		}
 		else if (g_ascii_strncasecmp (cur->param, "autolearn", sizeof ("autolearn") - 1) == 0) {
 			parse_autolearn_param (cur->param, cur->value, cfg);
+			cur_opt = g_list_next (cur_opt);
+			continue;
+		}
+		else if (g_ascii_strncasecmp (cur->param, "dynamic_rules", sizeof ("dynamic_rules") - 1) == 0) {
+			cur_opt = g_list_next (cur_opt);
 			continue;
 		}
 		cur_item = memory_pool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item));
diff --git a/src/symbols_cache.c b/src/symbols_cache.c
index 07aabf6d4..47ae81727 100644
--- a/src/symbols_cache.c
+++ b/src/symbols_cache.c
@@ -284,7 +284,7 @@ register_dynamic_symbol (memory_pool_t *dynamic_pool, struct symbols_cache **cac
 {
 	struct cache_item              *item = NULL;
 	struct symbols_cache           *pcache = *cache;
-	GList                         **target, *t, *cur;
+	GList                          *t, *cur;
 	uintptr_t                       r;
 	uint32_t                        mask = 0xFFFFFFFF;
 	struct dynamic_map_item        *it;
@@ -308,36 +308,70 @@ register_dynamic_symbol (memory_pool_t *dynamic_pool, struct symbols_cache **cac
 	set_counter (item->s->symbol, 0);
 	
 	if (networks == NULL) {
-		target = &pcache->dynamic_items;
+		pcache->dynamic_items = g_list_prepend (pcache->dynamic_items, item);
 	}
 	else {
 		if (pcache->dynamic_map == NULL) {
 			pcache->dynamic_map = radix_tree_create ();
+			pcache->negative_dynamic_map = radix_tree_create ();
 		}
 		cur = networks;
 		while (cur) {
 			it = cur->data;
 			mask = mask << (32 - it->mask);
 			r = ntohl (it->addr.s_addr & mask);
-			if ((r = radix32tree_find (pcache->dynamic_map, r)) != RADIX_NO_VALUE) {
-				t = (GList *)((gpointer)r);
-				target = &t;
+			if (it->negative) {
+				/* For negatve items insert into list and into negative cache map */
+				if ((r = radix32tree_find (pcache->negative_dynamic_map, r)) != RADIX_NO_VALUE) {
+					t = (GList *)((gpointer)r);
+					t = g_list_prepend (t, item);
+					/* Replace pointers in radix tree and in destructor function */
+					memory_pool_replace_destructor (dynamic_pool, (pool_destruct_func)g_list_free, (gpointer)r, t);
+					r = radix32tree_replace (pcache->negative_dynamic_map, ntohl (it->addr.s_addr), mask, (uintptr_t)t);
+					if (r == -1) {
+						msg_warn ("cannot replace ip to tree: %s, mask %X", inet_ntoa (it->addr), mask);
+					}
+				}
+				else {
+					t = g_list_prepend (NULL, item);
+					memory_pool_add_destructor (dynamic_pool, (pool_destruct_func)g_list_free, t);
+					r = radix32tree_insert (pcache->negative_dynamic_map, ntohl (it->addr.s_addr), mask, (uintptr_t)t);
+					if (r == -1) {
+						msg_warn ("cannot insert ip to tree: %s, mask %X", inet_ntoa (it->addr), mask);
+					}
+					else if (r == 1) {
+						msg_warn ("ip %s, mask %X, value already exists", inet_ntoa (it->addr), mask);
+					}
+				}
+				/* Insert into list */
+				pcache->dynamic_items = g_list_prepend (pcache->dynamic_items, item);
 			}
 			else {
-				t = g_list_prepend (NULL, item);
-				memory_pool_add_destructor (dynamic_pool, (pool_destruct_func)g_list_free, t);
-				r = radix32tree_insert (pcache->dynamic_map, ntohl (it->addr.s_addr), mask, (uintptr_t)t);
-				if (r == -1) {
-					msg_warn ("cannot insert ip to tree: %s, mask %X", inet_ntoa (it->addr), mask);
+				if ((r = radix32tree_find (pcache->dynamic_map, r)) != RADIX_NO_VALUE) {
+					t = (GList *)((gpointer)r);
+					t = g_list_prepend (t, item);
+					/* Replace pointers in radix tree and in destructor function */
+					memory_pool_replace_destructor (dynamic_pool, (pool_destruct_func)g_list_free, (gpointer)r, t);
+					r = radix32tree_replace (pcache->dynamic_map, ntohl (it->addr.s_addr), mask, (uintptr_t)t);
+					if (r == -1) {
+						msg_warn ("cannot replace ip to tree: %s, mask %X", inet_ntoa (it->addr), mask);
+					}
 				}
-				else if (r == 1) {
-					msg_warn ("ip %s, mask %X, value already exists", inet_ntoa (it->addr), mask);
+				else {
+					t = g_list_prepend (NULL, item);
+					memory_pool_add_destructor (dynamic_pool, (pool_destruct_func)g_list_free, t);
+					r = radix32tree_insert (pcache->dynamic_map, ntohl (it->addr.s_addr), mask, (uintptr_t)t);
+					if (r == -1) {
+						msg_warn ("cannot insert ip to tree: %s, mask %X", inet_ntoa (it->addr), mask);
+					}
+					else if (r == 1) {
+						msg_warn ("ip %s, mask %X, value already exists", inet_ntoa (it->addr), mask);
+					}
 				}
 			}
 			cur = g_list_next (cur);
 		}
 	}
-	*target = g_list_prepend (*target, item);
 }
 
 void
@@ -351,6 +385,9 @@ remove_dynamic_rules (struct symbols_cache *cache)
 	if (cache->dynamic_map) {
 		radix_tree_free (cache->dynamic_map);
 	}
+	if (cache->negative_dynamic_map) {
+		radix_tree_free (cache->negative_dynamic_map);
+	}
 }
 
 static void
@@ -374,6 +411,9 @@ free_cache (gpointer arg)
 	if (cache->dynamic_map) {
 		radix_tree_free (cache->dynamic_map);
 	}
+	if (cache->negative_dynamic_map) {
+		radix_tree_free (cache->negative_dynamic_map);
+	}
 
 	memory_pool_delete (cache->static_pool);
 
@@ -502,6 +542,27 @@ check_dynamic_item (struct worker_task *task, struct symbols_cache *cache)
 	return res;
 }
 
+static gboolean
+check_negative_dynamic_item (struct worker_task *task, struct symbols_cache *cache, struct cache_item *item)
+{
+	GList                          *res = NULL;
+	uintptr_t                       r;
+
+	if (cache->negative_dynamic_map != NULL && task->from_addr.s_addr != INADDR_NONE) {
+		if ((r = radix32tree_find (cache->negative_dynamic_map, ntohl (task->from_addr.s_addr))) != RADIX_NO_VALUE) {
+			res = (GList *)((gpointer)r);
+			while (res) {
+				if (res->data == (gpointer)item) {
+					return TRUE;
+				}
+				res = g_list_next (res);
+			}
+		}
+	}
+
+	return FALSE;
+}
+
 struct symbol_callback_data {
 	enum {
 		CACHE_STATE_NEGATIVE,
@@ -636,6 +697,23 @@ call_symbol_callback (struct worker_task * task, struct symbols_cache * cache, g
 				}
 				else {
 					s->saved_item = s->list_pointer->data;
+					/* Skip items that are in negative map */
+					while (s->list_pointer != NULL && check_negative_dynamic_item (task, cache, s->saved_item)) {
+						s->list_pointer = g_list_next (s->list_pointer);
+						if (s->list_pointer != NULL) {
+							s->saved_item = s->list_pointer->data;
+						}
+					}
+					if (s->list_pointer == NULL) {
+						s->state = CACHE_STATE_STATIC;
+						s->list_pointer = g_list_first (cache->static_items);
+						if (s->list_pointer) {
+							s->saved_item = s->list_pointer->data;
+						}
+						else {
+							return FALSE;
+						}
+					}
 				}
 				item = s->saved_item;
 				break;
diff --git a/src/symbols_cache.h b/src/symbols_cache.h
index dfd5672ad..9f67ca486 100644
--- a/src/symbols_cache.h
+++ b/src/symbols_cache.h
@@ -20,6 +20,7 @@ struct saved_cache_item {
 struct dynamic_map_item {
 	struct in_addr addr;
 	uint32_t mask;
+	gboolean negative;
 };
 
 struct cache_item {
@@ -46,6 +47,7 @@ struct symbols_cache {
 	
 	/* Radix map of dynamic rules with ip mappings */
 	radix_tree_t *dynamic_map;
+	radix_tree_t *negative_dynamic_map;
 
 	/* Common dynamic rules */
 	GList *dynamic_items;