* Fix shared usage of statfiles

* Add invalidation of statfiles in case of learning, so now statfiles
  are invalidated in about a minute after learning
* This should fix shared usage of statfile pool by several processes

5 files changed, 53 insertions, 5 deletions

diff --git a/src/controller.c b/src/controller.c
index bbf50d646..2f1473540 100644
--- a/src/controller.c
+++ b/src/controller.c
@@ -870,6 +870,7 @@ controller_read_socket (f_str_t * in, void *arg)
 		maybe_write_binlog (session->learn_classifier, st, statfile, tokens);
 		msg_info ("learn success for message <%s>, for statfile: %s, sum weight: %.2f",
 				task->message_id, session->learn_symbol, sum);
+		statfile_pool_plan_invalidate (session->worker->srv->statfile_pool, DEFAULT_STATFILE_INVALIDATE_TIME, DEFAULT_STATFILE_INVALIDATE_JITTER);
 		free_task (task, FALSE);
 		i = rspamd_snprintf (out_buf, sizeof (out_buf), "learn ok, sum weight: %.2f" CRLF, sum);
 		if (!rspamd_dispatcher_write (session->dispatcher, out_buf, i, FALSE, FALSE)) {
diff --git a/src/filter.c b/src/filter.c
index 7d1de3d20..f38831c00 100644
--- a/src/filter.c
+++ b/src/filter.c
@@ -444,6 +444,7 @@ process_autolearn (struct statfile *st, struct worker_task *task, GTree * tokens
 
 			classifier->learn_func (ctx, task->worker->srv->statfile_pool, st->symbol, tokens, TRUE, NULL, 1., NULL);
 			maybe_write_binlog (ctx->cfg, st, statfile, tokens);
+			statfile_pool_plan_invalidate (task->worker->srv->statfile_pool, DEFAULT_STATFILE_INVALIDATE_TIME, DEFAULT_STATFILE_INVALIDATE_JITTER);
 		}
 	}
 }
diff --git a/src/statfile.c b/src/statfile.c
index 0460a43c8..1660a29f4 100644
--- a/src/statfile.c
+++ b/src/statfile.c
@@ -209,11 +209,10 @@ statfile_pool_new (memory_pool_t *pool, size_t max_size)
 {
 	statfile_pool_t                *new;
 
-	new = memory_pool_alloc_shared (pool, sizeof (statfile_pool_t));
-	bzero (new, sizeof (statfile_pool_t));
+	new = memory_pool_alloc0 (pool, sizeof (statfile_pool_t));
 	new->pool = memory_pool_new (memory_pool_get_size ());
 	new->max = max_size;
-	new->files = memory_pool_alloc_shared (new->pool, STATFILES_MAX * sizeof (stat_file_t));
+	new->files = memory_pool_alloc0 (new->pool, STATFILES_MAX * sizeof (stat_file_t));
 	new->lock = memory_pool_get_mutex (new->pool);
 
 	return new;
@@ -813,3 +812,37 @@ statfile_get_total_blocks (stat_file_t *file)
 
 	return header->total_blocks;
 }
+
+static void
+statfile_pool_invalidate_callback (int fd, short what, void *ud)
+{
+	statfile_pool_t                *pool = ud;
+	stat_file_t                    *file;
+	int                             i;
+
+	msg_info ("invalidating %d statfiles", pool->opened);
+
+	for (i = 0; i < pool->opened; i ++) {
+		file = &pool->files[i];
+		msync (file->map, file->len, MS_ASYNC | MS_INVALIDATE);
+	}
+
+}
+
+
+void
+statfile_pool_plan_invalidate (statfile_pool_t *pool, time_t seconds, time_t jitter)
+{
+
+	if (pool->invalidate_event == NULL || ! evtimer_pending (pool->invalidate_event, NULL)) {
+
+		if (pool->invalidate_event == NULL) {
+			pool->invalidate_event = memory_pool_alloc (pool->pool, sizeof (struct event));
+		}
+		pool->invalidate_tv.tv_sec = seconds + g_random_int_range (0, jitter);
+		pool->invalidate_tv.tv_usec = 0;
+		evtimer_set (pool->invalidate_event, statfile_pool_invalidate_callback, pool);
+		evtimer_add (pool->invalidate_event, &pool->invalidate_tv);
+		msg_info ("invalidate of statfile pool is planned in %d seconds", (int)pool->invalidate_tv.tv_sec);
+	}
+}
diff --git a/src/statfile.h b/src/statfile.h
index 95f800ce6..dfdd16dbe 100644
--- a/src/statfile.h
+++ b/src/statfile.h
@@ -18,6 +18,9 @@
 #define STATFILE_SECTION_URLS 3
 #define STATFILE_SECTION_REGEXP 4
 
+#define DEFAULT_STATFILE_INVALIDATE_TIME 30
+#define DEFAULT_STATFILE_INVALIDATE_JITTER 30
+
 /**
  * Common statfile header
  */
@@ -89,6 +92,8 @@ typedef struct statfile_pool_s {
 	size_t occupied;						/**< current size						*/
 	memory_pool_t *pool;					/**< memory pool object					*/
 	memory_pool_mutex_t *lock;				/**< mutex								*/
+	struct event  *invalidate_event;        /**< event for pool invalidation        */
+	struct timeval invalidate_tv;
 } statfile_pool_t;
 
 /**
@@ -242,4 +247,10 @@ uint64_t statfile_get_used_blocks (stat_file_t *file);
  */
 uint64_t statfile_get_total_blocks (stat_file_t *file);
 
+
+/**
+ * Plan statfile pool invalidation
+ */
+void statfile_pool_plan_invalidate (statfile_pool_t *pool, time_t seconds, time_t jitter);
+
 #endif
diff --git a/src/tokenizers/tokenizers.c b/src/tokenizers/tokenizers.c
index 2bb478aaa..9ca690e47 100644
--- a/src/tokenizers/tokenizers.c
+++ b/src/tokenizers/tokenizers.c
@@ -196,15 +196,16 @@ tokenize_headers (memory_pool_t * pool, struct worker_task *task, GTree ** tree)
 	const char                     *value;
 
 	ls = GMIME_OBJECT (task->message)->headers;
+	iter = g_mime_header_iter_new ();
 
 	if (g_mime_header_list_get_iter (ls, iter)) {
 		while (g_mime_header_iter_is_valid (iter)) {
 			new = memory_pool_alloc (pool, sizeof (token_node_t));
 			name = g_mime_header_iter_get_name (iter);
 			value = g_mime_header_iter_get_value (iter);
-			headername.begin = name;
+			headername.begin = (u_char *)name;
 			headername.len = strlen (name);
-			headervalue.begin = value;
+			headervalue.begin = (u_char *)value;
 			headervalue.len = strlen (value);
 			new->h1 = fstrhash (&headername) * primes[0];
 			new->h2 = fstrhash (&headervalue) * primes[1];
@@ -216,6 +217,7 @@ tokenize_headers (memory_pool_t * pool, struct worker_task *task, GTree ** tree)
 			}
 		}
 	}
+	g_mime_header_iter_free (iter);
 #endif
 	return TRUE;
 }