From 18e0c770b951c5d8a1269b3829b73a416fc261fe Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Thu, 16 Sep 2010 19:18:23 +0400 Subject: [PATCH] * Fix shared usage of statfiles * Add invalidation of statfiles in case of learning, so now statfiles are invalidated in about a minute after learning * This should fix shared usage of statfile pool by several processes --- src/controller.c | 1 + src/filter.c | 1 + src/statfile.c | 39 ++++++++++++++++++++++++++++++++++--- src/statfile.h | 11 +++++++++++ src/tokenizers/tokenizers.c | 6 ++++-- 5 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/controller.c b/src/controller.c index bbf50d646..2f1473540 100644 --- a/src/controller.c +++ b/src/controller.c @@ -870,6 +870,7 @@ controller_read_socket (f_str_t * in, void *arg) maybe_write_binlog (session->learn_classifier, st, statfile, tokens); msg_info ("learn success for message <%s>, for statfile: %s, sum weight: %.2f", task->message_id, session->learn_symbol, sum); + statfile_pool_plan_invalidate (session->worker->srv->statfile_pool, DEFAULT_STATFILE_INVALIDATE_TIME, DEFAULT_STATFILE_INVALIDATE_JITTER); free_task (task, FALSE); i = rspamd_snprintf (out_buf, sizeof (out_buf), "learn ok, sum weight: %.2f" CRLF, sum); if (!rspamd_dispatcher_write (session->dispatcher, out_buf, i, FALSE, FALSE)) { diff --git a/src/filter.c b/src/filter.c index 7d1de3d20..f38831c00 100644 --- a/src/filter.c +++ b/src/filter.c @@ -444,6 +444,7 @@ process_autolearn (struct statfile *st, struct worker_task *task, GTree * tokens classifier->learn_func (ctx, task->worker->srv->statfile_pool, st->symbol, tokens, TRUE, NULL, 1., NULL); maybe_write_binlog (ctx->cfg, st, statfile, tokens); + statfile_pool_plan_invalidate (task->worker->srv->statfile_pool, DEFAULT_STATFILE_INVALIDATE_TIME, DEFAULT_STATFILE_INVALIDATE_JITTER); } } } diff --git a/src/statfile.c b/src/statfile.c index 0460a43c8..1660a29f4 100644 --- a/src/statfile.c +++ b/src/statfile.c @@ -209,11 +209,10 @@ statfile_pool_new (memory_pool_t *pool, size_t max_size) { statfile_pool_t *new; - new = memory_pool_alloc_shared (pool, sizeof (statfile_pool_t)); - bzero (new, sizeof (statfile_pool_t)); + new = memory_pool_alloc0 (pool, sizeof (statfile_pool_t)); new->pool = memory_pool_new (memory_pool_get_size ()); new->max = max_size; - new->files = memory_pool_alloc_shared (new->pool, STATFILES_MAX * sizeof (stat_file_t)); + new->files = memory_pool_alloc0 (new->pool, STATFILES_MAX * sizeof (stat_file_t)); new->lock = memory_pool_get_mutex (new->pool); return new; @@ -813,3 +812,37 @@ statfile_get_total_blocks (stat_file_t *file) return header->total_blocks; } + +static void +statfile_pool_invalidate_callback (int fd, short what, void *ud) +{ + statfile_pool_t *pool = ud; + stat_file_t *file; + int i; + + msg_info ("invalidating %d statfiles", pool->opened); + + for (i = 0; i < pool->opened; i ++) { + file = &pool->files[i]; + msync (file->map, file->len, MS_ASYNC | MS_INVALIDATE); + } + +} + + +void +statfile_pool_plan_invalidate (statfile_pool_t *pool, time_t seconds, time_t jitter) +{ + + if (pool->invalidate_event == NULL || ! evtimer_pending (pool->invalidate_event, NULL)) { + + if (pool->invalidate_event == NULL) { + pool->invalidate_event = memory_pool_alloc (pool->pool, sizeof (struct event)); + } + pool->invalidate_tv.tv_sec = seconds + g_random_int_range (0, jitter); + pool->invalidate_tv.tv_usec = 0; + evtimer_set (pool->invalidate_event, statfile_pool_invalidate_callback, pool); + evtimer_add (pool->invalidate_event, &pool->invalidate_tv); + msg_info ("invalidate of statfile pool is planned in %d seconds", (int)pool->invalidate_tv.tv_sec); + } +} diff --git a/src/statfile.h b/src/statfile.h index 95f800ce6..dfdd16dbe 100644 --- a/src/statfile.h +++ b/src/statfile.h @@ -18,6 +18,9 @@ #define STATFILE_SECTION_URLS 3 #define STATFILE_SECTION_REGEXP 4 +#define DEFAULT_STATFILE_INVALIDATE_TIME 30 +#define DEFAULT_STATFILE_INVALIDATE_JITTER 30 + /** * Common statfile header */ @@ -89,6 +92,8 @@ typedef struct statfile_pool_s { size_t occupied; /**< current size */ memory_pool_t *pool; /**< memory pool object */ memory_pool_mutex_t *lock; /**< mutex */ + struct event *invalidate_event; /**< event for pool invalidation */ + struct timeval invalidate_tv; } statfile_pool_t; /** @@ -242,4 +247,10 @@ uint64_t statfile_get_used_blocks (stat_file_t *file); */ uint64_t statfile_get_total_blocks (stat_file_t *file); + +/** + * Plan statfile pool invalidation + */ +void statfile_pool_plan_invalidate (statfile_pool_t *pool, time_t seconds, time_t jitter); + #endif diff --git a/src/tokenizers/tokenizers.c b/src/tokenizers/tokenizers.c index 2bb478aaa..9ca690e47 100644 --- a/src/tokenizers/tokenizers.c +++ b/src/tokenizers/tokenizers.c @@ -196,15 +196,16 @@ tokenize_headers (memory_pool_t * pool, struct worker_task *task, GTree ** tree) const char *value; ls = GMIME_OBJECT (task->message)->headers; + iter = g_mime_header_iter_new (); if (g_mime_header_list_get_iter (ls, iter)) { while (g_mime_header_iter_is_valid (iter)) { new = memory_pool_alloc (pool, sizeof (token_node_t)); name = g_mime_header_iter_get_name (iter); value = g_mime_header_iter_get_value (iter); - headername.begin = name; + headername.begin = (u_char *)name; headername.len = strlen (name); - headervalue.begin = value; + headervalue.begin = (u_char *)value; headervalue.len = strlen (value); new->h1 = fstrhash (&headername) * primes[0]; new->h2 = fstrhash (&headervalue) * primes[1]; @@ -216,6 +217,7 @@ tokenize_headers (memory_pool_t * pool, struct worker_task *task, GTree ** tree) } } } + g_mime_header_iter_free (iter); #endif return TRUE; } -- 2.39.5