From 2a45f0daa25c66db449741d9c93a7ea6ef868729 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Thu, 13 May 2010 17:07:20 +0400 Subject: [PATCH] * Fix config reloading * Add ability to register variables in memory pools (hash with known lifetime) * Avoid of using of some global variables --- src/expressions.c | 40 ++++++++++++++++++++-------------- src/expressions.h | 4 ++-- src/fuzzy_storage.c | 1 + src/main.c | 32 ++++++++++++++------------- src/mem_pool.c | 29 +++++++++++++++++++++++++ src/mem_pool.h | 19 +++++++++++++++++ src/plugins/regexp.c | 4 ++-- src/symbols_cache.c | 51 ++++++++++++++++++++++++++++++++++++++------ src/symbols_cache.h | 1 + 9 files changed, 139 insertions(+), 42 deletions(-) diff --git a/src/expressions.c b/src/expressions.c index e52745541..103da5553 100644 --- a/src/expressions.c +++ b/src/expressions.c @@ -90,23 +90,31 @@ fl_cmp (const void *s1, const void *s2) } /* Cache for regular expressions that are used in functions */ -static GHashTable *re_cache = NULL; - void * -re_cache_check (const char *line) +re_cache_check (const char *line, memory_pool_t *pool) { + GHashTable *re_cache; + + re_cache = memory_pool_get_variable (pool, "re_cache"); + if (re_cache == NULL) { re_cache = g_hash_table_new (g_str_hash, g_str_equal); + memory_pool_set_variable (pool, "re_cache", re_cache, (pool_destruct_func)g_hash_table_destroy); return NULL; } return g_hash_table_lookup (re_cache, line); } void -re_cache_add (char *line, void *pointer) +re_cache_add (char *line, void *pointer, memory_pool_t *pool) { + GHashTable *re_cache; + + re_cache = memory_pool_get_variable (pool, "re_cache"); + if (re_cache == NULL) { re_cache = g_hash_table_new (g_str_hash, g_str_equal); + memory_pool_set_variable (pool, "re_cache", re_cache, (pool_destruct_func)g_hash_table_destroy); } g_hash_table_insert (re_cache, line, pointer); @@ -677,7 +685,7 @@ parse_regexp (memory_pool_t * pool, char *line, gboolean raw_mode) } /* Avoid multiply regexp structures for similar regexps */ - if ((check = (struct rspamd_regexp *)re_cache_check (begin)) != NULL) { + if ((check = (struct rspamd_regexp *)re_cache_check (begin, pool)) != NULL) { /* Additional check for headers */ if (result->type == REGEXP_HEADER || result->type == REGEXP_RAW_HEADER) { if (result->header && check->header) { @@ -714,7 +722,7 @@ parse_regexp (memory_pool_t * pool, char *line, gboolean raw_mode) } /* Add to cache for further usage */ - re_cache_add (result->regexp_text, result); + re_cache_add (result->regexp_text, result, pool); return result; } @@ -986,13 +994,13 @@ rspamd_content_type_compare_param (struct worker_task * task, GList * args, void } if (*param_pattern == '/') { /* This is regexp, so compile and create g_regexp object */ - if ((re = re_cache_check (param_pattern)) == NULL) { + if ((re = re_cache_check (param_pattern, task->cfg->cfg_pool)) == NULL) { re = parse_regexp (task->cfg->cfg_pool, param_pattern, task->cfg->raw_mode); if (re == NULL) { msg_warn ("cannot compile regexp for function"); return FALSE; } - re_cache_add (param_pattern, re); + re_cache_add (param_pattern, re, task->cfg->cfg_pool); } if ((r = task_cache_check (task, re)) == -1) { if (g_regex_match (re->regexp, param_data, 0, NULL) == TRUE) { @@ -1084,13 +1092,13 @@ rspamd_content_type_is_subtype (struct worker_task *task, GList * args, void *un if (*param_pattern == '/') { /* This is regexp, so compile and create g_regexp object */ - if ((re = re_cache_check (param_pattern)) == NULL) { + if ((re = re_cache_check (param_pattern, task->cfg->cfg_pool)) == NULL) { re = parse_regexp (task->cfg->cfg_pool, param_pattern, task->cfg->raw_mode); if (re == NULL) { msg_warn ("cannot compile regexp for function"); return FALSE; } - re_cache_add (param_pattern, re); + re_cache_add (param_pattern, re, task->cfg->cfg_pool); } if ((r = task_cache_check (task, re)) == -1) { if (g_regex_match (re->regexp, ct->subtype, 0, NULL) == TRUE) { @@ -1143,13 +1151,13 @@ rspamd_content_type_is_type (struct worker_task * task, GList * args, void *unus if (*param_pattern == '/') { /* This is regexp, so compile and create g_regexp object */ - if ((re = re_cache_check (param_pattern)) == NULL) { + if ((re = re_cache_check (param_pattern, task->cfg->cfg_pool)) == NULL) { re = parse_regexp (task->cfg->cfg_pool, param_pattern, task->cfg->raw_mode); if (re == NULL) { msg_warn ("cannot compile regexp for function"); return FALSE; } - re_cache_add (param_pattern, re); + re_cache_add (param_pattern, re, task->cfg->cfg_pool); } if ((r = task_cache_check (task, re)) == -1) { if (g_regex_match (re->regexp, ct->type, 0, NULL) == TRUE) { @@ -1326,13 +1334,13 @@ compare_subtype (struct worker_task *task, const localContentType * ct, char *su if (*subtype == '/') { /* This is regexp, so compile and create g_regexp object */ - if ((re = re_cache_check (subtype)) == NULL) { + if ((re = re_cache_check (subtype, task->cfg->cfg_pool)) == NULL) { re = parse_regexp (task->cfg->cfg_pool, subtype, task->cfg->raw_mode); if (re == NULL) { msg_warn ("cannot compile regexp for function"); return FALSE; } - re_cache_add (subtype, re); + re_cache_add (subtype, re, task->cfg->cfg_pool); } if ((r = task_cache_check (task, re)) == -1) { if (g_regex_match (re->regexp, subtype, 0, NULL) == TRUE) { @@ -1393,14 +1401,14 @@ common_has_content_part (struct worker_task * task, char *param_type, char *para if (*param_type == '/') { /* This is regexp, so compile and create g_regexp object */ - if ((re = re_cache_check (param_type)) == NULL) { + if ((re = re_cache_check (param_type, task->cfg->cfg_pool)) == NULL) { re = parse_regexp (task->cfg->cfg_pool, param_type, task->cfg->raw_mode); if (re == NULL) { msg_warn ("cannot compile regexp for function"); cur = g_list_next (cur); continue; } - re_cache_add (param_type, re); + re_cache_add (param_type, re, task->cfg->cfg_pool); } if ((r = task_cache_check (task, re)) == -1) { if (g_regex_match (re->regexp, ct->type, 0, NULL) == TRUE) { diff --git a/src/expressions.h b/src/expressions.h index 519efc6da..14ae03f4f 100644 --- a/src/expressions.h +++ b/src/expressions.h @@ -87,14 +87,14 @@ void register_expression_function (const char *name, rspamd_internal_func_t func * @param line symbolic representation * @param pointer regexp data */ -void re_cache_add (char *line, void *pointer); +void re_cache_add (char *line, void *pointer, memory_pool_t *pool); /** * Check regexp in cache * @param line symbolic representation * @return pointer to regexp data or NULL if regexp is not found */ -void * re_cache_check (const char *line); +void * re_cache_check (const char *line, memory_pool_t *pool); /** * Add regexp to regexp task cache diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index 8957c9d35..62f2711d7 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -318,6 +318,7 @@ read_hashes_file (struct rspamd_worker *wrk) msg_err ("unsupported version of fuzzy hash file: %d", version); return FALSE; } + msg_info ("reading fuzzy hashes storage file of version %d of size %d", version, (int)(st.st_size - sizeof (header)) / sizeof (struct rspamd_fuzzy_node)); } else { /* Old version */ diff --git a/src/main.c b/src/main.c index e13ccb77c..5672817e4 100644 --- a/src/main.c +++ b/src/main.c @@ -52,7 +52,7 @@ rspamd_hash_t *counters; static struct rspamd_worker *fork_worker (struct rspamd_main *, struct worker_conf *); -static gboolean load_rspamd_config (struct config_file *cfg); +static gboolean load_rspamd_config (struct config_file *cfg, gboolean init_modules); static void init_metrics_cache (struct config_file *cfg); sig_atomic_t do_restart; @@ -266,7 +266,7 @@ reread_config (struct rspamd_main *rspamd) tmp_cfg->cfg_name = cfg_file; init_lua (tmp_cfg); - if (! load_rspamd_config (tmp_cfg)) { + if (! load_rspamd_config (tmp_cfg, FALSE)) { msg_err ("cannot parse new config file, revert to old one"); free_config (tmp_cfg); } @@ -276,7 +276,6 @@ reread_config (struct rspamd_main *rspamd) close_log (); g_free (rspamd->cfg); rspamd->cfg = tmp_cfg; - post_load_config (rspamd->cfg); config_logger (rspamd, FALSE); /* Perform modules configuring */ l = g_list_first (rspamd->cfg->filters); @@ -285,6 +284,7 @@ reread_config (struct rspamd_main *rspamd) filt = l->data; if (filt->module) { (void)filt->module->module_reconfig_func (rspamd->cfg); + msg_info ("reconfig of %s", filt->module->name); } l = g_list_next (l); } @@ -630,7 +630,7 @@ convert_old_config (struct rspamd_main *rspamd) } static gboolean -load_rspamd_config (struct config_file *cfg) +load_rspamd_config (struct config_file *cfg, gboolean init_modules) { GList *l; struct filter *filt; @@ -653,19 +653,21 @@ load_rspamd_config (struct config_file *cfg) /* Do post-load actions */ post_load_config (cfg); + + if (init_modules) { + /* Init C modules */ + l = g_list_first (cfg->filters); - /* Init C modules */ - l = g_list_first (cfg->filters); - - while (l) { - filt = l->data; - if (filt->module) { - cur_module = memory_pool_alloc (cfg->cfg_pool, sizeof (struct module_ctx)); - if (filt->module->module_init_func (cfg, &cur_module) == 0) { - g_hash_table_insert (cfg->c_modules, (gpointer) filt->module->name, cur_module); + while (l) { + filt = l->data; + if (filt->module) { + cur_module = memory_pool_alloc (cfg->cfg_pool, sizeof (struct module_ctx)); + if (filt->module->module_init_func (cfg, &cur_module) == 0) { + g_hash_table_insert (cfg->c_modules, (gpointer) filt->module->name, cur_module); + } } + l = g_list_next (l); } - l = g_list_next (l); } return TRUE; @@ -798,7 +800,7 @@ main (int argc, char **argv, char **env) } } - if (! load_rspamd_config (rspamd->cfg)) { + if (! load_rspamd_config (rspamd->cfg, TRUE)) { exit (EXIT_FAILURE); } diff --git a/src/mem_pool.c b/src/mem_pool.c index 23907d260..11f74f2f3 100644 --- a/src/mem_pool.c +++ b/src/mem_pool.c @@ -148,6 +148,8 @@ memory_pool_new (memory_pool_ssize_t size) new->shared_pool = NULL; new->first_pool = new->cur_pool; new->destructors = NULL; + /* Set it upon first call of set variable */ + new->variables = NULL; mem_pool_stat->pools_allocated++; @@ -460,6 +462,9 @@ memory_pool_delete (memory_pool_t * pool) mem_pool_stat->chunks_freed++; STAT_UNLOCK (); } + if (pool->variables) { + g_hash_table_destroy (pool->variables); + } mem_pool_stat->pools_freed++; g_slice_free (memory_pool_t, pool); @@ -570,6 +575,30 @@ memory_pool_wunlock_rwlock (memory_pool_rwlock_t * lock) memory_pool_unlock_mutex (lock->__w_lock); } +void +memory_pool_set_variable (memory_pool_t *pool, const gchar *name, gpointer value, pool_destruct_func destructor) +{ + if (pool->variables == NULL) { + pool->variables = g_hash_table_new (g_str_hash, g_str_equal); + } + + g_hash_table_insert (pool->variables, memory_pool_strdup (pool, name), value); + if (destructor != NULL) { + memory_pool_add_destructor (pool, destructor, value); + } +} + +gpointer +memory_pool_get_variable (memory_pool_t *pool, const gchar *name) +{ + if (pool->variables == NULL) { + return NULL; + } + + return g_hash_table_lookup (pool->variables, name); +} + + /* * vi:ts=4 */ diff --git a/src/mem_pool.h b/src/mem_pool.h index b55670c95..81d4a85fa 100644 --- a/src/mem_pool.h +++ b/src/mem_pool.h @@ -74,6 +74,7 @@ typedef struct memory_pool_s { struct _pool_chain *first_pool; /**< first page */ struct _pool_chain_shared *shared_pool; /**< shared chain */ struct _pool_destructors *destructors; /**< destructors chain */ + GHashTable *variables; /**< private memory pool variables */ } memory_pool_t; /** @@ -235,6 +236,24 @@ void memory_pool_stat (memory_pool_stat_t *st); */ memory_pool_ssize_t memory_pool_get_size (); +/** + * Set memory pool variable + * @param pool memory pool object + * @param name name of variable + * @param gpointer value value of variable + * @param destructor pointer to function-destructor + */ +void memory_pool_set_variable (memory_pool_t *pool, const gchar *name, gpointer value, pool_destruct_func destructor); + +/** + * Get memory pool variable + * @param pool memory pool object + * @param name name of variable + * @return NULL or pointer to variable data + */ +gpointer memory_pool_get_variable (memory_pool_t *pool, const gchar *name); + + /** * Macro that return free space in pool page * @param x pool page struct diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c index d129664fd..33aa60f29 100644 --- a/src/plugins/regexp.c +++ b/src/plugins/regexp.c @@ -78,7 +78,7 @@ regexp_module_init (struct config_file *cfg, struct module_ctx **ctx) regexp_module_ctx = g_malloc (sizeof (struct regexp_ctx)); regexp_module_ctx->filter = regexp_common_filter; - regexp_module_ctx->regexp_pool = memory_pool_new (1024); + regexp_module_ctx->regexp_pool = memory_pool_new (memory_pool_get_size ()); regexp_module_ctx->autolearn_symbols = g_hash_table_new (g_str_hash, g_str_equal); *ctx = (struct module_ctx *)regexp_module_ctx; @@ -231,7 +231,7 @@ int regexp_module_reconfig (struct config_file *cfg) { memory_pool_delete (regexp_module_ctx->regexp_pool); - regexp_module_ctx->regexp_pool = memory_pool_new (1024); + regexp_module_ctx->regexp_pool = memory_pool_new (memory_pool_get_size ()); return regexp_module_config (cfg); } diff --git a/src/symbols_cache.c b/src/symbols_cache.c index 060a04187..48eaf95ad 100644 --- a/src/symbols_cache.c +++ b/src/symbols_cache.c @@ -120,8 +120,18 @@ post_cache_init (struct symbols_cache *cache) qsort (cache->items, cache->used_items, sizeof (struct cache_item), cache_logic_cmp); } +/* Unmap cache file */ +static void +unmap_cache_file (gpointer arg) +{ + struct symbols_cache *cache = arg; + + /* A bit ugly usage */ + munmap (cache->map, cache->used_items * sizeof (struct saved_cache_item)); +} + static gboolean -mmap_cache_file (struct symbols_cache *cache, int fd) +mmap_cache_file (struct symbols_cache *cache, int fd, memory_pool_t *pool) { void *map; int i; @@ -134,19 +144,21 @@ mmap_cache_file (struct symbols_cache *cache, int fd) } /* Close descriptor as it would never be used */ close (fd); + cache->map = map; /* Now free old values for saved cache items and fill them with mmapped ones */ for (i = 0; i < cache->used_items; i++) { g_free (cache->items[i].s); cache->items[i].s = ((struct saved_cache_item *)map) + i; } - + post_cache_init (cache); + return TRUE; } /* Fd must be opened for writing, after creating file is mmapped */ static gboolean -create_cache_file (struct symbols_cache *cache, const char *filename, int fd) +create_cache_file (struct symbols_cache *cache, const char *filename, int fd, memory_pool_t *pool) { int i; GChecksum *cksum; @@ -193,7 +205,7 @@ create_cache_file (struct symbols_cache *cache, const char *filename, int fd) return FALSE; } - return mmap_cache_file (cache, fd); + return mmap_cache_file (cache, fd, pool); } void @@ -205,6 +217,7 @@ register_symbol (struct symbols_cache **cache, const char *name, double weight, if (*cache == NULL) { *cache = g_new0 (struct symbols_cache, 1); } + if ((*cache)->items == NULL) { (*cache)->cur_items = MIN_CACHE; (*cache)->used_items = 0; @@ -228,9 +241,29 @@ register_symbol (struct symbols_cache **cache, const char *name, double weight, item->user_data = user_data; item->s->weight = weight; (*cache)->used_items++; + msg_info ("used items: %d, added symbol: %s", (*cache)->used_items, name); set_counter (item->s->symbol, 0); } +static void +free_cache (gpointer arg) +{ + struct symbols_cache *cache = arg; + int i; + + if (cache->map == NULL) { + /* Free items in memory, otherwise */ + for (i = 0; i < cache->cur_items; i++) { + g_free (cache->items[i].s); + } + } + else { + unmap_cache_file (cache); + } + + g_free (cache); +} + gboolean init_symbols_cache (memory_pool_t * pool, struct symbols_cache *cache, const char *filename) { @@ -239,6 +272,7 @@ init_symbols_cache (memory_pool_t * pool, struct symbols_cache *cache, const cha GChecksum *cksum; u_char *mem_sum, *file_sum; gsize cklen; + gboolean res; if (cache == NULL || cache->items == NULL) { return FALSE; @@ -255,6 +289,7 @@ init_symbols_cache (memory_pool_t * pool, struct symbols_cache *cache, const cha post_cache_init (cache); return TRUE; } + /* First of all try to stat file */ if (stat (filename, &st) == -1) { @@ -266,7 +301,7 @@ init_symbols_cache (memory_pool_t * pool, struct symbols_cache *cache, const cha return FALSE; } else { - return create_cache_file (cache, filename, fd); + return create_cache_file (cache, filename, fd, pool); } } else { @@ -323,7 +358,7 @@ init_symbols_cache (memory_pool_t * pool, struct symbols_cache *cache, const cha return FALSE; } else { - return create_cache_file (cache, filename, fd); + return create_cache_file (cache, filename, fd, pool); } } @@ -331,7 +366,9 @@ init_symbols_cache (memory_pool_t * pool, struct symbols_cache *cache, const cha g_free (file_sum); g_checksum_free (cksum); /* MMap cache file and copy saved_cache structures */ - return mmap_cache_file (cache, fd); + res = mmap_cache_file (cache, fd, pool); + memory_pool_add_destructor (pool, (pool_destruct_func)free_cache, cache); + return res; } gboolean diff --git a/src/symbols_cache.h b/src/symbols_cache.h index 65028cfcc..246205260 100644 --- a/src/symbols_cache.h +++ b/src/symbols_cache.h @@ -27,6 +27,7 @@ struct symbols_cache { guint cur_items; guint used_items; guint uses; + gpointer map; memory_pool_rwlock_t *lock; }; -- 2.39.5