From: Vsevolod Stakhov Date: Fri, 23 Jan 2015 22:24:19 +0000 (+0000) Subject: Implement statfiles runtime initialization. X-Git-Tag: 0.9.0~839 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=d1a34b664dc79b8be41659a6d83ad873f623fcb2;p=rspamd.git Implement statfiles runtime initialization. --- diff --git a/src/libstat/backends/backends.h b/src/libstat/backends/backends.h index 45a8cd314..02041410c 100644 --- a/src/libstat/backends/backends.h +++ b/src/libstat/backends/backends.h @@ -38,9 +38,11 @@ struct rspamd_stat_ctx; struct rspamd_stat_backend { const char *name; gpointer (*init)(struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg); + gpointer (*runtime)(struct rspamd_statfile_config *stcf, gpointer ctx); gpointer ctx; }; gpointer rspamd_mmaped_file_init(struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg); +gpointer rspamd_mmaped_file_runtime (struct rspamd_statfile_config *stcf, gpointer ctx); #endif /* BACKENDS_H_ */ diff --git a/src/libstat/backends/mmaped_file.c b/src/libstat/backends/mmaped_file.c index f703f7f5b..b487f5c96 100644 --- a/src/libstat/backends/mmaped_file.c +++ b/src/libstat/backends/mmaped_file.c @@ -97,6 +97,7 @@ typedef struct { time_t access_time; /**< last access time */ size_t len; /**< length of file(in bytes) */ rspamd_mempool_mutex_t *lock; /**< mutex */ + struct rspamd_statfile_config *cf; } rspamd_mmaped_file_t; /** @@ -119,9 +120,250 @@ static void rspamd_mmaped_file_set_block_common ( gboolean from_now); rspamd_mmaped_file_t * rspamd_mmaped_file_is_open ( - rspamd_mmaped_file_ctx * pool, gchar *filename); + rspamd_mmaped_file_ctx * pool, struct rspamd_statfile_config *stcf); rspamd_mmaped_file_t * rspamd_mmaped_file_open (rspamd_mmaped_file_ctx * pool, - const gchar *filename, size_t size); + const gchar *filename, size_t size, struct rspamd_statfile_config *stcf); +gint rspamd_mmaped_file_create (rspamd_mmaped_file_ctx * pool, + const gchar *filename, size_t size, struct rspamd_statfile_config *stcf); + +void +rspamd_mmaped_file_lock_file (rspamd_mmaped_file_ctx * pool, rspamd_mmaped_file_t * file) +{ + rspamd_mempool_lock_mutex (file->lock); +} + +void +rspamd_mmaped_file_unlock_file (rspamd_mmaped_file_ctx * pool, rspamd_mmaped_file_t * file) +{ + + rspamd_mempool_unlock_mutex (file->lock); +} + +double +rspamd_mmaped_file_get_block (rspamd_mmaped_file_ctx * pool, + rspamd_mmaped_file_t * file, + guint32 h1, + guint32 h2, + time_t now) +{ + struct stat_file_block *block; + guint i, blocknum; + u_char *c; + + + file->access_time = now; + if (!file->map) { + return 0; + } + + blocknum = h1 % file->cur_section.length; + c = (u_char *) file->map + file->seek_pos + blocknum * + sizeof (struct stat_file_block); + block = (struct stat_file_block *)c; + + for (i = 0; i < CHAIN_LENGTH; i++) { + if (i + blocknum >= file->cur_section.length) { + break; + } + if (block->hash1 == h1 && block->hash2 == h2) { + return block->value; + } + c += sizeof (struct stat_file_block); + block = (struct stat_file_block *)c; + } + + + return 0; +} + +static void +rspamd_mmaped_file_set_block_common (rspamd_mmaped_file_ctx * pool, + rspamd_mmaped_file_t * file, + guint32 h1, + guint32 h2, + time_t t, + double value, + gboolean from_now) +{ + struct stat_file_block *block, *to_expire = NULL; + struct stat_file_header *header; + guint i, blocknum; + u_char *c; + double min = G_MAXDOUBLE; + + if (from_now) { + file->access_time = t; + } + if (!file->map) { + return; + } + + blocknum = h1 % file->cur_section.length; + header = (struct stat_file_header *)file->map; + c = (u_char *) file->map + file->seek_pos + blocknum * + sizeof (struct stat_file_block); + block = (struct stat_file_block *)c; + + for (i = 0; i < CHAIN_LENGTH; i++) { + if (i + blocknum >= file->cur_section.length) { + /* Need to expire some block in chain */ + msg_info ("chain %ud is full in statfile %s, starting expire", + blocknum, + file->filename); + break; + } + /* First try to find block in chain */ + if (block->hash1 == h1 && block->hash2 == h2) { + block->value = value; + return; + } + /* Check whether we have a free block in chain */ + if (block->hash1 == 0 && block->hash2 == 0) { + /* Write new block here */ + msg_debug ("found free block %ud in chain %ud, set h1=%ud, h2=%ud", + i, + blocknum, + h1, + h2); + block->hash1 = h1; + block->hash2 = h2; + block->value = value; + header->used_blocks++; + + return; + } + + /* Expire block with minimum value otherwise */ + if (block->value < min) { + to_expire = block; + min = block->value; + } + c += sizeof (struct stat_file_block); + block = (struct stat_file_block *)c; + } + + /* Try expire some block */ + if (to_expire) { + block = to_expire; + } + else { + /* Expire first block in chain */ + c = (u_char *) file->map + file->seek_pos + blocknum * + sizeof (struct stat_file_block); + block = (struct stat_file_block *)c; + } + + block->hash1 = h1; + block->hash2 = h2; + block->value = value; +} + +void +rspamd_mmaped_file_set_block (rspamd_mmaped_file_ctx * pool, + rspamd_mmaped_file_t * file, + guint32 h1, + guint32 h2, + time_t now, + double value) +{ + rspamd_mmaped_file_set_block_common (pool, file, h1, h2, now, value, TRUE); +} + +rspamd_mmaped_file_t * +rspamd_mmaped_file_is_open (rspamd_mmaped_file_ctx * pool, + struct rspamd_statfile_config *stcf) +{ + return g_hash_table_lookup (pool->files, stcf); +} + + + +gboolean +rspamd_mmaped_file_set_revision (rspamd_mmaped_file_t *file, guint64 rev, time_t time) +{ + struct stat_file_header *header; + + if (file == NULL || file->map == NULL) { + return FALSE; + } + + header = (struct stat_file_header *)file->map; + + header->revision = rev; + header->rev_time = time; + + return TRUE; +} + +gboolean +rspamd_mmaped_file_inc_revision (rspamd_mmaped_file_t *file) +{ + struct stat_file_header *header; + + if (file == NULL || file->map == NULL) { + return FALSE; + } + + header = (struct stat_file_header *)file->map; + + header->revision++; + + return TRUE; +} + +gboolean +rspamd_mmaped_file_get_revision (rspamd_mmaped_file_t *file, guint64 *rev, time_t *time) +{ + struct stat_file_header *header; + + if (file == NULL || file->map == NULL) { + return FALSE; + } + + header = (struct stat_file_header *)file->map; + + if (rev != NULL) { + *rev = header->revision; + } + if (time != NULL) { + *time = header->rev_time; + } + + return TRUE; +} + +guint64 +rspamd_mmaped_file_get_used (rspamd_mmaped_file_t *file) +{ + struct stat_file_header *header; + + if (file == NULL || file->map == NULL) { + return (guint64) - 1; + } + + header = (struct stat_file_header *)file->map; + + return header->used_blocks; +} + +guint64 +rspamd_mmaped_file_get_total (rspamd_mmaped_file_t *file) +{ + struct stat_file_header *header; + + if (file == NULL || file->map == NULL) { + return (guint64) - 1; + } + + header = (struct stat_file_header *)file->map; + + /* If total blocks is 0 we have old version of header, so set total blocks correctly */ + if (header->total_blocks == 0) { + header->total_blocks = file->cur_section.length; + } + + return header->total_blocks; +} /* Check whether specified file is statistic file and calculate its len in blocks */ static gint @@ -183,9 +425,10 @@ rspamd_mmaped_file_check (rspamd_mmaped_file_t * file) static rspamd_mmaped_file_t * rspamd_mmaped_file_reindex (rspamd_mmaped_file_ctx * pool, - gchar *filename, + const gchar *filename, size_t old_size, - size_t size) + size_t size, + struct rspamd_statfile_config *stcf) { gchar *backup; gint fd; @@ -218,14 +461,14 @@ rspamd_mmaped_file_reindex (rspamd_mmaped_file_ctx * pool, rspamd_mempool_unlock_mutex (pool->lock); /* Now create new file with required size */ - if (rspamd_mmaped_file_create (pool, filename, size) != 0) { + if (rspamd_mmaped_file_create (pool, filename, size, stcf) != 0) { msg_err ("cannot create new file"); g_free (backup); return NULL; } /* Now open new file and start copying */ fd = open (backup, O_RDONLY); - new = rspamd_mmaped_file_open (pool, filename, size); + new = rspamd_mmaped_file_open (pool, filename, size, stcf); if (fd == -1 || new == NULL) { msg_err ("cannot open file: %s", strerror (errno)); @@ -302,12 +545,13 @@ rspamd_mmaped_file_preload (rspamd_mmaped_file_t *file) rspamd_mmaped_file_t * rspamd_mmaped_file_open (rspamd_mmaped_file_ctx * pool, - const gchar *filename, size_t size) + const gchar *filename, size_t size, + struct rspamd_statfile_config *stcf) { struct stat st; rspamd_mmaped_file_t *new_file; - if ((new_file = rspamd_mmaped_file_is_open (pool, filename)) != NULL) { + if ((new_file = rspamd_mmaped_file_is_open (pool, stcf)) != NULL) { return new_file; } @@ -323,7 +567,7 @@ rspamd_mmaped_file_open (rspamd_mmaped_file_ctx * pool, rspamd_mempool_unlock_mutex (pool->lock); msg_warn ("need to reindex statfile old size: %Hz, new size: %Hz", (size_t)st.st_size, size); - return rspamd_mmaped_file_reindex (pool, filename, st.st_size, size); + return rspamd_mmaped_file_reindex (pool, filename, st.st_size, size, stcf); } else if (size < sizeof (struct stat_file)) { msg_err ("requested to shrink statfile to %Hz but it is too small", @@ -380,24 +624,24 @@ rspamd_mmaped_file_open (rspamd_mmaped_file_ctx * pool, new_file->open_time = time (NULL); new_file->access_time = new_file->open_time; new_file->lock = rspamd_mempool_get_mutex (pool->pool); + new_file->cf = stcf; rspamd_mmaped_file_preload (new_file); - g_hash_table_insert (pool->files, new_file->filename, new_file); + g_hash_table_insert (pool->files, stcf, new_file); rspamd_mempool_unlock_mutex (pool->lock); - return rspamd_mmaped_file_is_open (pool, filename); + return rspamd_mmaped_file_is_open (pool, stcf); } gint rspamd_mmaped_file_close (rspamd_mmaped_file_ctx * pool, - rspamd_mmaped_file_t * file, - gboolean keep_sorted) + rspamd_mmaped_file_t * file) { rspamd_mmaped_file_t *pos; - if ((pos = rspamd_mmaped_file_is_open (pool, file->filename)) == NULL) { + if ((pos = rspamd_mmaped_file_is_open (pool, file->cf)) == NULL) { msg_info ("file %s is not opened", file->filename); return -1; } @@ -413,7 +657,7 @@ rspamd_mmaped_file_close (rspamd_mmaped_file_ctx * pool, close (file->fd); } - g_hash_table_remove (pool->files, file->filename); + g_hash_table_remove (pool->files, file->cf); g_slice_free1 (sizeof (*file), file); @@ -423,7 +667,8 @@ rspamd_mmaped_file_close (rspamd_mmaped_file_ctx * pool, } gint -rspamd_mmaped_file_create (rspamd_mmaped_file_ctx * pool, gchar *filename, size_t size) +rspamd_mmaped_file_create (rspamd_mmaped_file_ctx * pool, const gchar *filename, + size_t size, struct rspamd_statfile_config *stcf) { struct stat_file_header header = { .magic = {'r', 's', 'd'}, @@ -441,7 +686,7 @@ rspamd_mmaped_file_create (rspamd_mmaped_file_ctx * pool, gchar *filename, size_ guint buflen = 0, nblocks; gchar *buf = NULL; - if (rspamd_mmaped_file_is_open (pool, filename) != NULL) { + if (rspamd_mmaped_file_is_open (pool, stcf) != NULL) { msg_info ("file %s is already opened", filename); return 0; } @@ -555,251 +800,13 @@ rspamd_mmaped_file_destroy (rspamd_mmaped_file_ctx * pool) g_hash_table_iter_init (&it, pool->files); while (g_hash_table_iter_next (&it, &k, &v)) { f = (rspamd_mmaped_file_t *)v; - rspamd_mmaped_file_close (pool, f, FALSE); + rspamd_mmaped_file_close (pool, f); } g_hash_table_destroy (pool->files); rspamd_mempool_delete (pool->pool); } -void -rspamd_mmaped_file_lock_file (rspamd_mmaped_file_ctx * pool, rspamd_mmaped_file_t * file) -{ - rspamd_mempool_lock_mutex (file->lock); -} - -void -rspamd_mmaped_file_unlock_file (rspamd_mmaped_file_ctx * pool, rspamd_mmaped_file_t * file) -{ - - rspamd_mempool_unlock_mutex (file->lock); -} - -double -rspamd_mmaped_file_get_block (rspamd_mmaped_file_ctx * pool, - rspamd_mmaped_file_t * file, - guint32 h1, - guint32 h2, - time_t now) -{ - struct stat_file_block *block; - guint i, blocknum; - u_char *c; - - - file->access_time = now; - if (!file->map) { - return 0; - } - - blocknum = h1 % file->cur_section.length; - c = (u_char *) file->map + file->seek_pos + blocknum * - sizeof (struct stat_file_block); - block = (struct stat_file_block *)c; - - for (i = 0; i < CHAIN_LENGTH; i++) { - if (i + blocknum >= file->cur_section.length) { - break; - } - if (block->hash1 == h1 && block->hash2 == h2) { - return block->value; - } - c += sizeof (struct stat_file_block); - block = (struct stat_file_block *)c; - } - - - return 0; -} - -static void -rspamd_mmaped_file_set_block_common (rspamd_mmaped_file_ctx * pool, - rspamd_mmaped_file_t * file, - guint32 h1, - guint32 h2, - time_t t, - double value, - gboolean from_now) -{ - struct stat_file_block *block, *to_expire = NULL; - struct stat_file_header *header; - guint i, blocknum; - u_char *c; - double min = G_MAXDOUBLE; - - if (from_now) { - file->access_time = t; - } - if (!file->map) { - return; - } - - blocknum = h1 % file->cur_section.length; - header = (struct stat_file_header *)file->map; - c = (u_char *) file->map + file->seek_pos + blocknum * - sizeof (struct stat_file_block); - block = (struct stat_file_block *)c; - - for (i = 0; i < CHAIN_LENGTH; i++) { - if (i + blocknum >= file->cur_section.length) { - /* Need to expire some block in chain */ - msg_info ("chain %ud is full in statfile %s, starting expire", - blocknum, - file->filename); - break; - } - /* First try to find block in chain */ - if (block->hash1 == h1 && block->hash2 == h2) { - block->value = value; - return; - } - /* Check whether we have a free block in chain */ - if (block->hash1 == 0 && block->hash2 == 0) { - /* Write new block here */ - msg_debug ("found free block %ud in chain %ud, set h1=%ud, h2=%ud", - i, - blocknum, - h1, - h2); - block->hash1 = h1; - block->hash2 = h2; - block->value = value; - header->used_blocks++; - - return; - } - - /* Expire block with minimum value otherwise */ - if (block->value < min) { - to_expire = block; - min = block->value; - } - c += sizeof (struct stat_file_block); - block = (struct stat_file_block *)c; - } - - /* Try expire some block */ - if (to_expire) { - block = to_expire; - } - else { - /* Expire first block in chain */ - c = (u_char *) file->map + file->seek_pos + blocknum * - sizeof (struct stat_file_block); - block = (struct stat_file_block *)c; - } - - block->hash1 = h1; - block->hash2 = h2; - block->value = value; -} - -void -rspamd_mmaped_file_set_block (rspamd_mmaped_file_ctx * pool, - rspamd_mmaped_file_t * file, - guint32 h1, - guint32 h2, - time_t now, - double value) -{ - rspamd_mmaped_file_set_block_common (pool, file, h1, h2, now, value, TRUE); -} - -rspamd_mmaped_file_t * -rspamd_mmaped_file_is_open (rspamd_mmaped_file_ctx * pool, gchar *filename) -{ - return g_hash_table_lookup (pool->files, filename); -} - - - -gboolean -rspamd_mmaped_file_set_revision (rspamd_mmaped_file_t *file, guint64 rev, time_t time) -{ - struct stat_file_header *header; - - if (file == NULL || file->map == NULL) { - return FALSE; - } - - header = (struct stat_file_header *)file->map; - - header->revision = rev; - header->rev_time = time; - - return TRUE; -} - -gboolean -rspamd_mmaped_file_inc_revision (rspamd_mmaped_file_t *file) -{ - struct stat_file_header *header; - - if (file == NULL || file->map == NULL) { - return FALSE; - } - - header = (struct stat_file_header *)file->map; - - header->revision++; - - return TRUE; -} - -gboolean -rspamd_mmaped_file_get_revision (rspamd_mmaped_file_t *file, guint64 *rev, time_t *time) -{ - struct stat_file_header *header; - - if (file == NULL || file->map == NULL) { - return FALSE; - } - - header = (struct stat_file_header *)file->map; - - if (rev != NULL) { - *rev = header->revision; - } - if (time != NULL) { - *time = header->rev_time; - } - - return TRUE; -} - -guint64 -rspamd_mmaped_file_get_used (rspamd_mmaped_file_t *file) -{ - struct stat_file_header *header; - - if (file == NULL || file->map == NULL) { - return (guint64) - 1; - } - - header = (struct stat_file_header *)file->map; - - return header->used_blocks; -} - -guint64 -rspamd_mmaped_file_get_total (rspamd_mmaped_file_t *file) -{ - struct stat_file_header *header; - - if (file == NULL || file->map == NULL) { - return (guint64) - 1; - } - - header = (struct stat_file_header *)file->map; - - /* If total blocks is 0 we have old version of header, so set total blocks correctly */ - if (header->total_blocks == 0) { - header->total_blocks = file->cur_section.length; - } - - return header->total_blocks; -} - gpointer rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg) { @@ -807,7 +814,6 @@ rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg) struct rspamd_classifier_config *clf; struct rspamd_statfile_config *stf; GList *cur, *curst; - rspamd_mmaped_file_t *mf; const ucl_object_t *filenameo, *sizeo; const gchar *filename; gsize size; @@ -815,7 +821,7 @@ rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg) new = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (rspamd_mmaped_file_ctx)); new->lock = rspamd_mempool_get_mutex (new->pool); new->mlock_ok = cfg->mlock_statfile_pool; - new->files = g_hash_table_new (g_str_hash, g_str_equal); + new->files = g_hash_table_new (g_direct_hash, g_direct_equal); /* Iterate over all classifiers and load matching statfiles */ cur = cfg->classifiers; @@ -851,7 +857,7 @@ rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg) size = ucl_object_toint (sizeo); - rspamd_mmaped_file_open (new, filename, size); + rspamd_mmaped_file_open (new, filename, size, stf); ctx->statfiles ++; } @@ -864,3 +870,16 @@ rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg) return (gpointer)new; } + +gpointer +rspamd_mmaped_file_runtime (struct rspamd_statfile_config *stcf, gpointer p) +{ + rspamd_mmaped_file_ctx *ctx = (rspamd_mmaped_file_ctx *)p; + rspamd_mmaped_file_t *mf; + + g_assert (ctx != NULL); + + mf = rspamd_mmaped_file_is_open (ctx, stcf); + + return (gpointer)mf; +} diff --git a/src/libstat/stat_internal.h b/src/libstat/stat_internal.h index 15b5f03af..6587f595e 100644 --- a/src/libstat/stat_internal.h +++ b/src/libstat/stat_internal.h @@ -31,6 +31,7 @@ struct rspamd_statfile_runtime { struct rspamd_statfile_config *st; + gpointer statfile_data; guint64 hits; guint64 total_hits; };