From f1fdc9c6c1fa897f6eac5abc4cd193b55d31e7bc Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Thu, 27 Nov 2008 19:46:15 +0300 Subject: [PATCH] * Add initial implementation of statistics files handling API --- configure | 2 +- src/statfile.c | 241 +++++++++++++++++++++++++++++++++++++++++++++++++ src/statfile.h | 61 +++++++++++++ 3 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 src/statfile.c create mode 100644 src/statfile.h diff --git a/configure b/configure index 2cd2af6f9..6c8eb4bf2 100755 --- a/configure +++ b/configure @@ -24,7 +24,7 @@ CACHE="config.cache" SRCDIR="src" OBJDIR="src/.obj" -SOURCES="upstream.c cfg_utils.c memcached.c main.c util.c controller.c worker.c fstring.c url.c perl.c protocol.c mem_pool.c filter.c plugins/regexp.c plugins/surbl.c tokenizers/tokenizers.c tokenizers/osb.c ${LEX_OUTPUT} ${YACC_OUTPUT}" +SOURCES="upstream.c cfg_utils.c memcached.c main.c util.c controller.c worker.c fstring.c url.c perl.c protocol.c mem_pool.c filter.c plugins/regexp.c plugins/surbl.c tokenizers/tokenizers.c tokenizers/osb.c statfile.c ${LEX_OUTPUT} ${YACC_OUTPUT}" MODULES="surbl regexp" CFLAGS="$CFLAGS -W -Wpointer-arith -Wno-unused-parameter" diff --git a/src/statfile.c b/src/statfile.c new file mode 100644 index 000000000..ba735bcbf --- /dev/null +++ b/src/statfile.c @@ -0,0 +1,241 @@ +#include +#include +#include +#include +#include +#include +#include +#include "config.h" +#include +#include +#include + +#include "statfile.h" +#include "main.h" + +/* Check whether specified file is statistic file and calculate its len in blocks */ +static int +statfile_pool_check (stat_file_t *file) +{ + struct stat_file *f; + char *c; + + if (!file || !file->map) { + return -1; + } + + if (file->len < sizeof (struct stat_file)) { + msg_info ("statfile_pool_check: file %s is too short to be stat file: %zd", file->filename, file->len); + return -1; + } + + f = (struct stat_file *)file->map; + c = f->header.magic; + /* Check magic and version */ + if (*c++ != 'r' || *c++ != 's' || *c++ != 'd' || + /* version */ *c++ != 1 || *c != 0) { + msg_info ("statfile_pool_check: file %s is invalid stat file", file->filename); + return -1; + } + + if (file->len - sizeof (f->header) % sizeof (struct stat_file_block) != 0) { + msg_info ("statfile_pool_check: file %s does not contain integer count of stat blocks", file->filename); + return -1; + } + + file->blocks = file->len - sizeof (f->header) / sizeof (struct stat_file_block); + return 0; +} + + +struct expiration_data { + statfile_pool_t *pool; + uint64_t oldest; + char *filename; +}; + +static void +pool_expiration_callback (gpointer key, gpointer value, void *data) +{ + struct expiration_data *exp = data; + stat_file_t *file = (stat_file_t *)value; + + if ((uint64_t)file->access_time < exp->oldest) { + exp->oldest = file->access_time; + exp->filename = file->filename; + } +} + +static int +statfile_pool_expire (statfile_pool_t *pool) +{ + struct expiration_data exp; + + if (g_hash_table_size (pool->files) == 0) { + return -1; + } + + exp.pool = pool; + exp.oldest = ULLONG_MAX; + exp.filename = NULL; + g_hash_table_foreach (pool->files, pool_expiration_callback, &exp); + + if (exp.filename) { + statfile_pool_close (pool, exp.filename); + } +} + +statfile_pool_t* +statfile_pool_new (size_t max_size) +{ + statfile_pool_t *new; + + new = g_malloc (sizeof (statfile_pool_t)); + bzero (new, sizeof (statfile_pool_t)); + new->pool = memory_pool_new (memory_pool_get_size ()); + new->max = max_size; + new->files = g_hash_table_new (g_str_hash, g_str_equal); + memory_pool_add_destructor (new->pool, (pool_destruct_func)g_hash_table_destroy, new->files); + + return new; +} + +int +statfile_pool_open (statfile_pool_t *pool, char *filename) +{ + struct stat st; + stat_file_t *new_file; + + if (g_hash_table_lookup (pool->files, filename) != NULL) { + msg_info ("statfile_pool_open: file %s is already opened", filename); + return 0; + } + + if (stat (filename, &st) == -1) { + msg_info ("statfile_pool_open: cannot stat file %s, error %m, %d", filename, errno); + return -1; + } + + if (st.st_size > pool->max) { + msg_info ("statfile_pool_open: cannot attach file to pool, too large: %zd", st.st_size); + return -1; + } + + while (pool->max <= pool->occupied + st.st_size) { + if (statfile_pool_expire (pool) == -1) { + /* Failed to find any more free space in pool */ + msg_info ("statfile_pool_open: expiration for pool failed, opening file %s failed", filename); + return -1; + } + } + + new_file = memory_pool_alloc (pool->pool, sizeof (stat_file_t)); + if ((new_file->fd = open (filename, O_RDWR)) == -1 ) { + msg_info ("statfile_pool_open: cannot open file %s, error %m, %d", filename, errno); + return -1; + } + + if ((new_file->map = mmap (NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, new_file->fd, 0)) == NULL) { + close (new_file->fd); + msg_info ("statfile_pool_open: cannot mmap file %s, error %m, %d", filename, errno); + return -1; + + } + + /* XXX: this is temporary copy of name to avoid strdup early */ + new_file->filename = filename; + new_file->len = st.st_size; + if (statfile_pool_check (new_file) == -1) { + return -1; + } + + pool->occupied += st.st_size; + new_file->filename = memory_pool_strdup (pool->pool, filename); + new_file->open_time = time (NULL); + new_file->access_time = new_file->open_time; + new_file->lock = memory_pool_get_mutex (pool->pool); + g_hash_table_insert (pool->files, new_file->filename, new_file); + + return 0; +} + +int +statfile_pool_close (statfile_pool_t *pool, char *filename) +{ + stat_file_t *file; + + if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) { + msg_info ("statfile_pool_open: file %s is not opened", filename); + return -1; + } + + if (file->lock) { + memory_pool_lock_mutex (file->lock); + } + + if (file->map) { + munmap (file->map, file->len); + } + if (file->fd != -1) { + close (file->fd); + } + pool->occupied -= file->len; + g_hash_table_remove (pool->files, file->filename); +} + +int +statfile_pool_create (statfile_pool_t *pool, char *filename, size_t blocks) +{ + static struct stat_file_header header = { + {'r', 's', 'd'}, + {1, 0}, + 0 + }; + static struct stat_file_block block = {0, 0, 0, 0}; + int fd; + + if (g_hash_table_lookup (pool->files, filename) != NULL) { + msg_info ("statfile_pool_create: file %s is already opened", filename); + return 0; + } + + if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT)) == -1 ) { + msg_info ("statfile_pool_create: cannot create file %s, error %m, %d", filename, errno); + return -1; + } + + header.create_time = (uint64_t)time (NULL); + if (write (fd, &header, sizeof (header)) == -1) { + msg_info ("statfile_pool_create: cannot write header to file %s, error %m, %d", filename, errno); + close (fd); + return -1; + } + + while (blocks --) { + if (write (fd, &block, sizeof (block)) == -1) { + msg_info ("statfile_pool_create: cannot write block to file %s, error %m, %d", filename, errno); + close (fd); + return -1; + } + } + + close (fd); + + return 0; +} + +static void +pool_delete_callback (gpointer key, gpointer value, void *data) +{ + statfile_pool_t *pool = (statfile_pool_t *)data; + + statfile_pool_close (pool, (char *)key); +} + +void +statfile_pool_delete (statfile_pool_t *pool) +{ + g_hash_table_foreach (pool->files, pool_delete_callback, pool); + memory_pool_delete (pool->pool); + g_free (pool); +} diff --git a/src/statfile.h b/src/statfile.h new file mode 100644 index 000000000..801d1dae2 --- /dev/null +++ b/src/statfile.h @@ -0,0 +1,61 @@ +/* + * Describes common methods in accessing statistics files and caching them in memory + */ + +#ifndef RSPAMD_STATFILE_H +#define RSPAMD_STATFILE_H + +#include "config.h" +#include +#include + +#ifdef HAVE_STDINT_H +#include +#endif +#include "mem_pool.h" + +struct stat_file_header { + u_char magic[3]; + u_char version[2]; + uint64_t create_time; +}; + +struct stat_file_block { + uint32_t hash1; + uint32_t hash2; + uint32_t value; /* In fact this is float */ + uint32_t last_access; +}; + +struct stat_file { + struct stat_file_header header; + struct stat_file_block blocks[1]; +}; + +typedef struct stat_file_s { + char *filename; + int fd; + void *map; + time_t open_time; + time_t access_time; + size_t len; + /* Length is in blocks */ + size_t blocks; + gint *lock; +} stat_file_t; + +typedef struct statfile_pool_s { + GHashTable *files; + int opened; + size_t max; + size_t occupied; + memory_pool_t *pool; +} statfile_pool_t; + +statfile_pool_t* statfile_pool_new (size_t max_size); +int statfile_pool_open (statfile_pool_t *pool, char *filename); +int statfile_pool_create (statfile_pool_t *pool, char *filename, size_t len); +int statfile_pool_close (statfile_pool_t *pool, char *filename); +void statfile_pool_delete (statfile_pool_t *pool); + +#endif -- 2.39.5