* Add initial implementation of statistics files handling API

diff --git a/configure b/configure
index 2cd2af6f97ff21bfc9697551db1e4f65c708764d..6c8eb4bf27e6b50a1471c7c4c5972263c2a896bf 100755 (executable)
--- a/configure
+++ b/configure
@@ -24,7 +24,7 @@ CACHE="config.cache"
 
 SRCDIR="src"
 OBJDIR="src/.obj"
-SOURCES="upstream.c cfg_utils.c memcached.c main.c util.c controller.c worker.c fstring.c url.c perl.c protocol.c mem_pool.c filter.c plugins/regexp.c plugins/surbl.c tokenizers/tokenizers.c tokenizers/osb.c ${LEX_OUTPUT} ${YACC_OUTPUT}"
+SOURCES="upstream.c cfg_utils.c memcached.c main.c util.c controller.c worker.c fstring.c url.c perl.c protocol.c mem_pool.c filter.c plugins/regexp.c plugins/surbl.c tokenizers/tokenizers.c tokenizers/osb.c statfile.c ${LEX_OUTPUT} ${YACC_OUTPUT}"
 MODULES="surbl regexp"
 
 CFLAGS="$CFLAGS -W -Wpointer-arith -Wno-unused-parameter"

diff --git a/src/statfile.c b/src/statfile.c
new file mode 100644 (file)
index 0000000..ba735bc
--- /dev/null
+++ b/src/statfile.c
@@ -0,0 +1,241 @@
+#include <sys/types.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <time.h>
+#include <limits.h>
+#include <sys/mman.h>
+#include "config.h"
+#include <string.h>
+#include <unistd.h>
+#include <glib.h>
+
+#include "statfile.h"
+#include "main.h"
+
+/* Check whether specified file is statistic file and calculate its len in blocks */
+static int
+statfile_pool_check (stat_file_t *file)
+{
+       struct stat_file *f;
+       char *c;
+
+       if (!file || !file->map) {
+               return -1;
+       }
+
+       if (file->len < sizeof (struct stat_file)) {
+               msg_info ("statfile_pool_check: file %s is too short to be stat file: %zd", file->filename, file->len);
+               return -1;
+       }
+
+       f = (struct stat_file *)file->map;
+       c = f->header.magic;
+       /* Check magic and version */
+       if (*c++ != 'r' || *c++ != 's' || *c++ != 'd' ||
+               /* version */ *c++ != 1 || *c != 0) {
+               msg_info ("statfile_pool_check: file %s is invalid stat file", file->filename);
+               return -1;
+       }
+       
+       if (file->len - sizeof (f->header) % sizeof (struct stat_file_block) != 0) {
+               msg_info ("statfile_pool_check: file %s does not contain integer count of stat blocks", file->filename);
+               return -1;
+       }
+
+       file->blocks = file->len - sizeof (f->header) / sizeof (struct stat_file_block);
+       return 0;
+}
+
+
+struct expiration_data {
+       statfile_pool_t *pool;
+       uint64_t oldest;
+       char *filename;
+};
+
+static void
+pool_expiration_callback (gpointer key, gpointer value, void *data)
+{
+       struct expiration_data *exp = data;
+       stat_file_t *file = (stat_file_t *)value;
+
+       if ((uint64_t)file->access_time < exp->oldest) {
+               exp->oldest = file->access_time;
+               exp->filename = file->filename;
+       }
+}
+
+static int
+statfile_pool_expire (statfile_pool_t *pool)
+{
+       struct expiration_data exp;
+
+       if (g_hash_table_size (pool->files) == 0) {
+               return -1;
+       }
+
+       exp.pool = pool;
+       exp.oldest = ULLONG_MAX;
+       exp.filename = NULL;
+       g_hash_table_foreach (pool->files, pool_expiration_callback, &exp);
+
+       if (exp.filename) {
+               statfile_pool_close (pool, exp.filename);
+       }
+}
+
+statfile_pool_t* 
+statfile_pool_new (size_t max_size)
+{
+       statfile_pool_t *new;
+
+       new = g_malloc (sizeof (statfile_pool_t));
+       bzero (new, sizeof (statfile_pool_t));
+       new->pool = memory_pool_new (memory_pool_get_size ());
+       new->max = max_size;
+       new->files = g_hash_table_new (g_str_hash, g_str_equal);
+       memory_pool_add_destructor (new->pool, (pool_destruct_func)g_hash_table_destroy, new->files);
+
+       return new;
+}
+
+int 
+statfile_pool_open (statfile_pool_t *pool, char *filename)
+{
+       struct stat st;
+       stat_file_t *new_file;  
+       
+       if (g_hash_table_lookup (pool->files, filename) != NULL) {
+               msg_info ("statfile_pool_open: file %s is already opened", filename);
+               return 0;
+       }
+
+       if (stat (filename, &st) == -1) {
+               msg_info ("statfile_pool_open: cannot stat file %s, error %m, %d", filename, errno);
+               return -1;
+       }
+       
+       if (st.st_size > pool->max) {
+               msg_info ("statfile_pool_open: cannot attach file to pool, too large: %zd", st.st_size);
+               return -1;
+       }
+
+       while (pool->max <= pool->occupied + st.st_size) {
+               if (statfile_pool_expire (pool) == -1) {
+                       /* Failed to find any more free space in pool */
+                       msg_info ("statfile_pool_open: expiration for pool failed, opening file %s failed", filename);
+                       return -1;
+               }
+       }
+
+       new_file = memory_pool_alloc (pool->pool, sizeof (stat_file_t));
+       if ((new_file->fd = open (filename, O_RDWR)) == -1 ) {
+               msg_info ("statfile_pool_open: cannot open file %s, error %m, %d", filename, errno);
+               return -1;
+       }
+       
+       if ((new_file->map = mmap (NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, new_file->fd, 0)) == NULL) {
+               close (new_file->fd);
+               msg_info ("statfile_pool_open: cannot mmap file %s, error %m, %d", filename, errno);
+               return -1;
+       
+       }
+       
+       /* XXX: this is temporary copy of name to avoid strdup early */
+       new_file->filename = filename;
+       new_file->len = st.st_size;
+       if (statfile_pool_check (new_file) == -1) {
+               return -1;
+       }
+
+       pool->occupied += st.st_size;
+       new_file->filename = memory_pool_strdup (pool->pool, filename);
+       new_file->open_time = time (NULL);
+       new_file->access_time = new_file->open_time;
+       new_file->lock = memory_pool_get_mutex (pool->pool);
+       g_hash_table_insert (pool->files, new_file->filename, new_file);
+
+       return 0;
+}
+
+int
+statfile_pool_close (statfile_pool_t *pool, char *filename) 
+{
+       stat_file_t *file;      
+       
+       if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) {
+               msg_info ("statfile_pool_open: file %s is not opened", filename);
+               return -1;
+       }
+       
+       if (file->lock) {
+               memory_pool_lock_mutex (file->lock);
+       }
+
+       if (file->map) {
+               munmap (file->map, file->len);
+       }
+       if (file->fd != -1) {
+               close (file->fd);
+       }
+       pool->occupied -= file->len;
+       g_hash_table_remove (pool->files, file->filename);
+}
+
+int
+statfile_pool_create (statfile_pool_t *pool, char *filename, size_t blocks)
+{
+       static struct stat_file_header header = {
+               {'r', 's', 'd'},
+               {1, 0},
+               0
+       };
+       static struct stat_file_block block = {0, 0, 0, 0};
+       int fd;
+       
+       if (g_hash_table_lookup (pool->files, filename) != NULL) {
+               msg_info ("statfile_pool_create: file %s is already opened", filename);
+               return 0;
+       }
+
+       if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT)) == -1 ) {
+               msg_info ("statfile_pool_create: cannot create file %s, error %m, %d", filename, errno);
+               return -1;
+       }
+
+       header.create_time = (uint64_t)time (NULL);
+       if (write (fd, &header, sizeof (header)) == -1) {
+               msg_info ("statfile_pool_create: cannot write header to file %s, error %m, %d", filename, errno);
+               close (fd);
+               return -1;
+       }
+       
+       while (blocks --) {
+               if (write (fd, &block, sizeof (block)) == -1) {
+                       msg_info ("statfile_pool_create: cannot write block to file %s, error %m, %d", filename, errno);
+                       close (fd);
+                       return -1;
+               }
+       }
+
+       close (fd);
+       
+       return 0;
+}
+
+static void
+pool_delete_callback (gpointer key, gpointer value, void *data)
+{
+       statfile_pool_t *pool = (statfile_pool_t *)data;
+
+       statfile_pool_close (pool, (char *)key);
+}
+
+void
+statfile_pool_delete (statfile_pool_t *pool)
+{
+       g_hash_table_foreach (pool->files, pool_delete_callback, pool);
+       memory_pool_delete (pool->pool);
+       g_free (pool);
+}

diff --git a/src/statfile.h b/src/statfile.h
new file mode 100644 (file)
index 0000000..801d1da
--- /dev/null
+++ b/src/statfile.h
@@ -0,0 +1,61 @@
+/*
+ * Describes common methods in accessing statistics files and caching them in memory
+ */
+
+#ifndef RSPAMD_STATFILE_H
+#define RSPAMD_STATFILE_H
+
+#include "config.h"
+#include <sys/types.h>
+#include <glib.h>
+
+#ifdef HAVE_STDINT_H
+#include <stdint.h>
+#endif
+#include "mem_pool.h"
+
+struct stat_file_header {
+       u_char magic[3];
+       u_char version[2];
+       uint64_t create_time;
+};
+
+struct stat_file_block {
+       uint32_t hash1;
+       uint32_t hash2;
+       uint32_t value; /* In fact this is float */
+       uint32_t last_access;
+};
+
+struct stat_file {
+       struct stat_file_header header;
+       struct stat_file_block blocks[1];
+};
+
+typedef struct stat_file_s {
+       char *filename;
+       int fd;
+       void *map;
+       time_t open_time;
+       time_t access_time;
+       size_t len;
+       /* Length is in blocks */
+       size_t blocks;
+       gint *lock;
+} stat_file_t;
+
+typedef struct statfile_pool_s {
+       GHashTable *files;
+       int opened;
+       size_t max;
+       size_t occupied;
+       memory_pool_t *pool;
+} statfile_pool_t;
+
+statfile_pool_t* statfile_pool_new (size_t max_size);
+int statfile_pool_open (statfile_pool_t *pool, char *filename);
+int statfile_pool_create (statfile_pool_t *pool, char *filename, size_t len);
+int statfile_pool_close (statfile_pool_t *pool, char *filename);
+void statfile_pool_delete (statfile_pool_t *pool);
+
+#endif