* Add initial implementation of statistics files handling API

3 files changed, 303 insertions, 1 deletions

diff --git a/configure b/configure
index 2cd2af6f9..6c8eb4bf2 100755
--- a/configure
+++ b/configure
@@ -24,7 +24,7 @@ CACHE="config.cache"
 
 SRCDIR="src"
 OBJDIR="src/.obj"
-SOURCES="upstream.c cfg_utils.c memcached.c main.c util.c controller.c worker.c fstring.c url.c perl.c protocol.c mem_pool.c filter.c plugins/regexp.c plugins/surbl.c tokenizers/tokenizers.c tokenizers/osb.c ${LEX_OUTPUT} ${YACC_OUTPUT}"
+SOURCES="upstream.c cfg_utils.c memcached.c main.c util.c controller.c worker.c fstring.c url.c perl.c protocol.c mem_pool.c filter.c plugins/regexp.c plugins/surbl.c tokenizers/tokenizers.c tokenizers/osb.c statfile.c ${LEX_OUTPUT} ${YACC_OUTPUT}"
 MODULES="surbl regexp"
 
 CFLAGS="$CFLAGS -W -Wpointer-arith -Wno-unused-parameter"
diff --git a/src/statfile.c b/src/statfile.c
new file mode 100644
index 000000000..ba735bcbf
--- /dev/null
+++ b/src/statfile.c
@@ -0,0 +1,241 @@
+#include <sys/types.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <time.h>
+#include <limits.h>
+#include <sys/mman.h>
+#include "config.h"
+#include <string.h>
+#include <unistd.h>
+#include <glib.h>
+
+#include "statfile.h"
+#include "main.h"
+
+/* Check whether specified file is statistic file and calculate its len in blocks */
+static int
+statfile_pool_check (stat_file_t *file)
+{
+	struct stat_file *f;
+	char *c;
+
+	if (!file || !file->map) {
+		return -1;
+	}
+
+	if (file->len < sizeof (struct stat_file)) {
+		msg_info ("statfile_pool_check: file %s is too short to be stat file: %zd", file->filename, file->len);
+		return -1;
+	}
+
+	f = (struct stat_file *)file->map;
+	c = f->header.magic;
+	/* Check magic and version */
+	if (*c++ != 'r' || *c++ != 's' || *c++ != 'd' ||
+		/* version */ *c++ != 1 || *c != 0) {
+		msg_info ("statfile_pool_check: file %s is invalid stat file", file->filename);
+		return -1;
+	}
+	
+	if (file->len - sizeof (f->header) % sizeof (struct stat_file_block) != 0) {
+		msg_info ("statfile_pool_check: file %s does not contain integer count of stat blocks", file->filename);
+		return -1;
+	}
+
+	file->blocks = file->len - sizeof (f->header) / sizeof (struct stat_file_block);
+	return 0;
+}
+
+
+struct expiration_data {
+	statfile_pool_t *pool;
+	uint64_t oldest;
+	char *filename;
+};
+
+static void
+pool_expiration_callback (gpointer key, gpointer value, void *data)
+{
+	struct expiration_data *exp = data;
+	stat_file_t *file = (stat_file_t *)value;
+
+	if ((uint64_t)file->access_time < exp->oldest) {
+		exp->oldest = file->access_time;
+		exp->filename = file->filename;
+	}
+}
+
+static int
+statfile_pool_expire (statfile_pool_t *pool)
+{
+	struct expiration_data exp;
+
+	if (g_hash_table_size (pool->files) == 0) {
+		return -1;
+	}
+
+	exp.pool = pool;
+	exp.oldest = ULLONG_MAX;
+	exp.filename = NULL;
+	g_hash_table_foreach (pool->files, pool_expiration_callback, &exp);
+
+	if (exp.filename) {
+		statfile_pool_close (pool, exp.filename);
+	}
+}
+
+statfile_pool_t* 
+statfile_pool_new (size_t max_size)
+{
+	statfile_pool_t *new;
+
+	new = g_malloc (sizeof (statfile_pool_t));
+	bzero (new, sizeof (statfile_pool_t));
+	new->pool = memory_pool_new (memory_pool_get_size ());
+	new->max = max_size;
+	new->files = g_hash_table_new (g_str_hash, g_str_equal);
+	memory_pool_add_destructor (new->pool, (pool_destruct_func)g_hash_table_destroy, new->files);
+
+	return new;
+}
+
+int 
+statfile_pool_open (statfile_pool_t *pool, char *filename)
+{
+	struct stat st;
+	stat_file_t *new_file;	
+	
+	if (g_hash_table_lookup (pool->files, filename) != NULL) {
+		msg_info ("statfile_pool_open: file %s is already opened", filename);
+		return 0;
+	}
+
+	if (stat (filename, &st) == -1) {
+		msg_info ("statfile_pool_open: cannot stat file %s, error %m, %d", filename, errno);
+		return -1;
+	}
+	
+	if (st.st_size > pool->max) {
+		msg_info ("statfile_pool_open: cannot attach file to pool, too large: %zd", st.st_size);
+		return -1;
+	}
+
+	while (pool->max <= pool->occupied + st.st_size) {
+		if (statfile_pool_expire (pool) == -1) {
+			/* Failed to find any more free space in pool */
+			msg_info ("statfile_pool_open: expiration for pool failed, opening file %s failed", filename);
+			return -1;
+		}
+	}
+
+	new_file = memory_pool_alloc (pool->pool, sizeof (stat_file_t));
+	if ((new_file->fd = open (filename, O_RDWR)) == -1 ) {
+		msg_info ("statfile_pool_open: cannot open file %s, error %m, %d", filename, errno);
+		return -1;
+	}
+	
+	if ((new_file->map = mmap (NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, new_file->fd, 0)) == NULL) {
+		close (new_file->fd);
+		msg_info ("statfile_pool_open: cannot mmap file %s, error %m, %d", filename, errno);
+		return -1;
+	
+	}
+	
+	/* XXX: this is temporary copy of name to avoid strdup early */
+	new_file->filename = filename;
+	new_file->len = st.st_size;
+	if (statfile_pool_check (new_file) == -1) {
+		return -1;
+	}
+
+	pool->occupied += st.st_size;
+	new_file->filename = memory_pool_strdup (pool->pool, filename);
+	new_file->open_time = time (NULL);
+	new_file->access_time = new_file->open_time;
+	new_file->lock = memory_pool_get_mutex (pool->pool);
+	g_hash_table_insert (pool->files, new_file->filename, new_file);
+
+	return 0;
+}
+
+int
+statfile_pool_close (statfile_pool_t *pool, char *filename) 
+{
+	stat_file_t *file;	
+	
+	if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) {
+		msg_info ("statfile_pool_open: file %s is not opened", filename);
+		return -1;
+	}
+	
+	if (file->lock) {
+		memory_pool_lock_mutex (file->lock);
+	}
+
+	if (file->map) {
+		munmap (file->map, file->len);
+	}
+	if (file->fd != -1) {
+		close (file->fd);
+	}
+	pool->occupied -= file->len;
+	g_hash_table_remove (pool->files, file->filename);
+}
+
+int
+statfile_pool_create (statfile_pool_t *pool, char *filename, size_t blocks)
+{
+	static struct stat_file_header header = {
+		{'r', 's', 'd'},
+		{1, 0},
+		0
+	};
+	static struct stat_file_block block = {0, 0, 0, 0};
+	int fd;
+	
+	if (g_hash_table_lookup (pool->files, filename) != NULL) {
+		msg_info ("statfile_pool_create: file %s is already opened", filename);
+		return 0;
+	}
+
+	if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT)) == -1 ) {
+		msg_info ("statfile_pool_create: cannot create file %s, error %m, %d", filename, errno);
+		return -1;
+	}
+
+	header.create_time = (uint64_t)time (NULL);
+	if (write (fd, &header, sizeof (header)) == -1) {
+		msg_info ("statfile_pool_create: cannot write header to file %s, error %m, %d", filename, errno);
+		close (fd);
+		return -1;
+	}
+	
+	while (blocks --) {
+		if (write (fd, &block, sizeof (block)) == -1) {
+			msg_info ("statfile_pool_create: cannot write block to file %s, error %m, %d", filename, errno);
+			close (fd);
+			return -1;
+		}
+	}
+
+	close (fd);
+	
+	return 0;
+}
+
+static void
+pool_delete_callback (gpointer key, gpointer value, void *data)
+{
+	statfile_pool_t *pool = (statfile_pool_t *)data;
+
+	statfile_pool_close (pool, (char *)key);
+}
+
+void
+statfile_pool_delete (statfile_pool_t *pool)
+{
+	g_hash_table_foreach (pool->files, pool_delete_callback, pool);
+	memory_pool_delete (pool->pool);
+	g_free (pool);
+}
diff --git a/src/statfile.h b/src/statfile.h
new file mode 100644
index 000000000..801d1dae2
--- /dev/null
+++ b/src/statfile.h
@@ -0,0 +1,61 @@
+/*
+ * Describes common methods in accessing statistics files and caching them in memory
+ */
+
+#ifndef RSPAMD_STATFILE_H
+#define RSPAMD_STATFILE_H
+
+#include "config.h"
+#include <sys/types.h>
+#include <glib.h>
+
+#ifdef HAVE_STDINT_H
+#include <stdint.h>
+#endif
+#include "mem_pool.h"
+
+struct stat_file_header {
+	u_char magic[3];
+	u_char version[2];
+	uint64_t create_time;
+};
+
+struct stat_file_block {
+	uint32_t hash1;
+	uint32_t hash2;
+	uint32_t value; /* In fact this is float */
+	uint32_t last_access;
+};
+
+struct stat_file {
+	struct stat_file_header header;
+	struct stat_file_block blocks[1];
+};
+
+typedef struct stat_file_s {
+	char *filename;
+	int fd;
+	void *map;
+	time_t open_time;
+	time_t access_time;
+	size_t len;
+	/* Length is in blocks */
+	size_t blocks;
+	gint *lock;
+} stat_file_t;
+
+typedef struct statfile_pool_s {
+	GHashTable *files;
+	int opened;
+	size_t max;
+	size_t occupied;
+	memory_pool_t *pool;
+} statfile_pool_t;
+
+statfile_pool_t* statfile_pool_new (size_t max_size);
+int statfile_pool_open (statfile_pool_t *pool, char *filename);
+int statfile_pool_create (statfile_pool_t *pool, char *filename, size_t len);
+int statfile_pool_close (statfile_pool_t *pool, char *filename);
+void statfile_pool_delete (statfile_pool_t *pool);
+
+#endif