* Write functions to operate blocks in stat files

* Write test case for statistics files API

6 files changed, 214 insertions, 4 deletions

diff --git a/src/statfile.c b/src/statfile.c
index ba735bcbf..fc7ebfc7a 100644
--- a/src/statfile.c
+++ b/src/statfile.c
@@ -38,12 +38,12 @@ statfile_pool_check (stat_file_t *file)
 		return -1;
 	}
 	
-	if (file->len - sizeof (f->header) % sizeof (struct stat_file_block) != 0) {
+	if ((file->len - sizeof (f->header)) % sizeof (struct stat_file_block) != 0) {
 		msg_info ("statfile_pool_check: file %s does not contain integer count of stat blocks", file->filename);
 		return -1;
 	}
 
-	file->blocks = file->len - sizeof (f->header) / sizeof (struct stat_file_block);
+	file->blocks = (file->len - sizeof (f->header)) / sizeof (struct stat_file_block);
 	return 0;
 }
 
@@ -189,6 +189,7 @@ statfile_pool_create (statfile_pool_t *pool, char *filename, size_t blocks)
 	static struct stat_file_header header = {
 		{'r', 's', 'd'},
 		{1, 0},
+		{0, 0, 0},
 		0
 	};
 	static struct stat_file_block block = {0, 0, 0, 0};
@@ -199,7 +200,7 @@ statfile_pool_create (statfile_pool_t *pool, char *filename, size_t blocks)
 		return 0;
 	}
 
-	if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT)) == -1 ) {
+	if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1 ) {
 		msg_info ("statfile_pool_create: cannot create file %s, error %m, %d", filename, errno);
 		return -1;
 	}
@@ -239,3 +240,140 @@ statfile_pool_delete (statfile_pool_t *pool)
 	memory_pool_delete (pool->pool);
 	g_free (pool);
 }
+
+void
+statfile_pool_lock_file (statfile_pool_t *pool, char *filename) 
+{
+	stat_file_t *file;
+
+	if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) {
+		msg_info ("statfile_pool_lock_file: file %s is not opened", filename);
+		return;
+	}
+
+	memory_pool_lock_mutex (file->lock);
+}
+
+void
+statfile_pool_unlock_file (statfile_pool_t *pool, char *filename) 
+{
+	stat_file_t *file;
+
+	if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) {
+		msg_info ("statfile_pool_unlock_file: file %s is not opened", filename);
+		return;
+	}
+
+	memory_pool_unlock_mutex (file->lock);
+}
+
+uint32_t 
+statfile_pool_get_block (statfile_pool_t *pool, char *filename, uint32_t h1, uint32_t h2, time_t now)
+{
+	stat_file_t *file;
+	struct stat_file_block *block;
+	struct stat_file_header *header;
+	unsigned int i, blocknum;
+	u_char *c;
+	
+	if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) {
+		msg_info ("statfile_pool_get_block: file %s is not opened", filename);
+		return 0;
+	}
+	
+	file->access_time = now;
+	if (!file->map) {
+		return 0;
+	}
+	
+	blocknum = h1 % file->blocks;
+	header = (struct stat_file_header *)file->map;
+	c = (u_char *)file->map + sizeof (struct stat_file_header) + blocknum * sizeof (struct stat_file_block);
+	block = (struct stat_file_block *)c;
+
+	for (i = 0; i < CHAIN_LENGTH; i ++) {
+		if (i + blocknum > file->blocks) {
+			break;
+		}
+		msg_debug ("statfile_pool_get_block: test block with h1=%u, h2=%u, number %u in chain %u", block->hash1, block->hash2, i, blocknum);
+		if (block->hash1 == h1 && block->hash2 == h2) {
+			msg_debug ("statfile_pool_get_block: found block with h1=%u, h2=%u, number %u in chain %u", h1, h2, i, blocknum);
+			block->last_access = now - (time_t)header->create_time;
+			return block->value;
+		}
+		c += sizeof (struct stat_file_block);
+		block = (struct stat_file_block *)c;
+	}
+
+	msg_debug ("statfile_pool_get_block: block with h1=%u, h2=%u, not found in chain %u", h1, h2, blocknum);
+
+	return 0;
+}
+
+void
+statfile_pool_set_block (statfile_pool_t *pool, char *filename, uint32_t h1, uint32_t h2, time_t now, uint32_t value)
+{
+	stat_file_t *file;
+	struct stat_file_block *block, *to_expire = NULL;
+	struct stat_file_header *header;
+	unsigned int i, blocknum, oldest = 0;
+	u_char *c;
+	
+	if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) {
+		msg_info ("statfile_pool_set_block: file %s is not opened", filename);
+		return;
+	}
+	
+	file->access_time = now;
+	if (!file->map) {
+		return;
+	}
+	
+	blocknum = h1 % file->blocks;
+	header = (struct stat_file_header *)file->map;
+	c = (u_char *)file->map + sizeof (struct stat_file_header) + blocknum * sizeof (struct stat_file_block);
+	block = (struct stat_file_block *)c;
+
+	for (i = 0; i < CHAIN_LENGTH; i ++) {
+		if (i + blocknum > file->blocks) {
+			/* Need to expire some block in chain */
+			msg_debug ("statfile_pool_set_block: chain %u is full, starting expire", blocknum);
+			break;
+		}
+		/* Check whether we have a free block in chain */
+		if (block->hash1 == 0 && block->hash2 == 0) {
+			/* Write new block here */
+			msg_debug ("statfile_pool_set_block: found free block %u in chain %u, set h1=%u, h2=%u", i, blocknum, h1, h2);
+			block->hash1 = h1;
+			block->hash2 = h2;
+			block->value = value;
+			block->last_access = now - (time_t)header->create_time;
+			return;
+		}
+		if (block->last_access > oldest) {
+			to_expire = block;
+		}
+		c += sizeof (struct stat_file_block);
+		block = (struct stat_file_block *)c;
+	}
+
+	/* Try expire some block */
+	if (to_expire) {
+		block = to_expire;
+	}
+	else {
+		/* Expire first block in chain */
+		c = (u_char *)file->map + sizeof (struct stat_file_header) + blocknum * sizeof (struct stat_file_block);
+		block = (struct stat_file_block *)c;
+	}
+	block->last_access = now - (time_t)header->create_time;
+	block->hash1 = h1;
+	block->hash2 = h2;
+	block->value = value;
+}
+
+int
+statfile_pool_is_open (statfile_pool_t *pool, char *filename)
+{
+	return g_hash_table_lookup (pool->files, filename) != NULL;
+}
diff --git a/src/statfile.h b/src/statfile.h
index 801d1dae2..624ea64ee 100644
--- a/src/statfile.h
+++ b/src/statfile.h
@@ -14,11 +14,14 @@
 #endif
 #include "mem_pool.h"
 
+#define CHAIN_LENGTH 128
+
 struct stat_file_header {
 	u_char magic[3];
 	u_char version[2];
+	u_char padding[3];
 	uint64_t create_time;
-};
+} __attribute__((__packed__));
 
 struct stat_file_block {
 	uint32_t hash1;
@@ -57,5 +60,10 @@ int statfile_pool_open (statfile_pool_t *pool, char *filename);
 int statfile_pool_create (statfile_pool_t *pool, char *filename, size_t len);
 int statfile_pool_close (statfile_pool_t *pool, char *filename);
 void statfile_pool_delete (statfile_pool_t *pool);
+void statfile_pool_lock_file (statfile_pool_t *pool, char *filename);
+void statfile_pool_unlock_file (statfile_pool_t *pool, char *filename);
+uint32_t statfile_pool_get_block (statfile_pool_t *pool, char *filename, uint32_t h1, uint32_t h2, time_t now);
+void statfile_pool_set_block (statfile_pool_t *pool, char *filename, uint32_t h1, uint32_t h2, time_t now, uint32_t value);
+int statfile_pool_is_open (statfile_pool_t *pool, char *filename);
 
 #endif
diff --git a/test/.depends b/test/.depends
index 2c4e7865e..984f4699c 100644
--- a/test/.depends
+++ b/test/.depends
@@ -5,3 +5,4 @@
 ../src/url.c
 ../src/util.c
 ../src/memcached.c
+../src/statfile.c
diff --git a/test/rspamd_statfile_test.c b/test/rspamd_statfile_test.c
new file mode 100644
index 000000000..6537bcfaf
--- /dev/null
+++ b/test/rspamd_statfile_test.c
@@ -0,0 +1,59 @@
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <syslog.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "../src/config.h"
+#include "../src/main.h"
+#include "../src/statfile.h"
+#include "tests.h"
+
+#define TEST_FILENAME "/tmp/rspamd_test.stat"
+#define HASHES_NUM 1024
+
+void 
+rspamd_statfile_test_func ()
+{
+	statfile_pool_t *pool;
+	uint32_t random_hashes[HASHES_NUM], i, v;
+	time_t now;
+	
+	umask (S_IWGRP | S_IWOTH);
+	pool = statfile_pool_new (10 * 1024 * 1024);
+
+	now = time (NULL);
+	/* Fill random array */
+	srand (now);
+	for (i = 0; i < HASHES_NUM; i ++) {
+		random_hashes[i] = rand ();
+	}
+
+	/* Create new file */
+	g_assert (statfile_pool_create (pool, TEST_FILENAME, 65535) != -1);
+	g_assert (statfile_pool_open (pool, TEST_FILENAME) != -1);
+	
+	/* Get and set random blocks */
+	statfile_pool_lock_file (pool, TEST_FILENAME);
+	for (i = 0; i < HASHES_NUM; i ++) {
+		statfile_pool_set_block (pool, TEST_FILENAME, random_hashes[i], random_hashes[i], now, random_hashes[i]);
+	}
+	statfile_pool_unlock_file (pool, TEST_FILENAME);
+
+	for (i = 0; i < HASHES_NUM; i ++) {
+		v = statfile_pool_get_block (pool, TEST_FILENAME, random_hashes[i], random_hashes[i], now);
+		g_assert(v == random_hashes[i]);
+	}
+
+	statfile_pool_delete (pool);
+	
+}
diff --git a/test/rspamd_test_suite.c b/test/rspamd_test_suite.c
index 08de6fcf4..81291aa06 100644
--- a/test/rspamd_test_suite.c
+++ b/test/rspamd_test_suite.c
@@ -24,6 +24,7 @@ main (int argc, char **argv)
 	g_test_add_func ("/rspamd/mem_pool", rspamd_mem_pool_test_func);
 	g_test_add_func ("/rspamd/url", rspamd_url_test_func);
 	g_test_add_func ("/rspamd/expression", rspamd_expression_test_func);
+	g_test_add_func ("/rspamd/statfile", rspamd_statfile_test_func);
 
 	g_test_run ();
 }
diff --git a/test/tests.h b/test/tests.h
index 7ff763623..ed852f2d3 100644
--- a/test/tests.h
+++ b/test/tests.h
@@ -17,4 +17,7 @@ void rspamd_mem_pool_test_func ();
 /* Expressions */
 void rspamd_expression_test_func ();
 
+/* Stat file */
+void rspamd_statfile_test_func ();
+
 #endif