* Write functions to operate blocks in stat files

* Write test case for statistics files API

diff --git a/src/statfile.c b/src/statfile.c
index ba735bcbf787d2db1b6885739d97f0fd3f36c2da..fc7ebfc7a5951dfd267c7a4e9568f7439f3aba09 100644 (file)
--- a/src/statfile.c
+++ b/src/statfile.c
@@ -38,12 +38,12 @@ statfile_pool_check (stat_file_t *file)
                return -1;
        }
        
-       if (file->len - sizeof (f->header) % sizeof (struct stat_file_block) != 0) {
+       if ((file->len - sizeof (f->header)) % sizeof (struct stat_file_block) != 0) {
                msg_info ("statfile_pool_check: file %s does not contain integer count of stat blocks", file->filename);
                return -1;
        }
 
-       file->blocks = file->len - sizeof (f->header) / sizeof (struct stat_file_block);
+       file->blocks = (file->len - sizeof (f->header)) / sizeof (struct stat_file_block);
        return 0;
 }
 
@@ -189,6 +189,7 @@ statfile_pool_create (statfile_pool_t *pool, char *filename, size_t blocks)
        static struct stat_file_header header = {
                {'r', 's', 'd'},
                {1, 0},
+               {0, 0, 0},
                0
        };
        static struct stat_file_block block = {0, 0, 0, 0};
@@ -199,7 +200,7 @@ statfile_pool_create (statfile_pool_t *pool, char *filename, size_t blocks)
                return 0;
        }
 
-       if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT)) == -1 ) {
+       if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1 ) {
                msg_info ("statfile_pool_create: cannot create file %s, error %m, %d", filename, errno);
                return -1;
        }
@@ -239,3 +240,140 @@ statfile_pool_delete (statfile_pool_t *pool)
        memory_pool_delete (pool->pool);
        g_free (pool);
 }
+
+void
+statfile_pool_lock_file (statfile_pool_t *pool, char *filename) 
+{
+       stat_file_t *file;
+
+       if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) {
+               msg_info ("statfile_pool_lock_file: file %s is not opened", filename);
+               return;
+       }
+
+       memory_pool_lock_mutex (file->lock);
+}
+
+void
+statfile_pool_unlock_file (statfile_pool_t *pool, char *filename) 
+{
+       stat_file_t *file;
+
+       if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) {
+               msg_info ("statfile_pool_unlock_file: file %s is not opened", filename);
+               return;
+       }
+
+       memory_pool_unlock_mutex (file->lock);
+}
+
+uint32_t 
+statfile_pool_get_block (statfile_pool_t *pool, char *filename, uint32_t h1, uint32_t h2, time_t now)
+{
+       stat_file_t *file;
+       struct stat_file_block *block;
+       struct stat_file_header *header;
+       unsigned int i, blocknum;
+       u_char *c;
+       
+       if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) {
+               msg_info ("statfile_pool_get_block: file %s is not opened", filename);
+               return 0;
+       }
+       
+       file->access_time = now;
+       if (!file->map) {
+               return 0;
+       }
+       
+       blocknum = h1 % file->blocks;
+       header = (struct stat_file_header *)file->map;
+       c = (u_char *)file->map + sizeof (struct stat_file_header) + blocknum * sizeof (struct stat_file_block);
+       block = (struct stat_file_block *)c;
+
+       for (i = 0; i < CHAIN_LENGTH; i ++) {
+               if (i + blocknum > file->blocks) {
+                       break;
+               }
+               msg_debug ("statfile_pool_get_block: test block with h1=%u, h2=%u, number %u in chain %u", block->hash1, block->hash2, i, blocknum);
+               if (block->hash1 == h1 && block->hash2 == h2) {
+                       msg_debug ("statfile_pool_get_block: found block with h1=%u, h2=%u, number %u in chain %u", h1, h2, i, blocknum);
+                       block->last_access = now - (time_t)header->create_time;
+                       return block->value;
+               }
+               c += sizeof (struct stat_file_block);
+               block = (struct stat_file_block *)c;
+       }
+
+       msg_debug ("statfile_pool_get_block: block with h1=%u, h2=%u, not found in chain %u", h1, h2, blocknum);
+
+       return 0;
+}
+
+void
+statfile_pool_set_block (statfile_pool_t *pool, char *filename, uint32_t h1, uint32_t h2, time_t now, uint32_t value)
+{
+       stat_file_t *file;
+       struct stat_file_block *block, *to_expire = NULL;
+       struct stat_file_header *header;
+       unsigned int i, blocknum, oldest = 0;
+       u_char *c;
+       
+       if ((file = g_hash_table_lookup (pool->files, filename)) == NULL) {
+               msg_info ("statfile_pool_set_block: file %s is not opened", filename);
+               return;
+       }
+       
+       file->access_time = now;
+       if (!file->map) {
+               return;
+       }
+       
+       blocknum = h1 % file->blocks;
+       header = (struct stat_file_header *)file->map;
+       c = (u_char *)file->map + sizeof (struct stat_file_header) + blocknum * sizeof (struct stat_file_block);
+       block = (struct stat_file_block *)c;
+
+       for (i = 0; i < CHAIN_LENGTH; i ++) {
+               if (i + blocknum > file->blocks) {
+                       /* Need to expire some block in chain */
+                       msg_debug ("statfile_pool_set_block: chain %u is full, starting expire", blocknum);
+                       break;
+               }
+               /* Check whether we have a free block in chain */
+               if (block->hash1 == 0 && block->hash2 == 0) {
+                       /* Write new block here */
+                       msg_debug ("statfile_pool_set_block: found free block %u in chain %u, set h1=%u, h2=%u", i, blocknum, h1, h2);
+                       block->hash1 = h1;
+                       block->hash2 = h2;
+                       block->value = value;
+                       block->last_access = now - (time_t)header->create_time;
+                       return;
+               }
+               if (block->last_access > oldest) {
+                       to_expire = block;
+               }
+               c += sizeof (struct stat_file_block);
+               block = (struct stat_file_block *)c;
+       }
+
+       /* Try expire some block */
+       if (to_expire) {
+               block = to_expire;
+       }
+       else {
+               /* Expire first block in chain */
+               c = (u_char *)file->map + sizeof (struct stat_file_header) + blocknum * sizeof (struct stat_file_block);
+               block = (struct stat_file_block *)c;
+       }
+       block->last_access = now - (time_t)header->create_time;
+       block->hash1 = h1;
+       block->hash2 = h2;
+       block->value = value;
+}
+
+int
+statfile_pool_is_open (statfile_pool_t *pool, char *filename)
+{
+       return g_hash_table_lookup (pool->files, filename) != NULL;
+}

diff --git a/src/statfile.h b/src/statfile.h
index 801d1dae214cb1e6b9cde804699a82fc8dea7d9f..624ea64ee62d9e4ff0a93bc32ea6f0adb868e237 100644 (file)
--- a/src/statfile.h
+++ b/src/statfile.h
@@ -14,11 +14,14 @@
 #endif
 #include "mem_pool.h"
 
+#define CHAIN_LENGTH 128
+
 struct stat_file_header {
        u_char magic[3];
        u_char version[2];
+       u_char padding[3];
        uint64_t create_time;
-};
+} __attribute__((__packed__));
 
 struct stat_file_block {
        uint32_t hash1;
@@ -57,5 +60,10 @@ int statfile_pool_open (statfile_pool_t *pool, char *filename);
 int statfile_pool_create (statfile_pool_t *pool, char *filename, size_t len);
 int statfile_pool_close (statfile_pool_t *pool, char *filename);
 void statfile_pool_delete (statfile_pool_t *pool);
+void statfile_pool_lock_file (statfile_pool_t *pool, char *filename);
+void statfile_pool_unlock_file (statfile_pool_t *pool, char *filename);
+uint32_t statfile_pool_get_block (statfile_pool_t *pool, char *filename, uint32_t h1, uint32_t h2, time_t now);
+void statfile_pool_set_block (statfile_pool_t *pool, char *filename, uint32_t h1, uint32_t h2, time_t now, uint32_t value);
+int statfile_pool_is_open (statfile_pool_t *pool, char *filename);
 
 #endif

diff --git a/test/.depends b/test/.depends
index 2c4e7865e690e58f6f25ea4ea6095cc8c52eb50e..984f4699ca10cc4741360c848ef2aa07dea4d09e 100644 (file)
--- a/test/.depends
+++ b/test/.depends
@@ -5,3 +5,4 @@
 ../src/url.c
 ../src/util.c
 ../src/memcached.c
+../src/statfile.c

diff --git a/test/rspamd_statfile_test.c b/test/rspamd_statfile_test.c
new file mode 100644 (file)
index 0000000..6537bcf
--- /dev/null
+++ b/test/rspamd_statfile_test.c
@@ -0,0 +1,59 @@
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <syslog.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "../src/config.h"
+#include "../src/main.h"
+#include "../src/statfile.h"
+#include "tests.h"
+
+#define TEST_FILENAME "/tmp/rspamd_test.stat"
+#define HASHES_NUM 1024
+
+void 
+rspamd_statfile_test_func ()
+{
+       statfile_pool_t *pool;
+       uint32_t random_hashes[HASHES_NUM], i, v;
+       time_t now;
+       
+       umask (S_IWGRP | S_IWOTH);
+       pool = statfile_pool_new (10 * 1024 * 1024);
+
+       now = time (NULL);
+       /* Fill random array */
+       srand (now);
+       for (i = 0; i < HASHES_NUM; i ++) {
+               random_hashes[i] = rand ();
+       }
+
+       /* Create new file */
+       g_assert (statfile_pool_create (pool, TEST_FILENAME, 65535) != -1);
+       g_assert (statfile_pool_open (pool, TEST_FILENAME) != -1);
+       
+       /* Get and set random blocks */
+       statfile_pool_lock_file (pool, TEST_FILENAME);
+       for (i = 0; i < HASHES_NUM; i ++) {
+               statfile_pool_set_block (pool, TEST_FILENAME, random_hashes[i], random_hashes[i], now, random_hashes[i]);
+       }
+       statfile_pool_unlock_file (pool, TEST_FILENAME);
+
+       for (i = 0; i < HASHES_NUM; i ++) {
+               v = statfile_pool_get_block (pool, TEST_FILENAME, random_hashes[i], random_hashes[i], now);
+               g_assert(v == random_hashes[i]);
+       }
+
+       statfile_pool_delete (pool);
+       
+}

diff --git a/test/rspamd_test_suite.c b/test/rspamd_test_suite.c
index 08de6fcf4674b7d5f3eeaa52c3f25ec7f5b96909..81291aa060472fb6c846bf14c5d85a6ac80584b5 100644 (file)
--- a/test/rspamd_test_suite.c
+++ b/test/rspamd_test_suite.c
@@ -24,6 +24,7 @@ main (int argc, char **argv)
        g_test_add_func ("/rspamd/mem_pool", rspamd_mem_pool_test_func);
        g_test_add_func ("/rspamd/url", rspamd_url_test_func);
        g_test_add_func ("/rspamd/expression", rspamd_expression_test_func);
+       g_test_add_func ("/rspamd/statfile", rspamd_statfile_test_func);
 
        g_test_run ();
 }

diff --git a/test/tests.h b/test/tests.h
index 7ff7636236b4764f4e66873bad5741010f60c03a..ed852f2d378fb6e752822ebaa3c7557d81ca15f4 100644 (file)
--- a/test/tests.h
+++ b/test/tests.h
@@ -17,4 +17,7 @@ void rspamd_mem_pool_test_func ();
 /* Expressions */
 void rspamd_expression_test_func ();
 
+/* Stat file */
+void rspamd_statfile_test_func ();
+
 #endif