* Rework structure and API of statfiles functions to improve performance and avoid missusage of hash table

* Correct url length calculation in urls command

1 files changed, 85 insertions, 112 deletions

diff --git a/src/statfile.c b/src/statfile.c
index 008154194..ac0c3bfaa 100644
--- a/src/statfile.c
+++ b/src/statfile.c
@@ -27,6 +27,17 @@
 #include "statfile.h"
 #include "main.h"
 
+/* Maximum number of statistics files */
+#define STATFILES_MAX 255
+
+static int
+cmpstatfile (const void *a, const void *b)
+{
+	const stat_file_t *s1 = a, *s2 = b;
+
+	return rspamd_strcase_equal (s1->filename, s2->filename);
+}
+
 /* Check whether specified file is statistic file and calculate its len in blocks */
 static int
 statfile_pool_check (stat_file_t *file)
@@ -67,34 +78,32 @@ struct expiration_data {
 	char *filename;
 };
 
-static void
-pool_expiration_callback (gpointer key, gpointer value, void *data)
-{
-	struct expiration_data *exp = data;
-	stat_file_t *file = (stat_file_t *)value;
-
-	if ((uint64_t)file->access_time < exp->oldest) {
-		exp->oldest = file->access_time;
-		exp->filename = file->filename;
-	}
-}
 
 static int
 statfile_pool_expire (statfile_pool_t *pool)
 {
 	struct expiration_data exp;
+	stat_file_t *file;
+	int i;
 
-	if (rspamd_hash_size (pool->files) == 0) {
+	if (pool->opened == 0) {
 		return -1;
 	}
 
 	exp.pool = pool;
 	exp.oldest = ULLONG_MAX;
 	exp.filename = NULL;
-	rspamd_hash_foreach (pool->files, pool_expiration_callback, &exp);
+
+	for (i = 0; i < pool->opened; i++) {
+		file = &pool->files[i];
+		if ((uint64_t)file->access_time < exp.oldest) {
+			exp.oldest = file->access_time;
+			exp.filename = file->filename;
+		}
+	}
 
 	if (exp.filename) {
-		statfile_pool_close (pool, exp.filename, TRUE);
+		statfile_pool_close (pool, file, TRUE);
 	}
 
 	return 0;
@@ -109,63 +118,66 @@ statfile_pool_new (size_t max_size)
 	bzero (new, sizeof (statfile_pool_t));
 	new->pool = memory_pool_new (memory_pool_get_size ());
 	new->max = max_size;
-	new->files = rspamd_hash_new (new->pool, g_str_hash, g_str_equal);
-	new->maps = rspamd_hash_new_shared (new->pool, g_str_hash, g_str_equal, 64);
+	new->files = memory_pool_alloc_shared (new->pool, STATFILES_MAX * sizeof (stat_file_t));
 
 	return new;
 }
 
-int 
+stat_file_t *
 statfile_pool_open (statfile_pool_t *pool, char *filename)
 {
 	struct stat st;
 	stat_file_t *new_file;	
 	
-	if (rspamd_hash_lookup (pool->files, filename) != NULL) {
+	if (statfile_pool_is_open (pool, filename) != NULL) {
 		msg_info ("statfile_pool_open: file %s is already opened", filename);
-		return 0;
+		return NULL;
+	}
+
+	if (pool->opened >= STATFILES_MAX - 1) {
+		msg_err ("sttafile_pool_open: reached hard coded limit of statfiles opened: %d", STATFILES_MAX); 
+		return NULL;
 	}
 
 	if (stat (filename, &st) == -1) {
 		msg_info ("statfile_pool_open: cannot stat file %s, error %s, %d", filename, strerror (errno), errno);
-		return -1;
+		return NULL;
 	}
 	
 	if (st.st_size > pool->max) {
 		msg_info ("statfile_pool_open: cannot attach file to pool, too large: %zd", (size_t)st.st_size);
-		return -1;
+		return NULL;
 	}
 
 	while (pool->max <= pool->occupied + st.st_size) {
 		if (statfile_pool_expire (pool) == -1) {
 			/* Failed to find any more free space in pool */
 			msg_info ("statfile_pool_open: expiration for pool failed, opening file %s failed", filename);
-			return -1;
+			return NULL;
 		}
 	}
 
-	new_file = memory_pool_alloc (pool->pool, sizeof (stat_file_t));
+	new_file = &pool->files[pool->opened ++];
 	if ((new_file->fd = open (filename, O_RDWR)) == -1 ) {
 		msg_info ("statfile_pool_open: cannot open file %s, error %d, %s", filename, errno, strerror (errno));
-		return -1;
+		pool->opened --;
+		return NULL;
 	}
 	
-	/* First try to search mmapped area in already opened areas */
-	if ((new_file->map = rspamd_hash_lookup (pool->maps, filename)) == NULL) {
-		if ((new_file->map = mmap (NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, new_file->fd, 0)) == NULL) {
-			close (new_file->fd);
-			msg_info ("statfile_pool_open: cannot mmap file %s, error %d, %s", filename, errno, strerror (errno));
-			return -1;
-		
-		}
-		rspamd_hash_insert (pool->maps, filename, new_file->map);
+	if ((new_file->map = mmap (NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, new_file->fd, 0)) == NULL) {
+		close (new_file->fd);
+		msg_info ("statfile_pool_open: cannot mmap file %s, error %d, %s", filename, errno, strerror (errno));
+		pool->opened --;
+		return NULL;
+	
 	}
 	
 	/* XXX: this is temporary copy of name to avoid strdup early */
 	new_file->filename = filename;
 	new_file->len = st.st_size;
 	if (statfile_pool_check (new_file) == -1) {
-		return -1;
+		pool->opened --;
+		return NULL;
 	}
 
 	pool->occupied += st.st_size;
@@ -173,18 +185,20 @@ statfile_pool_open (statfile_pool_t *pool, char *filename)
 	new_file->open_time = time (NULL);
 	new_file->access_time = new_file->open_time;
 	new_file->lock = memory_pool_get_mutex (pool->pool);
-	rspamd_hash_insert (pool->files, new_file->filename, new_file);
+	
+	/* Keep sorted */
+	qsort (pool->files, pool->opened, sizeof (stat_file_t), cmpstatfile);
 
-	return 0;
+	return new_file;
 }
 
 int
-statfile_pool_close (statfile_pool_t *pool, char *filename, gboolean remove_hash)
+statfile_pool_close (statfile_pool_t *pool, stat_file_t *file, gboolean keep_sorted)
 {
-	stat_file_t *file;	
-	
-	if ((file = rspamd_hash_lookup (pool->files, filename)) == NULL) {
-		msg_info ("statfile_pool_open: file %s is not opened", filename);
+	stat_file_t *pos;
+
+	if ((pos = statfile_pool_is_open (pool, file->filename)) == NULL) {
+		msg_info ("statfile_pool_open: file %s is not opened", file->filename);
 		return -1;
 	}
 	
@@ -194,14 +208,16 @@ statfile_pool_close (statfile_pool_t *pool, char *filename, gboolean remove_hash
 
 	if (file->map) {
 		munmap (file->map, file->len);
-		rspamd_hash_remove (pool->maps, filename);
 	}
 	if (file->fd != -1) {
 		close (file->fd);
 	}
 	pool->occupied -= file->len;
-	if (remove_hash) {
-		rspamd_hash_remove (pool->files, file->filename);
+	
+	if (keep_sorted) {
+		memmove (pos, &pool->files[pool->opened--], sizeof (stat_file_t));
+		/* Keep sorted */
+		qsort (pool->files, pool->opened, sizeof (stat_file_t), cmpstatfile);
 	}
 	return 0;
 }
@@ -220,8 +236,8 @@ statfile_pool_create (statfile_pool_t *pool, char *filename, size_t blocks)
 	struct stat_file_block block = {0, 0, 0, 0};
 	int fd;
 	
-	if (rspamd_hash_lookup (pool->files, filename) != NULL) {
-		msg_info ("statfile_pool_create: file %s is already opened", filename);
+	if (statfile_pool_is_open (pool, filename) != NULL) {
+		msg_info ("statfile_pool_open: file %s is already opened", filename);
 		return 0;
 	}
 
@@ -257,61 +273,40 @@ statfile_pool_create (statfile_pool_t *pool, char *filename, size_t blocks)
 	return 0;
 }
 
-static void
-pool_delete_callback (gpointer key, gpointer value, void *data)
-{
-	statfile_pool_t *pool = (statfile_pool_t *)data;
-
-	statfile_pool_close (pool, (char *)key, FALSE);
-}
-
 void
 statfile_pool_delete (statfile_pool_t *pool)
 {
-	rspamd_hash_foreach (pool->files, pool_delete_callback, pool);
+	int i;
+
+	for (i = 0; i < pool->opened; i ++) {
+		statfile_pool_close (pool, &pool->files[i], FALSE);
+	}
 	memory_pool_delete (pool->pool);
 	g_free (pool);
 }
 
 void
-statfile_pool_lock_file (statfile_pool_t *pool, char *filename) 
+statfile_pool_lock_file (statfile_pool_t *pool, stat_file_t *file) 
 {
-	stat_file_t *file;
-
-	if ((file = rspamd_hash_lookup (pool->files, filename)) == NULL) {
-		msg_info ("statfile_pool_lock_file: file %s is not opened", filename);
-		return;
-	}
 
 	memory_pool_lock_mutex (file->lock);
 }
 
 void
-statfile_pool_unlock_file (statfile_pool_t *pool, char *filename) 
+statfile_pool_unlock_file (statfile_pool_t *pool, stat_file_t *file) 
 {
-	stat_file_t *file;
-
-	if ((file = rspamd_hash_lookup (pool->files, filename)) == NULL) {
-		msg_info ("statfile_pool_unlock_file: file %s is not opened", filename);
-		return;
-	}
 
 	memory_pool_unlock_mutex (file->lock);
 }
 
 float
-statfile_pool_get_block (statfile_pool_t *pool, char *filename, uint32_t h1, uint32_t h2, time_t now)
+statfile_pool_get_block (statfile_pool_t *pool, stat_file_t *file, uint32_t h1, uint32_t h2, time_t now)
 {
-	stat_file_t *file;
 	struct stat_file_block *block;
 	struct stat_file_header *header;
 	unsigned int i, blocknum;
 	u_char *c;
 	
-	if ((file = rspamd_hash_lookup (pool->files, filename)) == NULL) {
-		msg_info ("statfile_pool_get_block: file %s is not opened", filename);
-		return 0;
-	}
 	
 	file->access_time = now;
 	if (!file->map) {
@@ -340,18 +335,13 @@ statfile_pool_get_block (statfile_pool_t *pool, char *filename, uint32_t h1, uin
 }
 
 void
-statfile_pool_set_block (statfile_pool_t *pool, char *filename, uint32_t h1, uint32_t h2, time_t now, float value)
+statfile_pool_set_block (statfile_pool_t *pool, stat_file_t *file, uint32_t h1, uint32_t h2, time_t now, float value)
 {
-	stat_file_t *file;
 	struct stat_file_block *block, *to_expire = NULL;
 	struct stat_file_header *header;
 	unsigned int i, blocknum, oldest = 0;
 	u_char *c;
 	
-	if ((file = rspamd_hash_lookup (pool->files, filename)) == NULL) {
-		msg_info ("statfile_pool_set_block: file %s is not opened", filename);
-		return;
-	}
 	
 	file->access_time = now;
 	if (!file->map) {
@@ -407,36 +397,27 @@ statfile_pool_set_block (statfile_pool_t *pool, char *filename, uint32_t h1, uin
 	block->value = value;
 }
 
-gboolean
+stat_file_t *
 statfile_pool_is_open (statfile_pool_t *pool, char *filename)
 {
-	return (rspamd_hash_lookup (pool->files, filename) != NULL);
+	static stat_file_t f;
+	f.filename = filename;
+	return bsearch (&f, pool->files, pool->opened, sizeof (stat_file_t), cmpstatfile);
 }
 
 uint32_t
-statfile_pool_get_section (statfile_pool_t *pool, char *filename)
+statfile_pool_get_section (statfile_pool_t *pool, stat_file_t *file)
 {
-	stat_file_t *file;
 
-	if ((file = rspamd_hash_lookup (pool->files, filename)) == NULL) {
-		msg_info ("statfile_pool_get_section: file %s is not opened", filename);
-		return 0;
-	}
-	
 	return file->cur_section.code;
 }
 
 gboolean 
-statfile_pool_set_section (statfile_pool_t *pool, char *filename, uint32_t code, gboolean from_begin)
+statfile_pool_set_section (statfile_pool_t *pool, stat_file_t *file, uint32_t code, gboolean from_begin)
 {
-	stat_file_t *file;
 	struct stat_file_section *sec;
 	off_t cur_offset;
 
-	if ((file = rspamd_hash_lookup (pool->files, filename)) == NULL) {
-		msg_info ("statfile_pool_set_section: file %s is not opened", filename);
-		return FALSE;
-	}
 	
 	/* Try to find section */
 	if (from_begin) {
@@ -460,19 +441,13 @@ statfile_pool_set_section (statfile_pool_t *pool, char *filename, uint32_t code,
 }
 
 gboolean 
-statfile_pool_add_section (statfile_pool_t *pool, char *filename, uint32_t code, uint64_t length)
+statfile_pool_add_section (statfile_pool_t *pool, stat_file_t *file, uint32_t code, uint64_t length)
 {
-	stat_file_t *file;
 	struct stat_file_section sect;
 	struct stat_file_block block = {0, 0, 0, 0};
 	
-	if ((file = rspamd_hash_lookup (pool->files, filename)) == NULL) {
-		msg_info ("statfile_pool_add_section: file %s is not opened", filename);
-		return FALSE;
-	}
-
 	if (lseek (file->fd, 0, SEEK_END) == -1) {
-		msg_info ("statfile_pool_add_section: cannot lseek file %s, error %d, %s", filename, errno, strerror (errno));
+		msg_info ("statfile_pool_add_section: cannot lseek file %s, error %d, %s", file->filename, errno, strerror (errno));
 		return FALSE;
 	}
 	
@@ -480,20 +455,19 @@ statfile_pool_add_section (statfile_pool_t *pool, char *filename, uint32_t code,
 	sect.length = length;
 
 	if (write (file->fd, &sect, sizeof (sect)) == -1) {
-		msg_info ("statfile_pool_add_section: cannot write block to file %s, error %d, %s", filename, errno, strerror (errno));
+		msg_info ("statfile_pool_add_section: cannot write block to file %s, error %d, %s", file->filename, errno, strerror (errno));
 		return FALSE;
 	}
 
 	while (length --) {
 		if (write (file->fd, &block, sizeof (block)) == -1) {
-			msg_info ("statfile_pool_add_section: cannot write block to file %s, error %d, %s", filename, errno, strerror (errno));
+			msg_info ("statfile_pool_add_section: cannot write block to file %s, error %d, %s", file->filename, errno, strerror (errno));
 			return FALSE;
 		}
 	}
 	
 	/* Lock statfile to remap memory */
-	statfile_pool_lock_file (pool, filename);
-	rspamd_hash_remove (pool->maps, filename);
+	statfile_pool_lock_file (pool, file);
 	munmap (file->map, file->len);
 	fsync (file->fd);
 	file->len += length;
@@ -506,16 +480,15 @@ statfile_pool_add_section (statfile_pool_t *pool, char *filename, uint32_t code,
 	while (pool->max <= pool->occupied + file->len) {
 		if (statfile_pool_expire (pool) == -1) {
 			/* Failed to find any more free space in pool */
-			msg_info ("statfile_pool_open: expiration for pool failed, opening file %s failed", filename);
+			msg_info ("statfile_pool_open: expiration for pool failed, opening file %s failed", file->filename);
 			return FALSE;
 		}
 	}
 	if ((file->map = mmap (NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0)) == NULL) {
-		msg_info ("statfile_pool_open: cannot mmap file %s, error %d, %s", filename, errno, strerror (errno));
+		msg_info ("statfile_pool_open: cannot mmap file %s, error %d, %s", file->filename, errno, strerror (errno));
 		return FALSE;
 	}
-	rspamd_hash_insert (pool->maps, filename, file->map);
-	statfile_pool_unlock_file (pool, filename);
+	statfile_pool_unlock_file (pool, file);
 
 	return TRUE;