Implement statfiles runtime initialization.

3 files changed, 281 insertions, 259 deletions

diff --git a/src/libstat/backends/backends.h b/src/libstat/backends/backends.h
index 45a8cd314..02041410c 100644
--- a/src/libstat/backends/backends.h
+++ b/src/libstat/backends/backends.h
@@ -38,9 +38,11 @@ struct rspamd_stat_ctx;
 struct rspamd_stat_backend {
 	const char *name;
 	gpointer (*init)(struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg);
+	gpointer (*runtime)(struct rspamd_statfile_config *stcf, gpointer ctx);
 	gpointer ctx;
 };
 
 gpointer rspamd_mmaped_file_init(struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg);
+gpointer rspamd_mmaped_file_runtime (struct rspamd_statfile_config *stcf, gpointer ctx);
 
 #endif /* BACKENDS_H_ */
diff --git a/src/libstat/backends/mmaped_file.c b/src/libstat/backends/mmaped_file.c
index f703f7f5b..b487f5c96 100644
--- a/src/libstat/backends/mmaped_file.c
+++ b/src/libstat/backends/mmaped_file.c
@@ -97,6 +97,7 @@ typedef struct {
 	time_t access_time;                     /**< last access time					*/
 	size_t len;                             /**< length of file(in bytes)			*/
 	rspamd_mempool_mutex_t *lock;               /**< mutex								*/
+	struct rspamd_statfile_config *cf;
 } rspamd_mmaped_file_t;
 
 /**
@@ -119,9 +120,250 @@ static void rspamd_mmaped_file_set_block_common (
 	gboolean from_now);
 
 rspamd_mmaped_file_t * rspamd_mmaped_file_is_open (
-		rspamd_mmaped_file_ctx * pool, gchar *filename);
+		rspamd_mmaped_file_ctx * pool, struct rspamd_statfile_config *stcf);
 rspamd_mmaped_file_t * rspamd_mmaped_file_open (rspamd_mmaped_file_ctx * pool,
-		const gchar *filename, size_t size);
+		const gchar *filename, size_t size, struct rspamd_statfile_config *stcf);
+gint rspamd_mmaped_file_create (rspamd_mmaped_file_ctx * pool,
+		const gchar *filename, size_t size, struct rspamd_statfile_config *stcf);
+
+void
+rspamd_mmaped_file_lock_file (rspamd_mmaped_file_ctx * pool, rspamd_mmaped_file_t * file)
+{
+	rspamd_mempool_lock_mutex (file->lock);
+}
+
+void
+rspamd_mmaped_file_unlock_file (rspamd_mmaped_file_ctx * pool, rspamd_mmaped_file_t * file)
+{
+
+	rspamd_mempool_unlock_mutex (file->lock);
+}
+
+double
+rspamd_mmaped_file_get_block (rspamd_mmaped_file_ctx * pool,
+	rspamd_mmaped_file_t * file,
+	guint32 h1,
+	guint32 h2,
+	time_t now)
+{
+	struct stat_file_block *block;
+	guint i, blocknum;
+	u_char *c;
+
+
+	file->access_time = now;
+	if (!file->map) {
+		return 0;
+	}
+
+	blocknum = h1 % file->cur_section.length;
+	c = (u_char *) file->map + file->seek_pos + blocknum *
+		sizeof (struct stat_file_block);
+	block = (struct stat_file_block *)c;
+
+	for (i = 0; i < CHAIN_LENGTH; i++) {
+		if (i + blocknum >= file->cur_section.length) {
+			break;
+		}
+		if (block->hash1 == h1 && block->hash2 == h2) {
+			return block->value;
+		}
+		c += sizeof (struct stat_file_block);
+		block = (struct stat_file_block *)c;
+	}
+
+
+	return 0;
+}
+
+static void
+rspamd_mmaped_file_set_block_common (rspamd_mmaped_file_ctx * pool,
+		rspamd_mmaped_file_t * file,
+	guint32 h1,
+	guint32 h2,
+	time_t t,
+	double value,
+	gboolean from_now)
+{
+	struct stat_file_block *block, *to_expire = NULL;
+	struct stat_file_header *header;
+	guint i, blocknum;
+	u_char *c;
+	double min = G_MAXDOUBLE;
+
+	if (from_now) {
+		file->access_time = t;
+	}
+	if (!file->map) {
+		return;
+	}
+
+	blocknum = h1 % file->cur_section.length;
+	header = (struct stat_file_header *)file->map;
+	c = (u_char *) file->map + file->seek_pos + blocknum *
+		sizeof (struct stat_file_block);
+	block = (struct stat_file_block *)c;
+
+	for (i = 0; i < CHAIN_LENGTH; i++) {
+		if (i + blocknum >= file->cur_section.length) {
+			/* Need to expire some block in chain */
+			msg_info ("chain %ud is full in statfile %s, starting expire",
+				blocknum,
+				file->filename);
+			break;
+		}
+		/* First try to find block in chain */
+		if (block->hash1 == h1 && block->hash2 == h2) {
+			block->value = value;
+			return;
+		}
+		/* Check whether we have a free block in chain */
+		if (block->hash1 == 0 && block->hash2 == 0) {
+			/* Write new block here */
+			msg_debug ("found free block %ud in chain %ud, set h1=%ud, h2=%ud",
+				i,
+				blocknum,
+				h1,
+				h2);
+			block->hash1 = h1;
+			block->hash2 = h2;
+			block->value = value;
+			header->used_blocks++;
+
+			return;
+		}
+
+		/* Expire block with minimum value otherwise */
+		if (block->value < min) {
+			to_expire = block;
+			min = block->value;
+		}
+		c += sizeof (struct stat_file_block);
+		block = (struct stat_file_block *)c;
+	}
+
+	/* Try expire some block */
+	if (to_expire) {
+		block = to_expire;
+	}
+	else {
+		/* Expire first block in chain */
+		c = (u_char *) file->map + file->seek_pos + blocknum *
+			sizeof (struct stat_file_block);
+		block = (struct stat_file_block *)c;
+	}
+
+	block->hash1 = h1;
+	block->hash2 = h2;
+	block->value = value;
+}
+
+void
+rspamd_mmaped_file_set_block (rspamd_mmaped_file_ctx * pool,
+	rspamd_mmaped_file_t * file,
+	guint32 h1,
+	guint32 h2,
+	time_t now,
+	double value)
+{
+	rspamd_mmaped_file_set_block_common (pool, file, h1, h2, now, value, TRUE);
+}
+
+rspamd_mmaped_file_t *
+rspamd_mmaped_file_is_open (rspamd_mmaped_file_ctx * pool,
+		struct rspamd_statfile_config *stcf)
+{
+	return g_hash_table_lookup (pool->files, stcf);
+}
+
+
+
+gboolean
+rspamd_mmaped_file_set_revision (rspamd_mmaped_file_t *file, guint64 rev, time_t time)
+{
+	struct stat_file_header *header;
+
+	if (file == NULL || file->map == NULL) {
+		return FALSE;
+	}
+
+	header = (struct stat_file_header *)file->map;
+
+	header->revision = rev;
+	header->rev_time = time;
+
+	return TRUE;
+}
+
+gboolean
+rspamd_mmaped_file_inc_revision (rspamd_mmaped_file_t *file)
+{
+	struct stat_file_header *header;
+
+	if (file == NULL || file->map == NULL) {
+		return FALSE;
+	}
+
+	header = (struct stat_file_header *)file->map;
+
+	header->revision++;
+
+	return TRUE;
+}
+
+gboolean
+rspamd_mmaped_file_get_revision (rspamd_mmaped_file_t *file, guint64 *rev, time_t *time)
+{
+	struct stat_file_header *header;
+
+	if (file == NULL || file->map == NULL) {
+		return FALSE;
+	}
+
+	header = (struct stat_file_header *)file->map;
+
+	if (rev != NULL) {
+		*rev = header->revision;
+	}
+	if (time != NULL) {
+		*time = header->rev_time;
+	}
+
+	return TRUE;
+}
+
+guint64
+rspamd_mmaped_file_get_used (rspamd_mmaped_file_t *file)
+{
+	struct stat_file_header *header;
+
+	if (file == NULL || file->map == NULL) {
+		return (guint64) - 1;
+	}
+
+	header = (struct stat_file_header *)file->map;
+
+	return header->used_blocks;
+}
+
+guint64
+rspamd_mmaped_file_get_total (rspamd_mmaped_file_t *file)
+{
+	struct stat_file_header *header;
+
+	if (file == NULL || file->map == NULL) {
+		return (guint64) - 1;
+	}
+
+	header = (struct stat_file_header *)file->map;
+
+	/* If total blocks is 0 we have old version of header, so set total blocks correctly */
+	if (header->total_blocks == 0) {
+		header->total_blocks = file->cur_section.length;
+	}
+
+	return header->total_blocks;
+}
 
 /* Check whether specified file is statistic file and calculate its len in blocks */
 static gint
@@ -183,9 +425,10 @@ rspamd_mmaped_file_check (rspamd_mmaped_file_t * file)
 
 static rspamd_mmaped_file_t *
 rspamd_mmaped_file_reindex (rspamd_mmaped_file_ctx * pool,
-	gchar *filename,
+	const gchar *filename,
 	size_t old_size,
-	size_t size)
+	size_t size,
+	struct rspamd_statfile_config *stcf)
 {
 	gchar *backup;
 	gint fd;
@@ -218,14 +461,14 @@ rspamd_mmaped_file_reindex (rspamd_mmaped_file_ctx * pool,
 	rspamd_mempool_unlock_mutex (pool->lock);
 
 	/* Now create new file with required size */
-	if (rspamd_mmaped_file_create (pool, filename, size) != 0) {
+	if (rspamd_mmaped_file_create (pool, filename, size, stcf) != 0) {
 		msg_err ("cannot create new file");
 		g_free (backup);
 		return NULL;
 	}
 	/* Now open new file and start copying */
 	fd = open (backup, O_RDONLY);
-	new = rspamd_mmaped_file_open (pool, filename, size);
+	new = rspamd_mmaped_file_open (pool, filename, size, stcf);
 
 	if (fd == -1 || new == NULL) {
 		msg_err ("cannot open file: %s", strerror (errno));
@@ -302,12 +545,13 @@ rspamd_mmaped_file_preload (rspamd_mmaped_file_t *file)
 
 rspamd_mmaped_file_t *
 rspamd_mmaped_file_open (rspamd_mmaped_file_ctx * pool,
-		const gchar *filename, size_t size)
+		const gchar *filename, size_t size,
+		struct rspamd_statfile_config *stcf)
 {
 	struct stat st;
 	rspamd_mmaped_file_t *new_file;
 
-	if ((new_file = rspamd_mmaped_file_is_open (pool, filename)) != NULL) {
+	if ((new_file = rspamd_mmaped_file_is_open (pool, stcf)) != NULL) {
 		return new_file;
 	}
 
@@ -323,7 +567,7 @@ rspamd_mmaped_file_open (rspamd_mmaped_file_ctx * pool,
 		rspamd_mempool_unlock_mutex (pool->lock);
 		msg_warn ("need to reindex statfile old size: %Hz, new size: %Hz",
 			(size_t)st.st_size, size);
-		return rspamd_mmaped_file_reindex (pool, filename, st.st_size, size);
+		return rspamd_mmaped_file_reindex (pool, filename, st.st_size, size, stcf);
 	}
 	else if (size < sizeof (struct stat_file)) {
 		msg_err ("requested to shrink statfile to %Hz but it is too small",
@@ -380,24 +624,24 @@ rspamd_mmaped_file_open (rspamd_mmaped_file_ctx * pool,
 	new_file->open_time = time (NULL);
 	new_file->access_time = new_file->open_time;
 	new_file->lock = rspamd_mempool_get_mutex (pool->pool);
+	new_file->cf = stcf;
 
 	rspamd_mmaped_file_preload (new_file);
 
-	g_hash_table_insert (pool->files, new_file->filename, new_file);
+	g_hash_table_insert (pool->files, stcf, new_file);
 
 	rspamd_mempool_unlock_mutex (pool->lock);
 
-	return rspamd_mmaped_file_is_open (pool, filename);
+	return rspamd_mmaped_file_is_open (pool, stcf);
 }
 
 gint
 rspamd_mmaped_file_close (rspamd_mmaped_file_ctx * pool,
-	rspamd_mmaped_file_t * file,
-	gboolean keep_sorted)
+	rspamd_mmaped_file_t * file)
 {
 	rspamd_mmaped_file_t *pos;
 
-	if ((pos = rspamd_mmaped_file_is_open (pool, file->filename)) == NULL) {
+	if ((pos = rspamd_mmaped_file_is_open (pool, file->cf)) == NULL) {
 		msg_info ("file %s is not opened", file->filename);
 		return -1;
 	}
@@ -413,7 +657,7 @@ rspamd_mmaped_file_close (rspamd_mmaped_file_ctx * pool,
 		close (file->fd);
 	}
 
-	g_hash_table_remove (pool->files, file->filename);
+	g_hash_table_remove (pool->files, file->cf);
 
 	g_slice_free1 (sizeof (*file), file);
 
@@ -423,7 +667,8 @@ rspamd_mmaped_file_close (rspamd_mmaped_file_ctx * pool,
 }
 
 gint
-rspamd_mmaped_file_create (rspamd_mmaped_file_ctx * pool, gchar *filename, size_t size)
+rspamd_mmaped_file_create (rspamd_mmaped_file_ctx * pool, const gchar *filename,
+		size_t size, struct rspamd_statfile_config *stcf)
 {
 	struct stat_file_header header = {
 		.magic = {'r', 's', 'd'},
@@ -441,7 +686,7 @@ rspamd_mmaped_file_create (rspamd_mmaped_file_ctx * pool, gchar *filename, size_
 	guint buflen = 0, nblocks;
 	gchar *buf = NULL;
 
-	if (rspamd_mmaped_file_is_open (pool, filename) != NULL) {
+	if (rspamd_mmaped_file_is_open (pool, stcf) != NULL) {
 		msg_info ("file %s is already opened", filename);
 		return 0;
 	}
@@ -555,251 +800,13 @@ rspamd_mmaped_file_destroy (rspamd_mmaped_file_ctx * pool)
 	g_hash_table_iter_init (&it, pool->files);
 	while (g_hash_table_iter_next (&it, &k, &v)) {
 		f = (rspamd_mmaped_file_t *)v;
-		rspamd_mmaped_file_close (pool, f, FALSE);
+		rspamd_mmaped_file_close (pool, f);
 	}
 
 	g_hash_table_destroy (pool->files);
 	rspamd_mempool_delete (pool->pool);
 }
 
-void
-rspamd_mmaped_file_lock_file (rspamd_mmaped_file_ctx * pool, rspamd_mmaped_file_t * file)
-{
-	rspamd_mempool_lock_mutex (file->lock);
-}
-
-void
-rspamd_mmaped_file_unlock_file (rspamd_mmaped_file_ctx * pool, rspamd_mmaped_file_t * file)
-{
-
-	rspamd_mempool_unlock_mutex (file->lock);
-}
-
-double
-rspamd_mmaped_file_get_block (rspamd_mmaped_file_ctx * pool,
-	rspamd_mmaped_file_t * file,
-	guint32 h1,
-	guint32 h2,
-	time_t now)
-{
-	struct stat_file_block *block;
-	guint i, blocknum;
-	u_char *c;
-
-
-	file->access_time = now;
-	if (!file->map) {
-		return 0;
-	}
-
-	blocknum = h1 % file->cur_section.length;
-	c = (u_char *) file->map + file->seek_pos + blocknum *
-		sizeof (struct stat_file_block);
-	block = (struct stat_file_block *)c;
-
-	for (i = 0; i < CHAIN_LENGTH; i++) {
-		if (i + blocknum >= file->cur_section.length) {
-			break;
-		}
-		if (block->hash1 == h1 && block->hash2 == h2) {
-			return block->value;
-		}
-		c += sizeof (struct stat_file_block);
-		block = (struct stat_file_block *)c;
-	}
-
-
-	return 0;
-}
-
-static void
-rspamd_mmaped_file_set_block_common (rspamd_mmaped_file_ctx * pool,
-		rspamd_mmaped_file_t * file,
-	guint32 h1,
-	guint32 h2,
-	time_t t,
-	double value,
-	gboolean from_now)
-{
-	struct stat_file_block *block, *to_expire = NULL;
-	struct stat_file_header *header;
-	guint i, blocknum;
-	u_char *c;
-	double min = G_MAXDOUBLE;
-
-	if (from_now) {
-		file->access_time = t;
-	}
-	if (!file->map) {
-		return;
-	}
-
-	blocknum = h1 % file->cur_section.length;
-	header = (struct stat_file_header *)file->map;
-	c = (u_char *) file->map + file->seek_pos + blocknum *
-		sizeof (struct stat_file_block);
-	block = (struct stat_file_block *)c;
-
-	for (i = 0; i < CHAIN_LENGTH; i++) {
-		if (i + blocknum >= file->cur_section.length) {
-			/* Need to expire some block in chain */
-			msg_info ("chain %ud is full in statfile %s, starting expire",
-				blocknum,
-				file->filename);
-			break;
-		}
-		/* First try to find block in chain */
-		if (block->hash1 == h1 && block->hash2 == h2) {
-			block->value = value;
-			return;
-		}
-		/* Check whether we have a free block in chain */
-		if (block->hash1 == 0 && block->hash2 == 0) {
-			/* Write new block here */
-			msg_debug ("found free block %ud in chain %ud, set h1=%ud, h2=%ud",
-				i,
-				blocknum,
-				h1,
-				h2);
-			block->hash1 = h1;
-			block->hash2 = h2;
-			block->value = value;
-			header->used_blocks++;
-
-			return;
-		}
-
-		/* Expire block with minimum value otherwise */
-		if (block->value < min) {
-			to_expire = block;
-			min = block->value;
-		}
-		c += sizeof (struct stat_file_block);
-		block = (struct stat_file_block *)c;
-	}
-
-	/* Try expire some block */
-	if (to_expire) {
-		block = to_expire;
-	}
-	else {
-		/* Expire first block in chain */
-		c = (u_char *) file->map + file->seek_pos + blocknum *
-			sizeof (struct stat_file_block);
-		block = (struct stat_file_block *)c;
-	}
-
-	block->hash1 = h1;
-	block->hash2 = h2;
-	block->value = value;
-}
-
-void
-rspamd_mmaped_file_set_block (rspamd_mmaped_file_ctx * pool,
-	rspamd_mmaped_file_t * file,
-	guint32 h1,
-	guint32 h2,
-	time_t now,
-	double value)
-{
-	rspamd_mmaped_file_set_block_common (pool, file, h1, h2, now, value, TRUE);
-}
-
-rspamd_mmaped_file_t *
-rspamd_mmaped_file_is_open (rspamd_mmaped_file_ctx * pool, gchar *filename)
-{
-	return g_hash_table_lookup (pool->files, filename);
-}
-
-
-
-gboolean
-rspamd_mmaped_file_set_revision (rspamd_mmaped_file_t *file, guint64 rev, time_t time)
-{
-	struct stat_file_header *header;
-
-	if (file == NULL || file->map == NULL) {
-		return FALSE;
-	}
-
-	header = (struct stat_file_header *)file->map;
-
-	header->revision = rev;
-	header->rev_time = time;
-
-	return TRUE;
-}
-
-gboolean
-rspamd_mmaped_file_inc_revision (rspamd_mmaped_file_t *file)
-{
-	struct stat_file_header *header;
-
-	if (file == NULL || file->map == NULL) {
-		return FALSE;
-	}
-
-	header = (struct stat_file_header *)file->map;
-
-	header->revision++;
-
-	return TRUE;
-}
-
-gboolean
-rspamd_mmaped_file_get_revision (rspamd_mmaped_file_t *file, guint64 *rev, time_t *time)
-{
-	struct stat_file_header *header;
-
-	if (file == NULL || file->map == NULL) {
-		return FALSE;
-	}
-
-	header = (struct stat_file_header *)file->map;
-
-	if (rev != NULL) {
-		*rev = header->revision;
-	}
-	if (time != NULL) {
-		*time = header->rev_time;
-	}
-
-	return TRUE;
-}
-
-guint64
-rspamd_mmaped_file_get_used (rspamd_mmaped_file_t *file)
-{
-	struct stat_file_header *header;
-
-	if (file == NULL || file->map == NULL) {
-		return (guint64) - 1;
-	}
-
-	header = (struct stat_file_header *)file->map;
-
-	return header->used_blocks;
-}
-
-guint64
-rspamd_mmaped_file_get_total (rspamd_mmaped_file_t *file)
-{
-	struct stat_file_header *header;
-
-	if (file == NULL || file->map == NULL) {
-		return (guint64) - 1;
-	}
-
-	header = (struct stat_file_header *)file->map;
-
-	/* If total blocks is 0 we have old version of header, so set total blocks correctly */
-	if (header->total_blocks == 0) {
-		header->total_blocks = file->cur_section.length;
-	}
-
-	return header->total_blocks;
-}
-
 gpointer
 rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg)
 {
@@ -807,7 +814,6 @@ rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg)
 	struct rspamd_classifier_config *clf;
 	struct rspamd_statfile_config *stf;
 	GList *cur, *curst;
-	rspamd_mmaped_file_t *mf;
 	const ucl_object_t *filenameo, *sizeo;
 	const gchar *filename;
 	gsize size;
@@ -815,7 +821,7 @@ rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg)
 	new = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (rspamd_mmaped_file_ctx));
 	new->lock = rspamd_mempool_get_mutex (new->pool);
 	new->mlock_ok = cfg->mlock_statfile_pool;
-	new->files = g_hash_table_new (g_str_hash, g_str_equal);
+	new->files = g_hash_table_new (g_direct_hash, g_direct_equal);
 
 	/* Iterate over all classifiers and load matching statfiles */
 	cur = cfg->classifiers;
@@ -851,7 +857,7 @@ rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg)
 
 				size = ucl_object_toint (sizeo);
 
-				rspamd_mmaped_file_open (new, filename, size);
+				rspamd_mmaped_file_open (new, filename, size, stf);
 
 				ctx->statfiles ++;
 			}
@@ -864,3 +870,16 @@ rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg)
 
 	return (gpointer)new;
 }
+
+gpointer
+rspamd_mmaped_file_runtime (struct rspamd_statfile_config *stcf, gpointer p)
+{
+	rspamd_mmaped_file_ctx *ctx = (rspamd_mmaped_file_ctx *)p;
+	rspamd_mmaped_file_t *mf;
+
+	g_assert (ctx != NULL);
+
+	mf = rspamd_mmaped_file_is_open (ctx, stcf);
+
+	return (gpointer)mf;
+}
diff --git a/src/libstat/stat_internal.h b/src/libstat/stat_internal.h
index 15b5f03af..6587f595e 100644
--- a/src/libstat/stat_internal.h
+++ b/src/libstat/stat_internal.h
@@ -31,6 +31,7 @@
 
 struct rspamd_statfile_runtime {
 	struct rspamd_statfile_config *st;
+	gpointer statfile_data;
 	guint64 hits;
 	guint64 total_hits;
 };