* Improve logic of reading list files, add automatic rereading of list files while changing

4 files changed, 143 insertions, 75 deletions

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 0fccb14b2..fdc6e7f8d 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -33,85 +33,12 @@
 
 #include "surbl.h"
 
-static char *hash_fill = "1";
 struct surbl_ctx *surbl_module_ctx;
 GRegex *extract_hoster_regexp, *extract_normal_regexp, *extract_numeric_regexp;
 
 static int surbl_test_url (struct worker_task *task);
 static void dns_callback (int result, char type, int count, int ttl, void *addresses, void *data);
 
-static void
-parse_host_list (GHashTable *tbl, const char *filename)
-{
-	int fd;
-	char buf[BUFSIZ], str[BUFSIZ], *s, *p;
-	ssize_t r;
-	enum {
-		READ_SYMBOL,
-		SKIP_COMMENT,
-	} state = READ_SYMBOL;
-
-	if ((fd = open (filename, O_RDONLY)) == -1) {
-		msg_warn ("parse_host_list: cannot open file '%s': %s", filename, strerror (errno));
-		return;
-	}
-
-	s = str;
-
-	while ((r = read (fd, buf, sizeof (buf) - 1)) > 0) {
-		buf[r] = '\0';
-		p = buf;
-		while (*p) {
-			switch (state) {
-				case READ_SYMBOL:
-					if (*p == '#') {
-						if (s != str) {
-							*s = '\0';
-							s = memory_pool_strdup (surbl_module_ctx->surbl_pool, str);
-							g_hash_table_insert (tbl, s, hash_fill);
-							s = str;
-						}
-						state = SKIP_COMMENT;
-					}
-					else if (*p == '\r' || *p == '\n') {
-						if (s != str) {
-							*s = '\0';
-							s = memory_pool_strdup (surbl_module_ctx->surbl_pool, str);
-							g_hash_table_insert (tbl, s, hash_fill);
-							s = str;
-						}
-						while (*p == '\r' || *p == '\n') {
-							p ++;
-						}
-					}
-					else if (g_ascii_isspace (*p)) {
-						p ++;
-					}
-					else {
-						*s = *p;
-						s ++;
-						p ++;
-					}
-					break;
-				case SKIP_COMMENT:
-					if (*p == '\r' || *p == '\n') {
-						while (*p == '\r' || *p == '\n') {
-							p ++;
-						}
-						s = str;
-						state = READ_SYMBOL;
-					}
-					else {
-						p ++;
-					}
-					break;
-			}
-		}
-	}
-
-	close (fd);
-}
-
 int
 surbl_module_init (struct config_file *cfg, struct module_ctx **ctx)
 {
@@ -128,6 +55,8 @@ surbl_module_init (struct config_file *cfg, struct module_ctx **ctx)
 	surbl_module_ctx->bits = NULL;
 	surbl_module_ctx->surbl_pool = memory_pool_new (1024);
 	
+	surbl_module_ctx->tld2_file = NULL;
+	surbl_module_ctx->whitelist_file = NULL;
 	surbl_module_ctx->tld2 = g_hash_table_new (g_str_hash, g_str_equal);
 	/* Register destructors */
 	memory_pool_add_destructor (surbl_module_ctx->surbl_pool, (pool_destruct_func)g_hash_table_remove_all, surbl_module_ctx->tld2);
@@ -217,12 +146,16 @@ surbl_module_config (struct config_file *cfg)
 	}
 	if ((value = get_module_opt (cfg, "surbl", "2tld")) != NULL) {
 		if (g_ascii_strncasecmp (value, "file://", sizeof ("file://") - 1) == 0) {
-			parse_host_list (surbl_module_ctx->tld2, value + sizeof ("file://") - 1);
+			if (parse_host_list (surbl_module_ctx->surbl_pool, surbl_module_ctx->tld2, value + sizeof ("file://") - 1)) {
+				surbl_module_ctx->tld2_file = memory_pool_strdup (surbl_module_ctx->surbl_pool, value + sizeof ("file://") - 1);
+			}
 		}
 	}
 	if ((value = get_module_opt (cfg, "surbl", "whitelist")) != NULL) {
 		if (g_ascii_strncasecmp (value, "file://", sizeof ("file://") - 1) == 0) {
-			parse_host_list (surbl_module_ctx->whitelist, value + sizeof ("file://") - 1);
+			if (parse_host_list (surbl_module_ctx->surbl_pool, surbl_module_ctx->whitelist, value + sizeof ("file://") - 1)) {
+				surbl_module_ctx->whitelist_file = memory_pool_strdup (surbl_module_ctx->surbl_pool, value + sizeof ("file://") - 1);
+			}
 		}
 	}
 	
@@ -724,6 +657,14 @@ surbl_test_url (struct worker_task *task)
 	struct mime_text_part *part;
 	struct redirector_param param;
 
+	/* Try to check lists */
+	if (surbl_module_ctx->tld2_file) {
+		maybe_parse_host_list (surbl_module_ctx->surbl_pool, surbl_module_ctx->tld2, surbl_module_ctx->tld2_file, task->ts.tv_sec);
+	}
+	if (surbl_module_ctx->whitelist_file) {
+		maybe_parse_host_list (surbl_module_ctx->surbl_pool, surbl_module_ctx->whitelist, surbl_module_ctx->whitelist_file, task->ts.tv_sec);
+	}
+
 	url_tree = g_tree_new ((GCompareFunc)g_ascii_strcasecmp);
 	
 	param.tree = url_tree;
diff --git a/src/plugins/surbl.h b/src/plugins/surbl.h
index 4849e29f0..3c24f83f2 100644
--- a/src/plugins/surbl.h
+++ b/src/plugins/surbl.h
@@ -32,6 +32,8 @@ struct surbl_ctx {
 	GList *suffixes;
 	GList *bits;
 	char *metric;
+	const char *tld2_file;
+	const char *whitelist_file;
 	GHashTable *tld2;
 	GHashTable *whitelist;
 	unsigned use_redirector;
diff --git a/src/util.c b/src/util.c
index eb57d6c88..3b151ed63 100644
--- a/src/util.c
+++ b/src/util.c
@@ -28,8 +28,19 @@
 #include "cfg_file.h"
 #include "main.h"
 
+/* Value in seconds after whitch we would try to do stat on list file */
+#define MON_TIMEOUT 10
+
 sig_atomic_t do_reopen_log = 0;
 extern rspamd_hash_t *counters;
+static char *hash_fill = "1";
+
+struct list_file {
+	char *filename;
+	struct stat st;
+};
+
+static GHashTable *listfiles = NULL;
 
 int
 make_socket_nonblocking (int fd)
@@ -839,6 +850,116 @@ set_counter (const char *name, long int value)
 	}
 }
 
+gboolean
+parse_host_list (memory_pool_t *pool, GHashTable *tbl, const char *filename)
+{
+	int fd;
+	char buf[BUFSIZ], str[BUFSIZ], *s, *p;
+	ssize_t r;
+	enum {
+		READ_SYMBOL,
+		SKIP_COMMENT,
+	} state = READ_SYMBOL;
+
+	struct list_file *new_file;
+
+	if (listfiles == NULL) {
+		listfiles = g_hash_table_new (g_str_hash, g_str_equal);
+	}
+	
+	if ((fd = open (filename, O_RDONLY)) == -1) {
+		msg_warn ("parse_host_list: cannot open file '%s': %s", filename, strerror (errno));
+		return FALSE;
+	}
+
+	new_file = memory_pool_alloc (pool, sizeof (struct list_file));
+	new_file->filename = memory_pool_strdup (pool, filename);
+	fstat (fd, &new_file->st);
+	g_hash_table_insert (listfiles, new_file->filename, new_file);
+
+	s = str;
+
+	while ((r = read (fd, buf, sizeof (buf) - 1)) > 0) {
+		buf[r] = '\0';
+		p = buf;
+		while (*p) {
+			switch (state) {
+				case READ_SYMBOL:
+					if (*p == '#') {
+						if (s != str) {
+							*s = '\0';
+							s = memory_pool_strdup (pool, str);
+							g_hash_table_insert (tbl, s, hash_fill);
+							s = str;
+						}
+						state = SKIP_COMMENT;
+					}
+					else if (*p == '\r' || *p == '\n') {
+						if (s != str) {
+							*s = '\0';
+							s = memory_pool_strdup (pool, str);
+							g_hash_table_insert (tbl, s, hash_fill);
+							s = str;
+						}
+						while (*p == '\r' || *p == '\n') {
+							p ++;
+						}
+					}
+					else if (g_ascii_isspace (*p)) {
+						p ++;
+					}
+					else {
+						*s = *p;
+						s ++;
+						p ++;
+					}
+					break;
+				case SKIP_COMMENT:
+					if (*p == '\r' || *p == '\n') {
+						while (*p == '\r' || *p == '\n') {
+							p ++;
+						}
+						s = str;
+						state = READ_SYMBOL;
+					}
+					else {
+						p ++;
+					}
+					break;
+			}
+		}
+	}
+
+	close (fd);
+
+	return TRUE;
+}
+
+gboolean
+maybe_parse_host_list (memory_pool_t *pool, GHashTable *tbl, const char *filename, time_t tv)
+{
+	struct list_file *lf;
+	struct stat cur_st;
+
+	if (listfiles == NULL || (lf = g_hash_table_lookup (listfiles, filename)) == NULL) {
+		/* Do not try to parse unknown files */
+		return FALSE;
+	}
+	
+	if (tv - lf->st.st_mtime > MON_TIMEOUT) {
+		/* Try to stat */
+		if (stat (lf->filename, &cur_st) != -1) {
+			if (cur_st.st_mtime > lf->st.st_mtime) {
+				g_hash_table_remove (listfiles, lf->filename);
+				g_hash_table_remove_all (tbl);
+				parse_host_list (pool, tbl, filename);
+			}
+		}
+	}
+
+	return TRUE;
+}
+
 /*
  * vi:ts=4
  */
diff --git a/src/util.h b/src/util.h
index 2727ef4a5..d771e74d8 100644
--- a/src/util.h
+++ b/src/util.h
@@ -59,4 +59,8 @@ const char* calculate_check_time (struct timespec *begin, int resolution);
 
 void set_counter (const char *name, long int value);
 
+gboolean parse_host_list (memory_pool_t *pool, GHashTable *tbl, const char *filename);
+gboolean maybe_parse_host_list (memory_pool_t *pool, GHashTable *tbl, const char *filename, time_t tv);
+
+
 #endif