* Improve logic of reading list files, add automatic rereading of list files while...

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 0fccb14b25de7552be85e332d9bf877a352265f5..fdc6e7f8dc2b97a18bb7c413d8f7c717c4b22979 100644 (file)
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -33,85 +33,12 @@
 
 #include "surbl.h"
 
-static char *hash_fill = "1";
 struct surbl_ctx *surbl_module_ctx;
 GRegex *extract_hoster_regexp, *extract_normal_regexp, *extract_numeric_regexp;
 
 static int surbl_test_url (struct worker_task *task);
 static void dns_callback (int result, char type, int count, int ttl, void *addresses, void *data);
 
-static void
-parse_host_list (GHashTable *tbl, const char *filename)
-{
-       int fd;
-       char buf[BUFSIZ], str[BUFSIZ], *s, *p;
-       ssize_t r;
-       enum {
-               READ_SYMBOL,
-               SKIP_COMMENT,
-       } state = READ_SYMBOL;
-
-       if ((fd = open (filename, O_RDONLY)) == -1) {
-               msg_warn ("parse_host_list: cannot open file '%s': %s", filename, strerror (errno));
-               return;
-       }
-
-       s = str;
-
-       while ((r = read (fd, buf, sizeof (buf) - 1)) > 0) {
-               buf[r] = '\0';
-               p = buf;
-               while (*p) {
-                       switch (state) {
-                               case READ_SYMBOL:
-                                       if (*p == '#') {
-                                               if (s != str) {
-                                                       *s = '\0';
-                                                       s = memory_pool_strdup (surbl_module_ctx->surbl_pool, str);
-                                                       g_hash_table_insert (tbl, s, hash_fill);
-                                                       s = str;
-                                               }
-                                               state = SKIP_COMMENT;
-                                       }
-                                       else if (*p == '\r' || *p == '\n') {
-                                               if (s != str) {
-                                                       *s = '\0';
-                                                       s = memory_pool_strdup (surbl_module_ctx->surbl_pool, str);
-                                                       g_hash_table_insert (tbl, s, hash_fill);
-                                                       s = str;
-                                               }
-                                               while (*p == '\r' || *p == '\n') {
-                                                       p ++;
-                                               }
-                                       }
-                                       else if (g_ascii_isspace (*p)) {
-                                               p ++;
-                                       }
-                                       else {
-                                               *s = *p;
-                                               s ++;
-                                               p ++;
-                                       }
-                                       break;
-                               case SKIP_COMMENT:
-                                       if (*p == '\r' || *p == '\n') {
-                                               while (*p == '\r' || *p == '\n') {
-                                                       p ++;
-                                               }
-                                               s = str;
-                                               state = READ_SYMBOL;
-                                       }
-                                       else {
-                                               p ++;
-                                       }
-                                       break;
-                       }
-               }
-       }
-
-       close (fd);
-}
-
 int
 surbl_module_init (struct config_file *cfg, struct module_ctx **ctx)
 {
@@ -128,6 +55,8 @@ surbl_module_init (struct config_file *cfg, struct module_ctx **ctx)
        surbl_module_ctx->bits = NULL;
        surbl_module_ctx->surbl_pool = memory_pool_new (1024);
        
+       surbl_module_ctx->tld2_file = NULL;
+       surbl_module_ctx->whitelist_file = NULL;
        surbl_module_ctx->tld2 = g_hash_table_new (g_str_hash, g_str_equal);
        /* Register destructors */
        memory_pool_add_destructor (surbl_module_ctx->surbl_pool, (pool_destruct_func)g_hash_table_remove_all, surbl_module_ctx->tld2);
@@ -217,12 +146,16 @@ surbl_module_config (struct config_file *cfg)
        }
        if ((value = get_module_opt (cfg, "surbl", "2tld")) != NULL) {
                if (g_ascii_strncasecmp (value, "file://", sizeof ("file://") - 1) == 0) {
-                       parse_host_list (surbl_module_ctx->tld2, value + sizeof ("file://") - 1);
+                       if (parse_host_list (surbl_module_ctx->surbl_pool, surbl_module_ctx->tld2, value + sizeof ("file://") - 1)) {
+                               surbl_module_ctx->tld2_file = memory_pool_strdup (surbl_module_ctx->surbl_pool, value + sizeof ("file://") - 1);
+                       }
                }
        }
        if ((value = get_module_opt (cfg, "surbl", "whitelist")) != NULL) {
                if (g_ascii_strncasecmp (value, "file://", sizeof ("file://") - 1) == 0) {
-                       parse_host_list (surbl_module_ctx->whitelist, value + sizeof ("file://") - 1);
+                       if (parse_host_list (surbl_module_ctx->surbl_pool, surbl_module_ctx->whitelist, value + sizeof ("file://") - 1)) {
+                               surbl_module_ctx->whitelist_file = memory_pool_strdup (surbl_module_ctx->surbl_pool, value + sizeof ("file://") - 1);
+                       }
                }
        }
        
@@ -724,6 +657,14 @@ surbl_test_url (struct worker_task *task)
        struct mime_text_part *part;
        struct redirector_param param;
 
+       /* Try to check lists */
+       if (surbl_module_ctx->tld2_file) {
+               maybe_parse_host_list (surbl_module_ctx->surbl_pool, surbl_module_ctx->tld2, surbl_module_ctx->tld2_file, task->ts.tv_sec);
+       }
+       if (surbl_module_ctx->whitelist_file) {
+               maybe_parse_host_list (surbl_module_ctx->surbl_pool, surbl_module_ctx->whitelist, surbl_module_ctx->whitelist_file, task->ts.tv_sec);
+       }
+
        url_tree = g_tree_new ((GCompareFunc)g_ascii_strcasecmp);
        
        param.tree = url_tree;

diff --git a/src/plugins/surbl.h b/src/plugins/surbl.h
index 4849e29f092dc9be5b4b835ab5e06fb6edb0e629..3c24f83f2d64a466ac8533d6860bcc45a67acd1c 100644 (file)
--- a/src/plugins/surbl.h
+++ b/src/plugins/surbl.h
@@ -32,6 +32,8 @@ struct surbl_ctx {
        GList *suffixes;
        GList *bits;
        char *metric;
+       const char *tld2_file;
+       const char *whitelist_file;
        GHashTable *tld2;
        GHashTable *whitelist;
        unsigned use_redirector;

diff --git a/src/util.c b/src/util.c
index eb57d6c884244736477cd882cc5f8c8cfe1401ca..3b151ed63e7934318aa41d8ce4fcfff3f3178b74 100644 (file)
--- a/src/util.c
+++ b/src/util.c
@@ -28,8 +28,19 @@
 #include "cfg_file.h"
 #include "main.h"
 
+/* Value in seconds after whitch we would try to do stat on list file */
+#define MON_TIMEOUT 10
+
 sig_atomic_t do_reopen_log = 0;
 extern rspamd_hash_t *counters;
+static char *hash_fill = "1";
+
+struct list_file {
+       char *filename;
+       struct stat st;
+};
+
+static GHashTable *listfiles = NULL;
 
 int
 make_socket_nonblocking (int fd)
@@ -839,6 +850,116 @@ set_counter (const char *name, long int value)
        }
 }
 
+gboolean
+parse_host_list (memory_pool_t *pool, GHashTable *tbl, const char *filename)
+{
+       int fd;
+       char buf[BUFSIZ], str[BUFSIZ], *s, *p;
+       ssize_t r;
+       enum {
+               READ_SYMBOL,
+               SKIP_COMMENT,
+       } state = READ_SYMBOL;
+
+       struct list_file *new_file;
+
+       if (listfiles == NULL) {
+               listfiles = g_hash_table_new (g_str_hash, g_str_equal);
+       }
+       
+       if ((fd = open (filename, O_RDONLY)) == -1) {
+               msg_warn ("parse_host_list: cannot open file '%s': %s", filename, strerror (errno));
+               return FALSE;
+       }
+
+       new_file = memory_pool_alloc (pool, sizeof (struct list_file));
+       new_file->filename = memory_pool_strdup (pool, filename);
+       fstat (fd, &new_file->st);
+       g_hash_table_insert (listfiles, new_file->filename, new_file);
+
+       s = str;
+
+       while ((r = read (fd, buf, sizeof (buf) - 1)) > 0) {
+               buf[r] = '\0';
+               p = buf;
+               while (*p) {
+                       switch (state) {
+                               case READ_SYMBOL:
+                                       if (*p == '#') {
+                                               if (s != str) {
+                                                       *s = '\0';
+                                                       s = memory_pool_strdup (pool, str);
+                                                       g_hash_table_insert (tbl, s, hash_fill);
+                                                       s = str;
+                                               }
+                                               state = SKIP_COMMENT;
+                                       }
+                                       else if (*p == '\r' || *p == '\n') {
+                                               if (s != str) {
+                                                       *s = '\0';
+                                                       s = memory_pool_strdup (pool, str);
+                                                       g_hash_table_insert (tbl, s, hash_fill);
+                                                       s = str;
+                                               }
+                                               while (*p == '\r' || *p == '\n') {
+                                                       p ++;
+                                               }
+                                       }
+                                       else if (g_ascii_isspace (*p)) {
+                                               p ++;
+                                       }
+                                       else {
+                                               *s = *p;
+                                               s ++;
+                                               p ++;
+                                       }
+                                       break;
+                               case SKIP_COMMENT:
+                                       if (*p == '\r' || *p == '\n') {
+                                               while (*p == '\r' || *p == '\n') {
+                                                       p ++;
+                                               }
+                                               s = str;
+                                               state = READ_SYMBOL;
+                                       }
+                                       else {
+                                               p ++;
+                                       }
+                                       break;
+                       }
+               }
+       }
+
+       close (fd);
+
+       return TRUE;
+}
+
+gboolean
+maybe_parse_host_list (memory_pool_t *pool, GHashTable *tbl, const char *filename, time_t tv)
+{
+       struct list_file *lf;
+       struct stat cur_st;
+
+       if (listfiles == NULL || (lf = g_hash_table_lookup (listfiles, filename)) == NULL) {
+               /* Do not try to parse unknown files */
+               return FALSE;
+       }
+       
+       if (tv - lf->st.st_mtime > MON_TIMEOUT) {
+               /* Try to stat */
+               if (stat (lf->filename, &cur_st) != -1) {
+                       if (cur_st.st_mtime > lf->st.st_mtime) {
+                               g_hash_table_remove (listfiles, lf->filename);
+                               g_hash_table_remove_all (tbl);
+                               parse_host_list (pool, tbl, filename);
+                       }
+               }
+       }
+
+       return TRUE;
+}
+
 /*
  * vi:ts=4
  */

diff --git a/src/util.h b/src/util.h
index 2727ef4a5b235c6f02bf2d019bce8de2e8d09d14..d771e74d83628a6b42725d94ad0b0632cb176983 100644 (file)
--- a/src/util.h
+++ b/src/util.h
@@ -59,4 +59,8 @@ const char* calculate_check_time (struct timespec *begin, int resolution);
 
 void set_counter (const char *name, long int value);
 
+gboolean parse_host_list (memory_pool_t *pool, GHashTable *tbl, const char *filename);
+gboolean maybe_parse_host_list (memory_pool_t *pool, GHashTable *tbl, const char *filename, time_t tv);
+
+
 #endif