* Add ability to check bits from surbl replies

diff --git a/rspamd.conf.sample b/rspamd.conf.sample
index 591f8876c3f88d1ea27f0e2ca59d5dbbc923f961..bac7dc9228e77e657776813a440904cbb578170c 100644 (file)
--- a/rspamd.conf.sample
+++ b/rspamd.conf.sample
@@ -107,8 +107,20 @@ delivery {
        redirector_connect_timeout = "1s";
        # IO timeout for redirector (may be usefull to set this value rather big)
        redirector_read_timeout = "10s";
-       # This is suffix for surbl dns requests
-       suffix_SURBL_MULTI = "multi.surbl.org";
+       # This is suffix for surbl dns requests,
+       # %b is replaced with bit metric if it is found
+       suffix_%b_SURBL_MULTI = "multi.surbl.org";
+
+       # Bits that are used to determine specific URI black list
+       # details are at http://www.surbl.org/lists.html#multi
+       # sytax is: bit_{number} = "SYMBOL"
+       bit_2 = "SC"; # sc.surbl.org
+       bit_4 = "WS"; # ws.surbl.org
+       bit_8 = "PH"; # ph.surbl.org
+       bit_16 = "OB"; # ob.surbl.org
+       bit_32 = "AB"; # ab.surbl.org
+       bit_64 = "JP"; # jp.surbl.org
+
        # Metric for surbl module
        metric = "default";
        # List of public known hostings (for which we should use 3 components of domain name instead of 2)

diff --git a/src/cfg_file.l b/src/cfg_file.l
index b28bbd26365309a5e2b8d6ca39ca8e4961a75216..7216cda7ea82c4034782dfbaa56b3b327fae45a1 100644 (file)
--- a/src/cfg_file.l
+++ b/src/cfg_file.l
@@ -19,7 +19,7 @@ extern struct config_file *cfg;
 %option yylineno
 
 %%
-^[ \t]*#.*                                             /* ignore comments */;
+[ \t]*#.*                                              /* ignore comments */;
 .include                                               BEGIN(incl);
 .module                                                        BEGIN(module);
 composites                                             return COMPOSITES;
@@ -133,13 +133,13 @@ yes|YES|no|NO|[yY]|[nN]                   yylval.flag=parse_flag(yytext); return FLAG;
 
 <module>\n                                                             /* ignore EOL */;
 <module>[ \t]+                                                 /* ignore whitespace */;
-<module>^[ \t]*#.*                                             /* ignore comments */;
+<module>[ \t]*#.*                                              /* ignore comments */;
 <module>\'[a-zA-Z0-9_-]+\'     yylval.string=strdup(yytext + 1); yylval.string[strlen(yylval.string) - 1] = '\0'; return MODULE_OPT; 
 <module>\{     return OBRACE;
 <module>\}  BEGIN(INITIAL); return EBRACE;
 <module>\;     return SEMICOLON;
 <module>=      return EQSIGN;
-<module>[a-zA-Z0-9_-]+ yylval.string=strdup(yytext); return PARAM;
+<module>[a-zA-Z0-9_%-]+        yylval.string=strdup(yytext); return PARAM;
 <module>\$[a-zA-Z_][a-zA-Z0-9_]+               yylval.string=strdup(yytext + 1); return VARIABLE;
 <module>\"[^"]+\"      yylval.string=strdup(yytext + 1); yylval.string[strlen(yylval.string) - 1] = '\0'; return QUOTEDSTRING;
 

diff --git a/src/lmtp.c b/src/lmtp.c
index 9503255a4794ac0ca0573389460e65915a462dc9..aa5c034ff2377f4e763c9d18fe4c97a23bdfe485 100644 (file)
--- a/src/lmtp.c
+++ b/src/lmtp.c
@@ -269,12 +269,6 @@ start_lmtp_worker (struct rspamd_worker *worker)
 
        init_signals (&signals, sig_handler);
        sigprocmask (SIG_UNBLOCK, &signals.sa_mask, NULL);
-       /* Ignore SIGPIPE for further use in LDA delivery */
-       sigemptyset (&signals.sa_mask);
-       sigaddset (&signals.sa_mask, SIGPIPE);
-       signals.sa_handler = SIG_IGN;
-       sigaction (SIGPIPE, &signals, NULL);
-       sigprocmask (SIG_UNBLOCK, &signals.sa_mask, NULL);
 
        /* SIGUSR2 handler */
        signal_set (&worker->sig_ev, SIGUSR2, sigusr_handler, (void *) worker);

diff --git a/src/mem_pool.c b/src/mem_pool.c
index eb722fb9254a1036c3bea23eecc9cfad5ca80ec8..23f25b5f39aeae8198d9f610bdc807585644ec3b 100644 (file)
--- a/src/mem_pool.c
+++ b/src/mem_pool.c
@@ -351,7 +351,10 @@ memory_pool_delete (memory_pool_t *pool)
        
        /* Call all pool destructors */
        while (destructor) {
-               destructor->func (destructor->data);
+               /* Avoid calling destructors for NULL pointers */
+               if (destructor->data != NULL) { 
+                       destructor->func (destructor->data);
+               }
                destructor = destructor->prev;
        }
 

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 5b9b355466ef440d613e9c9dd8a7c1d4c9637836..44b5d3fb8fa83e1766bffdbf4a8dc016eb8e30fd 100644 (file)
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -51,6 +51,7 @@ surbl_module_init (struct config_file *cfg, struct module_ctx **ctx)
        surbl_module_ctx->url_filter = surbl_test_url;
        surbl_module_ctx->use_redirector = 0;
        surbl_module_ctx->suffixes = NULL;
+       surbl_module_ctx->bits = NULL;
        surbl_module_ctx->surbl_pool = memory_pool_new (1024);
        
        surbl_module_ctx->hosters = g_hash_table_new (g_str_hash, g_str_equal);
@@ -60,6 +61,8 @@ surbl_module_init (struct config_file *cfg, struct module_ctx **ctx)
        surbl_module_ctx->whitelist = g_hash_table_new (g_str_hash, g_str_equal);
        /* Register destructors */
        memory_pool_add_destructor (surbl_module_ctx->surbl_pool, (pool_destruct_func)g_hash_table_remove_all, surbl_module_ctx->whitelist);
+       memory_pool_add_destructor (surbl_module_ctx->surbl_pool, (pool_destruct_func)g_list_free, surbl_module_ctx->suffixes);
+       memory_pool_add_destructor (surbl_module_ctx->surbl_pool, (pool_destruct_func)g_list_free, surbl_module_ctx->bits);
                
        /* Init matching regexps */
        extract_hoster_regexp = g_regex_new ("([^.]+)\\.([^.]+)\\.([^.]+)$", G_REGEX_RAW, 0, &err);
@@ -78,8 +81,10 @@ surbl_module_config (struct config_file *cfg)
        LIST_HEAD (moduleoptq, module_opt) *opt = NULL;
        struct module_opt *cur;
        struct suffix_item *new_suffix;
+       struct surbl_bit_item *new_bit;
 
        char *value, *cur_tok, *str;
+       uint32_t bit;
 
        evdns_init ();
 
@@ -131,12 +136,6 @@ surbl_module_config (struct config_file *cfg)
        else {
                surbl_module_ctx->max_urls = DEFAULT_SURBL_MAX_URLS;
        }
-       if ((value = get_module_opt (cfg, "surbl", "symbol")) != NULL) {
-               surbl_module_ctx->symbol = memory_pool_strdup (surbl_module_ctx->surbl_pool, value);
-       }
-       else {
-               surbl_module_ctx->symbol = DEFAULT_SURBL_SYMBOL;
-       }
        if ((value = get_module_opt (cfg, "surbl", "metric")) != NULL) {
                surbl_module_ctx->metric = memory_pool_strdup (surbl_module_ctx->surbl_pool, value);
        }
@@ -197,6 +196,19 @@ surbl_module_config (struct config_file *cfg)
                                        surbl_module_ctx->suffixes = g_list_prepend (surbl_module_ctx->suffixes, new_suffix);
                                }
                        }
+                       if (!g_strncasecmp (cur->param, "bit", sizeof ("bit") - 1)) {
+                               if ((str = strchr (cur->param, '_')) != NULL) {
+                                       bit = strtoul (str + 1, NULL, 10);
+                                       if (bit != 0) {
+                                               new_bit = memory_pool_alloc (surbl_module_ctx->surbl_pool, sizeof (struct surbl_bit_item));
+                                               new_bit->bit = bit;
+                                               new_bit->symbol = memory_pool_strdup (surbl_module_ctx->surbl_pool, cur->value);
+                                               msg_debug ("surbl_module_config: add new bit suffix: %d with symbol: %s", 
+                                                                       (int)new_bit->bit, new_bit->symbol);
+                                               surbl_module_ctx->bits = g_list_prepend (surbl_module_ctx->bits, new_bit);
+                                       }
+                               }
+                       }
                }
        }
        /* Add default suffix */
@@ -312,6 +324,42 @@ make_surbl_requests (struct uri* url, struct worker_task *task)
        }
 }
 
+static void
+process_dns_results (struct worker_task *task, struct suffix_item *suffix, uint32_t addr)
+{
+       char *c, *symbol;
+       GList *cur;
+       struct surbl_bit_item *bit;
+       int len, found = 0;
+
+       if ((c = strchr (suffix->symbol, '%')) != NULL && *(c + 1) == 'b') {
+               cur = g_list_first (surbl_module_ctx->bits);
+
+               while (cur) {
+                       bit = (struct surbl_bit_item *)cur->data;
+                       msg_debug ("process_dns_results: got result(%d) AND bit(%d): %d", (int)addr, (int)ntohl(bit->bit), 
+                                                       (int)bit->bit & (int)ntohl (addr));
+                       if (((int)bit->bit & (int)ntohl (addr)) != 0) {
+                               len = strlen (suffix->symbol) - 2 + strlen (bit->symbol) + 1;
+                               *c = '\0';
+                               symbol = memory_pool_alloc (task->task_pool, len);
+                               snprintf (symbol, len, "%s%s%s", suffix->symbol, bit->symbol, c + 2);
+                               *c = '%';
+                               insert_result (task, surbl_module_ctx->metric, symbol, 1);
+                               found = 1;
+                       }
+                       cur = g_list_next (cur);
+               }
+
+               if (!found) {
+                       insert_result (task, surbl_module_ctx->metric, suffix->symbol, 1);
+               }
+       }
+       else {
+               insert_result (task, surbl_module_ctx->metric, suffix->symbol, 1);
+       }
+}
+
 static void 
 dns_callback (int result, char type, int count, int ttl, void *addresses, void *data)
 {
@@ -324,7 +372,7 @@ dns_callback (int result, char type, int count, int ttl, void *addresses, void *
        /* If we have result from DNS server, this url exists in SURBL, so increase score */
        if (result == DNS_ERR_NONE && type == DNS_IPv4_A) {
                msg_info ("surbl_check: url %s is in surbl %s", param->url->host, param->suffix->suffix);
-               insert_result (param->task, surbl_module_ctx->metric, param->suffix->symbol, 1);
+               process_dns_results (param->task, param->suffix, (uint32_t)(((in_addr_t *)addresses)[0]));
        }
        else {
                msg_debug ("surbl_check: url %s is not in surbl %s", param->url->host, param->suffix->suffix);
@@ -379,7 +427,10 @@ memcached_callback (memcached_ctx_t *ctx, memc_error_t error, void *data)
                                /* Do not check DNS for urls that have count more than max_urls */
                                if (*url_count > surbl_module_ctx->max_urls) {
                                        msg_info ("memcached_callback: url '%s' has count %d, max: %d", struri (param->url), *url_count, surbl_module_ctx->max_urls);
-                                       insert_result (param->task, surbl_module_ctx->metric, surbl_module_ctx->symbol, 1);
+                                       /* 
+                                        * XXX: try to understand why we should use memcached here
+                                        * insert_result (param->task, surbl_module_ctx->metric, surbl_module_ctx->symbol, 1);
+                                        */
                                }
                                (*url_count) ++;
                                memc_set (param->ctx, param->ctx->param, surbl_module_ctx->url_expire);
@@ -474,6 +525,7 @@ redirector_callback (int fd, short what, void *arg)
                                        msg_err ("redirector_callback: write failed %s", strerror (errno));
                                        event_del (&param->ev);
                                        param->task->save.saved --;
+                                       make_surbl_requests (param->url, param->task);
                                        if (param->task->save.saved == 0) {
                                                /* Call other filters */
                                                param->task->save.saved = 1;

diff --git a/src/plugins/surbl.h b/src/plugins/surbl.h
index 196f4517c14b4acb49319db893830571c3b2484d..21d77c25b32056e92348cd0ea20700a591ae0458 100644 (file)
--- a/src/plugins/surbl.h
+++ b/src/plugins/surbl.h
@@ -30,7 +30,7 @@ struct surbl_ctx {
        unsigned int max_urls;
        unsigned int url_expire;
        GList *suffixes;
-       char *symbol;
+       GList *bits;
        char *metric;
        GHashTable *hosters;
        GHashTable *whitelist;
@@ -66,4 +66,9 @@ struct memcached_param {
        memcached_ctx_t *ctx;
 };
 
+struct surbl_bit_item {
+       uint32_t bit;
+       const char *symbol;
+};
+
 #endif