* Make surbl checks working

diff --git a/rspamd.conf.sample b/rspamd.conf.sample
index 24bcda5989879947b2b268a507db60ce5e06e9dc..794638a5ea7c06a9e8c54d625e7bd9bd1efa3e43 100644 (file)
--- a/rspamd.conf.sample
+++ b/rspamd.conf.sample
@@ -46,7 +46,7 @@ logging {
 
 # Limit for statfile pool size
 # Default: 100M
-statfile_pool_size = 20M;
+statfile_pool_size = 40M;
 
 
 # Sample statfile definition
@@ -69,6 +69,8 @@ statfile {
 statfile {
        alias = "test.ham";
        pattern = "./test.ham";
-       weight = -1.0;
+       weight = -2.0;
        size = 10M;
 };
+
+url_filters = "surbl";

diff --git a/src/cfg_utils.c b/src/cfg_utils.c
index 310c54fd259a3c6126edf91d138fded19a88af72..99e7881b841fd1744459189e468f41899db9e09b 100644 (file)
--- a/src/cfg_utils.c
+++ b/src/cfg_utils.c
@@ -394,13 +394,14 @@ parse_filters_str (struct config_file *cfg, const char *str, enum script_type ty
        }
 
        p = strvec;
-       while (*p++) {
+       while (*p) {
                cur = NULL;
                /* Search modules from known C modules */
                for (i = 0; i < MODULES_NUM; i++) {
                        if (strcasecmp (modules[i].name, *p) == 0) {
                                cur = memory_pool_alloc (cfg->cfg_pool, sizeof (struct filter));
                                cur->type = C_FILTER;
+                               msg_debug ("parse_filters_str: found C filter %s", *p);
                                switch (type) {
                                        case SCRIPT_HEADER:
                                                cur->func_name = memory_pool_strdup (cfg->cfg_pool, *p);
@@ -424,6 +425,7 @@ parse_filters_str (struct config_file *cfg, const char *str, enum script_type ty
                }
                if (cur != NULL) {
                        /* Go to next iteration */
+                       p++;
                        continue;
                }
                cur = memory_pool_alloc (cfg->cfg_pool, sizeof (struct filter));
@@ -446,6 +448,7 @@ parse_filters_str (struct config_file *cfg, const char *str, enum script_type ty
                                LIST_INSERT_HEAD (&cfg->url_filters, cur, next);
                                break;
                }
+               p ++;
        }
 
        g_strfreev (strvec);

diff --git a/src/filter.c b/src/filter.c
index d4953caf2e9ea91913c6751dfacc52b541e2aec7..651aac26774082403c00a9790eb4311e11714e19 100644 (file)
--- a/src/filter.c
+++ b/src/filter.c
@@ -77,15 +77,17 @@ factor_consolidation_func (struct worker_task *task, const char *metric_name)
  * Call perl or C module function for specified part of message 
  */
 static void
-call_filter_by_name (struct worker_task *task, const char *name, enum script_type sc_type, enum filter_type filt_type)
+call_filter_by_name (struct worker_task *task, const char *name, enum filter_type filt_type, enum script_type sc_type)
 {
        struct module_ctx *c_module;
-
+       int res = 0;
+       
        switch (filt_type) {
                case C_FILTER:
                        c_module = g_hash_table_lookup (task->worker->srv->cfg->c_modules, name);
                        if (c_module) {
-                               switch (filt_type) {
+                               res = 1;
+                               switch (sc_type) {
                                        case SCRIPT_HEADER:
                                                c_module->header_filter (task);
                                                break;
@@ -100,9 +102,13 @@ call_filter_by_name (struct worker_task *task, const char *name, enum script_typ
                                                break;
                                }
                        }
+                       else {
+                               msg_debug ("call_filter_by_name: %s is not a C module", name);
+                       }
                        break;
                case PERL_FILTER:
-                       switch (filt_type) {
+                       res = 1;
+                       switch (sc_type) {
                                case SCRIPT_HEADER:
                                        perl_call_header_filter (name, task);
                                        break;
@@ -118,6 +124,8 @@ call_filter_by_name (struct worker_task *task, const char *name, enum script_typ
                        }
                        break;
        }
+
+       msg_debug ("call_filter_by_name: filter name: %s, result: %d", name, (int)res);
 }
 
 static void
@@ -403,12 +411,12 @@ statfiles_callback (gpointer key, gpointer value, void *arg)
                if ((res = g_hash_table_lookup (data->metrics, st->metric)) == NULL) {
                        res = memory_pool_alloc (task->task_pool, sizeof (struct statfile_result));
                        res->symbols = g_list_prepend (NULL, st->alias);
-                       res->weight = st->classifier->add_result_func (0, weight);
+                       res->weight = st->classifier->add_result_func (0, weight * st->weight);
                        g_hash_table_insert (data->metrics, st->metric, res);
                }
                else {
                        res->symbols = g_list_prepend (NULL, st->alias);
-                       res->weight = st->classifier->add_result_func (res->weight, weight);
+                       res->weight = st->classifier->add_result_func (res->weight, weight * st->weight);
                }
        }
        

diff --git a/src/main.c b/src/main.c
index c903d3ede0808576d80d3e6bd86acc41fb7fd6d4..f315367d7bd087ee8efcfdbb3fb43f0ea9852410 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -227,8 +227,10 @@ main (int argc, char **argv, char **env)
 
        rspamd->cfg->cfg_name = memory_pool_strdup (rspamd->cfg->cfg_pool, FIXED_CONFIG_FILE);
        read_cmd_line (argc, argv, rspamd->cfg);
-
-    msg_warn ("(main) starting...");
+    
+    /* First set logger to console logger */
+    cfg->log_fd = 2;
+       g_log_set_default_handler (file_log_function, cfg);
 
        #ifndef HAVE_SETPROCTITLE
        init_title (argc, argv, environ);
@@ -236,26 +238,27 @@ main (int argc, char **argv, char **env)
        
        f = fopen (rspamd->cfg->cfg_name , "r");
        if (f == NULL) {
-               msg_warn ("cannot open file: %s", rspamd->cfg->cfg_name );
+               msg_err ("main: cannot open file: %s", rspamd->cfg->cfg_name );
                return EBADF;
        }
        yyin = f;
 
        if (yyparse() != 0 || yynerrs > 0) {
-               msg_warn ("yyparse: cannot parse config file, %d errors", yynerrs);
+               msg_err ("main: cannot parse config file, %d errors", yynerrs);
                return EBADF;
        }
 
        fclose (f);
+    msg_info ("main: starting...");
        rspamd->cfg->cfg_name = memory_pool_strdup (rspamd->cfg->cfg_pool, rspamd->cfg->cfg_name );
 
        /* Strictly set temp dir */
     if (!rspamd->cfg->temp_dir) {
-               msg_warn ("tempdir is not set, trying to use $TMPDIR");
+               msg_warn ("main: tempdir is not set, trying to use $TMPDIR");
                rspamd->cfg->temp_dir = memory_pool_strdup (rspamd->cfg->cfg_pool, getenv ("TMPDIR"));
 
                if (!rspamd->cfg->temp_dir) {
-                       msg_warn ("$TMPDIR is empty too, using /tmp as default");
+                       msg_warn ("main: $TMPDIR is empty too, using /tmp as default");
                rspamd->cfg->temp_dir = memory_pool_strdup (rspamd->cfg->cfg_pool, "/tmp");
                }
     }

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 07e2376bfb4b8fdeaec1077d6c87d20adb2bf9ee..dc790f1bfbb158f9a67eeeb1f00a5e13a80a66f4 100644 (file)
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -238,25 +238,25 @@ surbl_module_reconfig (struct config_file *cfg)
 }
 
 static char *
-format_surbl_request (char *hostname) 
+format_surbl_request (memory_pool_t *pool, f_str_t *hostname) 
 {
        GMatchInfo *info;
-       char *result;
-
-       result = g_malloc (strlen (hostname) + strlen (surbl_module_ctx->suffix) + 1); 
+       char *result = NULL;
+    int len;
+    
+    len = hostname->len + strlen (surbl_module_ctx->suffix) + 2;
 
        /* First try to match numeric expression */
-       if (g_regex_match (extract_numeric_regexp, hostname, 0, &info) == TRUE) {
+       if (g_regex_match_full (extract_numeric_regexp, hostname->begin, hostname->len, 0, 0, &info, NULL) == TRUE) {
                gchar *octet1, *octet2, *octet3, *octet4;
-               octet1 = g_match_info_fetch (info, 0);
-               g_match_info_next (info, NULL);
-               octet2 = g_match_info_fetch (info, 0);
-               g_match_info_next (info, NULL);
-               octet3 = g_match_info_fetch (info, 0);
-               g_match_info_next (info, NULL);
-               octet4 = g_match_info_fetch (info, 0);
+               octet1 = g_match_info_fetch (info, 1);
+               octet2 = g_match_info_fetch (info, 2);
+               octet3 = g_match_info_fetch (info, 3);
+               octet4 = g_match_info_fetch (info, 4);
                g_match_info_free (info);
-               sprintf (result, "%s.%s.%s.%s.%s", octet4, octet3, octet2, octet1, surbl_module_ctx->suffix);
+               result = memory_pool_alloc (pool, len); 
+               msg_debug ("format_surbl_request: got numeric host for check: %s.%s.%s.%s", octet1, octet2, octet3, octet4);
+               snprintf (result, len, "%s.%s.%s.%s.%s", octet4, octet3, octet2, octet1, surbl_module_ctx->suffix);
                g_free (octet1);
                g_free (octet2);
                g_free (octet3);
@@ -265,26 +265,25 @@ format_surbl_request (char *hostname)
        }
        g_match_info_free (info);
        /* Try to match normal domain */
-       if (g_regex_match (extract_normal_regexp, hostname, 0, &info) == TRUE) {
+       if (g_regex_match_full (extract_normal_regexp, hostname->begin, hostname->len, 0, 0, &info, NULL) == TRUE) {
                gchar *part1, *part2;
-               part1 = g_match_info_fetch (info, 0);
-               g_match_info_next (info, NULL);
-               part2 = g_match_info_fetch (info, 0);
+               part1 = g_match_info_fetch (info, 1);
+               part2 = g_match_info_fetch (info, 2);
                g_match_info_free (info);
-               sprintf (result, "%s.%s", part1, part2);
+               result = memory_pool_alloc (pool, len); 
+               msg_debug ("format_surbl_request: got normal 2-d level domain %s.%s", part1, part2);
                if (g_hash_table_lookup (surbl_module_ctx->hosters, result) != NULL) {
                        /* Match additional part for hosters */
                        g_free (part1);
                        g_free (part2);
-                       if (g_regex_match (extract_hoster_regexp, hostname, 0, &info) == TRUE) {
+                       if (g_regex_match_full (extract_hoster_regexp, hostname->begin, hostname->len, 0, 0, &info, NULL) == TRUE) {
                                gchar *hpart1, *hpart2, *hpart3;
-                               hpart1 = g_match_info_fetch (info, 0);
-                               g_match_info_next (info, NULL);
-                               hpart2 = g_match_info_fetch (info, 0);
-                               g_match_info_next (info, NULL);
-                               hpart3 = g_match_info_fetch (info, 0);
+                               hpart1 = g_match_info_fetch (info, 1);
+                               hpart2 = g_match_info_fetch (info, 2);
+                               hpart3 = g_match_info_fetch (info, 3);
                                g_match_info_free (info);
-                               sprintf (result, "%s.%s.%s.%s", hpart1, hpart2, hpart3, surbl_module_ctx->suffix);
+                               msg_debug ("format_surbl_request: got hoster 3-d level domain %s.%s.%s", hpart1, hpart2, hpart3);
+                               snprintf (result, len, "%s.%s.%s.%s", hpart1, hpart2, hpart3, surbl_module_ctx->suffix);
                                g_free (hpart1);
                                g_free (hpart2);
                                g_free (hpart3);
@@ -292,6 +291,7 @@ format_surbl_request (char *hostname)
                        }
                        return NULL;
                }
+               snprintf (result, len, "%s.%s.%s", part1, part2, surbl_module_ctx->suffix);
                g_free (part1);
                g_free (part2);
                return result;
@@ -305,12 +305,14 @@ dns_callback (int result, char type, int count, int ttl, void *addresses, void *
 {
        struct memcached_param *param = (struct memcached_param *)data;
        
+       msg_debug ("dns_callback: in surbl request callback");
        /* If we have result from DNS server, this url exists in SURBL, so increase score */
-       if (result != DNS_ERR_NONE || type != DNS_IPv4_A) {
+       if (result == DNS_ERR_NONE && type == DNS_IPv4_A) {
                msg_info ("surbl_check: url %s is in surbl %s", param->url->host, surbl_module_ctx->suffix);
                insert_result (param->task, surbl_module_ctx->metric, surbl_module_ctx->symbol, 1);
        }
        else {
+               msg_debug ("surbl_check: url %s is not in surbl %s", param->url->host, surbl_module_ctx->suffix);
                insert_result (param->task, surbl_module_ctx->metric, surbl_module_ctx->symbol, 0);
        }
 
@@ -321,10 +323,6 @@ dns_callback (int result, char type, int count, int ttl, void *addresses, void *
                process_filters (param->task);
        }
 
-       g_free (param->ctx->param->buf);
-       g_free (param->ctx->param);
-       g_free (param->ctx);
-       g_free (param);
 }
 
 static void 
@@ -333,6 +331,7 @@ memcached_callback (memcached_ctx_t *ctx, memc_error_t error, void *data)
        struct memcached_param *param = (struct memcached_param *)data;
        int *url_count;
        char *surbl_req;
+       f_str_t c;
 
        switch (ctx->op) {
                case CMD_CONNECT:
@@ -345,10 +344,6 @@ memcached_callback (memcached_ctx_t *ctx, memc_error_t error, void *data)
                                        param->task->save.saved = 1;
                                        process_filters (param->task);
                                }
-                               g_free (param->ctx->param->buf);
-                               g_free (param->ctx->param);
-                               g_free (param->ctx);
-                               g_free (param);
                        }
                        else {
                                memc_get (param->ctx, param->ctx->param);
@@ -364,10 +359,6 @@ memcached_callback (memcached_ctx_t *ctx, memc_error_t error, void *data)
                                        param->task->save.saved = 1;
                                        process_filters (param->task);
                                }
-                               g_free (param->ctx->param->buf);
-                               g_free (param->ctx->param);
-                               g_free (param->ctx);
-                               g_free (param);
                        }
                        else {
                                url_count = (int *)param->ctx->param->buf;
@@ -391,15 +382,14 @@ memcached_callback (memcached_ctx_t *ctx, memc_error_t error, void *data)
                                param->task->save.saved = 1;
                                process_filters (param->task);
                        }
-                       if ((surbl_req = format_surbl_request (param->url->host)) != NULL) {
+                       c.begin = param->url->host;
+                       c.len = param->url->hostlen;
+                       if ((surbl_req = format_surbl_request (param->task->task_pool, &c)) != NULL) {
+                               msg_debug ("surbl_memcached: send surbl dns request %s", surbl_req);
                                param->task->save.saved ++;
                                evdns_resolve_ipv4 (surbl_req, DNS_QUERY_NO_SEARCH, dns_callback, (void *)param);
                                return;
                        }
-                       g_free (param->ctx->param->buf);
-                       g_free (param->ctx->param);
-                       g_free (param->ctx);
-                       g_free (param);
                        break;
        }
 }
@@ -413,16 +403,14 @@ register_memcached_call (struct uri *url, struct worker_task *task)
        gchar *sum_str;
        int *url_count;
 
-       param = g_malloc (sizeof (struct memcached_param));
-       cur_param = g_malloc (sizeof (memcached_param_t));
-       url_count = g_malloc (sizeof (int));
+       param = memory_pool_alloc (task->task_pool, sizeof (struct memcached_param));
+       cur_param = memory_pool_alloc0 (task->task_pool, sizeof (memcached_param_t));
+       url_count = memory_pool_alloc (task->task_pool, sizeof (int));
 
        param->url = url;
        param->task = task;
 
-       param->ctx = g_malloc (sizeof (memcached_ctx_t));
-       bzero (param->ctx, sizeof (memcached_ctx_t));
-       bzero (cur_param, sizeof (memcached_param_t));
+       param->ctx = memory_pool_alloc0 (task->task_pool, sizeof (memcached_ctx_t));
 
        cur_param->buf = (u_char *)url_count;
        cur_param->bufsize = sizeof (int);
@@ -435,6 +423,10 @@ register_memcached_call (struct uri *url, struct worker_task *task)
                                                                                        task->cfg->memcached_servers_num, sizeof (struct memcached_server),
                                                                                        time (NULL), task->cfg->memcached_error_time, task->cfg->memcached_dead_time, task->cfg->memcached_maxerrors,
                                                                                        cur_param->key, strlen(cur_param->key));
+       if (selected == NULL) {
+               msg_err ("surbl_register_memcached_call: no memcached servers can be selected");
+               return;
+       }
        param->ctx->callback = memcached_callback;
        param->ctx->callback_data = (void *)param;
        param->ctx->protocol = task->cfg->memcached_protocol;
@@ -480,7 +472,6 @@ redirector_callback (int fd, short what, void *arg)
                                                param->task->save.saved = 1;
                                                process_filters (param->task);
                                        }
-                                       g_free (param);
                                        return;
                                }
                                param->state = STATE_READ;
@@ -494,7 +485,6 @@ redirector_callback (int fd, short what, void *arg)
                                        param->task->save.saved = 1;
                                        process_filters (param->task);
                                }
-                               g_free (param);
                        }
                        break;
                case STATE_READ:
@@ -523,7 +513,6 @@ redirector_callback (int fd, short what, void *arg)
                                        param->task->save.saved = 1;
                                        process_filters (param->task);
                                }
-                               g_free (param);
                        }
                        else {
                                event_del (&param->ev);
@@ -534,7 +523,6 @@ redirector_callback (int fd, short what, void *arg)
                                        param->task->save.saved = 1;
                                        process_filters (param->task);
                                }
-                               g_free (param);
                        }
                        break;
        }
@@ -571,7 +559,7 @@ register_redirector_call (struct uri *url, struct worker_task *task)
                        msg_info ("register_redirector_call: connect() failed: %m");
                }
        }
-       param = g_malloc (sizeof (struct redirector_param));
+       param = memory_pool_alloc (task->task_pool, sizeof (struct redirector_param));
        param->url = url;
        param->task = task;
        param->state = STATE_READ;
@@ -586,13 +574,34 @@ static int
 surbl_test_url (struct worker_task *task)
 {
        struct uri *url;
+       char *surbl_req;
+       struct memcached_param *param;
+       f_str_t c;
 
        TAILQ_FOREACH (url, &task->urls, next) {
+               msg_debug ("surbl_test_url: check url %s", struri (url));
                if (surbl_module_ctx->use_redirector) {
                        register_redirector_call (url, task);
                }
                else {
-                       register_memcached_call (url, task);
+                       if (task->worker->srv->cfg->memcached_servers_num > 0) {
+                               register_memcached_call (url, task);
+                       }
+                       else {
+                               c.begin = url->host;
+                               c.len = url->hostlen;
+                               if ((surbl_req = format_surbl_request (task->task_pool, &c)) != NULL) {
+                                       param = memory_pool_alloc (task->task_pool, sizeof (struct memcached_param));
+                                       param->task = task;
+                                       param->url = url;
+                                       msg_debug ("surbl_test_url: send surbl dns request %s", surbl_req);
+                                       evdns_resolve_ipv4 (surbl_req, DNS_QUERY_NO_SEARCH, dns_callback, (void *)param);
+                               }
+                               else {
+                                       msg_info ("surbl_test_url: cannot format url string for surbl %s", struri (url));
+                                       return;
+                               }
+                       }
                }
                task->save.saved++;
        }