* Avoid extra requests to surbls saving them into requests cache

* Lower debug level for some errors (INFO to DEBUG)

diff --git a/src/message.c b/src/message.c
index c2acaa0cd8ffd214797159bf752fed839c7d5097..d1f79ebfb37ec639719142c1000f7552851574d4 100644 (file)
--- a/src/message.c
+++ b/src/message.c
@@ -339,7 +339,7 @@ process_message (struct worker_task *task)
         * This causes g_mime_stream not to free memory by itself as it is memory allocated by
         * pool allocator
         */
-       g_mime_stream_mem_set_owner (stream, FALSE);
+       g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE);
 
        msg_debug ("process_message: construct mime parser from string length %ld", (long int)task->msg->len);
        /* create a new parser object to parse the stream */
@@ -458,7 +458,7 @@ process_learn (struct controller_session *session)
         * This causes g_mime_stream not to free memory by itself as it is memory allocated by
         * pool allocator
         */
-       g_mime_stream_mem_set_owner (stream, FALSE);
+       g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE);
 
        /* create a new parser object to parse the stream */
        parser = g_mime_parser_new_with_stream (stream);

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 1185da68fafe2a75db9895967747d14e024687c9..74fd56894ae6412e8678fcf1cf75b593210b4ebb 100644 (file)
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -301,7 +301,7 @@ format_surbl_request (memory_pool_t *pool, f_str_t *hostname, struct suffix_item
 }
 
 static void 
-make_surbl_requests (struct uri* url, struct worker_task *task)
+make_surbl_requests (struct uri* url, struct worker_task *task, GTree *tree)
 {      
        char *surbl_req;
        f_str_t f;
@@ -318,8 +318,14 @@ make_surbl_requests (struct uri* url, struct worker_task *task)
                param->task = task;
                param->suffix = (struct suffix_item *)cur->data;
                if ((surbl_req = format_surbl_request (task->task_pool, &f, (struct suffix_item *)cur->data)) != NULL) {
-                       msg_debug ("surbl_test_url: send surbl dns request %s", surbl_req);
-                       evdns_resolve_ipv4 (surbl_req, DNS_QUERY_NO_SEARCH, dns_callback, (void *)param);
+                       if (g_tree_lookup (tree, surbl_req) == NULL) {
+                               g_tree_insert (tree, surbl_req, surbl_req);
+                               msg_debug ("surbl_test_url: send surbl dns request %s", surbl_req);
+                               evdns_resolve_ipv4 (surbl_req, DNS_QUERY_NO_SEARCH, dns_callback, (void *)param);
+                       }
+                       else {
+                               msg_debug ("make_surbl_requests: request %s is already sent", surbl_req);
+                       }
                }
                else {
                        msg_info ("surbl_test_url: cannot format url string for surbl %s", struri (url));
@@ -387,7 +393,7 @@ dns_callback (int result, char type, int count, int ttl, void *addresses, void *
                msg_debug ("surbl_check: url %s is not in surbl %s", param->url->host, param->suffix->suffix);
        }
        *(param->url->host + param->url->hostlen) = c;
-
+       
        param->task->save.saved --;
        if (param->task->save.saved == 0) {
                /* Call other filters */
@@ -456,13 +462,13 @@ memcached_callback (memcached_ctx_t *ctx, memc_error_t error, void *data)
                                param->task->save.saved = 1;
                                process_filters (param->task);
                        }
-                       make_surbl_requests (param->url, param->task);
+                       make_surbl_requests (param->url, param->task, param->tree);
                        break;
        }
 }
 
 static void
-register_memcached_call (struct uri *url, struct worker_task *task) 
+register_memcached_call (struct uri *url, struct worker_task *task, GTree *url_tree) 
 {
        struct memcached_param *param;
        struct memcached_server *selected;
@@ -476,6 +482,7 @@ register_memcached_call (struct uri *url, struct worker_task *task)
 
        param->url = url;
        param->task = task;
+       param->tree = url_tree;
 
        param->ctx = memory_pool_alloc0 (task->task_pool, sizeof (memcached_ctx_t));
 
@@ -534,7 +541,7 @@ redirector_callback (int fd, short what, void *arg)
                                        msg_err ("redirector_callback: write failed %s", strerror (errno));
                                        event_del (&param->ev);
                                        param->task->save.saved --;
-                                       make_surbl_requests (param->url, param->task);
+                                       make_surbl_requests (param->url, param->task, param->tree);
                                        if (param->task->save.saved == 0) {
                                                /* Call other filters */
                                                param->task->save.saved = 1;
@@ -548,7 +555,7 @@ redirector_callback (int fd, short what, void *arg)
                                event_del (&param->ev);
                                msg_info ("redirector_callback: connection to redirector timed out while waiting for write");
                                param->task->save.saved --;
-                               make_surbl_requests (param->url, param->task);
+                               make_surbl_requests (param->url, param->task, param->tree);
 
                                if (param->task->save.saved == 0) {
                                        /* Call other filters */
@@ -576,7 +583,7 @@ redirector_callback (int fd, short what, void *arg)
                                }
                                event_del (&param->ev);
                                param->task->save.saved --;
-                               make_surbl_requests (param->url, param->task);
+                               make_surbl_requests (param->url, param->task, param->tree);
                                if (param->task->save.saved == 0) {
                                        /* Call other filters */
                                        param->task->save.saved = 1;
@@ -587,7 +594,7 @@ redirector_callback (int fd, short what, void *arg)
                                event_del (&param->ev);
                                msg_info ("redirector_callback: reading redirector timed out, while waiting for read");
                                param->task->save.saved --;
-                               make_surbl_requests (param->url, param->task);
+                               make_surbl_requests (param->url, param->task, param->tree);
                                if (param->task->save.saved == 0) {
                                        /* Call other filters */
                                        param->task->save.saved = 1;
@@ -600,7 +607,7 @@ redirector_callback (int fd, short what, void *arg)
 
 
 static void
-register_redirector_call (struct uri *url, struct worker_task *task) 
+register_redirector_call (struct uri *url, struct worker_task *task, GTree *url_tree) 
 {
        int s;
        struct redirector_param *param;
@@ -611,7 +618,7 @@ register_redirector_call (struct uri *url, struct worker_task *task)
        if (s == -1) {
                msg_info ("register_redirector_call: cannot create tcp socket failed: %s", strerror (errno));
                task->save.saved --;
-               make_surbl_requests (url, task);
+               make_surbl_requests (url, task, url_tree);
                return; 
        }
 
@@ -620,6 +627,7 @@ register_redirector_call (struct uri *url, struct worker_task *task)
        param->task = task;
        param->state = STATE_CONNECT;
        param->sock = s;
+       param->tree = url_tree;
        timeout.tv_sec = surbl_module_ctx->connect_timeout / 1000;
        timeout.tv_usec = surbl_module_ctx->connect_timeout - timeout.tv_sec * 1000;
        event_set (&param->ev, s, EV_WRITE, redirector_callback, (void *)param);
@@ -631,23 +639,28 @@ surbl_test_url (struct worker_task *task)
 {
        struct uri *url;
        struct memcached_param *param;
+       GTree *url_tree;
+
+       url_tree = g_tree_new ((GCompareFunc)g_strcasecmp);
 
        TAILQ_FOREACH (url, &task->urls, next) {
                msg_debug ("surbl_test_url: check url %s", struri (url));
                if (surbl_module_ctx->use_redirector) {
-                       register_redirector_call (url, task);
+                       register_redirector_call (url, task, url_tree);
                        task->save.saved++;
                }
                else {
                        if (task->worker->srv->cfg->memcached_servers_num > 0) {
-                               register_memcached_call (url, task);
+                               register_memcached_call (url, task, url_tree);
                                task->save.saved++;
                        }
                        else {
-                               make_surbl_requests (url, task);
+                               make_surbl_requests (url, task, url_tree);
                        }
                }
        }
+
+       g_tree_destroy (url_tree);
        return 0;
 }
 

diff --git a/src/plugins/surbl.h b/src/plugins/surbl.h
index 21d77c25b32056e92348cd0ea20700a591ae0458..90c86a490b5ea079b7924cc332de2389a928bc62 100644 (file)
--- a/src/plugins/surbl.h
+++ b/src/plugins/surbl.h
@@ -58,12 +58,14 @@ struct redirector_param {
        } state;
        struct event ev;
        int sock;
+       GTree *tree;
 };
 
 struct memcached_param {
        struct uri *url;
        struct worker_task *task;
        memcached_ctx_t *ctx;
+       GTree *tree;
 };
 
 struct surbl_bit_item {

diff --git a/src/url.c b/src/url.c
index c2575f3c8af5b7efa85b99e38821702001d5eb17..ff08615aa5908796fb8a4d578d264bf376f515ae 100644 (file)
--- a/src/url.c
+++ b/src/url.c
@@ -878,7 +878,7 @@ url_parse_text (struct worker_task *task, GByteArray *content)
                                                if (new != NULL) {
                                                        rc = parse_uri (new, url_str, task->task_pool);
                                                        if (rc != URI_ERRNO_OK) {
-                                                               msg_info ("url_parse_html: error while parsing url %s: %s", url_str, url_strerror (rc));
+                                                               msg_debug ("url_parse_html: error while parsing url %s: %s", url_str, url_strerror (rc));
                                                        }
                                                        if (rc != URI_ERRNO_EMPTY && rc != URI_ERRNO_NO_HOST) {
                                                                TAILQ_INSERT_TAIL (&task->urls, new, next);
@@ -928,7 +928,7 @@ url_parse_html (struct worker_task *task, GByteArray *content)
                                                if (new != NULL) {
                                                        rc = parse_uri (new, url_str, task->task_pool);
                                                        if (rc != URI_ERRNO_OK) {
-                                                               msg_info ("url_parse_html: error while parsing url %s: %s", url_str, url_strerror (rc));
+                                                               msg_debug ("url_parse_html: error while parsing url %s: %s", url_str, url_strerror (rc));
                                                        }
                                                        if (rc != URI_ERRNO_EMPTY && rc != URI_ERRNO_NO_HOST) {
                                                                TAILQ_INSERT_TAIL (&task->urls, new, next);