* Add ability to extract urls from subject field

Fix phishing plugin.
* Important fix for multimap/cdb handling
* Important fix for phishing detector

diff --git a/src/html.c b/src/html.c
index 31863ba9c095c4b75d63764789511c6f00d23340..b18914e417c0d209ae8a9c30135bd6cd924f82db 100644 (file)
--- a/src/html.c
+++ b/src/html.c
@@ -728,7 +728,13 @@ check_phishing (struct worker_task *task, struct uri *href_url, const gchar *url
                                                        }
                                                }
                                                /* Compare parts and check for phished hostname */
-                                               if (c != NULL && g_ascii_strncasecmp (p, c, len) != 0) {
+                                               if (c != NULL) {
+                                                       if (g_ascii_strncasecmp (p, c, len) != 0) {
+                                                               href_url->is_phished = TRUE;
+                                                               href_url->phished_url = new;
+                                                       }
+                                               }
+                                               else {
                                                        href_url->is_phished = TRUE;
                                                        href_url->phished_url = new;
                                                }

diff --git a/src/main.c b/src/main.c
index 64357c26494e8fbd6f5a1eef6822500bce18e1cf..cef64d8a140334f8df062c72b8a16651a0a24c65 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -605,8 +605,8 @@ wait_for_workers (gpointer key, gpointer value, gpointer unused)
 
        if (got_alarm) {
                got_alarm = 0;
-               /* Set alarm for hard termination */
-               set_alarm (HARD_TERMINATION_TIME);
+               /* Set alarm for hard termination but with less time */
+               set_alarm (HARD_TERMINATION_TIME / 10);
        }
 
        if (waitpid (w->pid, &res, 0) == -1) {

diff --git a/src/message.c b/src/message.c
index 4f462735124d0ff4efaaf50d428cab091833fa72..48eb4d89cafb7c91d65fb745fb12d8dccdafb400 100644 (file)
--- a/src/message.c
+++ b/src/message.c
@@ -945,7 +945,10 @@ process_message (struct worker_task *task)
        GMimePart                      *part;
        GMimeDataWrapper               *wrapper;
        struct received_header         *recv;
-       gchar                           *mid;
+       gchar                          *mid, *url_str, *p, *end;
+       struct uri                     *subject_url;
+       gsize                           len;
+       gint                            pos, rc;
 
        tmp = memory_pool_alloc (task->task_pool, sizeof (GByteArray));
        tmp->data = task->msg->begin;
@@ -1090,6 +1093,44 @@ process_message (struct worker_task *task)
 #endif
        }
 
+       /* Parse urls inside Subject header */
+       cur = message_get_header (task->task_pool, task->message, "Subject", FALSE);
+       if (cur) {
+               p = cur->data;
+               len = strlen (p);
+               end = p + len;
+
+               while (p < end) {
+                       /* Search to the end of url */
+                       if (url_try_text (task->task_pool, p, end - p, &pos, &url_str)) {
+                               if (url_str != NULL) {
+                                       subject_url = memory_pool_alloc0 (task->task_pool, sizeof (struct uri));
+                                       if (subject_url != NULL) {
+                                               /* Try to parse url */
+                                               rc = parse_uri (subject_url, url_str, task->task_pool);
+                                               if ((rc == URI_ERRNO_OK || rc == URI_ERRNO_NO_SLASHES || rc == URI_ERRNO_NO_HOST_SLASH) &&
+                                                               subject_url->hostlen > 0) {
+                                                       if (subject_url->protocol != PROTOCOL_MAILTO) {
+                                                               if (!g_tree_lookup (task->urls, subject_url)) {
+                                                                       g_tree_insert (task->urls, subject_url, subject_url);
+                                                               }
+                                                       }
+                                               }
+                                               else if (rc != URI_ERRNO_OK) {
+                                                       msg_info ("extract of url '%s' failed: %s", url_str, url_strerror (rc));
+                                               }
+                                       }
+                               }
+                       }
+                       else {
+                               break;
+                       }
+                       p += pos;
+               }
+               /* Free header's list */
+               g_list_free (cur);
+       }
+
        return 0;
 }
 

diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua
index 98ebeb35361d67005c53c439444e682f45f7ee44..5d1312dc25becdcc799dd6c6c2f91563bd8b3876 100644 (file)
--- a/src/plugins/lua/multimap.lua
+++ b/src/plugins/lua/multimap.lua
@@ -283,13 +283,13 @@ local function add_multimap_rule(params)
        if string.find(newrule['map'], '^cdb://.*$') then
                local test = cdb.create(newrule['map'])
                newrule['hash'] = cdb.create(newrule['map'])
+               newrule['cdb'] = true
                if newrule['hash'] then
                        table.insert(rules, newrule)
                        return newrule
                else
                        rspamd_logger.warn('Cannot add rule: map doesn\'t exists: ' .. newrule['map'])
                end
-               newrule['cdb'] = true
        else
                if newrule['type'] == 'ip' then
                        newrule['ips'] = rspamd_config:add_radix_map (newrule['map'])

diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua
index 1e648768ace9427043b1261e3591ff1058d72d65..e3def6fd93c56387c73ae6f938ac1f9544d711c3 100644 (file)
--- a/src/plugins/lua/phishing.lua
+++ b/src/plugins/lua/phishing.lua
@@ -11,10 +11,10 @@ function phishing_cb (task)
        if urls then
                for _,url in ipairs(urls) do
                        if url:is_phished() then
+                               local found = false
                                local purl = url:get_phished()
                                if table.maxn(strict_domains) > 0 then
                                        local _,_,tld = string.find(purl:get_host(), '([a-zA-Z0-9%-]+\.[a-zA-Z0-9%-]+)$')
-                                       local found = false
                                        if tld then
                                                for _,rule in ipairs(strict_domains) do
                                                        if rule['map']:get_key(tld) then
@@ -22,20 +22,19 @@ function phishing_cb (task)
                                                                found = true
                                                        end
                                                end
-                                               if found then
-                                                       return
-                                               end
                                        end
                                end
-                               if domains then
-                                       local _,_,tld = string.find(purl:get_host(), '([a-zA-Z0-9%-]+\.[a-zA-Z0-9%-]+)$')
-                                       if tld then
-                                               if domains:get_key(tld) then
-                                                       task:insert_result(symbol, 1, purl:get_host())
+                               if not found then
+                                       if domains then
+                                               local _,_,tld = string.find(purl:get_host(), '([a-zA-Z0-9%-]+\.[a-zA-Z0-9%-]+)$')
+                                               if tld then
+                                                       if domains:get_key(tld) then
+                                                               task:insert_result(symbol, 1, purl:get_host())
+                                                       end
                                                end
+                                       else            
+                                               task:insert_result(symbol, 1, purl:get_host())
                                        end
-                               else            
-                                       task:insert_result(symbol, 1, purl:get_host())
                                end
                        end
                end
@@ -70,7 +69,7 @@ if opts then
                        sd[1] = opts['strict_domains']
                end
                for _,d in ipairs(sd) do
-                       local s, _ = string.find(d, ':')
+                       local s, _ = string.find(d, ':[^:]+$')
                        if s then
                                local sym = string.sub(d, s + 1, -1)
                                local map = string.sub(d, 1, s - 1)