* Add ability to obtain phished url from lua

* Add ability to specify check domains for phishing check with 'domains' option

diff --git a/src/html.c b/src/html.c
index dee38a86b74f9edba686ebf0e2062eae224e5557..810ba963b6b94b6638ec8c654872df0e1b552089 100644 (file)
--- a/src/html.c
+++ b/src/html.c
@@ -697,6 +697,7 @@ check_phishing (struct worker_task *task, struct uri *href_url, const gchar *url
                                if (g_ascii_strncasecmp (href_url->host, new->host,
                                                MAX (href_url->hostlen, new->hostlen)) != 0) {
                                        href_url->is_phished = TRUE;
+                                       href_url->phished_url = new;
                                }
                        }
                        else {

diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
index 7102940aab6806515a3ab0f9776fa26e3161c8bd..67fd3e7b99d3ba2e7f6133cef96a98d212c3c7e9 100644 (file)
--- a/src/lua/lua_task.c
+++ b/src/lua/lua_task.c
@@ -128,6 +128,7 @@ LUA_FUNCTION_DEF (url, get_user);
 LUA_FUNCTION_DEF (url, get_path);
 LUA_FUNCTION_DEF (url, get_text);
 LUA_FUNCTION_DEF (url, is_phished);
+LUA_FUNCTION_DEF (url, get_phished);
 
 static const struct luaL_reg    urllib_m[] = {
        LUA_INTERFACE_DEF (url, get_host),
@@ -135,6 +136,7 @@ static const struct luaL_reg    urllib_m[] = {
        LUA_INTERFACE_DEF (url, get_path),
        LUA_INTERFACE_DEF (url, get_text),
        LUA_INTERFACE_DEF (url, is_phished),
+       LUA_INTERFACE_DEF (url, get_phished),
        {"__tostring", lua_class_tostring},
        {NULL, NULL}
 };
@@ -1024,6 +1026,25 @@ lua_url_is_phished (lua_State *L)
        return 1;
 }
 
+static gint
+lua_url_get_phished (lua_State *L)
+{
+       struct uri                    **purl, *url = lua_check_url (L);
+
+       if (url) {
+               if (url->is_phished && url->phished_url != NULL) {
+                       purl = lua_newuserdata (L, sizeof (struct uri *));
+                       lua_setclass (L, "rspamd{url}", -1);
+                       *purl = url->phished_url;
+
+                       return 1;
+               }
+       }
+
+       lua_pushnil (L);
+       return 1;
+}
+
 /* Init part */
 gint
 luaopen_task (lua_State * L)

diff --git a/src/map.c b/src/map.c
index 2dd898f02e9b7edc85ab507134150da92238975d..b5825206a49e2a9d504ad1b58c82100f1406bc6b 100644 (file)
--- a/src/map.c
+++ b/src/map.c
@@ -424,7 +424,7 @@ add_map (const gchar *map_line, map_cb_t read_callback, map_fin_cb_t fin_callbac
                def = map_line + sizeof ("file://") - 1;
        }
        else {
-               msg_debug ("invalid map fetching protocol: %s", map_line);
+               msg_warn ("invalid map fetching protocol: %s", map_line);
                return FALSE;
        }
        /* Constant pool */

diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua
index 8021a006912e94df2def786d65ce8c65a9e0bc00..70bb67a520d5f429af4cc1a74b3f619b2189d77b 100644 (file)
--- a/src/plugins/lua/phishing.lua
+++ b/src/plugins/lua/phishing.lua
@@ -2,6 +2,7 @@
 --
 --
 local symbol = 'PHISHED_URL'
+local domains = nil
 
 function phishing_cb (task)
        local urls = task:get_urls();
@@ -9,7 +10,18 @@ function phishing_cb (task)
        if urls then
                for _,url in ipairs(urls) do
                        if url:is_phished() then
-                               task:insert_result(symbol, 1, url:get_host())
+                               if domains then
+                                       local _,_,tld = string.find(url:get_phished():get_host(), '([a-zA-Z0-9%-]+\.[a-zA-Z0-9%-]+)$')
+                                       if tld then
+                                               if domains:get_key(tld) then
+                                                       if url:is_phished() then
+                                                               task:insert_result(symbol, 1, url:get_host())
+                                                       end
+                                               end
+                                       end
+                               else            
+                                       task:insert_result(symbol, 1, url:get_phished():get_host())
+                               end
                        end
                end
        end
@@ -24,5 +36,8 @@ if opts then
         -- Register symbol's callback
         rspamd_config:register_symbol(symbol, 1.0, 'phishing_cb')
     end
+       if opts['domains'] then
+               domains = rspamd_config:add_hash_map (opts['domains'])
+       end
     -- If no symbol defined, do not register this module
 end

diff --git a/src/url.h b/src/url.h
index 105466aa592e835af0c26729bfa394dd0a91e1cf..6b08682ba92bfaa64745975c4ff7edf812611efa 100644 (file)
--- a/src/url.h
+++ b/src/url.h
@@ -30,6 +30,8 @@ struct uri {
         * POST_CHAR in the uri string. */
        gchar *post;
 
+       struct uri *phished_url;
+
        /* @protocollen should only be usable if @protocol is either
         * PROTOCOL_USER or an uri string should be composed. */
        guint protocollen;