* Add ability to obtain phished url from lua

* Add ability to specify check domains for phishing check with 'domains' option

5 files changed, 41 insertions, 2 deletions

diff --git a/src/html.c b/src/html.c
index dee38a86b..810ba963b 100644
--- a/src/html.c
+++ b/src/html.c
@@ -697,6 +697,7 @@ check_phishing (struct worker_task *task, struct uri *href_url, const gchar *url
 				if (g_ascii_strncasecmp (href_url->host, new->host,
 						MAX (href_url->hostlen, new->hostlen)) != 0) {
 					href_url->is_phished = TRUE;
+					href_url->phished_url = new;
 				}
 			}
 			else {
diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
index 7102940aa..67fd3e7b9 100644
--- a/src/lua/lua_task.c
+++ b/src/lua/lua_task.c
@@ -128,6 +128,7 @@ LUA_FUNCTION_DEF (url, get_user);
 LUA_FUNCTION_DEF (url, get_path);
 LUA_FUNCTION_DEF (url, get_text);
 LUA_FUNCTION_DEF (url, is_phished);
+LUA_FUNCTION_DEF (url, get_phished);
 
 static const struct luaL_reg    urllib_m[] = {
 	LUA_INTERFACE_DEF (url, get_host),
@@ -135,6 +136,7 @@ static const struct luaL_reg    urllib_m[] = {
 	LUA_INTERFACE_DEF (url, get_path),
 	LUA_INTERFACE_DEF (url, get_text),
 	LUA_INTERFACE_DEF (url, is_phished),
+	LUA_INTERFACE_DEF (url, get_phished),
 	{"__tostring", lua_class_tostring},
 	{NULL, NULL}
 };
@@ -1024,6 +1026,25 @@ lua_url_is_phished (lua_State *L)
 	return 1;
 }
 
+static gint
+lua_url_get_phished (lua_State *L)
+{
+	struct uri                    **purl, *url = lua_check_url (L);
+
+	if (url) {
+		if (url->is_phished && url->phished_url != NULL) {
+			purl = lua_newuserdata (L, sizeof (struct uri *));
+			lua_setclass (L, "rspamd{url}", -1);
+			*purl = url->phished_url;
+
+			return 1;
+		}
+	}
+
+	lua_pushnil (L);
+	return 1;
+}
+
 /* Init part */
 gint
 luaopen_task (lua_State * L)
diff --git a/src/map.c b/src/map.c
index 2dd898f02..b5825206a 100644
--- a/src/map.c
+++ b/src/map.c
@@ -424,7 +424,7 @@ add_map (const gchar *map_line, map_cb_t read_callback, map_fin_cb_t fin_callbac
 		def = map_line + sizeof ("file://") - 1;
 	}
 	else {
-		msg_debug ("invalid map fetching protocol: %s", map_line);
+		msg_warn ("invalid map fetching protocol: %s", map_line);
 		return FALSE;
 	}
 	/* Constant pool */
diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua
index 8021a0069..70bb67a52 100644
--- a/src/plugins/lua/phishing.lua
+++ b/src/plugins/lua/phishing.lua
@@ -2,6 +2,7 @@
 --
 --
 local symbol = 'PHISHED_URL'
+local domains = nil
 
 function phishing_cb (task)
 	local urls = task:get_urls();
@@ -9,7 +10,18 @@ function phishing_cb (task)
 	if urls then
 		for _,url in ipairs(urls) do
 			if url:is_phished() then
-				task:insert_result(symbol, 1, url:get_host())
+				if domains then
+					local _,_,tld = string.find(url:get_phished():get_host(), '([a-zA-Z0-9%-]+\.[a-zA-Z0-9%-]+)$')
+					if tld then
+						if domains:get_key(tld) then
+							if url:is_phished() then
+								task:insert_result(symbol, 1, url:get_host())
+							end
+						end
+					end
+				else		
+					task:insert_result(symbol, 1, url:get_phished():get_host())
+				end
 			end
 		end
 	end
@@ -24,5 +36,8 @@ if opts then
         -- Register symbol's callback
         rspamd_config:register_symbol(symbol, 1.0, 'phishing_cb')
     end
+	if opts['domains'] then
+		domains = rspamd_config:add_hash_map (opts['domains'])
+	end
     -- If no symbol defined, do not register this module
 end
diff --git a/src/url.h b/src/url.h
index 105466aa5..6b08682ba 100644
--- a/src/url.h
+++ b/src/url.h
@@ -30,6 +30,8 @@ struct uri {
 	 * POST_CHAR in the uri string. */
 	gchar *post;
 
+	struct uri *phished_url;
+
 	/* @protocollen should only be usable if @protocol is either
 	 * PROTOCOL_USER or an uri string should be composed. */
 	guint protocollen;