From e372a9ac428a070e4444df249648a371d22d44fa Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 1 Jul 2016 11:55:47 +0100 Subject: [PATCH] [Feature] Improve phishtank and openphish support --- src/plugins/lua/phishing.lua | 143 ++++++++++++++++++++++++++++------- 1 file changed, 115 insertions(+), 28 deletions(-) diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua index b3b3da12e..ee42b9068 100644 --- a/src/plugins/lua/phishing.lua +++ b/src/plugins/lua/phishing.lua @@ -30,42 +30,69 @@ local phishtank_enabled = false local openphish_premium = false local openphish_hash local phishtank_hash -local openphish_json = {} +local openphish_data = {} local phishtank_data = {} local rspamd_logger = require "rspamd_logger" local util = require "rspamd_util" local opts = rspamd_config:get_all_opt('phishing') local function phishing_cb(task) - local urls = task:get_urls() + local function check_phishing_map(map, url, symbol) + local host = url:get_host() - if urls then - for _,url in ipairs(urls) do - if openphish_hash then - local t = url:get_text() - - if openphish_premium then - local elt = openphish_json[t] - if elt then - task:insert_result(openphish_symbol, 1.0, { - elt['tld'], - elt['sector'], - elt['brand'], - }) + if host then + local elt = map[host] + local found = false + local data = nil + + if elt then + local path = url:get_path() + + if path then + for _,d in ipairs(elt) do + if d['path'] == path then + found = true + data = d['data'] + end end + end + + + if found then + local args = nil + + if type(data) == 'table' then + args = { + data['tld'], + data['sector'], + data['brand'], + } + elseif type(data) == 'string' then + args = data + else + args = host + end + + task:insert_result(symbol, 1.0, args) else - if openphish_hash:get_key(t) then - task:insert_result(openphish_symbol, 1.0, url:get_tld()) + if url:is_phished() and not url:is_redirected() then + task:insert_result(symbol, 0.7, host) end end end + end + end + + local urls = task:get_urls() + + if urls then + for _,url in ipairs(urls) do + if openphish_hash then + check_phishing_map(openphish_data, url, openphish_symbol) + end if phishtank_hash then - local t = url:get_text() - local elt = phishtank_data[t] - if elt then - task:insert_result(phishtank_symbol, 1.0, elt) - end + check_phishing_map(phishtank_data, url, phishtank_symbol) end if url:is_phished() and not url:is_redirected() then @@ -158,12 +185,41 @@ local function rspamd_str_split_fun(s, sep, func) return lpeg.match(p, s) end +local function insert_url_from_string(pool, tbl, str, data) + local rspamd_url = require "rspamd_url" + + local u = rspamd_url.create(pool, str) + + if u then + local host = u:get_host() + if host then + local elt = { + data = data, + path = u:get_path() + } + + if tbl[host] then + table.insert(tbl[host], elt) + else + tbl[host] = {elt} + end + + return true + end + end + + return false +end + local function openphish_json_cb(string) local ucl = require "ucl" + local rspamd_mempool = require "rspamd_mempool" local nelts = 0 local new_json_map = {} local valid = true + local pool = rspamd_mempool.create() + local function openphish_elt_parser(cap) if valid then local parser = ucl.parser() @@ -175,8 +231,9 @@ local function openphish_json_cb(string) local obj = parser:get_object() if obj['url'] then - new_json_map[obj['url']] = obj - nelts = nelts + 1 + if insert_url_from_string(pool, new_json_map, obj['url'], obj) then + nelts = nelts + 1 + end end end end @@ -185,10 +242,32 @@ local function openphish_json_cb(string) rspamd_str_split_fun(string, '\n', openphish_elt_parser) if valid then - openphish_json = new_json_map + openphish_data = new_json_map rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed", nelts) end + + pool:destroy() +end + +local function openphish_plain_cb(string) + local nelts = 0 + local new_data = {} + local rspamd_mempool = require "rspamd_mempool" + local pool = rspamd_mempool.create() + + local function openphish_elt_parser(cap) + if insert_url_from_string(pool, new_data, cap, nil) then + nelts = nelts + 1 + end + end + + rspamd_str_split_fun(string, '\n', openphish_elt_parser) + + openphish_data = new_data + rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed", + nelts) + pool:destroy() end local function phishtank_json_cb(string) @@ -198,6 +277,8 @@ local function phishtank_json_cb(string) local valid = true local parser = ucl.parser() local res,err = parser:parse_string(string) + local rspamd_mempool = require "rspamd_mempool" + local pool = rspamd_mempool.create() if not res then valid = false @@ -207,8 +288,10 @@ local function phishtank_json_cb(string) for _,elt in ipairs(obj) do if elt['url'] then - new_data[elt['url']] = elt['phish_detail_url'] - nelts = nelts + 1 + if insert_url_from_string(pool, new_data, elt['url'], + elt['phish_detail_url']) then + nelts = nelts + 1 + end end end end @@ -218,6 +301,9 @@ local function phishtank_json_cb(string) rspamd_logger.infox(phishtank_hash, "parsed %s elements from phishtank feed", nelts) end + + + pool:destroy() end if opts then @@ -243,8 +329,9 @@ if opts then if opts['openphish_enabled'] then if not openphish_premium then openphish_hash = rspamd_config:add_map({ - type = 'set', + type = 'callback', url = openphish_map, + callback = openphish_plain_cb, description = 'Open phishing feed map (see https://www.openphish.com for details)' }) else -- 2.39.5