[Feature] Add preliminary phishtank support

diff --git a/conf/metrics.conf b/conf/metrics.conf
index 17c060d189cf15461eac9bc9da6a8d7996572230..03dc0f4890905ceb7a4a2b71de8368532d52e676 100644 (file)
--- a/conf/metrics.conf
+++ b/conf/metrics.conf
@@ -841,6 +841,10 @@ metric {
             weight = 7.0;
             description = "Phished URL found in openphish.com";
         }
+        symbol "PHISHED_PHISHTANK" {
+            weight = 7.0;
+            description = "Phished URL found in phishtank.com";
+        }
     }
 
     group "date" {

diff --git a/conf/modules.d/phishing.conf b/conf/modules.d/phishing.conf
index 392708cde274eb30618870e98ffca9c1a995874b..861aee7ae2de4fb3413263059291d0da72f59e93 100644 (file)
--- a/conf/modules.d/phishing.conf
+++ b/conf/modules.d/phishing.conf
@@ -19,6 +19,9 @@ phishing {
     .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/phishing.conf"
     symbol = "PHISHING";
     openphish_map = "https://www.openphish.com/feed.txt";
+    # Disabled by default
+    phishtank_enabled = false;
+    phishtank_map = "http://data.phishtank.com/data/online-valid.json";
 
     # Make exclusions for known redirectors
     redirector_domains = [

diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua
index f09cf53e9f64b96e3a68e51f541f9afa5ca434ce..04a0fe9b79b172c5479e5236bf270864a84ea2e9 100644 (file)
--- a/src/plugins/lua/phishing.lua
+++ b/src/plugins/lua/phishing.lua
@@ -19,13 +19,19 @@ limitations under the License.
 --
 local symbol = 'PHISHED_URL'
 local openphish_symbol = 'PHISHED_OPENPHISH'
+local phishtank_symbol = 'PHISHED_PHISHTANK'
 local domains = nil
 local strict_domains = {}
 local redirector_domains = {}
 local openphish_map = 'https://www.openphish.com/feed.txt'
+local phishtank_map = 'http://data.phishtank.com/data/online-valid.json'
+-- Not enabled by default as their feed is quite large
+local phishtank_enabled = false
 local openphish_premium = false
 local openphish_hash
+local phishtank_hash
 local openphish_json = {}
+local phishtank_data = {}
 local rspamd_logger = require "rspamd_logger"
 local util = require "rspamd_util"
 local opts = rspamd_config:get_all_opt('phishing')
@@ -54,6 +60,14 @@ local function phishing_cb(task)
         end
       end
 
+      if phishtank_hash then
+        local t = url:get_text()
+        local elt = phishtank_data[t]
+        if elt then
+          task:insert_result(phishtank_symbol, 1.0, elt)
+        end
+      end
+
       if url:is_phished() and not url:is_redirected() then
         local found = false
         local purl = url:get_phished()
@@ -177,6 +191,35 @@ local function openphish_json_cb(string)
   end
 end
 
+local function phishtank_json_cb(string)
+  local ucl = require "ucl"
+  local nelts = 0
+  local new_data = {}
+  local valid = true
+  local parser = ucl.parser()
+  local res,err = parser:parse_string(string)
+
+  if not res then
+    valid = false
+    rspamd_logger.warnx(rspamd_config, 'cannot parse openphish map: ' .. err)
+  else
+    local obj = parser:get_object()
+
+    for _,elt in ipairs(obj) do
+      if elt['url'] then
+        new_data[elt['url']] = elt['phish_detail_url']
+        nelts = nelts + 1
+      end
+    end
+  end
+
+  if valid then
+    phishtank_data = new_data
+    rspamd_logger.infox(phishtank_hash, "parsed %s elements from phishtank feed",
+      nelts)
+  end
+end
+
 if opts then
   if opts['symbol'] then
     symbol = opts['symbol']
@@ -189,6 +232,9 @@ if opts then
     if opts['openphish_map'] then
       openphish_map = opts['openphish_map']
     end
+    if opts['openphish_url'] then
+      openphish_map = opts['openphish_url']
+    end
 
     if opts['openphish_premium'] then
       openphish_premium = true
@@ -209,6 +255,22 @@ if opts then
         })
     end
 
+    if opts['phihtank_map'] then
+      phihtank_map = opts['openphish_map']
+    end
+    if opts['phihtank_url'] then
+      phihtank_map = opts['phihtank_url']
+    end
+
+    if opts['phishtank_enabled'] then
+      phishtank_hash = rspamd_config:add_map({
+          type = 'callback',
+          url = openphish_map,
+          callback = phishtank_json_cb,
+          description = 'Phishtank feed (see https://www.phishtank.com for details)'
+        })
+    end
+
     if openphish_hash then
       rspamd_config:register_symbol({
         type = 'virtual',
@@ -216,6 +278,14 @@ if opts then
         name = openphish_symbol,
       })
     end
+
+    if phishtank_hash then
+      rspamd_config:register_symbol({
+        type = 'virtual',
+        parent = id,
+        name = phishtank_symbol,
+      })
+    end
   end
   if opts['domains'] and type(opt['domains']) == 'string' then
     domains = rspamd_config:add_map({