Add support to a generic phishing feed

diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua
index 3f81cd0366a82ecd534f36a9e5d60fd88dc3c86b..a717e1b7ec2387c8d0c7128b096f3b9fa4226267 100644 (file)
--- a/src/plugins/lua/phishing.lua
+++ b/src/plugins/lua/phishing.lua
@@ -23,17 +23,22 @@ end
 --
 local N = 'phishing'
 local symbol = 'PHISHED_URL'
+local generic_service_symbol = 'PHISHED_GENERIC_SERVICE'
 local openphish_symbol = 'PHISHED_OPENPHISH'
 local phishtank_symbol = 'PHISHED_PHISHTANK'
+local generic_service_name = 'generic service'
 local domains = nil
 local strict_domains = {}
 local redirector_domains = {}
+local generic_service_map = nil
 local openphish_map = 'https://www.openphish.com/feed.txt'
 local phishtank_map = 'http://data.phishtank.com/data/online-valid.json'
 -- Not enabled by default as their feed is quite large
 local openphish_premium = false
+local generic_service_hash
 local openphish_hash
 local phishtank_hash
+local generic_service_data = {}
 local openphish_data = {}
 local phishtank_data = {}
 local rspamd_logger = require "rspamd_logger"
@@ -124,6 +129,10 @@ local function phishing_cb(task)
 
   if urls then
     for _,url in ipairs(urls) do
+      if generic_service_hash then
+        check_phishing_map(generic_service_data, url, generic_service_symbol)
+      end
+
       if openphish_hash then
         check_phishing_map(openphish_data, url, openphish_symbol)
       end
@@ -300,6 +309,26 @@ local function insert_url_from_string(pool, tbl, str, data)
   return false
 end
 
+local function generic_service_plain_cb(string)
+  local nelts = 0
+  local new_data = {}
+  local rspamd_mempool = require "rspamd_mempool"
+  local pool = rspamd_mempool.create()
+
+  local function generic_service_elt_parser(cap)
+    if insert_url_from_string(pool, new_data, cap, nil) then
+      nelts = nelts + 1
+    end
+  end
+
+  rspamd_str_split_fun(string, '\n', generic_service_elt_parser)
+
+  generic_service_data = new_data
+  rspamd_logger.infox(generic_service_hash, "parsed %s elements from %s feed",
+    nelts, opts['generic_service_name'])
+  pool:destroy()
+end
+
 local function openphish_json_cb(string)
   local ucl = require "ucl"
   local rspamd_mempool = require "rspamd_mempool"
@@ -405,6 +434,25 @@ if opts then
       callback = phishing_cb
     })
 
+    if opts['generic_service_symbol'] then
+      generic_service_symbol = opts['generic_service_symbol']
+    end
+    if opts['generic_service_map'] then
+      generic_service_map = opts['generic_service_map']
+    end
+    if opts['generic_service_url'] then
+      generic_service_map = opts['generic_service_url']
+    end
+
+    if opts['generic_service_enabled'] then
+      generic_service_hash = rspamd_config:add_map({
+          type = 'callback',
+          url = generic_service_map,
+          callback = generic_service_plain_cb,
+          description = 'Generic feed'
+        })
+    end
+
     if opts['openphish_map'] then
       openphish_map = opts['openphish_map']
     end
@@ -450,6 +498,12 @@ if opts then
         })
     end
 
+    rspamd_config:register_symbol({
+      type = 'virtual',
+      parent = id,
+      name = generic_service_symbol,
+    })
+
     rspamd_config:register_symbol({
       type = 'virtual',
       parent = id,