Add redirector_domains setting

doc/markdown/modules/phishing.md

@@ -20,9 +20,6 @@ And the following URLs are considered as phished:
     <a href="http://t.co/xxx">http://example.com</a>
     <a href="http://redir.to/example.com">http://example.com</a>
 
-Unfortunately, rspamd can generate false positives for different redirectors or
-URL shorteners. In future rspamd releases, this issue is going to be fixed.
-
 ## Configuration of phishing module
 
 Here is an example of full module configuraition.
@@ -34,15 +31,29 @@ phishing {
 	# Check only domains from this list
 	domains = "file:///path/to/map";
 	
+	# Make exclusions for known redirectors
+	redirector_domains = [
+		# URL/path for map, colon, name of symbol
+		"${CONFDIR}/redirectors.map:REDIRECTOR_FALSE"
+	];
 	# For certain domains from the specified strict maps
 	# use another symbol for phishing plugin
 	strict_domains = [
-		"${CONFDIR}/paypal.map:PAYPAL_PHISHING",
-		"${CONFDIR}/redirectors.map:REDIRECTOR_FALSE"
+		"${CONFDIR}/paypal.map:PAYPAL_PHISHING"
 	];
 }
 ~~~
 
-If `domains` is unspecified then rspamd checks all domains for phishing. `strict_domains`
-allows fine-grained control to avoid false positives and enforce some really bad phishing
-mails, such as bank phishing or other payments system phishing.
+If an anchoring (actual as opposed to phished) domain is found in a map
+referenced by the `redirector_domains` setting then the related symbol is
+yielded and the URL is not checked further. This allows making exclusions
+for known redirectors, especially ESPs.
+
+Further to this, if the phished domain is found in a map referenced by
+`strict_domains` the related symbol is yielded and the URL not checked
+further. This allows fine-grained control to avoid false positives and
+enforce some really bad phishing mails, such as bank phishing or other
+payments system phishing.
+
+Finally, the default symbol is yielded- if `domains` is specified then
+only if the phished domain is found in the related map.

src/plugins/lua/phishing.lua

@@ -30,86 +30,110 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 local symbol = 'PHISHED_URL'
 local domains = nil
 local strict_domains = {}
+local redirector_domains = {}
 local rspamd_logger = require "rspamd_logger"
+local opts = rspamd_config:get_all_opt('phishing')
 
 local function phishing_cb(task)
-	local urls = task:get_urls();
+  local urls = task:get_urls()
+
+  if urls then
+    for _,url in ipairs(urls) do
+      if url:is_phished() then
+        local found = false
+        local purl = url:get_phished()
+        if table.maxn(redirector_domains) > 0 then
+          local tld = url:get_tld()
+          if tld then
+            for _,rule in ipairs(redirector_domains) do
+              if rule['map']:get_key(url:get_tld()) then
+                task:insert_result(rule['symbol'], 1, url:get_host())
+                found = true
+              end
+            end
+          end
+        end
+        if not found and table.maxn(strict_domains) > 0 then
+          local tld = purl:get_tld()
+          if tld then
+            for _,rule in ipairs(redirector_domains) do
+              if rule['map']:get_key(url:get_tld()) then
+                task:insert_result(rule['symbol'], 1, purl:get_host())
+                found = true
+              end
+            end
+            for _,rule in ipairs(strict_domains) do
+              if rule['map']:get_key(tld) then
+                task:insert_result(rule['symbol'], 1, purl:get_host())
+                found = true
+              end
+            end
+          end
+        end
+        if not found then
+          if domains then
+            local tld = purl:get_tld()
+            if tld then
+              if domains:get_key(tld) then
+                task:insert_result(symbol, 1, purl:get_host())
+              end
+            end
+          else
+            task:insert_result(symbol, 1, purl:get_host())
+          end
+        end
+      end
+    end
+  end
+end
 
-	if urls then
-		for _,url in ipairs(urls) do
-			if url:is_phished() then
-				local found = false
-				local purl = url:get_phished()
-				if table.maxn(strict_domains) > 0 then
-					local tld = purl:get_tld()
-					if tld then
-						for _,rule in ipairs(strict_domains) do
-							if rule['map']:get_key(tld) then
-								task:insert_result(rule['symbol'], 1, purl:get_host())
-								found = true
-							end
-						end
-					end
-				end
-				if not found then
-					if domains then
-						local tld = purl:get_tld()
-						if tld then
-							if domains:get_key(tld) then
-								task:insert_result(symbol, 1, purl:get_host())
-							end
-						end
-					else		
-						task:insert_result(symbol, 1, purl:get_host())
-					end
-				end
-			end
-		end
-	end
+local function phishing_map(mapname, phishmap)
+  if opts[mapname] then
+    local xd = {}
+    if type(opts[mapname]) == 'table' then
+      xd = opts[mapname]
+    else
+      xd[1] = opts[mapname]
+    end
+    for _,d in ipairs(xd) do
+      local s, _ = string.find(d, ':[^:]+$')
+      if s then
+        local sym = string.sub(d, s + 1, -1)
+        local map = string.sub(d, 1, s - 1)
+        rspamd_config:register_virtual_symbol(sym, 1, id)
+        local rmap = rspamd_config:add_hash_map (map, 'Phishing ' .. mapname .. ' map')
+        if rmap then
+          local rule = {symbol = sym, map = rmap}
+          table.insert(phishmap, rule)
+        else
+          rspamd_logger.info('cannot add map: ' .. map .. ' for symbol: ' .. sym)
+        end
+      else
+        rspamd_logger.info(mapname .. ' option must be in format <map>:<symbol>')
+      end
+    end
+  end
 end
 
 -- Registration
 if type(rspamd_config.get_api_version) ~= 'nil' then
-	if rspamd_config:get_api_version() >= 1 then
-		rspamd_config:register_module_option('phishing', 'symbol', 'string')
-		rspamd_config:register_module_option('phishing', 'domains', 'map')
-		rspamd_config:register_module_option('phishing', 'strict_domains', 'string')
-	end
+  if rspamd_config:get_api_version() >= 1 then
+    rspamd_config:register_module_option('phishing', 'symbol', 'string')
+    rspamd_config:register_module_option('phishing', 'domains', 'map')
+    rspamd_config:register_module_option('phishing', 'strict_domains', 'string')
+    rspamd_config:register_module_option('phishing', 'redirector_domains', 'string')
+  end
 end
 
-local opts = rspamd_config:get_all_opt('phishing')
 if opts then
   if opts['symbol'] then
     symbol = opts['symbol']
     -- Register symbol's callback
-    local id = rspamd_config:register_symbol(symbol, 1.0, phishing_cb)
-    if opts['domains'] and type(opt['domains']) == 'string' then
-      domains = rspamd_config:add_hash_map (opts['domains'])
-    end
-    if opts['strict_domains'] then
-      local sd = {}
-      if type(opts['strict_domains']) == 'table' then
-        sd = opts['strict_domains']
-      else
-        sd[1] = opts['strict_domains']
-      end
-      for _,d in ipairs(sd) do
-        local s, _ = string.find(d, ':[^:]+$')
-        if s then
-          local sym = string.sub(d, s + 1, -1)
-          local map = string.sub(d, 1, s - 1)
-          rspamd_config:register_virtual_symbol(sym, 1, id)
-          local rmap = rspamd_config:add_hash_map (map, 'Phishing strict domains map')
-          if rmap then
-            local rule = {symbol = sym, map = rmap}
-            table.insert(strict_domains, rule)
-          else
-            rspamd_logger.info('cannot add map: ' .. map .. ' for symbol: ' .. sym)
-          end
-        else
-          rspamd_logger.info('strict_domains option must be in format <map>:<symbol>')
-        end
-      end
-    end
+    rspamd_config:register_symbol(symbol, 1.0, phishing_cb)
+  end
+  if opts['domains'] and type(opt['domains']) == 'string' then
+    domains = rspamd_config:add_hash_map (opts['domains'])
   end
+  phishing_map('strict_domains', strict_domains)
+  phishing_map('redirector_domains', redirector_domains)
 end