Add redirector_domains setting

diff --git a/doc/markdown/modules/phishing.md b/doc/markdown/modules/phishing.md
index bca4211af71ef43f97ba1de785c7a728b12ddcb2..12e10fde4f959422c14609f0e64c126f4fb7f0fe 100644 (file)
--- a/doc/markdown/modules/phishing.md
+++ b/doc/markdown/modules/phishing.md
@@ -20,9 +20,6 @@ And the following URLs are considered as phished:
     <a href="http://t.co/xxx">http://example.com</a>
     <a href="http://redir.to/example.com">http://example.com</a>
 
-Unfortunately, rspamd can generate false positives for different redirectors or
-URL shorteners. In future rspamd releases, this issue is going to be fixed.
-
 ## Configuration of phishing module
 
 Here is an example of full module configuraition.
@@ -34,15 +31,29 @@ phishing {
        # Check only domains from this list
        domains = "file:///path/to/map";
        
+       # Make exclusions for known redirectors
+       redirector_domains = [
+               # URL/path for map, colon, name of symbol
+               "${CONFDIR}/redirectors.map:REDIRECTOR_FALSE"
+       ];
        # For certain domains from the specified strict maps
        # use another symbol for phishing plugin
        strict_domains = [
-               "${CONFDIR}/paypal.map:PAYPAL_PHISHING",
-               "${CONFDIR}/redirectors.map:REDIRECTOR_FALSE"
+               "${CONFDIR}/paypal.map:PAYPAL_PHISHING"
        ];
 }
 ~~~
 
-If `domains` is unspecified then rspamd checks all domains for phishing. `strict_domains`
-allows fine-grained control to avoid false positives and enforce some really bad phishing
-mails, such as bank phishing or other payments system phishing.
+If an anchoring (actual as opposed to phished) domain is found in a map
+referenced by the `redirector_domains` setting then the related symbol is
+yielded and the URL is not checked further. This allows making exclusions
+for known redirectors, especially ESPs.
+
+Further to this, if the phished domain is found in a map referenced by
+`strict_domains` the related symbol is yielded and the URL not checked
+further. This allows fine-grained control to avoid false positives and
+enforce some really bad phishing mails, such as bank phishing or other
+payments system phishing.
+
+Finally, the default symbol is yielded- if `domains` is specified then
+only if the phished domain is found in the related map.

diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua
index a96977d05136ff2ef21de0460c965b67ddc0a670..22c0c3cc102f4036f297188b41c25c02b0748b65 100644 (file)
--- a/src/plugins/lua/phishing.lua
+++ b/src/plugins/lua/phishing.lua
@@ -30,86 +30,110 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 local symbol = 'PHISHED_URL'
 local domains = nil
 local strict_domains = {}
+local redirector_domains = {}
 local rspamd_logger = require "rspamd_logger"
+local opts = rspamd_config:get_all_opt('phishing')
 
 local function phishing_cb(task)
-       local urls = task:get_urls();
+  local urls = task:get_urls()
+
+  if urls then
+    for _,url in ipairs(urls) do
+      if url:is_phished() then
+        local found = false
+        local purl = url:get_phished()
+        if table.maxn(redirector_domains) > 0 then
+          local tld = url:get_tld()
+          if tld then
+            for _,rule in ipairs(redirector_domains) do
+              if rule['map']:get_key(url:get_tld()) then
+                task:insert_result(rule['symbol'], 1, url:get_host())
+                found = true
+              end
+            end
+          end
+        end
+        if not found and table.maxn(strict_domains) > 0 then
+          local tld = purl:get_tld()
+          if tld then
+            for _,rule in ipairs(redirector_domains) do
+              if rule['map']:get_key(url:get_tld()) then
+                task:insert_result(rule['symbol'], 1, purl:get_host())
+                found = true
+              end
+            end
+            for _,rule in ipairs(strict_domains) do
+              if rule['map']:get_key(tld) then
+                task:insert_result(rule['symbol'], 1, purl:get_host())
+                found = true
+              end
+            end
+          end
+        end
+        if not found then
+          if domains then
+            local tld = purl:get_tld()
+            if tld then
+              if domains:get_key(tld) then
+                task:insert_result(symbol, 1, purl:get_host())
+              end
+            end
+          else
+            task:insert_result(symbol, 1, purl:get_host())
+          end
+        end
+      end
+    end
+  end
+end
 
-       if urls then
-               for _,url in ipairs(urls) do
-                       if url:is_phished() then
-                               local found = false
-                               local purl = url:get_phished()
-                               if table.maxn(strict_domains) > 0 then
-                                       local tld = purl:get_tld()
-                                       if tld then
-                                               for _,rule in ipairs(strict_domains) do
-                                                       if rule['map']:get_key(tld) then
-                                                               task:insert_result(rule['symbol'], 1, purl:get_host())
-                                                               found = true
-                                                       end
-                                               end
-                                       end
-                               end
-                               if not found then
-                                       if domains then
-                                               local tld = purl:get_tld()
-                                               if tld then
-                                                       if domains:get_key(tld) then
-                                                               task:insert_result(symbol, 1, purl:get_host())
-                                                       end
-                                               end
-                                       else            
-                                               task:insert_result(symbol, 1, purl:get_host())
-                                       end
-                               end
-                       end
-               end
-       end
+local function phishing_map(mapname, phishmap)
+  if opts[mapname] then
+    local xd = {}
+    if type(opts[mapname]) == 'table' then
+      xd = opts[mapname]
+    else
+      xd[1] = opts[mapname]
+    end
+    for _,d in ipairs(xd) do
+      local s, _ = string.find(d, ':[^:]+$')
+      if s then
+        local sym = string.sub(d, s + 1, -1)
+        local map = string.sub(d, 1, s - 1)
+        rspamd_config:register_virtual_symbol(sym, 1, id)
+        local rmap = rspamd_config:add_hash_map (map, 'Phishing ' .. mapname .. ' map')
+        if rmap then
+          local rule = {symbol = sym, map = rmap}
+          table.insert(phishmap, rule)
+        else
+          rspamd_logger.info('cannot add map: ' .. map .. ' for symbol: ' .. sym)
+        end
+      else
+        rspamd_logger.info(mapname .. ' option must be in format <map>:<symbol>')
+      end
+    end
+  end
 end
 
 -- Registration
 if type(rspamd_config.get_api_version) ~= 'nil' then
-       if rspamd_config:get_api_version() >= 1 then
-               rspamd_config:register_module_option('phishing', 'symbol', 'string')
-               rspamd_config:register_module_option('phishing', 'domains', 'map')
-               rspamd_config:register_module_option('phishing', 'strict_domains', 'string')
-       end
+  if rspamd_config:get_api_version() >= 1 then
+    rspamd_config:register_module_option('phishing', 'symbol', 'string')
+    rspamd_config:register_module_option('phishing', 'domains', 'map')
+    rspamd_config:register_module_option('phishing', 'strict_domains', 'string')
+    rspamd_config:register_module_option('phishing', 'redirector_domains', 'string')
+  end
 end
 
-local opts = rspamd_config:get_all_opt('phishing')
 if opts then
   if opts['symbol'] then
     symbol = opts['symbol']
     -- Register symbol's callback
-    local id = rspamd_config:register_symbol(symbol, 1.0, phishing_cb)
-    if opts['domains'] and type(opt['domains']) == 'string' then
-      domains = rspamd_config:add_hash_map (opts['domains'])
-    end
-    if opts['strict_domains'] then
-      local sd = {}
-      if type(opts['strict_domains']) == 'table' then
-        sd = opts['strict_domains']
-      else
-        sd[1] = opts['strict_domains']
-      end
-      for _,d in ipairs(sd) do
-        local s, _ = string.find(d, ':[^:]+$')
-        if s then
-          local sym = string.sub(d, s + 1, -1)
-          local map = string.sub(d, 1, s - 1)
-          rspamd_config:register_virtual_symbol(sym, 1, id)
-          local rmap = rspamd_config:add_hash_map (map, 'Phishing strict domains map')
-          if rmap then
-            local rule = {symbol = sym, map = rmap}
-            table.insert(strict_domains, rule)
-          else
-            rspamd_logger.info('cannot add map: ' .. map .. ' for symbol: ' .. sym)
-          end
-        else
-          rspamd_logger.info('strict_domains option must be in format <map>:<symbol>')
-        end
-      end
-    end
+    rspamd_config:register_symbol(symbol, 1.0, phishing_cb)
+  end
+  if opts['domains'] and type(opt['domains']) == 'string' then
+    domains = rspamd_config:add_hash_map (opts['domains'])
   end
+  phishing_map('strict_domains', strict_domains)
+  phishing_map('redirector_domains', redirector_domains)
 end