Hfilter: more fixes for helo checks, new metric

diff --git a/conf/lua/hfilter.lua b/conf/lua/hfilter.lua
index ad4488ae1f9504365a25c9c66c836fe3c08b5b25..93fb7272cf414bffa528a2da8862d55712f5e6a0 100644 (file)
--- a/conf/lua/hfilter.lua
+++ b/conf/lua/hfilter.lua
@@ -32,9 +32,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --local dumper = require 'pl.pretty'.dump
 local rspamd_regexp = require "rspamd_regexp"
-local rspamd_ip = require "rspamd_ip"
 
 local checks_hellohost = {
+  --Bad hosts
   ['[.-]gprs[.-]'] = 5, ['gprs[.-][0-9]'] = 5, ['[0-9][.-]?gprs'] = 5, 
   ['[.-]cdma[.-]'] = 5, ['cdma[.-][0-9]'] = 5, ['[0-9][.-]?cdma'] = 5, 
   ['[.-]homeuser[.-]'] = 5, ['homeuser[.-][0-9]'] = 5, ['[0-9][.-]?homeuser'] = 5, 
@@ -57,15 +57,18 @@ local checks_hellohost = {
   ['[.-]cablemodem[.-]'] = 5, ['cablemodem[.-][0-9]'] = 5, ['[0-9][.-]?cablemodem'] = 5, 
   ['[.-]comcast[.-]'] = 5, ['comcast[.-][0-9]'] = 5, ['[0-9][.-]?comcast'] = 5, 
   ['[.-][a|x]?dsl-dynamic[.-]'] = 5, ['[a|x]?dsl-dynamic[.-]?[0-9]'] = 5, ['[0-9][.-]?[a|x]?dsl-dynamic'] = 5, 
+
   ['[.-][a|x]?dsl[.-]'] = 4, ['[a|x]?dsl[.-]?[0-9]'] = 4, ['[0-9][.-]?[a|x]?dsl'] = 4, 
   ['[.-][a|x]?dsl-line[.-]'] = 4, ['[a|x]?dsl-line[.-]?[0-9]'] = 4, ['[0-9][.-]?[a|x]?dsl-line'] = 4, 
   ['[.-]in-?addr[.-]'] = 4, ['in-?addr[.-][0-9]'] = 4, ['[0-9][.-]?in-?addr'] = 4, 
   ['[.-]pool[.-]'] = 4, ['pool[.-][0-9]'] = 4, ['[0-9][.-]?pool'] = 4, 
   ['[.-]fibertel[.-]'] = 4, ['fibertel[.-][0-9]'] = 4, ['[0-9][.-]?fibertel'] = 4, 
   ['[.-]fbx[.-]'] = 4, ['fbx[.-][0-9]'] = 4, ['[0-9][.-]?fbx'] = 4, 
+
   ['[.-]unused-addr[.-]'] = 3, ['unused-addr[.-][0-9]'] = 3, ['[0-9][.-]?unused-addr'] = 3, 
   ['[.-]cable[.-]'] = 3, ['cable[.-][0-9]'] = 3, ['[0-9][.-]?cable'] = 3,
   ['[.-]kabel[.-]'] = 3, ['kabel[.-][0-9]'] = 3, ['[0-9][.-]?kabel'] = 3,
+
   ['[.-]host[.-]'] = 2, ['host[.-][0-9]'] = 2, ['[0-9][.-]?host'] = 2,
   ['[.-]customers?[.-]'] = 1, ['customers?[.-][0-9]'] = 1, ['[0-9][.-]?customers?'] = 1,
   ['[.-]user[.-]'] = 1, ['user[.-][0-9]'] = 1, ['[0-9][.-]?user'] = 1,
@@ -73,15 +76,30 @@ local checks_hellohost = {
 }
 
 local checks_hello = {
+  ['^[^\\.]+$'] = 5, -- for helo=COMPUTER, ANNA, etc... Without dot in helo
   ['localhost$'] = 5,
   ['^(dsl)?(device|speedtouch)\\.lan$'] = 5,
   ['\\.(lan|local|home|localdomain|intra|in-addr.arpa|priv|online|user|veloxzon)$'] = 5,
-  ['^\\[*0\\.'] = 5, ['^\\[*::1\\]*'] = 5, --loopback ipv4, ipv6
-  ['^\\[*127\\.'] = 5, ['^\\[*10\\.'] = 5, ['^\\[*172\\.16\\.'] = 5, ['^\\[*192\\.168\\.'] = 5, --local ipv4
-  ['^\\[*fe[89ab][0-9a-f]::'] = 5, ['^\\[*fe[cdf][0-9a-f]:'] = 5, --local ipv6 (fe80:: - febf::, fec0:: - feff::)
-  ['^\\[*2001:db8::'] = 5, --reserved RFC 3849 for ipv6
-  ['^\\[*fc00::'] = 5, ['^\\[*ffxx::'] = 5, --unicast, multicast ipv6
---['^\\[*\\d+[x.-]\\d+[x.-]\\d+[x.-]\\d+\\]*$'] = 4, ['^\\[*\\d+:'] = 4 --bareip ipv4, ipv6
+}
+
+local checks_hello_badip = {
+  ['^0\\.'] = 5, ['^::1$'] = 5, --loopback ipv4, ipv6
+  ['^127\\.'] = 5, ['^10\\.'] = 5, ['^192\\.168\\.'] = 5, --local ipv4
+  ['^172\\.1[6-9]\\.'] = 5, ['^172\\.2[0-9]\\.'] = 5, ['^172\\.3[01]\\.'] = 5,  --local ipv4
+  ['^169\\.254\\.'] = 5, --chanel ipv4
+  ['^192\\.0\\.0\\.'] = 5, --IETF Protocol
+  ['^192\\.88\\.99\\.'] = 5, --RFC3068
+  ['^100.6[4-9]\\.'] = 5, ['^100.[7-9]\\d\\.'] = 5, ['^100.1[01]\\d\\.'] = 5, ['^100.12[0-7]\\d\\.'] = 5, --RFC6598
+  ['^\\d\\.\\d\\.\\d\\.255$'] = 5, --multicast ipv4
+  ['^192\\.0\\.2\\.'] = 5, ['^198\\.51\\.100\\.'] = 5, ['^203\\.0\\.113\\.'] = 5,  --sample
+  ['^fe[89ab][0-9a-f]::'] = 5, ['^fe[cdf][0-9a-f]:'] = 5, --local ipv6 (fe80:: - febf::, fec0:: - feff::)
+  ['^2001:db8::'] = 5, --reserved RFC 3849 for ipv6
+  ['^fc00::'] = 5, ['^ffxx::'] = 5, --unicast, multicast ipv6
+}
+
+local checks_hello_bareip = {
+  '^\\d+[x.-]\\d+[x.-]\\d+[x.-]\\d+$', --bareip ipv4, 
+  '^[0-9a-f]+:', --bareip ipv6
 }
 
 local config = {
@@ -197,7 +215,6 @@ end
 
 --
 local function hfilter(task)
-
   -- Links checks
   if config['url_enabled'] then
     local parts = task:get_text_parts()
@@ -236,62 +253,78 @@ local function hfilter(task)
       end
     end
   end
-
+  
+  --No more checks for auth user
   if task:get_user() ~= nil then
-    return
+    return false
   end
-
-  --IP--
+  
+  --local message = task:get_message()
   local ip = false
   local rip = task:get_from_ip()
   if rip and rip:is_valid() then
     ip = rip:to_string()
   end
-
-  --HOSTNAME--
-  local hostname = task:get_hostname()
-
-  --HELO--
-  local helo = task:get_helo()
-
-  --RULES--RULES--RULES--
-
+  
   -- Check's HELO
   local weight_helo = 0
-  if config['helo_enabled'] then
-    if helo then  
-      if string.sub(helo,1,1) == '[' or rspamd_ip.from_string(helo):is_valid() then
-        task:insert_result('HFILTER_HELO_BAREIP', 1.0)
-      else
+  if config['helo_enabled'] then  
+    local helo = task:get_helo()
+    if helo then
+      helo = string.gsub(helo, '[%[%]]', '')
+      -- Regexp check HELO (checks_hello_badip)
+      local find_badip = false
+      for regexp,weight in pairs(checks_hello_badip) do
+        if check_regexp(helo, regexp) then
+          task:insert_result('HFILTER_HELO_BADIP', 1.0)
+          find_badip = true
+          break
+        end
+      end
+      
+      -- Regexp check HELO (checks_hello_bareip)
+      local find_bareip = false
+      if not find_badip then
+        for _,regexp in pairs(checks_hello_bareip) do
+          if check_regexp(helo, regexp) then
+            task:insert_result('HFILTER_HELO_BAREIP', 1.0)
+            find_bareip = true
+            break
+          end
+        end
+      end    
+      
+      if not find_badip and not find_bareip then
         -- Regexp check HELO (checks_hello)
         for regexp,weight in pairs(checks_hello) do
           if check_regexp(helo, regexp) then
             weight_helo = weight
             break
           end
-        end
-      end
-      -- Regexp check HELO (checks_hellohost)
-      for regexp,weight in pairs(checks_hellohost) do
-        if check_regexp(helo, regexp) then
-          if weight > weight_helo then
-            weight_helo = weight
+        end        
+        -- Regexp check HELO (checks_hellohost)
+        for regexp,weight in pairs(checks_hellohost) do
+          if check_regexp(helo, regexp) then
+            if weight > weight_helo then
+              weight_helo = weight
+            end
+            break
           end
-          break
+        end        
+        --FQDN check HELO
+        if ip and helo and weight_helo == 0 then
+          check_host(task, helo, 'HELO', ip, hostname)
         end
       end
-      --FQDN check HELO
-      if ip and helo then
-        check_host(task, helo, 'HELO', ip, hostname)
-      end
     else
       task:insert_result('HFILTER_HELO_UNKNOWN', 1.0)
     end
   end
   
   -- Check's HOSTNAME
+  local weight_hostname = 0
   if config['hostname_enabled'] then
-    local weight_hostname = 0
+    local hostname = task:get_hostname()  
     if hostname then
       -- Check regexp HOSTNAME
       if hostname == 'unknown' then
@@ -305,17 +338,17 @@ local function hfilter(task)
         end
       end
     else
-    task:insert_result('HFILTER_HOSTNAME_UNKNOWN', 1.00)
-    end
-
-    --Insert weight's for HELO or HOSTNAME
-    if weight_helo > 0 and weight_helo >= weight_hostname then
-      task:insert_result('HFILTER_HELO_' .. weight_helo, 1.0)
-    elseif weight_hostname > 0 and weight_hostname > weight_helo then
-      task:insert_result('HFILTER_HOSTNAME_' .. weight_hostname, 1.0)
+      task:insert_result('HFILTER_HOSTNAME_UNKNOWN', 1.00)
     end
   end
   
+  --Insert weight's for HELO or HOSTNAME
+  if weight_helo > 0 and weight_helo >= weight_hostname then
+    task:insert_result('HFILTER_HELO_' .. weight_helo, 1.0)
+  elseif weight_hostname > 0 and weight_hostname > weight_helo then
+    task:insert_result('HFILTER_HOSTNAME_' .. weight_hostname, 1.0)
+  end  
+  
   if config['from_enabled'] then
     -- MAILFROM checks --
     local from = task:get_from(1)
@@ -348,6 +381,7 @@ local symbols_enabled = {}
 
 local symbols_helo = {
   "HFILTER_HELO_BAREIP",
+  "HFILTER_HELO_BADIP",
   "HFILTER_HELO_UNKNOWN",
   "HFILTER_HELO_1",
   "HFILTER_HELO_2",