[Minor] Move rules from misc lua to headers and subject checks

4 files changed, 633 insertions, 532 deletions

diff --git a/rules/headers_checks.lua b/rules/headers_checks.lua
new file mode 100644
index 000000000..765643c72
--- /dev/null
+++ b/rules/headers_checks.lua
@@ -0,0 +1,562 @@
+--[[
+Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local util = require "rspamd_util"
+local ipairs = ipairs
+local table = table
+local fun = require "fun"
+local E = {}
+
+rspamd_config.CHECK_RECEIVED = {
+  callback = function (task)
+    local received = task:get_received_headers()
+    received = fun.filter(function(h)
+      return not h['artificial']
+    end, received):totable()
+    task:insert_result('RCVD_COUNT_' .. #received, 1.0)
+  end
+}
+
+rspamd_config.HAS_X_PRIO = {
+  callback = function (task)
+    local xprio = task:get_header('X-Priority');
+    if not xprio then return false end
+    local _,_,x = xprio:find('^%s?(%d+)');
+    if (x) then
+      task:insert_result('HAS_X_PRIO_' .. x, 1.0)
+    end
+  end
+}
+
+local check_replyto_id = rspamd_config:register_callback_symbol('CHECK_REPLYTO', 1.0,
+  function (task)
+    local replyto = task:get_header('Reply-To')
+    if not replyto then return false end
+    local rt = util.parse_mail_address(replyto)
+    if not (rt and rt[1]) then
+      task:insert_result('REPLYTO_UNPARSEABLE', 1.0)
+      return false
+    else
+      task:insert_result('HAS_REPLYTO', 1.0)
+      local rta = rt[1].addr
+      if rta then
+        -- Check if Reply-To address starts with title seen in display name
+        local sym = task:get_symbol('FROM_NAME_HAS_TITLE')
+        local title = (((sym or E)[1] or E).options or E)[1]
+        if title then
+          rta = rta:lower()
+          if rta:find('^' .. title) then
+            task:insert_result('REPLYTO_EMAIL_HAS_TITLE', 1.0)
+          end
+        end
+      end
+    end
+
+    -- See if Reply-To matches From in some way
+    local from = task:get_from(2)
+    local from_h = task:get_header('From')
+    if not (from and from[1]) then return false end
+    if (from_h and from_h == replyto) then
+      -- From and Reply-To are identical
+      task:insert_result('REPLYTO_EQ_FROM', 1.0)
+    else
+      if (from and from[1]) then
+        -- See if From and Reply-To addresses match
+        if (from[1].addr:lower() == rt[1].addr:lower()) then
+          task:insert_result('REPLYTO_ADDR_EQ_FROM', 1.0)
+        elseif from[1].domain and rt[1].domain then
+          if (from[1].domain:lower() == rt[1].domain:lower()) then
+            task:insert_result('REPLYTO_DOM_EQ_FROM_DOM', 1.0)
+          else
+            task:insert_result('REPLYTO_DOM_NEQ_FROM_DOM', 1.0)
+          end
+        end
+        -- See if the Display Names match
+        if (from[1].name and rt[1].name and from[1].name:lower() == rt[1].name:lower()) then
+          task:insert_result('REPLYTO_DN_EQ_FROM_DN', 1.0)
+        end
+      end
+    end
+  end
+)
+
+rspamd_config:register_symbol{
+  name = 'REPLYTO_UNPARSEABLE',
+  score = 1.0,
+  parent = check_replyto_id,
+  type = 'virtual',
+  description = 'Reply-To header could not be parsed',
+  group = 'header',
+}
+rspamd_config:register_symbol{
+  name = 'HAS_REPLYTO',
+  score = 0.0,
+  parent = check_replyto_id,
+  type = 'virtual',
+  description = 'Has Reply-To header',
+  group = 'header',
+}
+rspamd_config:register_symbol{
+  name = 'REPLYTO_EQ_FROM',
+  score = 0.0,
+  parent = check_replyto_id,
+  type = 'virtual',
+  description = 'Reply-To header is identical to From header',
+  group = 'header',
+}
+rspamd_config:register_symbol{
+  name = 'REPLYTO_ADDR_EQ_FROM',
+  score = 0.0,
+  parent = check_replyto_id,
+  type = 'virtual',
+  description = 'Reply-To header is identical to SMTP From',
+  group = 'header',
+}
+rspamd_config:register_symbol{
+  name = 'REPLYTO_DOM_EQ_FROM_DOM',
+  score = 0.0,
+  parent = check_replyto_id,
+  type = 'virtual',
+  description = 'Reply-To domain matches the From domain',
+  group = 'header',
+}
+rspamd_config:register_symbol{
+  name = 'REPLYTO_DOM_NEQ_FROM_DOM',
+  score = 0.0,
+  parent = check_replyto_id,
+  type = 'virtual',
+  description = 'Reply-To domain does not match the From domain',
+  group = 'header',
+}
+rspamd_config:register_symbol{
+  name = 'REPLYTO_DN_EQ_FROM_DN',
+  score = 0.0,
+  parent = check_replyto_id,
+  type = 'virtual',
+  description = 'Reply-To display name matches From',
+  group = 'header',
+}
+rspamd_config:register_symbol{
+  name = 'FROM_NAME_HAS_TITLE',
+  score = 2.0,
+  parent = check_replyto_id,
+  type = 'virtual',
+  description = 'Reply-To header has title',
+  group = 'header',
+}
+rspamd_config:register_dependency(check_replyto_id, 'FROM_NAME_HAS_TITLE')
+
+local check_mime_id = rspamd_config:register_callback_symbol('CHECK_MIME', 1.0,
+  function (task)
+    local parts = task:get_parts()
+    if not parts then return false end
+
+    -- Make sure there is a MIME-Version header
+    local mv = task:get_header('MIME-Version')
+    local missing_mime = false
+    if (not mv) then
+      missing_mime = true
+    end
+
+    local found_ma = false
+    local found_plain = false
+    local found_html = false
+    local cte_7bit = false
+
+    for _,p in ipairs(parts) do
+      local mtype,subtype = p:get_type()
+      local ctype = mtype:lower() .. '/' .. subtype:lower()
+      if (ctype == 'multipart/alternative') then
+        found_ma = true
+      end
+      if (ctype == 'text/plain') then
+        if p:get_cte() == '7bit' then
+          cte_7bit = true
+        end
+        found_plain = true
+      end
+      if (ctype == 'text/html') then
+        if p:get_cte() == '7bit' then
+          cte_7bit = true
+        end
+        found_html = true
+      end
+    end
+
+    if missing_mime then
+      if not (not found_ma and ((found_plain or found_html) and cte_7bit)) then
+        task:insert_result('MISSING_MIME_VERSION', 1.0)
+      end
+    end
+
+    if (found_ma) then
+      if (not found_plain) then
+        task:insert_result('MIME_MA_MISSING_TEXT', 1.0)
+      end
+      if (not found_html) then
+        task:insert_result('MIME_MA_MISSING_HTML', 1.0)
+      end
+    end
+  end
+)
+
+rspamd_config:register_symbol{
+  name = 'MISSING_MIME_VERSION',
+  score = 2.0,
+  parent = check_mime_id,
+  type = 'virtual',
+  description = 'MIME-Version header is missing',
+  group = 'header',
+}
+rspamd_config:register_symbol{
+  name = 'MIME_MA_MISSING_TEXT',
+  score = 2.0,
+  parent = check_mime_id,
+  type = 'virtual',
+  description = 'MIME multipart/alternative missing text/plain part',
+  group = 'header',
+}
+rspamd_config:register_symbol{
+  name = 'MIME_MA_MISSING_HTML',
+  score = 1.0,
+  parent = check_mime_id,
+  type = 'virtual',
+  description = 'MIME multipart/alternative missing text/html part',
+  group = 'header',
+}
+
+-- Used to be called IS_LIST
+rspamd_config.PREVIOUSLY_DELIVERED = {
+  callback = function(task)
+    if not task:has_recipients(2) then return false end
+    local to = task:get_recipients(2)
+    local rcvds = task:get_header_full('Received')
+    if not rcvds then return false end
+    for _, rcvd in ipairs(rcvds) do
+      local _,_,addr = rcvd['decoded']:lower():find("%sfor%s<(.-)>")
+      if addr then
+        for _, toa in ipairs(to) do
+          if toa and toa.addr:lower() == addr then
+            return true, addr
+          end
+        end
+        return false
+      end
+    end
+  end,
+  description = 'Message either to a list or was forwarded',
+  score = 0.0
+}
+rspamd_config.BROKEN_HEADERS = {
+  callback = function(task)
+    return task:has_flag('broken_headers')
+  end,
+  score = 10.0,
+  group = 'header',
+  description = 'Headers structure is likely broken'
+}
+
+rspamd_config.BROKEN_CONTENT_TYPE = {
+  callback = function(task)
+    return fun.any(function(p) return p:is_broken() end,
+      task:get_parts())
+  end,
+  score = 1.5,
+  group = 'header',
+  description = 'Message has part with broken content type'
+}
+
+rspamd_config.HEADER_RCONFIRM_MISMATCH = {
+  callback = function (task)
+    local header_from = nil
+    local cread = task:get_header('X-Confirm-Reading-To')
+
+    if task:has_from('mime') then
+      header_from  = task:get_from('mime')[1]
+    end
+
+    local header_cread = nil
+    if cread then
+      local headers_cread = util.parse_mail_address(cread)
+      if headers_cread then header_cread = headers_cread[1] end
+    end
+
+    if header_from and header_cread then
+      if not string.find(header_from['addr'], header_cread['addr']) then
+        return true
+      end
+    end
+
+    return false
+  end,
+
+  score = 2.0,
+  group = 'header',
+  description = 'Read confirmation address is different to from address'
+}
+
+rspamd_config.HEADER_FORGED_MDN = {
+  callback = function (task)
+    local mdn = task:get_header('Disposition-Notification-To')
+    if not mdn then return false end
+    local header_rp = nil
+
+    if task:has_from('smtp') then
+      header_rp = task:get_from('smtp')[1]
+    end
+
+    -- Parse mail addr
+    local headers_mdn = util.parse_mail_address(mdn)
+
+    if headers_mdn and not header_rp  then return true end
+    if header_rp  and not headers_mdn then return false end
+    if not headers_mdn and not header_rp then return false end
+
+    local found_match = false
+    for _, h in ipairs(headers_mdn) do
+      if util.strequal_caseless(h['addr'], header_rp['addr']) then
+        found_match = true
+        break
+      end
+    end
+
+    return (not found_match)
+  end,
+
+  score = 2.0,
+  group = 'header',
+  description = 'Read confirmation address is different to return path'
+}
+
+local headers_unique = {
+  'Content-Type',
+  'Content-Transfer-Encoding',
+  -- https://tools.ietf.org/html/rfc5322#section-3.6
+  'Date',
+  'From',
+  'Sender',
+  'Reply-To',
+  'To',
+  'Cc',
+  'Bcc',
+  'Message-ID',
+  'In-Reply-To',
+  'References',
+  'Subject'
+}
+
+rspamd_config.MULTIPLE_UNIQUE_HEADERS = {
+  callback = function(task)
+    local res = 0
+    local res_tbl = {}
+
+    for _,hdr in ipairs(headers_unique) do
+      local h = task:get_header_full(hdr)
+
+      if h and #h > 1 then
+        res = res + 1
+        table.insert(res_tbl, hdr)
+      end
+    end
+
+    if res > 0 then
+      return true,res,table.concat(res_tbl, ',')
+    end
+
+    return false
+  end,
+
+  score = 5.0,
+  group = 'header',
+  description = 'Repeated unique headers'
+}
+
+rspamd_config.MISSING_FROM = {
+  callback = function(task)
+    local from = task:get_header('From')
+    if from == nil or from == '' then
+      return true
+    end
+    return false
+  end,
+  score = 2.0,
+  group = 'header',
+  description = 'Missing From: header'
+}
+rspamd_config.MV_CASE = {
+  callback = function (task)
+    local mv = task:get_header('Mime-Version', true)
+    if (mv) then return true end
+  end,
+  description = 'Mime-Version .vs. MIME-Version',
+  score = 0.5
+}
+
+rspamd_config.FAKE_REPLY = {
+  callback = function (task)
+    local subject = task:get_header('Subject')
+    if (subject and subject:lower():find('^re:')) then
+      local ref = task:get_header('References')
+      local rt  = task:get_header('In-Reply-To')
+      if (not (ref or rt)) then return true end
+    end
+    return false
+  end,
+  description = 'Fake reply',
+  score = 1.0
+}
+
+local check_from_id = rspamd_config:register_callback_symbol('CHECK_FROM', 1.0,
+  function(task)
+    local envfrom = task:get_from(1)
+    local from = task:get_from(2)
+    if (from and from[1] and not from[1].name) then
+      task:insert_result('FROM_NO_DN', 1.0)
+    elseif (from and from[1] and from[1].name and
+            from[1].name:lower() == from[1].addr:lower()) then
+      task:insert_result('FROM_DN_EQ_ADDR', 1.0)
+    elseif (from and from[1] and from[1].name) then
+      task:insert_result('FROM_HAS_DN', 1.0)
+      -- Look for Mr/Mrs/Dr titles
+      local n = from[1].name:lower()
+      local match, match_end
+      match, match_end = n:find('^mrs?[%.%s]')
+      if match then
+        task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end-1))
+      end
+      match, match_end = n:find('^dr[%.%s]')
+      if match then
+        task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end-1))
+      end
+      -- Check for excess spaces
+      if n:find('%s%s') then
+        task:insert_result('FROM_NAME_EXCESS_SPACE', 1.0)
+      end
+    end
+    if (envfrom and from and envfrom[1] and from[1] and
+        envfrom[1].addr:lower() == from[1].addr:lower())
+    then
+      task:insert_result('FROM_EQ_ENVFROM', 1.0)
+    elseif (envfrom and envfrom[1] and envfrom[1].addr) then
+      task:insert_result('FROM_NEQ_ENVFROM', 1.0, from and from[1].addr or '', envfrom[1].addr)
+    end
+
+    local to = task:get_recipients(2)
+    if not (to and to[1] and #to == 1 and from) then return false end
+    -- Check if FROM == TO
+    if (to[1].addr:lower() == from[1].addr:lower()) then
+      task:insert_result('TO_EQ_FROM', 1.0)
+    elseif (to[1].domain and from[1].domain and
+        to[1].domain:lower() == from[1].domain:lower()) then
+      task:insert_result('TO_DOM_EQ_FROM_DOM', 1.0)
+    end
+  end
+)
+
+rspamd_config:register_virtual_symbol('FROM_NO_DN', 1.0, check_from_id)
+rspamd_config:set_metric_symbol('FROM_NO_DN', 0, 'From header does not have a display name')
+rspamd_config:register_virtual_symbol('FROM_DN_EQ_ADDR', 1.0, check_from_id)
+rspamd_config:set_metric_symbol('FROM_DN_EQ_ADDR', 1.0, 'From header display name is the same as the address')
+rspamd_config:register_virtual_symbol('FROM_HAS_DN', 1.0, check_from_id)
+rspamd_config:set_metric_symbol('FROM_HAS_DN', 0, 'From header has a display name')
+rspamd_config:register_virtual_symbol('FROM_NAME_EXCESS_SPACE', 1.0, check_from_id)
+rspamd_config:set_metric_symbol('FROM_NAME_EXCESS_SPACE', 1.0, 'From header display name contains excess whitespace')
+rspamd_config:register_virtual_symbol('FROM_NAME_HAS_TITLE', 1.0, check_from_id)
+rspamd_config:set_metric_symbol('FROM_NAME_HAS_TITLE', 1.0, 'From header display name has a title (Mr/Mrs/Dr)')
+rspamd_config:register_virtual_symbol('FROM_EQ_ENVFROM', 1.0, check_from_id)
+rspamd_config:set_metric_symbol('FROM_EQ_ENVFROM', 0, 'From address is the same as the envelope')
+rspamd_config:register_virtual_symbol('FROM_NEQ_ENVFROM', 1.0, check_from_id)
+rspamd_config:set_metric_symbol('FROM_NEQ_ENVFROM', 0, 'From address is different to the envelope')
+rspamd_config:register_virtual_symbol('TO_EQ_FROM', 1.0, check_from_id)
+rspamd_config:set_metric_symbol('TO_EQ_FROM', 0, 'To address matches the From address')
+rspamd_config:register_virtual_symbol('TO_DOM_EQ_FROM_DOM', 1.0, check_from_id)
+rspamd_config:set_metric_symbol('TO_DOM_EQ_FROM_DOM', 0, 'To domain is the same as the From domain')
+
+local check_to_cc_id = rspamd_config:register_callback_symbol('CHECK_TO_CC', 1.0,
+  function(task)
+    local rcpts = task:get_recipients(1)
+    local to = task:get_recipients(2)
+    local to_match_envrcpt = 0
+    if (not to) then return false end
+    -- Add symbol for recipient count
+    if (#to > 50) then
+      task:insert_result('RCPT_COUNT_GT_50', 1.0)
+    else
+      task:insert_result('RCPT_COUNT_' .. #to, 1.0)
+    end
+    -- Check for display names
+    local to_dn_count = 0
+    local to_dn_eq_addr_count = 0
+    for _, toa in ipairs(to) do
+      -- To: Recipients <noreply@dropbox.com>
+      if (toa['name'] and (toa['name']:lower() == 'recipient'
+          or toa['name']:lower() == 'recipients')) then
+        task:insert_result('TO_DN_RECIPIENTS', 1.0)
+      end
+      if (toa['name'] and toa['name']:lower() == toa['addr']:lower()) then
+        to_dn_eq_addr_count = to_dn_eq_addr_count + 1
+      elseif (toa['name']) then
+        to_dn_count = to_dn_count + 1
+      end
+      -- See if header recipients match envrcpts
+      if (rcpts) then
+        for _, rcpt in ipairs(rcpts) do
+          if (toa and toa['addr'] and rcpt and rcpt['addr'] and
+              rcpt['addr']:lower() == toa['addr']:lower())
+          then
+            to_match_envrcpt = to_match_envrcpt + 1
+          end
+        end
+      end
+    end
+    if (to_dn_count == 0 and to_dn_eq_addr_count == 0) then
+      task:insert_result('TO_DN_NONE', 1.0)
+    elseif (to_dn_count == #to) then
+      task:insert_result('TO_DN_ALL', 1.0)
+    elseif (to_dn_count > 0) then
+      task:insert_result('TO_DN_SOME', 1.0)
+    end
+    if (to_dn_eq_addr_count == #to) then
+      task:insert_result('TO_DN_EQ_ADDR_ALL', 1.0)
+    elseif (to_dn_eq_addr_count > 0) then
+      task:insert_result('TO_DN_EQ_ADDR_SOME', 1.0)
+    end
+
+    -- See if header recipients match envelope recipients
+    if (to_match_envrcpt == #to) then
+      task:insert_result('TO_MATCH_ENVRCPT_ALL', 1.0)
+    elseif (to_match_envrcpt > 0) then
+      task:insert_result('TO_MATCH_ENVRCPT_SOME', 1.0)
+    end
+  end
+)
+
+rspamd_config:register_virtual_symbol('TO_DN_RECIPIENTS', 1.0, check_to_cc_id)
+rspamd_config:set_metric_symbol('TO_DN_RECIPIENTS', 2.0, 'To header display name is "Recipients"')
+rspamd_config:register_virtual_symbol('TO_DN_NONE', 1.0, check_to_cc_id)
+rspamd_config:set_metric_symbol('TO_DN_NONE', 0, 'None of the recipients have display names')
+rspamd_config:register_virtual_symbol('TO_DN_ALL', 1.0, check_to_cc_id)
+rspamd_config:set_metric_symbol('TO_DN_ALL', 0, 'All of the recipients have display names')
+rspamd_config:register_virtual_symbol('TO_DN_SOME', 1.0, check_to_cc_id)
+rspamd_config:set_metric_symbol('TO_DN_SOME', 0, 'Some of the recipients have display names')
+rspamd_config:register_virtual_symbol('TO_DN_EQ_ADDR_ALL', 1.0, check_to_cc_id)
+rspamd_config:set_metric_symbol('TO_DN_EQ_ADDR_ALL', 0, 'All of the recipients have display names that are the same as their address')
+rspamd_config:register_virtual_symbol('TO_DN_EQ_ADDR_SOME', 1.0, check_to_cc_id)
+rspamd_config:set_metric_symbol('TO_DN_EQ_ADDR_SOME', 0, 'Some of the recipients have display names that are the same as their address')
+rspamd_config:register_virtual_symbol('TO_MATCH_ENVRCPT_ALL', 1.0, check_to_cc_id)
+rspamd_config:set_metric_symbol('TO_MATCH_ENVRCPT_ALL', 0, 'All of the recipients match the envelope')
+rspamd_config:register_virtual_symbol('TO_MATCH_ENVRCPT_SOME', 1.0, check_to_cc_id)
+rspamd_config:set_metric_symbol('TO_MATCH_ENVRCPT_SOME', 0, 'Some of the recipients match the envelope')
\ No newline at end of file
diff --git a/rules/misc.lua b/rules/misc.lua
index 1c69b7243..30a3d57ba 100644
--- a/rules/misc.lua
+++ b/rules/misc.lua
@@ -1,5 +1,5 @@
 --[[
-Copyright (c) 2011-2015, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2011-2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,61 +19,6 @@ limitations under the License.
 local E = {}
 local fun = require "fun"
 local util = require "rspamd_util"
-local rspamd_regexp = require "rspamd_regexp"
-
--- Uncategorized rules
-local subject_re = rspamd_regexp.create('/^(?:(?:Re|Fwd|Fw|Aw|Antwort|Sv):\\s*)+(.+)$/i')
-
--- Local functions
-
-
--- Subject issues
-local function test_subject(task, check_function, rate)
-  local function normalize_linear(a, x)
-      local f = a * x
-      return true, (( f < 1 ) and f or 1), tostring(x)
-  end
-
-  local sbj = task:get_header('Subject')
-
-  if sbj then
-    local stripped_subject = subject_re:search(sbj, false, true)
-    if stripped_subject and stripped_subject[1] and stripped_subject[1][2] then
-      sbj = stripped_subject[1][2]
-    end
-
-    local l = util.strlen_utf8(sbj)
-    if check_function(sbj, l) then
-      return normalize_linear(rate, l)
-    end
-  end
-
-  return false
-end
-
-rspamd_config.SUBJ_ALL_CAPS = {
-  callback = function(task)
-    local caps_test = function(sbj)
-      return util.is_uppercase(sbj)
-    end
-    return test_subject(task, caps_test, 1.0/40.0)
-  end,
-  score = 3.0,
-  group = 'subject',
-  description = 'All capital letters in subject'
-}
-
-rspamd_config.LONG_SUBJ = {
-  callback = function(task)
-    local length_test = function(_, len)
-      return len > 200
-    end
-    return test_subject(task, length_test, 1.0/400.0)
-  end,
-  score = 3.0,
-  group = 'subject',
-  description = 'Subject is too long'
-}
 
 -- Different text parts
 rspamd_config.R_PARTS_DIFFER = {
@@ -174,129 +119,6 @@ rspamd_config.R_SUSPICIOUS_URL = {
   group = 'url'
 }
 
-rspamd_config.BROKEN_HEADERS = {
-  callback = function(task)
-    return task:has_flag('broken_headers')
-  end,
-  score = 10.0,
-  group = 'header',
-  description = 'Headers structure is likely broken'
-}
-
-rspamd_config.BROKEN_CONTENT_TYPE = {
-  callback = function(task)
-    return fun.any(function(p) return p:is_broken() end,
-      task:get_parts())
-  end,
-  score = 1.5,
-  group = 'header',
-  description = 'Message has part with broken content type'
-}
-
-rspamd_config.HEADER_RCONFIRM_MISMATCH = {
-  callback = function (task)
-    local header_from = nil
-    local cread = task:get_header('X-Confirm-Reading-To')
-
-    if task:has_from('mime') then
-      header_from  = task:get_from('mime')[1]
-    end
-
-    local header_cread = nil
-    if cread then
-      local headers_cread = util.parse_mail_address(cread)
-      if headers_cread then header_cread = headers_cread[1] end
-    end
-
-    if header_from and header_cread then
-      if not string.find(header_from['addr'], header_cread['addr']) then
-        return true
-      end
-    end
-
-    return false
-  end,
-
-  score = 2.0,
-  group = 'header',
-  description = 'Read confirmation address is different to from address'
-}
-
-rspamd_config.HEADER_FORGED_MDN = {
-  callback = function (task)
-    local mdn = task:get_header('Disposition-Notification-To')
-    if not mdn then return false end
-    local header_rp = nil
-
-    if task:has_from('smtp') then
-      header_rp = task:get_from('smtp')[1]
-    end
-
-    -- Parse mail addr
-    local headers_mdn = util.parse_mail_address(mdn)
-
-    if headers_mdn and not header_rp  then return true end
-    if header_rp  and not headers_mdn then return false end
-    if not headers_mdn and not header_rp then return false end
-
-    local found_match = false
-    for _, h in ipairs(headers_mdn) do
-      if util.strequal_caseless(h['addr'], header_rp['addr']) then
-        found_match = true
-        break
-      end
-    end
-
-    return (not found_match)
-  end,
-
-  score = 2.0,
-  group = 'header',
-  description = 'Read confirmation address is different to return path'
-}
-
-local headers_unique = {
-  'Content-Type',
-  'Content-Transfer-Encoding',
-  -- https://tools.ietf.org/html/rfc5322#section-3.6
-  'Date',
-  'From',
-  'Sender',
-  'Reply-To',
-  'To',
-  'Cc',
-  'Bcc',
-  'Message-ID',
-  'In-Reply-To',
-  'References',
-  'Subject'
-}
-
-rspamd_config.MULTIPLE_UNIQUE_HEADERS = {
-  callback = function (task)
-    local res = 0
-    local res_tbl = {}
-
-    for _,hdr in ipairs(headers_unique) do
-      local h = task:get_header_full(hdr)
-
-      if h and #h > 1 then
-        res = res + 1
-        table.insert(res_tbl, hdr)
-      end
-    end
-
-    if res > 0 then
-      return true,res,table.concat(res_tbl, ',')
-    end
-
-    return false
-  end,
-
-  score = 5.0,
-  group = 'header',
-  description = 'Repeated unique headers'
-}
 
 rspamd_config.ENVFROM_PRVS = {
     callback = function (task)
@@ -391,19 +213,6 @@ rspamd_config.RCVD_TLS_ALL = {
     group = "encryption"
 }
 
-rspamd_config.MISSING_FROM = {
-    callback = function(task)
-      local from = task:get_header('From')
-      if from == nil or from == '' then
-        return true
-      end
-      return false
-    end,
-    score = 2.0,
-    group = 'header',
-    description = 'Missing From: header'
-}
-
 rspamd_config.RCVD_HELO_USER = {
   callback = function (task)
     -- Check HELO argument from MTA
@@ -453,346 +262,6 @@ rspamd_config.HAS_ATTACHMENT = {
   description = 'Message contains attachments'
 }
 
-rspamd_config.MV_CASE = {
-  callback = function (task)
-    local mv = task:get_header('Mime-Version', true)
-    if (mv) then return true end
-  end,
-  description = 'Mime-Version .vs. MIME-Version',
-  score = 0.5
-}
-
-rspamd_config.FAKE_REPLY = {
-  callback = function (task)
-    local subject = task:get_header('Subject')
-    if (subject and subject:lower():find('^re:')) then
-      local ref = task:get_header('References')
-      local rt  = task:get_header('In-Reply-To')
-      if (not (ref or rt)) then return true end
-    end
-    return false
-  end,
-  description = 'Fake reply',
-  score = 1.0
-}
-
-local check_from_id = rspamd_config:register_callback_symbol('CHECK_FROM', 1.0,
-  function(task)
-    local envfrom = task:get_from(1)
-    local from = task:get_from(2)
-    if (from and from[1] and not from[1].name) then
-      task:insert_result('FROM_NO_DN', 1.0)
-    elseif (from and from[1] and from[1].name and
-            from[1].name:lower() == from[1].addr:lower()) then
-      task:insert_result('FROM_DN_EQ_ADDR', 1.0)
-    elseif (from and from[1] and from[1].name) then
-      task:insert_result('FROM_HAS_DN', 1.0)
-      -- Look for Mr/Mrs/Dr titles
-      local n = from[1].name:lower()
-      local match, match_end
-      match, match_end = n:find('^mrs?[%.%s]')
-      if match then
-        task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end-1))
-      end
-      match, match_end = n:find('^dr[%.%s]')
-      if match then
-        task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end-1))
-      end
-      -- Check for excess spaces
-      if n:find('%s%s') then
-        task:insert_result('FROM_NAME_EXCESS_SPACE', 1.0)
-      end
-    end
-    if (envfrom and from and envfrom[1] and from[1] and
-        envfrom[1].addr:lower() == from[1].addr:lower())
-    then
-      task:insert_result('FROM_EQ_ENVFROM', 1.0)
-    elseif (envfrom and envfrom[1] and envfrom[1].addr) then
-      task:insert_result('FROM_NEQ_ENVFROM', 1.0, from and from[1].addr or '', envfrom[1].addr)
-    end
-
-    local to = task:get_recipients(2)
-    if not (to and to[1] and #to == 1 and from) then return false end
-    -- Check if FROM == TO
-    if (to[1].addr:lower() == from[1].addr:lower()) then
-      task:insert_result('TO_EQ_FROM', 1.0)
-    elseif (to[1].domain and from[1].domain and
-        to[1].domain:lower() == from[1].domain:lower()) then
-      task:insert_result('TO_DOM_EQ_FROM_DOM', 1.0)
-    end
-  end
-)
-
-rspamd_config:register_virtual_symbol('FROM_NO_DN', 1.0, check_from_id)
-rspamd_config:set_metric_symbol('FROM_NO_DN', 0, 'From header does not have a display name')
-rspamd_config:register_virtual_symbol('FROM_DN_EQ_ADDR', 1.0, check_from_id)
-rspamd_config:set_metric_symbol('FROM_DN_EQ_ADDR', 1.0, 'From header display name is the same as the address')
-rspamd_config:register_virtual_symbol('FROM_HAS_DN', 1.0, check_from_id)
-rspamd_config:set_metric_symbol('FROM_HAS_DN', 0, 'From header has a display name')
-rspamd_config:register_virtual_symbol('FROM_NAME_EXCESS_SPACE', 1.0, check_from_id)
-rspamd_config:set_metric_symbol('FROM_NAME_EXCESS_SPACE', 1.0, 'From header display name contains excess whitespace')
-rspamd_config:register_virtual_symbol('FROM_NAME_HAS_TITLE', 1.0, check_from_id)
-rspamd_config:set_metric_symbol('FROM_NAME_HAS_TITLE', 1.0, 'From header display name has a title (Mr/Mrs/Dr)')
-rspamd_config:register_virtual_symbol('FROM_EQ_ENVFROM', 1.0, check_from_id)
-rspamd_config:set_metric_symbol('FROM_EQ_ENVFROM', 0, 'From address is the same as the envelope')
-rspamd_config:register_virtual_symbol('FROM_NEQ_ENVFROM', 1.0, check_from_id)
-rspamd_config:set_metric_symbol('FROM_NEQ_ENVFROM', 0, 'From address is different to the envelope')
-rspamd_config:register_virtual_symbol('TO_EQ_FROM', 1.0, check_from_id)
-rspamd_config:set_metric_symbol('TO_EQ_FROM', 0, 'To address matches the From address')
-rspamd_config:register_virtual_symbol('TO_DOM_EQ_FROM_DOM', 1.0, check_from_id)
-rspamd_config:set_metric_symbol('TO_DOM_EQ_FROM_DOM', 0, 'To domain is the same as the From domain')
-
-local check_to_cc_id = rspamd_config:register_callback_symbol('CHECK_TO_CC', 1.0,
-  function(task)
-    local rcpts = task:get_recipients(1)
-    local to = task:get_recipients(2)
-    local to_match_envrcpt = 0
-    if (not to) then return false end
-    -- Add symbol for recipient count
-    if (#to > 50) then
-      task:insert_result('RCPT_COUNT_GT_50', 1.0)
-    else
-      task:insert_result('RCPT_COUNT_' .. #to, 1.0)
-    end
-    -- Check for display names
-    local to_dn_count = 0
-    local to_dn_eq_addr_count = 0
-    for _, toa in ipairs(to) do
-      -- To: Recipients <noreply@dropbox.com>
-      if (toa['name'] and (toa['name']:lower() == 'recipient'
-          or toa['name']:lower() == 'recipients')) then
-        task:insert_result('TO_DN_RECIPIENTS', 1.0)
-      end
-      if (toa['name'] and toa['name']:lower() == toa['addr']:lower()) then
-        to_dn_eq_addr_count = to_dn_eq_addr_count + 1
-      elseif (toa['name']) then
-        to_dn_count = to_dn_count + 1
-      end
-      -- See if header recipients match envrcpts
-      if (rcpts) then
-        for _, rcpt in ipairs(rcpts) do
-          if (toa and toa['addr'] and rcpt and rcpt['addr'] and
-              rcpt['addr']:lower() == toa['addr']:lower())
-          then
-            to_match_envrcpt = to_match_envrcpt + 1
-          end
-        end
-      end
-    end
-    if (to_dn_count == 0 and to_dn_eq_addr_count == 0) then
-      task:insert_result('TO_DN_NONE', 1.0)
-    elseif (to_dn_count == #to) then
-      task:insert_result('TO_DN_ALL', 1.0)
-    elseif (to_dn_count > 0) then
-      task:insert_result('TO_DN_SOME', 1.0)
-    end
-    if (to_dn_eq_addr_count == #to) then
-      task:insert_result('TO_DN_EQ_ADDR_ALL', 1.0)
-    elseif (to_dn_eq_addr_count > 0) then
-      task:insert_result('TO_DN_EQ_ADDR_SOME', 1.0)
-    end
-
-    -- See if header recipients match envelope recipients
-    if (to_match_envrcpt == #to) then
-      task:insert_result('TO_MATCH_ENVRCPT_ALL', 1.0)
-    elseif (to_match_envrcpt > 0) then
-      task:insert_result('TO_MATCH_ENVRCPT_SOME', 1.0)
-    end
-  end
-)
-
-rspamd_config:register_virtual_symbol('TO_DN_RECIPIENTS', 1.0, check_to_cc_id)
-rspamd_config:set_metric_symbol('TO_DN_RECIPIENTS', 2.0, 'To header display name is "Recipients"')
-rspamd_config:register_virtual_symbol('TO_DN_NONE', 1.0, check_to_cc_id)
-rspamd_config:set_metric_symbol('TO_DN_NONE', 0, 'None of the recipients have display names')
-rspamd_config:register_virtual_symbol('TO_DN_ALL', 1.0, check_to_cc_id)
-rspamd_config:set_metric_symbol('TO_DN_ALL', 0, 'All of the recipients have display names')
-rspamd_config:register_virtual_symbol('TO_DN_SOME', 1.0, check_to_cc_id)
-rspamd_config:set_metric_symbol('TO_DN_SOME', 0, 'Some of the recipients have display names')
-rspamd_config:register_virtual_symbol('TO_DN_EQ_ADDR_ALL', 1.0, check_to_cc_id)
-rspamd_config:set_metric_symbol('TO_DN_EQ_ADDR_ALL', 0, 'All of the recipients have display names that are the same as their address')
-rspamd_config:register_virtual_symbol('TO_DN_EQ_ADDR_SOME', 1.0, check_to_cc_id)
-rspamd_config:set_metric_symbol('TO_DN_EQ_ADDR_SOME', 0, 'Some of the recipients have display names that are the same as their address')
-rspamd_config:register_virtual_symbol('TO_MATCH_ENVRCPT_ALL', 1.0, check_to_cc_id)
-rspamd_config:set_metric_symbol('TO_MATCH_ENVRCPT_ALL', 0, 'All of the recipients match the envelope')
-rspamd_config:register_virtual_symbol('TO_MATCH_ENVRCPT_SOME', 1.0, check_to_cc_id)
-rspamd_config:set_metric_symbol('TO_MATCH_ENVRCPT_SOME', 0, 'Some of the recipients match the envelope')
-
-rspamd_config.CHECK_RECEIVED = {
-  callback = function (task)
-    local received = task:get_received_headers()
-    received = fun.filter(function(h)
-      return not h['artificial']
-    end, received):totable()
-    task:insert_result('RCVD_COUNT_' .. #received, 1.0)
-  end
-}
-
-rspamd_config.HAS_X_PRIO = {
-  callback = function (task)
-    local xprio = task:get_header('X-Priority');
-    if not xprio then return false end
-    local _,_,x = xprio:find('^%s?(%d+)');
-    if (x) then
-      task:insert_result('HAS_X_PRIO_' .. x, 1.0)
-    end
-  end
-}
-
-local check_replyto_id = rspamd_config:register_callback_symbol('CHECK_REPLYTO', 1.0,
-  function (task)
-    local replyto = task:get_header('Reply-To')
-    if not replyto then return false end
-    local rt = util.parse_mail_address(replyto)
-    if not (rt and rt[1]) then
-      task:insert_result('REPLYTO_UNPARSEABLE', 1.0)
-      return false
-    else
-      task:insert_result('HAS_REPLYTO', 1.0)
-      local rta = rt[1].addr
-      if rta then
-        -- Check if Reply-To address starts with title seen in display name
-        local sym = task:get_symbol('FROM_NAME_HAS_TITLE')
-        local title = (((sym or E)[1] or E).options or E)[1]
-        if title then
-          rta = rta:lower()
-          if rta:find('^' .. title) then
-            task:insert_result('REPLYTO_EMAIL_HAS_TITLE', 1.0)
-          end
-        end
-      end
-    end
-
-    -- See if Reply-To matches From in some way
-    local from = task:get_from(2)
-    local from_h = task:get_header('From')
-    if not (from and from[1]) then return false end
-    if (from_h and from_h == replyto) then
-      -- From and Reply-To are identical
-      task:insert_result('REPLYTO_EQ_FROM', 1.0)
-    else
-      if (from and from[1]) then
-        -- See if From and Reply-To addresses match
-        if (from[1].addr:lower() == rt[1].addr:lower()) then
-          task:insert_result('REPLYTO_ADDR_EQ_FROM', 1.0)
-        elseif from[1].domain and rt[1].domain then
-          if (from[1].domain:lower() == rt[1].domain:lower()) then
-            task:insert_result('REPLYTO_DOM_EQ_FROM_DOM', 1.0)
-          else
-            task:insert_result('REPLYTO_DOM_NEQ_FROM_DOM', 1.0)
-          end
-        end
-        -- See if the Display Names match
-        if (from[1].name and rt[1].name and from[1].name:lower() == rt[1].name:lower()) then
-          task:insert_result('REPLYTO_DN_EQ_FROM_DN', 1.0)
-        end
-      end
-    end
-  end
-)
-
-rspamd_config:register_virtual_symbol('REPLYTO_UNPARSEABLE', 1.0, check_replyto_id)
-rspamd_config:set_metric_symbol('REPLYTO_UNPARSEABLE', 1.0, 'Reply-To header could not be parsed')
-rspamd_config:register_virtual_symbol('HAS_REPLYTO', 1.0, check_replyto_id)
-rspamd_config:set_metric_symbol('HAS_REPLYTO', 0, 'Has Reply-To header')
-rspamd_config:register_virtual_symbol('REPLYTO_EQ_FROM', 1.0, check_replyto_id)
-rspamd_config:set_metric_symbol('REPLYTO_EQ_FROM', 0, 'Reply-To header is identical to From header')
-rspamd_config:register_virtual_symbol('REPLYTO_ADDR_EQ_FROM', 1.0, check_replyto_id)
-rspamd_config:set_metric_symbol('REPLYTO_ADDR_EQ_FROM', 0, 'Reply-To address is the same as From')
-rspamd_config:register_virtual_symbol('REPLYTO_DOM_EQ_FROM_DOM', 1.0, check_replyto_id)
-rspamd_config:set_metric_symbol('REPLYTO_DOM_EQ_FROM_DOM', 0, 'Reply-To domain matches the From domain')
-rspamd_config:register_virtual_symbol('REPLYTO_DOM_NEQ_FROM_DOM', 1.0, check_replyto_id)
-rspamd_config:set_metric_symbol('REPLYTO_DOM_NEQ_FROM_DOM', 0, 'Reply-To domain does not match the From domain')
-rspamd_config:register_virtual_symbol('REPLYTO_DN_EQ_FROM_DN', 1.0, check_replyto_id)
-rspamd_config:set_metric_symbol('REPLYTO_DN_EQ_FROM_DN', 0, 'Reply-To display name matches From')
-rspamd_config:register_virtual_symbol('REPLYTO_EMAIL_HAS_TITLE', 1.0, check_replyto_id)
-rspamd_config:set_metric_symbol('REPLYTO_EMAIL_HAS_TITLE', 2.0, check_replyto_id)
-rspamd_config:register_dependency(check_replyto_id, 'FROM_NAME_HAS_TITLE')
-
-local check_mime_id = rspamd_config:register_callback_symbol('CHECK_MIME', 1.0,
-  function (task)
-    local parts = task:get_parts()
-    if not parts then return false end
-
-    -- Make sure there is a MIME-Version header
-    local mv = task:get_header('MIME-Version')
-    local missing_mime = false
-    if (not mv) then
-      missing_mime = true
-    end
-
-    local found_ma = false
-    local found_plain = false
-    local found_html = false
-    local cte_7bit = false
-
-    for _,p in ipairs(parts) do
-      local mtype,subtype = p:get_type()
-      local ctype = mtype:lower() .. '/' .. subtype:lower()
-      if (ctype == 'multipart/alternative') then
-        found_ma = true
-      end
-      if (ctype == 'text/plain') then
-        if p:get_cte() == '7bit' then
-          cte_7bit = true
-        end
-        found_plain = true
-      end
-      if (ctype == 'text/html') then
-        if p:get_cte() == '7bit' then
-          cte_7bit = true
-        end
-        found_html = true
-      end
-    end
-
-    if missing_mime then
-      if not (not found_ma and ((found_plain or found_html) and cte_7bit)) then
-        task:insert_result('MISSING_MIME_VERSION', 1.0)
-      end
-    end
-
-    if (found_ma) then
-      if (not found_plain) then
-        task:insert_result('MIME_MA_MISSING_TEXT', 1.0)
-      end
-      if (not found_html) then
-        task:insert_result('MIME_MA_MISSING_HTML', 1.0)
-      end
-    end
-  end
-)
-
-rspamd_config:register_virtual_symbol('MISSING_MIME_VERSION', 1.0, check_mime_id)
-rspamd_config:set_metric_symbol('MISSING_MIME_VERSION', 2.0, 'MIME-Version header is missing')
-rspamd_config:register_virtual_symbol('MIME_MA_MISSING_TEXT', 1.0, check_mime_id)
-rspamd_config:set_metric_symbol('MIME_MA_MISSING_TEXT', 2.0, 'MIME multipart/alternative missing text/plain part')
-rspamd_config:register_virtual_symbol('MIME_MA_MISSING_HTML', 1.0, check_mime_id)
-rspamd_config:set_metric_symbol('MIME_MA_MISSING_HTML', 1.0, 'multipart/alternative missing text/html part')
-
--- Used to be called IS_LIST
-rspamd_config.PREVIOUSLY_DELIVERED = {
-  callback = function(task)
-    if not task:has_recipients(2) then return false end
-    local to = task:get_recipients(2)
-    local rcvds = task:get_header_full('Received')
-    if not rcvds then return false end
-    for _, rcvd in ipairs(rcvds) do
-      local _,_,addr = rcvd['decoded']:lower():find("%sfor%s<(.-)>")
-      if addr then
-        for _, toa in ipairs(to) do
-          if toa and toa.addr:lower() == addr then
-            return true, addr
-          end
-        end
-        return false
-      end
-    end
-  end,
-  description = 'Message either to a list or was forwarded',
-  score = 0.0
-}
-
 -- Requires freemail maps loaded in multimap
 local function freemail_reply_neq_from(task)
   local frt = task:get_symbol('FREEMAIL_REPLYTO')
diff --git a/rules/rspamd.lua b/rules/rspamd.lua
index a89f0a002..c17b8380e 100644
--- a/rules/rspamd.lua
+++ b/rules/rspamd.lua
@@ -30,6 +30,8 @@ dofile(local_rules .. '/regexp/misc.lua')
 dofile(local_rules .. '/regexp/upstream_spam_filters.lua')
 dofile(local_rules .. '/regexp/compromised_hosts.lua')
 dofile(local_rules .. '/html.lua')
+dofile(local_rules .. '/headers_checks.lua')
+dofile(local_rules .. '/subject_checks.lua')
 dofile(local_rules .. '/misc.lua')
 dofile(local_rules .. '/http_headers.lua')
 dofile(local_rules .. '/forwarding.lua')
diff --git a/rules/subject_checks.lua b/rules/subject_checks.lua
new file mode 100644
index 000000000..d5760ed25
--- /dev/null
+++ b/rules/subject_checks.lua
@@ -0,0 +1,68 @@
+--[[
+Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_regexp = require "rspamd_regexp"
+local util = require "rspamd_util"
+
+-- Uncategorized rules
+local subject_re = rspamd_regexp.create('/^(?:(?:Re|Fwd|Fw|Aw|Antwort|Sv):\\s*)+(.+)$/i')
+
+local function test_subject(task, check_function, rate)
+  local function normalize_linear(a, x)
+      local f = a * x
+      return true, (( f < 1 ) and f or 1), tostring(x)
+  end
+
+  local sbj = task:get_header('Subject')
+
+  if sbj then
+    local stripped_subject = subject_re:search(sbj, false, true)
+    if stripped_subject and stripped_subject[1] and stripped_subject[1][2] then
+      sbj = stripped_subject[1][2]
+    end
+
+    local l = util.strlen_utf8(sbj)
+    if check_function(sbj, l) then
+      return normalize_linear(rate, l)
+    end
+  end
+
+  return false
+end
+
+rspamd_config.SUBJ_ALL_CAPS = {
+  callback = function(task)
+    local caps_test = function(sbj)
+      return util.is_uppercase(sbj)
+    end
+    return test_subject(task, caps_test, 1.0/40.0)
+  end,
+  score = 3.0,
+  group = 'subject',
+  description = 'All capital letters in subject'
+}
+
+rspamd_config.LONG_SUBJ = {
+  callback = function(task)
+    local length_test = function(_, len)
+      return len > 200
+    end
+    return test_subject(task, length_test, 1.0/400.0)
+  end,
+  score = 3.0,
+  group = 'subject',
+  description = 'Subject is too long'
+}
\ No newline at end of file