From 05fd471df54b690e74125cf7ba2ff345f3be8bbf Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Sat, 10 Sep 2022 14:43:36 +0100
Subject: [PATCH] [Rework] Reiterate on priorities

---
 lualib/lua_util.lua                   |  8 +++++++
 rules/misc.lua                        | 32 +++++++++++++--------------
 src/plugins/lua/antivirus.lua         |  2 +-
 src/plugins/lua/asn.lua               |  2 +-
 src/plugins/lua/aws_s3.lua            |  2 +-
 src/plugins/lua/external_relay.lua    |  2 +-
 src/plugins/lua/external_services.lua |  2 +-
 src/plugins/lua/force_actions.lua     |  2 +-
 src/plugins/lua/greylist.lua          |  4 ++--
 src/plugins/lua/neural.lua            |  2 +-
 src/plugins/lua/p0f.lua               |  2 +-
 src/plugins/lua/ratelimit.lua         |  2 +-
 src/plugins/lua/replies.lua           |  4 ++--
 src/plugins/lua/settings.lua          |  4 ++--
 14 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/lualib/lua_util.lua b/lualib/lua_util.lua
index 454629bba..b4707c4bd 100644
--- a/lualib/lua_util.lua
+++ b/lualib/lua_util.lua
@@ -1532,4 +1532,12 @@ end
 
 exports.dns_timeout_augmentation = dns_timeout_augmentation
 
+-- Defines symbols priorities for common usage in prefilters/postfilters
+exports.symbols_priorities = {
+  top = 10, -- Symbols must be executed first (or last), such as settings
+  high = 9, -- Example: asn
+  medium = 5, -- Everything should use this as default
+  low = 0,
+}
+
 return exports
diff --git a/rules/misc.lua b/rules/misc.lua
index 1b2f3fa7b..eff49864f 100644
--- a/rules/misc.lua
+++ b/rules/misc.lua
@@ -18,10 +18,10 @@ limitations under the License.
 
 local E = {}
 local fun = require "fun"
-local util = require "rspamd_util"
+local rspamd_util = require "rspamd_util"
 local rspamd_parsers = require "rspamd_parsers"
 local rspamd_regexp = require "rspamd_regexp"
-local rspamd_lua_utils = require "lua_util"
+local lua_util = require "lua_util"
 local bit = require "bit"
 local rspamd_url = require "rspamd_url"
 local url_flags_tab = rspamd_url.flags
@@ -409,7 +409,7 @@ rspamd_config.OMOGRAPH_URL = {
           end
           if h1 and h2 then
             local selt = string.format('%s->%s', h1, h2)
-            if not seen[selt] and util.is_utf_spoofed(h1, h2) then
+            if not seen[selt] and rspamd_util.is_utf_spoofed(h1, h2) then
               bad_urls[#bad_urls + 1] = selt
               bad_omographs = bad_omographs + 1
             end
@@ -420,7 +420,7 @@ rspamd_config.OMOGRAPH_URL = {
           local h = u:get_tld()
 
           if h then
-            if not seen[h] and util.is_utf_spoofed(h) then
+            if not seen[h] and rspamd_util.is_utf_spoofed(h) then
               bad_urls[#bad_urls + 1] = h
               single_bad_omograps = single_bad_omograps + 1
             end
@@ -482,7 +482,7 @@ local aliases_id = rspamd_config:register_symbol{
     local function check_from(type)
       if task:has_from(type) then
         local addr = task:get_from(type)[1]
-        local na,tags = rspamd_lua_utils.remove_email_aliases(addr)
+        local na,tags = lua_util.remove_email_aliases(addr)
         if na then
           task:set_from(type, addr, 'alias')
           task:insert_result('TAGGED_FROM', 1.0, fun.totable(
@@ -501,7 +501,7 @@ local aliases_id = rspamd_config:register_symbol{
         local addrs = task:get_recipients(type)
 
         for _, addr in ipairs(addrs) do
-          local na,tags = rspamd_lua_utils.remove_email_aliases(addr)
+          local na,tags = lua_util.remove_email_aliases(addr)
           if na then
             modified = true
             fun.each(function(t) table.insert(all_tags, t) end,
@@ -519,7 +519,7 @@ local aliases_id = rspamd_config:register_symbol{
     check_rcpt('smtp')
     check_rcpt('mime')
   end,
-  priority = 150,
+  priority = lua_util.symbols_priorities.top + 1,
   description = 'Removes plus aliases from the email',
   group = 'headers',
 }
@@ -554,13 +554,13 @@ local check_from_display_name = rspamd_config:register_symbol{
     -- Make sure we did not mistake e.g. <something>@<name> for an email address
     if not parsed[1]['domain'] or not parsed[1]['domain']:find('%.') then return false end
     -- See if the parsed domains differ
-    if not util.strequal_caseless(from[1]['domain'], parsed[1]['domain']) then
+    if not rspamd_util.strequal_caseless(from[1]['domain'], parsed[1]['domain']) then
       -- See if the destination domain is the same as the spoof
       local mto = task:get_recipients(2)
       local sto = task:get_recipients(1)
       if mto then
         for _, to in ipairs(mto) do
-          if to['domain'] ~= '' and util.strequal_caseless(to['domain'], parsed[1]['domain']) then
+          if to['domain'] ~= '' and rspamd_util.strequal_caseless(to['domain'], parsed[1]['domain']) then
             task:insert_result('SPOOF_DISPLAY_NAME', 1.0, from[1]['domain'], parsed[1]['domain'])
             return false
           end
@@ -568,7 +568,7 @@ local check_from_display_name = rspamd_config:register_symbol{
       end
       if sto then
         for _, to in ipairs(sto) do
-          if to['domain'] ~= '' and util.strequal_caseless(to['domain'], parsed[1]['domain']) then
+          if to['domain'] ~= '' and rspamd_util.strequal_caseless(to['domain'], parsed[1]['domain']) then
             task:insert_result('SPOOF_DISPLAY_NAME', 1.0, from[1]['domain'], parsed[1]['domain'])
             return false
           end
@@ -611,7 +611,7 @@ rspamd_config.SPOOF_REPLYTO = {
     if not (from and from[1] and from[1].addr) then return false end
     if (to and to[1] and to[1].addr) then
       -- Handle common case for Web Contact forms of From = To
-      if util.strequal_caseless(from[1].addr, to[1].addr) then
+      if rspamd_util.strequal_caseless(from[1].addr, to[1].addr) then
         return false
       end
     end
@@ -619,10 +619,10 @@ rspamd_config.SPOOF_REPLYTO = {
     to = task:get_recipients(1)
     if not to then return false end
     -- Try mitigate some possible FPs on mailing list posts
-    if #to == 1 and util.strequal_caseless(to[1].addr, from[1].addr) then return false end
+    if #to == 1 and rspamd_util.strequal_caseless(to[1].addr, from[1].addr) then return false end
     local found_fromdom = false
     for _, t in ipairs(to) do
-      if util.strequal_caseless(t.domain, from[1].domain) then
+      if rspamd_util.strequal_caseless(t.domain, from[1].domain) then
         found_fromdom = true
         break
       end
@@ -632,7 +632,7 @@ rspamd_config.SPOOF_REPLYTO = {
     local parsed = ((rspamd_parsers.parse_mail_address(rt, task:get_mempool()) or E)[1] or E).domain
     if not parsed then return false end
     -- Reply-To domain must be different to From domain
-    if not util.strequal_caseless(parsed, from[1].domain) then
+    if not rspamd_util.strequal_caseless(parsed, from[1].domain) then
       return true, from[1].domain, parsed
     end
     return false
@@ -649,14 +649,14 @@ rspamd_config.INFO_TO_INFO_LU = {
       return false
     end
     local from = task:get_from('mime')
-    if not (from and from[1] and util.strequal_caseless(from[1].user, 'info')) then
+    if not (from and from[1] and rspamd_util.strequal_caseless(from[1].user, 'info')) then
       return false
     end
     local to = task:get_recipients('smtp')
     if not to then return false end
     local found = false
     for _,r in ipairs(to) do
-      if util.strequal_caseless(r['user'], 'info') then
+      if rspamd_util.strequal_caseless(r['user'], 'info') then
         found = true
       end
     end
diff --git a/src/plugins/lua/antivirus.lua b/src/plugins/lua/antivirus.lua
index dba472c50..968b24adc 100644
--- a/src/plugins/lua/antivirus.lua
+++ b/src/plugins/lua/antivirus.lua
@@ -192,7 +192,7 @@ if opts and type(opts) == 'table' then
 
         if m.symbol_type == 'postfilter' then
           t.type = 'postfilter'
-          t.priority = 3
+          t.priority = lua_util.symbols_priorities.medium
         else
           t.type = 'normal'
         end
diff --git a/src/plugins/lua/asn.lua b/src/plugins/lua/asn.lua
index 7f2bfea6b..e870fdb3b 100644
--- a/src/plugins/lua/asn.lua
+++ b/src/plugins/lua/asn.lua
@@ -143,7 +143,7 @@ if configure_asn_module() then
     name = 'ASN_CHECK',
     type = 'prefilter',
     callback = asn_check,
-    priority = 8,
+    priority = lua_util.symbols_priorities.high,
     flags = 'empty,nostat',
     augmentations = {lua_util.dns_timeout_augmentation(rspamd_config)},
   })
diff --git a/src/plugins/lua/aws_s3.lua b/src/plugins/lua/aws_s3.lua
index a1b30e4ed..bd53dac50 100644
--- a/src/plugins/lua/aws_s3.lua
+++ b/src/plugins/lua/aws_s3.lua
@@ -261,6 +261,6 @@ rspamd_config:register_symbol({
   name = 'EXPORT_AWS_S3',
   type = settings.fail_action and 'postfilter' or 'idempotent',
   callback = s3_aws_callback,
-  priority = settings.fail_action and 10 or nil,
+  priority = settings.fail_action and lua_util.symbols_priorities.high or nil,
   flags = 'empty,explicit_disable,ignore_passthrough,nostat',
 })
\ No newline at end of file
diff --git a/src/plugins/lua/external_relay.lua b/src/plugins/lua/external_relay.lua
index 5c0084bc9..841eca32f 100644
--- a/src/plugins/lua/external_relay.lua
+++ b/src/plugins/lua/external_relay.lua
@@ -229,7 +229,7 @@ if opts then
       rspamd_config:register_symbol({
         name = rule.symbol,
         type = 'prefilter',
-        priority = rule.priority or 20,
+        priority = rule.priority or lua_util.symbols_priorities.top + 1,
         group = N,
         callback = cb,
       })
diff --git a/src/plugins/lua/external_services.lua b/src/plugins/lua/external_services.lua
index 96e6b9e94..d6fedeece 100644
--- a/src/plugins/lua/external_services.lua
+++ b/src/plugins/lua/external_services.lua
@@ -222,7 +222,7 @@ if opts and type(opts) == 'table' then
 
         if m.symbol_type == 'postfilter' then
           t.type = 'postfilter'
-          t.priority = 3
+          t.priority = lua_util.symbols_priorities.medium
         else
           t.type = 'normal'
         end
diff --git a/src/plugins/lua/force_actions.lua b/src/plugins/lua/force_actions.lua
index c0c023fae..5f4620e6d 100644
--- a/src/plugins/lua/force_actions.lua
+++ b/src/plugins/lua/force_actions.lua
@@ -192,7 +192,7 @@ local function configure_module()
           local t = {}
           if (raction or honor) then
             t.type = 'postfilter'
-            t.priority = 10
+            t.priority = lua_util.symbols_priorities.high
           else
             t.type = 'normal'
             if not sett.least then
diff --git a/src/plugins/lua/greylist.lua b/src/plugins/lua/greylist.lua
index 04e003a1b..db1afeed7 100644
--- a/src/plugins/lua/greylist.lua
+++ b/src/plugins/lua/greylist.lua
@@ -514,14 +514,14 @@ if opts then
       name = 'GREYLIST_SAVE',
       type = 'postfilter',
       callback = greylist_set,
-      priority = 6,
+      priority = lua_util.symbols_priorities.medium,
       augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)},
     })
     local id = rspamd_config:register_symbol({
       name = 'GREYLIST_CHECK',
       type = 'prefilter',
       callback = greylist_check,
-      priority = 6,
+      priority = lua_util.symbols_priorities.medium,
       augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}
     })
     rspamd_config:register_symbol({
diff --git a/src/plugins/lua/neural.lua b/src/plugins/lua/neural.lua
index 85894389e..33361b7df 100644
--- a/src/plugins/lua/neural.lua
+++ b/src/plugins/lua/neural.lua
@@ -894,7 +894,7 @@ local id = rspamd_config:register_symbol({
   name = 'NEURAL_CHECK',
   type = 'postfilter,callback',
   flags = 'nostat',
-  priority = 6,
+  priority = lua_util.symbols_priorities.medium,
   callback = ann_scores_filter
 })
 
diff --git a/src/plugins/lua/p0f.lua b/src/plugins/lua/p0f.lua
index a07b1ee00..d597ee25d 100644
--- a/src/plugins/lua/p0f.lua
+++ b/src/plugins/lua/p0f.lua
@@ -87,7 +87,7 @@ if rule then
     name = 'P0F_CHECK',
     type = 'prefilter',
     callback = check_p0f,
-    priority = 8,
+    priority = lua_util.symbols_priorities.medium,
     flags = 'empty,nostat',
     group = N,
     augmentations = {string.format("timeout=%f", rule.timeout or 0.0)},
diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua
index 2b61b606f..34d6788c9 100644
--- a/src/plugins/lua/ratelimit.lua
+++ b/src/plugins/lua/ratelimit.lua
@@ -868,7 +868,7 @@ if opts then
     local s = {
       type = settings.prefilter and 'prefilter' or 'callback',
       name = 'RATELIMIT_CHECK',
-      priority = 7,
+      priority = lua_util.symbols_priorities.medium,
       callback = ratelimit_cb,
       flags = 'empty,nostat',
       augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)},
diff --git a/src/plugins/lua/replies.lua b/src/plugins/lua/replies.lua
index ef3643f66..8debe3271 100644
--- a/src/plugins/lua/replies.lua
+++ b/src/plugins/lua/replies.lua
@@ -286,7 +286,7 @@ if opts then
         type = 'prefilter',
         callback = replies_check_cookie,
         flags = 'nostat',
-        priority = 10,
+        priority = lua_util.symbols_priorities.medium,
         group = "replies"
       })
       rspamd_config:register_symbol({
@@ -310,7 +310,7 @@ if opts then
       type = 'prefilter',
       flags = 'nostat',
       callback = replies_check,
-      priority = 9,
+      priority = lua_util.symbols_priorities.medium,
       group = "replies"
     })
     rspamd_config:register_symbol({
diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua
index d8f0997f1..a00bc32ef 100644
--- a/src/plugins/lua/settings.lua
+++ b/src/plugins/lua/settings.lua
@@ -1228,7 +1228,7 @@ if redis_section then
       name = 'REDIS_SETTINGS' .. tostring(id),
       type = 'prefilter',
       callback = gen_redis_callback(h, id),
-      priority = 10,
+      priority = lua_util.symbols_priorities.top,
       flags = 'empty,nostat',
       augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)},
     })
@@ -1239,7 +1239,7 @@ module_sym_id = rspamd_config:register_symbol({
   name = 'SETTINGS_CHECK',
   type = 'prefilter',
   callback = check_settings,
-  priority = 10,
+  priority = lua_util.symbols_priorities.top,
   flags = 'empty,nostat,explicit_disable,ignore_passthrough',
 })
 
-- 
2.39.5