Merge branch 'master' into vstakhov-log-tag-passthrough

6 files changed, 200 insertions, 0 deletions

diff --git a/conf/groups.conf b/conf/groups.conf
index 2aeb4ed5d..4f40d865c 100644
--- a/conf/groups.conf
+++ b/conf/groups.conf
@@ -39,6 +39,13 @@ group "rbl" {
     .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/rbl_group.conf"
 }
 
+# Limits the maximum score when both bl.score.senderscore.com and score.senderscore.com RBLs are enabled.
+group "senderscore" {
+    max_score = 4.0;
+    .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/senderscore_group.conf"
+    .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/senderscore_group.conf"
+}
+
 group "statistics" {
     .include "$CONFDIR/scores.d/statistics_group.conf"
     .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/statistics_group.conf"
diff --git a/conf/modules.d/rbl.conf b/conf/modules.d/rbl.conf
index d8db05ecc..2a718e5a4 100644
--- a/conf/modules.d/rbl.conf
+++ b/conf/modules.d/rbl.conf
@@ -79,6 +79,11 @@ rbl {
     }
 
     senderscore {
+      # Disabled by default to prioritize the use of score.senderscore.com.
+      # Note: The free query limit applies to both bl.score.senderscore.com and score.senderscore.com RBLs
+      # (see https://knowledge.validity.com/hc/en-us/articles/20961730681243).
+      # Enabling this RBL is recommended for low-traffic systems or MyValidity account users who benefit from using both RBLs.
+      enabled = false;
       symbol = "RBL_SENDERSCORE_UNKNOWN";
       checks = ['from'];
       rbl = "bl.score.senderscore.com";
@@ -104,6 +109,27 @@ rbl {
       }
     }
 
+    senderscore_reputation {
+      symbol = "RBL_SENDERSCORE_REPUT_UNKNOWN";
+      checks = ['from'];
+      rbl = "score.senderscore.com";
+      returncodes_matcher = "luapattern";
+
+      returncodes {
+        RBL_SENDERSCORE_REPUT_0 = "127%.0%.4%.%d";
+        RBL_SENDERSCORE_REPUT_1 = "127%.0%.4%.1%d";
+        RBL_SENDERSCORE_REPUT_2 = "127%.0%.4%.2%d";
+        RBL_SENDERSCORE_REPUT_3 = "127%.0%.4%.3%d";
+        RBL_SENDERSCORE_REPUT_4 = "127%.0%.4%.4%d";
+        RBL_SENDERSCORE_REPUT_5 = "127%.0%.4%.5%d";
+        RBL_SENDERSCORE_REPUT_6 = "127%.0%.4%.6%d";
+        RBL_SENDERSCORE_REPUT_7 = "127%.0%.4%.7%d";
+        RBL_SENDERSCORE_REPUT_8 = "127%.0%.4%.8%d";  # Neutral reputation (80-89).
+        RBL_SENDERSCORE_REPUT_9 = ["127%.0%.4%.9%d", "127%.0%.4%.100"];  # Good reputation (90-100).
+        RBL_SENDERSCORE_REPUT_BLOCKED = "127%.255%.255%.255";
+      }
+    }
+
     sem {
       symbol = "RBL_SEM";
       rbl = "bl.spameatingmonkey.net";
diff --git a/conf/scores.d/rbl_group.conf b/conf/scores.d/rbl_group.conf
index 741552a17..6a59b865f 100644
--- a/conf/scores.d/rbl_group.conf
+++ b/conf/scores.d/rbl_group.conf
@@ -257,6 +257,67 @@ symbols = {
         groups = ["senderscore", "blocked"];
     }
 
+    "RBL_SENDERSCORE_REPUT_UNKNOWN" {
+        weight = 0.0;
+        description = "Unrecognized result from SenderScore Reputation list.";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_0" {
+        weight = 4.0;
+        description = "SenderScore Reputation: Very Bad (0-9).";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_1" {
+        weight = 3.5;
+        description = "SenderScore Reputation: Bad (10-19).";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_2" {
+        weight = 3.0;
+        description = "SenderScore Reputation: Bad (20-29).";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_3" {
+        weight = 2.5;
+        description = "SenderScore Reputation: Bad (30-39).";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_4" {
+        weight = 2.0;
+        description = "SenderScore Reputation: Bad (40-49).";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_5" {
+        weight = 1.5;
+        description = "SenderScore Reputation: Bad (50-59).";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_6" {
+        weight = 1.0;
+        description = "SenderScore Reputation: Bad (60-69).";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_7" {
+        weight = 0.5;
+        description = "SenderScore Reputation: Bad (70-79).";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_8" {
+        weight = 0.0;
+        description = "SenderScore Reputation: Neutral (80-89).";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_9" {
+        weight = -1.0;
+        description = "SenderScore Reputation: Good (90-100).";
+        groups = ["senderscore"];
+    }
+    "RBL_SENDERSCORE_REPUT_BLOCKED" {
+        weight = 0.0;
+        description = "Excessive number of queries to SenderScore RPBL, more info: https://knowledge.validity.com/hc/en-us/articles/20961730681243";
+        groups = ["senderscore", "blocked"];
+    }
+
     "MAILSPIKE" {
         weight = 0.0;
         description = "Unrecognised result from Mailspike";
diff --git a/test/functional/cases/550_milter_headers.robot b/test/functional/cases/550_milter_headers.robot
new file mode 100644
index 000000000..80471b83c
--- /dev/null
+++ b/test/functional/cases/550_milter_headers.robot
@@ -0,0 +1,39 @@
+*** Settings ***
+Suite Setup     Rspamd Setup
+Suite Teardown  Rspamd Teardown
+Library         ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource        ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables       ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG}                ${RSPAMD_TESTDIR}/configs/milter_headers.conf
+${MESSAGE}               ${RSPAMD_TESTDIR}/messages/zip.eml
+${RSPAMD_SCOPE}          Suite
+${RSPAMD_URL_TLD}        ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat
+${SETTINGS_NOSYMBOLS}    {symbols_enabled = []}
+${SETTINGS_TEST}         {SIMPLE_TEST = 2.0, symbols_enabled = [SIMPLE_TEST]}
+
+*** Test Cases ***
+CHECK HEADERS WITH TEST SYMBOL
+  Scan File  ${MESSAGE}  Settings=${SETTINGS_TEST}
+  # Check X-Virus header
+  Expect Removed Header  X-Virus
+  Expect Added Header  X-Virus  Fires always
+  # Check My-Spamd-Bar header
+  Expect Added Header  My-Spamd-Bar  ++
+  Do Not Expect Removed Header  My-Spamd-Bar
+  # Check X-Spam-Level header
+  Expect Added Header  X-Spam-Level  **
+  Expect Removed Header  X-Spam-Level
+
+CHECK HEADERS WITHOUT TEST SYMBOL
+  Scan File  ${MESSAGE}  Settings=${SETTINGS_NOSYMBOLS}
+  # Check X-Virus header
+  Expect Removed Header  X-Virus
+  Do Not Expect Added Header  X-Virus
+  # Check My-Spamd-Bar header
+  Expect Added Header  My-Spamd-Bar  /
+  Do Not Expect Removed Header  My-Spamd-Bar
+  # Check X-Spam-Level header
+  Do Not Expect Added Header  X-Spam-Level
+  Expect Removed Header  X-Spam-Level
diff --git a/test/functional/configs/milter_headers.conf b/test/functional/configs/milter_headers.conf
new file mode 100644
index 000000000..947bc28dd
--- /dev/null
+++ b/test/functional/configs/milter_headers.conf
@@ -0,0 +1,24 @@
+lua = "{= env.TESTDIR =}/lua/simple.lua"
+
+.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf"
+
+milter_headers {
+
+  use = ["remove-headers", "x-spam-level", "x-spamd-bar", "x-virus"];
+
+  routines {
+    remove-headers {
+      headers {
+        "X-Spam-Level" = 0,
+      }
+    }
+    x-spamd-bar {
+      header = "My-Spamd-Bar";
+      remove = null;
+    }
+    x-virus {
+      symbols = ["SIMPLE_TEST"];
+    }
+  }
+
+}
diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot
index 254a6f3ca..c45a19908 100644
--- a/test/functional/lib/rspamd.robot
+++ b/test/functional/lib/rspamd.robot
@@ -60,6 +60,28 @@ Check Rspamc Match String
     Should Not Contain  ${subject}  ${str}
   END
 
+Do Not Expect Added Header
+  [Arguments]  ${header_name}
+  IF  'milter' not in ${SCAN_RESULT}
+    RETURN
+  END
+  IF  'add_headers' not in ${SCAN_RESULT}[milter]
+    RETURN
+  END
+  Dictionary Should Not Contain Key  ${SCAN_RESULT}[milter][add_headers]  ${header_name}
+  ...  msg=${header_name} was added
+
+Do Not Expect Removed Header
+  [Arguments]  ${header_name}
+  IF  'milter' not in ${SCAN_RESULT}
+    RETURN
+  END
+  IF  'remove_headers' not in ${SCAN_RESULT}[milter]
+    RETURN
+  END
+  Dictionary Should Not Contain Key  ${SCAN_RESULT}[milter][remove_headers]  ${header_name}
+  ...  msg=${header_name} was removed
+
 Do Not Expect Symbol
   [Arguments]  ${symbol}
   Dictionary Should Not Contain Key  ${SCAN_RESULT}[symbols]  ${symbol}
@@ -76,10 +98,31 @@ Expect Action
   [Arguments]  ${action}
   Should Be Equal  ${SCAN_RESULT}[action]  ${action}
 
+Expect Added Header
+  [Arguments]  ${header_name}  ${header_value}  ${pos}=-1
+  Dictionary Should Contain Key  ${SCAN_RESULT}  milter
+  ...  msg=milter block was not present in protocol response
+  Dictionary Should Contain Key  ${SCAN_RESULT}[milter]  add_headers
+  ...  msg=add_headers block was not present in protocol response
+  Dictionary Should Contain Key  ${SCAN_RESULT}[milter][add_headers]  ${header_name}
+  ...  msg=${header_name} was not added
+  Should Be Equal  ${SCAN_RESULT}[milter][add_headers][${header_name}][value]  ${header_value}
+  Should Be Equal as Numbers  ${SCAN_RESULT}[milter][add_headers][${header_name}][order]  ${pos}
+
 Expect Email
   [Arguments]  ${email}
   List Should Contain Value  ${SCAN_RESULT}[emails]  ${email}
 
+Expect Removed Header
+  [Arguments]  ${header_name}  ${pos}=0
+  Dictionary Should Contain Key  ${SCAN_RESULT}  milter
+  ...  msg=milter block was not present in protocol response
+  Dictionary Should Contain Key  ${SCAN_RESULT}[milter]  remove_headers
+  ...  msg=remove_headers block was not present in protocol response
+  Dictionary Should Contain Key  ${SCAN_RESULT}[milter][remove_headers]  ${header_name}
+  ...  msg=${header_name} was not removed
+  Should Be Equal as Numbers  ${SCAN_RESULT}[milter][remove_headers][${header_name}]  ${pos}
+
 Expect Required Score
   [Arguments]  ${required_score}
   Should Be Equal As Numbers  ${SCAN_RESULT}[required_score]  ${required_score}