Merge pull request #3 from vstakhov/master

merge upstream into local master

14 files changed, 99 insertions, 19 deletions

diff --git a/conf/dmarc_whitelist.inc b/conf/dmarc_whitelist.inc
index 989bd418e..ca8c2148d 100644
--- a/conf/dmarc_whitelist.inc
+++ b/conf/dmarc_whitelist.inc
@@ -3,6 +3,7 @@
 
 4chan.org
 adp.com
+advice.hmrc.gov.uk
 airbnb.com
 airtel.in
 alibaba.com
@@ -29,9 +30,11 @@ force.com
 garant.ru
 gosuslugi.ru
 hh.ru
+hmrc.gov.uk
 instagram.com
 linkedin.com
 livejournal.com
+lufthansa-group.com
 mackeeper.com
 megafon.ru
 mercadolibre.com.ar
@@ -46,6 +49,7 @@ paypal.com
 pch.com
 pinterest.com
 rostelecom.ru
+rt.ru
 sberbank.ru
 sportmaster.ru
 squarespace.com
@@ -56,6 +60,7 @@ uber.com
 ulmart.ru
 ups.com
 usps.com
+utair.ru
 verizonwireless.com
 vk.com
 vkrugudruzei.ru
diff --git a/conf/modules.d/clickhouse.conf b/conf/modules.d/clickhouse.conf
index 3dbc3b60c..c35352a03 100644
--- a/conf/modules.d/clickhouse.conf
+++ b/conf/modules.d/clickhouse.conf
@@ -51,6 +51,18 @@ clickhouse {
   #dmarc_allow_symbols = ["DMARC_POLICY_ALLOW"];
   #dmarc_reject_symbols = ["DMARC_POLICY_REJECT", "DMARC_POLICY_QUARANTINE"];
 
+  #retention {
+  #  # disabled by default
+  #  enable = true;
+  #  # drop | detach, please refer to ClickHouse docs for details
+  #  # http://clickhouse-docs.readthedocs.io/en/latest/query_language/queries.html#manipulations-with-partitions-and-parts
+  #  method = "drop";
+  #  # how many month the data should be kept in ClickHouse
+  #  period_months = 3;
+  #  # how often run the cleanup process
+  #  run_every = "7d";
+  #}
+
   .include(try=true,priority=5) "${DBDIR}/dynamic/clickhouse.conf"
   .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/clickhouse.conf"
   .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/clickhouse.conf"
diff --git a/conf/modules.d/fuzzy_check.conf b/conf/modules.d/fuzzy_check.conf
index 4dc7b2994..2569a6881 100644
--- a/conf/modules.d/fuzzy_check.conf
+++ b/conf/modules.d/fuzzy_check.conf
@@ -19,7 +19,7 @@ fuzzy_check {
     retransmits = 1;
     rule "rspamd.com" {
         algorithm = "mumhash";
-        servers = "fuzzy.rspamd.com:11335";
+        servers = "round-robin:fuzzy1.rspamd.com:11335,fuzzy2.rspamd.com:11335";
         encryption_key = "icy63itbhhni8bq15ntp5n5symuixf73s1kpjh6skaq4e7nx5fiy";
         symbol = "FUZZY_UNKNOWN";
         mime_types = ["*"];
diff --git a/conf/modules.d/mid.conf b/conf/modules.d/mid.conf
index db1a5dafb..589ce746a 100644
--- a/conf/modules.d/mid.conf
+++ b/conf/modules.d/mid.conf
@@ -16,8 +16,9 @@
 mid = {
     source = {
         url = [
-            "${CONFDIR}/mid.inc",
-            "$LOCAL_CONFDIR/local.d/mid.inc"
+            "https://maps.rspamd.com/rspamd/mid.inc.zst",
+            "$LOCAL_CONFDIR/local.d/mid.inc",
+            "fallback+file://${CONFDIR}/mid.inc"
         ];
     }
 
diff --git a/conf/modules.d/mime_types.conf b/conf/modules.d/mime_types.conf
index 04117e543..7bbc0aa08 100644
--- a/conf/modules.d/mime_types.conf
+++ b/conf/modules.d/mime_types.conf
@@ -15,8 +15,9 @@
 
 mime_types {
     file = [
-        "${CONFDIR}/mime_types.inc",
-        "${DBDIR}/mime_types.inc.local"
+        "https://maps.rspamd.com/rspamd/mime_types.inc.zst",
+        "${DBDIR}/mime_types.inc.local",
+        "fallback+file://${CONFDIR}/mime_types.inc"
     ]
 
     # Match specific extensions to specific content types
diff --git a/conf/modules.d/phishing.conf b/conf/modules.d/phishing.conf
index 61bcfb1ac..dd77832a8 100644
--- a/conf/modules.d/phishing.conf
+++ b/conf/modules.d/phishing.conf
@@ -21,12 +21,13 @@ phishing {
     openphish_map = "https://www.openphish.com/feed.txt";
     # Disabled by default
     phishtank_enabled = false;
-    phishtank_map = "https://rspamd.com/phishtank/online-valid.json.zst";
+    phishtank_map = "https://maps.rspamd.com/phishtank/online-valid.json.zst";
 
     # Make exclusions for known redirectors
     redirector_domains = [
-        "${CONFDIR}/redirectors.inc:REDIRECTOR_FALSE",
-        "$LOCAL_CONFDIR/local.d/redirectors.inc:LOCAL_REDIRECTOR_FALSE"
+        "https://maps.rspamd.com/rspamd/redirectors.inc.zst:REDIRECTOR_FALSE",
+        "$LOCAL_CONFDIR/local.d/redirectors.inc:LOCAL_REDIRECTOR_FALSE",
+        "fallback+file://${CONFDIR}/redirectors.inc:REDIRECTOR_FALSE"
     ];
 
     .include(try=true,priority=5) "${DBDIR}/dynamic/phishing.conf"
diff --git a/conf/modules.d/rbl.conf b/conf/modules.d/rbl.conf
index 8a5b2dab8..f3373c2ae 100644
--- a/conf/modules.d/rbl.conf
+++ b/conf/modules.d/rbl.conf
@@ -115,6 +115,16 @@ rbl {
                 DNSWL_BLOCKED = "127.0.0.255";
             }
         }
+
+        # Provided by https://virusfree.cz
+        virusfree {
+            symbol = "RBL_VIRUSFREE_UNKNOWN";
+            rbl = "bip.virusfree.cz";
+            ipv6 = true;
+            returncodes {
+                RBL_VIRUSFREE_BOTNET = "127.0.0.2";
+            }
+        }
     }
 
     .include(try=true,priority=5) "${DBDIR}/dynamic/rbl.conf"
diff --git a/conf/modules.d/reputation.conf b/conf/modules.d/reputation.conf
new file mode 100644
index 000000000..412341f2c
--- /dev/null
+++ b/conf/modules.d/reputation.conf
@@ -0,0 +1,35 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+
+# Reputation module is experimental! (you need to enable experimental modules
+# explicitly in options by setting `enable_experimental = true`)
+
+reputation {
+#  rules {
+#    SPF_REPUTATION =  {
+#      selector {
+#        type = "spf";
+#      }
+#      backend {
+#        type = "redis";
+#      }
+#      symbol = "SPF_REPUTATION";
+#    }
+#  }
+    .include(try=true,priority=5) "${DBDIR}/dynamic/reputation.conf"
+    .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/reputation.conf"
+    .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/reputation.conf"
+}
diff --git a/conf/modules.d/surbl.conf b/conf/modules.d/surbl.conf
index 446cf616f..34fdf8ed2 100644
--- a/conf/modules.d/surbl.conf
+++ b/conf/modules.d/surbl.conf
@@ -15,12 +15,14 @@
 
 surbl {
     whitelist = [
-        "${CONFDIR}/surbl-whitelist.inc",
-        "${DBDIR}/surbl-whitelist.inc.local"
+        "https://maps.rspamd.com/rspamd/surbl-whitelist.inc.zst",
+        "${DBDIR}/surbl-whitelist.inc.local",
+        "fallback+file://${CONFDIR}/surbl-whitelist.inc"
     ];
     exceptions = [
-        "${CONFDIR}/2tld.inc",
-        "${DBDIR}/2tld.inc.local"
+        "https://maps.rspamd.com/rspamd/2tld.inc.zst",
+        "${DBDIR}/2tld.inc.local",
+        "fallback+file://${CONFDIR}/2tld.inc"
     ];
 
     rules {
@@ -48,7 +50,7 @@ surbl {
             process_script =<<EOD
 function(url, suffix)
   local cr = require "rspamd_cryptobox_hash"
-  h = cr.create(url):base32():sub(1, 32)
+  local h = cr.create(url):base32():sub(1, 32)
   return string.format("%s.%s", h, suffix)
 end
 EOD;
diff --git a/conf/modules.d/whitelist.conf b/conf/modules.d/whitelist.conf
index de8b9b516..96c9ed02e 100644
--- a/conf/modules.d/whitelist.conf
+++ b/conf/modules.d/whitelist.conf
@@ -18,8 +18,9 @@ whitelist {
         "WHITELIST_SPF" = {
             valid_spf = true;
             domains = [
-                "${CONFDIR}/spf_whitelist.inc",
+                #"https://maps.rspamd.com/rspamd/spf_whitelist.inc.zst", # Missing now
                 "${DBDIR}/spf_whitelist.inc.local",
+                "fallback+file://${CONFDIR}/spf_whitelist.inc"
             ];
             score = -1.0
             description = "Mail comes from the whitelisted domain and has a valid SPF policy";
@@ -28,8 +29,9 @@ whitelist {
         "WHITELIST_DKIM" = {
             valid_dkim = true;
             domains = [
-                "${CONFDIR}/dkim_whitelist.inc",
+                #"https://maps.rspamd.com/rspamd/dkim_whitelist.inc.zst", # Missing now
                 "${DBDIR}/dkim_whitelist.inc.local",
+                "fallback+file://${CONFDIR}/dkim_whitelist.inc"
             ];
             description = "Mail comes from the whitelisted domain and has a valid DKIM signature";
             score = -1.0
@@ -38,8 +40,9 @@ whitelist {
             valid_spf = true;
             valid_dkim = true;
             domains = [
-                "${CONFDIR}/spf_dkim_whitelist.inc",
+                "https://maps.rspamd.com/rspamd/spf_dkim_whitelist.inc.zst",
                 "${DBDIR}/spf_dkim_whitelist.inc.local",
+                "fallback+file://${CONFDIR}/spf_dkim_whitelist.inc"
             ];
             score = -3.0;
             description = "Mail comes from the whitelisted domain and has valid SPF and DKIM policies";
@@ -47,8 +50,9 @@ whitelist {
         "WHITELIST_DMARC" = {
             valid_dmarc = true;
             domains = [
-                "${CONFDIR}/dmarc_whitelist.inc",
+                "https://maps.rspamd.com/rspamd/dmarc_whitelist.inc.zst",
                 "${DBDIR}/dmarc_whitelist.inc.local",
+                "fallback+file://${CONFDIR}/dmarc_whitelist.inc"
             ];
             score = -7.0;
             description = "Mail comes from the whitelisted domain and has valid DMARC and DKIM policies";
diff --git a/conf/redirectors.inc b/conf/redirectors.inc
index 44829d577..812f40539 100644
--- a/conf/redirectors.inc
+++ b/conf/redirectors.inc
@@ -183,6 +183,7 @@ cort.as
 cortas.elpais.com
 cot.ag
 cowurl.com
+cp.bitrix.ru
 cr.am
 createurl.com
 crks.me
@@ -418,6 +419,7 @@ linkde.info
 linkee.com
 link.hhut.ru
 linkl.ru
+link.mail.e.glavbukh-mail.ru
 link.rengo.ru
 link.sendsay.ru
 linkslash.ca
diff --git a/conf/scores.d/policies_group.conf b/conf/scores.d/policies_group.conf
index b7ca9f6f6..e432053ec 100644
--- a/conf/scores.d/policies_group.conf
+++ b/conf/scores.d/policies_group.conf
@@ -98,7 +98,7 @@ symbols = {
 
     "ARC_REJECT" {
         weight = 2.0;
-        description = "ARC checks success";
+        description = "ARC checks failed";
         groups = ["arc"];
     }
 
@@ -119,4 +119,4 @@ symbols = {
         description = "ARC signature absent";
         groups = ["arc"];
     }
-}
\ No newline at end of file
+}
diff --git a/conf/scores.d/rbl_group.conf b/conf/scores.d/rbl_group.conf
index be145e350..6c48cf3bb 100644
--- a/conf/scores.d/rbl_group.conf
+++ b/conf/scores.d/rbl_group.conf
@@ -149,4 +149,9 @@ symbols = {
         weight = 1.0;
         description = "Address is listed in Spameatingmonkey RBL (ipv6)";
     }
+
+    "RBL_VIRUSFREE_BOTNET" {
+        weight = 2.0;
+        description = "Source IP is listed in virusfree.cz BL";
+    }
 }
diff --git a/conf/spf_dkim_whitelist.inc b/conf/spf_dkim_whitelist.inc
index 2759e6b7f..fe0ddbbaa 100644
--- a/conf/spf_dkim_whitelist.inc
+++ b/conf/spf_dkim_whitelist.inc
@@ -176,6 +176,7 @@ southwest.com
 spotify.com
 springer.com
 squarespace.com
+stalker.com
 steampowered.com
 stumbleupon.com
 surveymonkey.com
@@ -185,6 +186,7 @@ taleo.net
 taobao.com
 target.com
 taringa.net
+taxi.yandex.ru
 tele2.ru
 thekitchn.com
 tokopedia.com