Merge branch 'master' into temp-propose-alternative-solution-to-xbl-any-hack

cmake/Toolset.cmake

@@ -10,6 +10,8 @@ elseif(CMAKE_C_COMPILER_ID MATCHES "Clang|AppleClang")
     SET (COMPILER_CLANG 1)
 endif()
 
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
 SET (COMPILER_FAST_MATH "")
 if (ENABLE_FAST_MATH MATCHES "ON")
     # We need to keep nans and infinities, so cannot keep all fast math there
@@ -41,10 +43,29 @@ elseif (COMPILER_CLANG)
     if (CMAKE_C_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
         message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.")
     endif ()
-    ADD_COMPILE_OPTIONS(-Wno-unused-command-line-argument)
+    # Hack to fix try_compile
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-command-line-argument")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-command-line-argument")
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-ignored-optimization-argument")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-ignored-optimization-argument")
+    if (ENABLE_LIBCXX MATCHES "AUTO")
+        include(CheckCXXSourceCompiles)
+        set(OLD_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+        check_cxx_source_compiles("
+#include <version>
+int main() { return 0; }
+        " HAVE_LIBCXX )
+        if (HAVE_LIBCXX)
+            SET(ENABLE_LIBCXX "ON")
+        else()
+            SET(ENABLE_LIBCXX "OFF")
+        endif()
+        set(CMAKE_CXX_FLAGS "${OLD_CMAKE_CXX_FLAGS}")
+    endif()
     if (ENABLE_LIBCXX MATCHES "ON")
-        # Use libc++ as libstdc++ is buggy in many cases
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+        set(CLANG_DEFAULT_CXX_STDLIB "libc++")
     endif ()
 else ()
     message (WARNING "You are using an unsupported compiler ${CMAKE_C_COMPILER_ID}. Compilation has only been tested with Clang 4+ and GCC 4+.")

conf/actions.conf

@@ -26,4 +26,4 @@ actions {
 
     .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/actions.conf"
     .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/actions.conf"
-}
+}

conf/composites.conf

@@ -94,7 +94,7 @@ composites {
   }
   RCVD_UNAUTH_PBL {
     expression = "RECEIVED_PBL & !RCVD_VIA_SMTP_AUTH";
-    description = "Relayed through ZEN PBL IP without sufficient authentication (possible indicating an open relay)";
+    description = "Relayed through Spamhaus PBL IP without sufficient authentication (possible indicating an open relay)";
     score = 2.0;
     policy = "leave";
   }
@@ -129,18 +129,16 @@ composites {
     policy = "leave";
   }
   BAD_REP_POLICIES {
-    description = "Contains valid policies but are also marked by fuzzy/bayes/surbl/rbl";
+    description = "Contains valid policies but are also marked by fuzzy/bayes/SURBL/RBL";
     expression = "(~g-:policies) & (-g+:fuzzy | -g+:statistics | -g+:surbl | -g+:rbl)";
     score = 0.1;
   }
-
   VIOLATED_DIRECT_SPF {
     description = "Has no Received (or no trusted received relays) and SPF policy fails or soft fails";
     expression = "(R_SPF_FAIL | R_SPF_SOFTFAIL) & (RCVD_COUNT_ZERO | RCVD_NO_TLS_LAST)";
     policy = "leave";
     score = 3.5;
   }
-
   IP_SCORE_FREEMAIL {
     description = "Negate IP_SCORE when message comes from FreeMail";
     expression = "FREEMAIL_FROM & SENDER_REP_SPAM";
@@ -160,12 +158,11 @@ composites {
     score = 7.0;
     group = "scams";
   }
-  
   FREEMAIL_AFF {
-	  expression = "(FREEMAIL_FROM | FREEMAIL_ENVFROM | FREEMAIL_REPLYTO) & R_UNDISC_RCPT & (INTRODUCTION | FROM_NAME_HAS_TITLE | FREEMAIL_REPLYTO_NEQ_FROM_DOM)";
-	  score = 4.0;
-	  policy = "leave";
-	  description = "Message exhibits strong characteristics of advance fee fraud (AFF a/k/a '419' spam) involving freemail addresses";
+    expression = "(FREEMAIL_FROM | FREEMAIL_ENVFROM | FREEMAIL_REPLYTO) & R_UNDISC_RCPT & (INTRODUCTION | FROM_NAME_HAS_TITLE | FREEMAIL_REPLYTO_NEQ_FROM_DOM)";
+    score = 4.0;
+    policy = "leave";
+    description = "Message exhibits strong characteristics of advance fee fraud (AFF a/k/a '419' spam) involving freemail addresses";
   }
 
   .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/composites.conf"

conf/maps.d/maillist.inc

@@ -1,208 +1,176 @@
-usndr.com
-subscribe.ru
-smartsndr.com
-
-hh.ru
-free-lance.ru
-superjob.ru
-rabota.ru
-job.ru
-odesk.com
-jobinmoscow.ru
- 
-russianpost.ru
-shopotam.ru
-
-ebay.com
+1c-bitrix.ru
+360.cn
+360.com
+activeby.net
+adobe.com
+aeroflot.ru
 alibaba.com
 aliexpress.com
 alipay.com
-
-github.net
-github.com
-molotok.ru
-
+amazon.co.jp
+amazon.com
+amazon.co.uk
+amazon.de
+amazon.in
+apple.com
+ask.com
+avito.ru
+b2b-center.ru
+baby.ru
+babysfera.ru
+baidu.com
+beeline.ru
+bing.com
+blogger.com
+blogspot.com
+booking.com
+chase.com
+cnn.com
+comodo.com
+comodogroup.com
+comodo.net
+dating.ru
+dmir.ru
+dropbox.com
+ebay.com
+electrozon.ru
+e-moskva.ru
+etp-micex.ru
+exist.ru
+fabrikant.ru
 facebook.com
-vk.com
-odnoklassniki.ru
-vkrugudruzei.ru
+fc2.com
+flipkart.com
+free-lance.ru
+github.com
+github.net
+gmw.cn
+go.com
+google.ca
+google.co.id
+google.co.in
+google.co.jp
+google.com.au
+google.com.br
+google.com.hk
+google.com.mx
+google.com.tr
+google.co.uk
+google.de
+google.fr
+google.it
+google.pl
+google.ru
+googleusercontent.com
+gosuslugi.ru
+gov.ru
+habramail.net
+hao123.com
+hh.ru
+imdb.com
+imgur.com
+instagram.com
+ispsystem.com
+ispsystem.net
+ispsystem.ru
+jobinmoscow.ru
+job.ru
 linkedin.com
-professionali.ru
-mail.mtml.ru
+live.com
 livejournal.com
-twitter.com
-
-avito.ru
-dmir.ru
+mailgun.com
+mailgun.net
+mail.mtml.ru
+mamba.ru
+megafon.ru
+microsoft.com
+microsoftonline.com
 mnogo.ru
+mobilelement.ru
+moesk.ru
+molotok.ru
+mos.ru
+msn.com
+mts.ru
+nalog.ru
+naukanet.ru
+netflix.com
+nic.ru
+nix.ru
+nytimes.com
+odesk.com
+odnoklassniki.ru
+office.com
+ok.ru
+osmp.ru
+outbrain.com
+ozon.ru
 paypal.com
-roboxchange.com
-sberbank.ru
-
+pinterest.com
+pixnet.net
+pornhub.com
+professionali.ru
+psport.ru
 qiwi.com
 qiwi.ru
-osmp.ru
-mobilelement.ru
-rp-system.ru
+qq.com
 quickpay.ru
+r01.ru
+rabota.ru
 rbkmoney.ru
-
-gosuslugi.ru
-rostelecom.ru
-mos.ru
-gov.ru
-nalog.ru
-sitesoft.ru
-e-moskva.ru
-rosreestr.ru
+reddit.com
+reg.ru
+regtime.net
+returnpath.net
+roboxchange.com
 roseltorg.ru
-sberbank-ast.ru
-etp-micex.ru
-zakazrf.ru
-rtstender.ru
+rosreestr.ru
+rostelecom.ru
+rp-system.ru
 rts-tender.ru
-b2b-center.ru
-yamoney.ru
-
-fabrikant.ru
-
-apple.com
-dropbox.com
+rtstender.ru
+russianpost.ru
+sberbank-ast.ru
+sberbank.ru
+senderscore.net
+shopotam.ru
+sipnet.ru
+sitesoft.ru
 skype.com
-
-habramail.net
-mamba.ru
-dating.ru
+smartsndr.com
+sohu.com
+soso.com
+stackoverflow.com
+startcomca.com
+subscribe.ru
+superjob.ru
+taobao.com
+tario.ru
+t.co
+tks.ru
+tmall.com
 topface.com
-
-ulmart.ru
-electrozon.ru
-nix.ru
-ozon.ru
-
-beeline.ru
-mts.ru
-megafon.ru
-
-booking.com
+tumblr.com
 tutu.ru
-aeroflot.ru
-
+twitter.com
+ulmart.ru
+usndr.com
 vedomosti.ru
-
-1c-bitrix.ru
-
-moesk.ru
-
-exist.ru
-tks.ru
-zzap.ru
-
-activeby.net
-babysfera.ru
-baby.ru
-
-wordpress.com
-
-ispsystem.net
-ispsystem.com
-ispsystem.ru
-
-naukanet.ru
-
-startcomca.com
-
-wmtransfer.com
-
-sipnet.ru
-tario.ru
-
-mailgun.com
-mailgun.net
-
-psport.ru
-
-returnpath.net
-senderscore.net
-
+vk.com
+vkrugudruzei.ru
 webnames.ru
-regtime.net
-
-nic.ru
-r01.ru
-reg.ru
-
-ztel.ru
-
-youtube.com
-baidu.com
-yahoo.com
-amazon.com
-wikipedia.org
-qq.com
-google.co.in
-live.com
-taobao.com
-msn.com
-yahoo.co.jp
-google.co.jp
 weibo.com
-bing.com
-hao123.com
-instagram.com
-google.de
-amazon.co.jp
-360.cn
-tmall.com
-google.co.uk
-pinterest.com
-google.ru
-reddit.com
-google.com.br
-t.co
-netflix.com
-google.fr
-sohu.com
-microsoft.com
-google.it
-blogspot.com
-tumblr.com
-ok.ru
-gmw.cn
-imgur.com
-stackoverflow.com
-xvideos.com
-google.com.mx
-fc2.com
-imdb.com
-google.com.hk
-amazon.de
-ask.com
-google.com.tr
-google.ca
-office.com
-pornhub.com
-google.co.id
-soso.com
-go.com
-pixnet.net
-amazon.co.uk
-googleusercontent.com
-outbrain.com
-amazon.in
-blogger.com
-cnn.com
-google.pl
-google.com.au
-360.com
-xhamster.com
-adobe.com
-flipkart.com
-microsoftonline.com
 whatsapp.com
-nytimes.com
-chase.com
+wikipedia.org
+wmtransfer.com
+wordpress.com
 wosign.com
-comodo.com
-comodogroup.com
-comodo.net
+xhamster.com
+xvideos.com
+yahoo.co.jp
+yahoo.com
+yamoney.ru
+youtube.com
+zakazrf.ru
+ztel.ru
+zzap.ru

conf/maps.d/redirectors.inc

@@ -233,8 +233,8 @@ email.account.2gis.com
 email.mail.ostrovok.ru
 email.news.ostrovok.ru
 e.mail.ru
-em.digium.com
 emap.ws
+em.digium.com
 etdurl.com
 eweri.com
 exa.im

conf/maps.d/surbl-whitelist.inc

@@ -34,6 +34,7 @@ americanexpress.ch
 americanexpress.com
 anadolubank.nl
 ancestry.com
+anpdm.com
 anz.com
 anz.co.nz
 aol.com
@@ -294,6 +295,7 @@ discovery.co.za
 dnbnord.lt
 domain.com
 doubleclick.com
+dovecot.org
 dresdner-bank.de
 dsbbank.sr
 dsbl.org
@@ -314,6 +316,7 @@ egroups.com
 e-gulfbank.com
 emode.com
 esunbank.com.tw
+exacttarget.com
 example.com
 example.net
 example.org
@@ -349,6 +352,7 @@ generali.es
 genevoise.ch
 gentoo.org
 geocities.com
+github.com
 gkb.ch
 gmail.com
 gmx.net
@@ -416,6 +420,7 @@ isbank.de
 isbank.ge
 isbank.iq
 isbankkosova.com
+isc.org
 itau.com.br
 ivillage.com
 joingevalia.com
@@ -444,6 +449,8 @@ lcl.com
 lcl.fr
 li.ru
 list.ru
+lists.isc.org
+lists.roundcube.net
 liveinternet.ru
 livejournal.com
 lloydsbank.com
@@ -697,6 +704,7 @@ subscribe.ru
 sun.com
 suncorpbank.com.au
 suntrust.com
+svn.apache.org
 swedbank.com
 swedbank.ee
 swedbank.lt
@@ -707,6 +715,7 @@ swisscaution.ch
 swissquote.ch
 sydbank.dk
 sympatico.ca
+taggedmail.com
 tails.nl
 tangerine.ca
 tcb-bank.com.tw
@@ -726,6 +735,7 @@ top4top.ru
 tsbbank.co.nz
 tsb.co.nz
 tsb.co.uk
+tumblr.com
 tux.org
 twitter.com
 ubibanca.com
@@ -759,6 +769,8 @@ visa.com.br
 visaeurope.ch
 visaeurope.com
 viseca.ch
+vistaprint.com
+vistaprint.dk
 volksbank.de
 volkswagenbank.de
 vpbank.com
@@ -813,18 +825,6 @@ zdnet.com
 zenithbank.com
 zkb.ch
 zugerkb.ch
-vistaprint.dk
-vistaprint.com
-anpdm.com
-dovecot.org
-exacttarget.com
-github.com
-isc.org
 # list-manage1.com # grey
 # list-manage2.com # grey
 # list-manage.com # grey
-lists.isc.org
-lists.roundcube.net
-svn.apache.org
-taggedmail.com
-tumblr.com

conf/modules.d/bimi.conf

@@ -26,4 +26,4 @@ bimi {
   .include(try=true,priority=5) "${DBDIR}/dynamic/bimi.conf"
   .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/bimi.conf"
   .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/bimi.conf"
-}
+}

conf/modules.d/clickhouse.conf

@@ -12,7 +12,6 @@
 #
 # Module documentation can be found at  https://rspamd.com/doc/modules/clickhouse.html
 
-
 clickhouse {
   # Push update when 1000 records are collected (1000 if unset)
   limit = 1000;

conf/modules.d/multimap.conf

@@ -164,4 +164,3 @@ url_tld_re {
           symbol = "URL_MAP_RE";
 }
 */
-

conf/modules.d/rbl.conf

@@ -46,8 +46,7 @@ rbl {
       returncodes {
         SPAMHAUS_SBL = "127.0.0.2";
         SPAMHAUS_CSS = "127.0.0.3";
-        SPAMHAUS_XBL = ["127.0.0.4", "127.0.0.5",
-            "127.0.0.6", "127.0.0.7"];
+        SPAMHAUS_XBL = ["127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7"];
         SPAMHAUS_PBL = ["127.0.0.10", "127.0.0.11"];
         SPAMHAUS_DROP = "127.0.0.9";
         SPAMHAUS_BLOCKED_OPENRESOLVER = "127.255.255.254";
@@ -178,6 +177,7 @@ rbl {
         RSPAMD_EMAILBL = "127.0.0.2";
       }
     }
+
     MSBL_EBL {
       ignore_whitelist = true;
       ignore_defaults = true;
@@ -197,7 +197,7 @@ rbl {
         ];
       }
     }
-    # Old SURBL module
+
     "SURBL_MULTI" {
       ignore_defaults = true;
       rbl = "multi.surbl.org";
@@ -206,7 +206,7 @@ rbl {
       exclude_users = false;
 
       returnbits = {
-        CRACKED_SURBL = 128; # From February 2016
+        CRACKED_SURBL = 128;
         ABUSE_SURBL = 64;
         MW_SURBL_MULTI = 16;
         PH_SURBL_MULTI = 8;
@@ -334,13 +334,6 @@ rbl {
         SEM_URIBL_FRESH15 = 2;
       }
     }
-
-    # Proved to be broken
-    #"RBL_SARBL_BAD" {
-    #  suffix = "public.sarbl.org";
-    #  noip   = true;
-    #  images = true;
-    #}
   }
 
   .include(try=true,priority=5) "${DBDIR}/dynamic/rbl.conf"

conf/modules.d/redis.conf

@@ -23,4 +23,4 @@ redis {
   .include(try=true,priority=5) "${DBDIR}/dynamic/redis.conf"
   .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/redis.conf"
   .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/redis.conf"
-}
+}

conf/modules.d/spamassassin.conf

@@ -23,4 +23,4 @@ spamassassin {
   .include(try=true,priority=5) "${DBDIR}/dynamic/spamassassin.conf"
   .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/spamassassin.conf"
   .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/spamassassin.conf"
-}
+}

conf/scores.d/content_group.conf

@@ -35,18 +35,17 @@ symbols = {
   }
   "PDF_LONG_TRAILER" {
     weight = 0.2;
-    description = "There is an PDF with a long trailer";
+    description = "There is an PDF with a long trailer in the message";
     one_shot = true;
   }
   "PDF_MANY_OBJECTS" {
     weight = 0;
-    description = "There is a PDF file with too many objects";
+    description = "There is a PDF with too many objects in the message";
     one_shot = true;
   }
   "PDF_TIMEOUT" {
     weight = 0;
-    description = "There is a PDF file that caused timeout in processing";
+    description = "There is a PDF in the message that caused timeout in processing";
     one_shot = true;
   }
 }
-

conf/scores.d/headers_group.conf

@@ -16,6 +16,7 @@
 # See https://rspamd.com/doc/tutorials/writing_rules.html for details
 
 description = "Various headers checks";
+
 max_score = 8.0;
 
 symbols = {

conf/scores.d/mime_types_group.conf

@@ -16,9 +16,8 @@
 # See https://rspamd.com/doc/tutorials/writing_rules.html for details
 
 description = "Mime attachments rules";
-# Define some limit for this group
-max_score = 10.0;
 
+max_score = 10.0;
 
 symbols = {
     "MIME_GOOD" {
@@ -76,4 +75,4 @@ symbols = {
         description = "Filename with known obscured unicode characters";
         one_shot = true;
     }
-}
+}

conf/scores.d/policies_group.conf

@@ -124,25 +124,21 @@ symbols = {
         description = "ARC checks success";
         groups = ["arc"];
     }
-
     "ARC_REJECT" {
         weight = 1.0;
         description = "ARC checks failed";
         groups = ["arc"];
     }
-
     "ARC_INVALID" {
         weight = 0.5;
         description = "ARC structure invalid";
         groups = ["arc"];
     }
-
     "ARC_DNSFAIL" {
         weight = 0.0;
         description = "ARC DNS error";
         groups = ["arc"];
     }
-
     "ARC_NA" {
         weight = 0.0;
         description = "ARC signature absent";

conf/scores.d/rbl_group.conf

@@ -21,7 +21,7 @@ symbols = {
 
     "DNSWL_BLOCKED" {
         weight = 0.0;
-        description = "Resolver blocked due to excessive queries";
+        description = "https://www.dnswl.org: Resolver blocked due to excessive queries";
         groups = ["dnswl", "blocked"];
     }
     "RCVD_IN_DNSWL" {
@@ -52,12 +52,12 @@ symbols = {
 
     "DWL_DNSWL_BLOCKED" {
         weight = 0.0;
-        description = "Resolver blocked due to excessive queries (dwl)";
+        description = "https://www.dnswl.org: Resolver blocked due to excessive queries (DWL)";
         groups = ["dnswl", "blocked"];
     }
     "DWL_DNSWL" {
         weight = 0.0;
-        description = "Unrecognised result from https://www.dnswl.org (dwl)";
+        description = "Unrecognised result from https://www.dnswl.org (DWL)";
         groups = ["dnswl"];
     }
     "DWL_DNSWL_NONE" {
@@ -88,48 +88,48 @@ symbols = {
     }
     "RBL_SPAMHAUS_SBL" {
         weight = 4.0;
-        description = "From address is listed in ZEN SBL";
+        description = "From address is listed in Spamhaus SBL";
         groups = ["spamhaus"];
     }
     "RBL_SPAMHAUS_CSS" {
         weight = 2.0;
-        description = "From address is listed in ZEN CSS";
+        description = "From address is listed in Spamhaus CSS";
         groups = ["spamhaus"];
     }
     "RBL_SPAMHAUS_XBL" {
         weight = 4.0;
-        description = "From address is listed in ZEN XBL";
+        description = "From address is listed in Spamhaus XBL";
         groups = ["spamhaus"];
     }
     "RBL_SPAMHAUS_PBL" {
         weight = 2.0;
-        description = "From address is listed in ZEN PBL (ISP list)";
+        description = "From address is listed in Spamhaus PBL";
         groups = ["spamhaus"];
     }
     "RBL_SPAMHAUS_DROP" {
         weight = 7.0;
-        description = "From address is listed in ZEN DROP BL";
+        description = "From address is listed in Spamhaus DROP";
         groups = ["spamhaus"];
     }
     "RBL_SPAMHAUS_BLOCKED_OPENRESOLVER" {
         weight = 0.0;
         description = "You are querying Spamhaus from an open resolver, please see https://www.spamhaus.org/returnc/pub/";
-        groups = ["spamhaus"];
+        groups = ["spamhaus", "blocked"];
     }
     "RBL_SPAMHAUS_BLOCKED" {
         weight = 0.0;
         description = "You are exceeding the query limit, please see https://www.spamhaus.org/returnc/vol/";
-        groups = ["spamhaus"];
+        groups = ["spamhaus", "blocked"];
     }
     "RECEIVED_SPAMHAUS_SBL" {
         weight = 3.0;
-        description = "Received address is listed in ZEN SBL";
+        description = "Received address is listed in Spamhaus SBL";
         groups = ["spamhaus"];
         one_shot = true;
     }
     "RECEIVED_SPAMHAUS_CSS" {
         weight = 1.0;
-        description = "Received address is listed in ZEN CSS";
+        description = "Received address is listed in Spamhaus CSS";
         groups = ["spamhaus"];
         one_shot = true;
     }
@@ -141,31 +141,32 @@ symbols = {
     }
     "RECEIVED_SPAMHAUS_PBL" {
         weight = 0.0;
-        description = "Received address is listed in ZEN PBL (ISP list)";
+        description = "Received address is listed in Spamhaus PBL";
         groups = ["spamhaus"];
         one_shot = true;
     }
     "RECEIVED_SPAMHAUS_DROP" {
         weight = 6.0;
-        description = "Received address is listed in ZEN DROP BL";
+        description = "Received address is listed in Spamhaus DROP";
         groups = ["spamhaus"];
         one_shot = true;
     }
     "RECEIVED_SPAMHAUS_BLOCKED_OPENRESOLVER" {
         weight = 0.0;
         description = "You are querying Spamhaus from an open resolver, please see https://www.spamhaus.org/returnc/pub/";
-        groups = ["spamhaus"];
+        groups = ["spamhaus", "blocked"];
     }
     "RECEIVED_SPAMHAUS_BLOCKED" {
         weight = 0.0;
         description = "You are exceeding the query limit, please see https://www.spamhaus.org/returnc/vol/";
-        groups = ["spamhaus"];
+        groups = ["spamhaus", "blocked"];
     }
 
     "RBL_SENDERSCORE" {
         weight = 2.0;
         description = "From address is listed in senderscore.com BL";
     }
+
     "MAILSPIKE" {
         weight = 0.0;
         description = "Unrecognised result from Mailspike";
@@ -178,37 +179,37 @@ symbols = {
     }
     "RBL_MAILSPIKE_WORST" {
         weight = 2.0;
-        description = "From address is listed in RBL - worst possible reputation";
+        description = "From address is listed in Mailspike RBL - worst possible reputation";
         groups = ["mailspike"];
     }
     "RBL_MAILSPIKE_VERYBAD" {
         weight = 1.5;
-        description = "From address is listed in RBL - very bad reputation";
+        description = "From address is listed in Mailspike RBL - very bad reputation";
         groups = ["mailspike"];
     }
     "RBL_MAILSPIKE_BAD" {
         weight = 1.0;
-        description = "From address is listed in RBL - bad reputation";
+        description = "From address is listed in Mailspike RBL - bad reputation";
         groups = ["mailspike"];
     }
     "RWL_MAILSPIKE_POSSIBLE" {
         weight = 0.0;
-        description = "From address is listed in RWL - possibly legit";
+        description = "From address is listed in Mailspike RWL - possibly legit";
         groups = ["mailspike"];
     }
     "RWL_MAILSPIKE_GOOD" {
         weight = -0.1;
-        description = "From address is listed in RWL - good reputation";
+        description = "From address is listed in Mailspike RWL - good reputation";
         groups = ["mailspike"];
     }
     "RWL_MAILSPIKE_VERYGOOD" {
         weight = -0.2;
-        description = "From address is listed in RWL - very good reputation";
+        description = "From address is listed in Mailspike RWL - very good reputation";
         groups = ["mailspike"];
     }
     "RWL_MAILSPIKE_EXCELLENT" {
         weight = -0.4;
-        description = "From address is listed in RWL - excellent reputation";
+        description = "From address is listed in Mailspike RWL - excellent reputation";
         groups = ["mailspike"];
     }
 
@@ -231,7 +232,7 @@ symbols = {
 
     "RBL_NIXSPAM" {
         weight = 4.0;
-        description = "From address is listed in NiX Spam (http://www.dnsbl.manitu.net/)";
+        description = "From address is listed in NiX Spam (https://www.nixspam.net/)";
     }
 
     "RBL_BLOCKLISTDE" {

conf/scores.d/subject_group.conf

@@ -17,7 +17,7 @@
 
 description = "Subject filters";
 
+max_score = 6.0;
+
 symbols = {
 }
-
-max_score = 6.0;

conf/scores.d/surbl_group.conf

@@ -22,41 +22,41 @@ max_score = 12.5;
 symbols = {
     "SURBL_BLOCKED" {
         weight = 0.0;
-        description = "SURBL: blocked by policy/overusage";
+        description = "SURBL: query blocked by policy/overusage";
         one_shot = true;
         groups = ["surblorg", "blocked"];
     }
     "PH_SURBL_MULTI" {
         weight = 5.5;
-        description = "SURBL: Phishing sites";
+        description = "A domain in the message is listed in SURBL as phishing";
         one_shot = true;
         groups = ["surblorg", "phishing"];
     }
     "MW_SURBL_MULTI" {
         weight = 5.5;
-        description = "SURBL: Malware sites";
+        description = "A domain in the message is listed in SURBL as malware";
         one_shot = true;
         groups = ["surblorg"];
     }
     "ABUSE_SURBL" {
         weight = 5.5;
-        description = "SURBL: ABUSE";
+        description = "A domain in the message is listed in SURBL as abused";
         one_shot = true;
         groups = ["surblorg"];
     }
     "CRACKED_SURBL" {
         weight = 4.0;
-        description = "SURBL: cracked site";
+        description = "A domain in the message is listed in as SURBL cracked";
         one_shot = true;
         groups = ["surblorg"];
     }
+
     "RSPAMD_URIBL" {
         weight = 4.5;
         description = "Rspamd uribl, bl.rspamd.com";
         one_shot = true;
         groups = ["rspamdbl"];
     }
-
     "RSPAMD_EMAILBL" {
         weight = 2.5;
         description = "Rspamd emailbl, bl.rspamd.com";
@@ -66,101 +66,101 @@ symbols = {
 
     "MSBL_EBL" {
         weight = 7.5;
-        description = "MSBL emailbl";
+        description = "MSBL emailbl (https://www.msbl.org/)";
         one_shot = true;
         groups = ["ebl"];
     }
 
     "MSBL_EBL_GREY" {
         weight = 0.5; # TODO: test it
-        description = "MSBL emailbl grey list";
+        description = "MSBL emailbl grey list (https://www.msbl.org/)";
         one_shot = true;
         groups = ["ebl"];
     }
 
     "SEM_URIBL_UNKNOWN" {
         weight = 0.0;
-        description = "Spameatingmonkey uribl: unknown result";
+        description = "Unrecognised result from Spameatingmonkey URIBL";
         one_shot = true;
         groups = ["sem"];
     }
     "SEM_URIBL" {
         weight = 3.5;
-        description = "Spameatingmonkey uribl";
+        description = "A domain in the message is listed in Spameatingmonkey URIBL";
         one_shot = true;
         groups = ["sem"];
     }
 
     "SEM_URIBL_FRESH15_UNKNOWN" {
         weight = 0.0;
-        description = "Spameatingmonkey Fresh15 uribl: unknown result";
+        description = "Unrecognised result from Spameatingmonkey Fresh15 URIBL";
         one_shot = true;
         groups = ["sem"];
     }
     "SEM_URIBL_FRESH15" {
         weight = 3.0;
-        description = "Spameatingmonkey uribl. Domains registered in the last 15 days (.AERO,.BIZ,.COM,.INFO,.NAME,.NET,.PRO,.SK,.TEL,.US)";
+        description = "A domain in the message is listed in Spameatingmonkey Fresh15 URIBL (registered in the past 15 days, .AERO,.BIZ,.COM,.INFO,.NAME,.NET,.PRO,.SK,.TEL,.US only)";
         one_shot = true;
         groups = ["sem"];
     }
 
     "DBL" {
         weight = 0.0;
-        description = "DBL unknown result";
+        description = "Unrecognised result from Spamhaus DBL";
         one_shot = true;
         groups = ["spamhaus"];
     }
     "DBL_SPAM" {
         weight = 6.5;
-        description = "DBL uribl spam";
+        description = "A domain in the message is listed in Spamhaus DBL as spam";
         one_shot = true;
         groups = ["spamhaus"];
     }
     "DBL_PHISH" {
         weight = 6.5;
-        description = "DBL uribl phishing";
+        description = "A domain in the message is listed in Spamhaus DBL as phishing";
         one_shot = true;
         groups = ["spamhaus"];
     }
     "DBL_MALWARE" {
         weight = 6.5;
-        description = "DBL uribl malware";
+        description = "A domain in the message is listed in Spamhaus DBL as malware";
         one_shot = true;
         groups = ["spamhaus"];
     }
     "DBL_BOTNET" {
         weight = 5.5;
-        description = "DBL uribl botnet C&C domain";
+        description = "A domain in the message is listed in Spamhaus DBL as botnet C&C";
         one_shot = true;
         groups = ["spamhaus"];
     }
     "DBL_ABUSE" {
         weight = 6.5;
-        description = "DBL uribl abused legit spam";
+        description = "A domain in the message is listed in Spamhaus DBL as abused legit spam";
         one_shot = true;
         groups = ["spamhaus"];
     }
     "DBL_ABUSE_REDIR" {
         weight = 1.5;
-        description = "DBL uribl abused spammed redirector domain";
+        description = "A domain in the message is listed in Spamhaus DBL as spammed redirector domain";
         one_shot = true;
         groups = ["spamhaus"];
     }
     "DBL_ABUSE_PHISH" {
         weight = 7.5;
-        description = "DBL uribl abused legit phish";
+        description = "A domain in the message is listed in Spamhaus DBL as abused legit phish";
         one_shot = true;
         groups = ["spamhaus"];
     }
     "DBL_ABUSE_MALWARE" {
         weight = 7.5;
-        description = "DBL uribl abused legit malware";
+        description = "A domain in the message is listed in Spamhaus DBL as abused legit malware";
         one_shot = true;
         groups = ["spamhaus"];
     }
     "DBL_ABUSE_BOTNET" {
         weight = 5.5;
-        description = "DBL uribl abused legit botnet C&C";
+        description = "A domain in the message is listed in Spamhaus DBL as abused legit botnet C&C";
         one_shot = true;
         groups = ["spamhaus"];
     }
@@ -174,48 +174,50 @@ symbols = {
         weight = 0.0;
         description = "You are querying Spamhaus from an open resolver, please see https://www.spamhaus.org/returnc/pub/";
         one_shot = true;
-        groups = ["spamhaus"];
+        groups = ["spamhaus", "blocked"];
     }
     "DBL_BLOCKED" {
         weight = 0.0;
         description = "You are exceeding the query limit, please see https://www.spamhaus.org/returnc/vol/";
         one_shot = true;
-        groups = ["spamhaus"];
+        groups = ["spamhaus", "blocked"];
     }
+
     "URIBL_MULTI" {
         weight = 0.0;
-        description = "uribl.com: unrecognised result";
+        description = "Unrecognised result from URIBL.com";
         one_shot = true;
         groups = ["uribl"];
     }
     "URIBL_BLOCKED" {
         weight = 0.0;
-        description = "uribl.com: query refused";
+        description = "URIBL.com: query refused, likely due to policy/overusage";
         one_shot = true;
         groups = ["uribl", "blocked"];
     }
     "URIBL_BLACK" {
         weight = 7.5;
-        description = "uribl.com black url";
+        description = "A domain in the message is listed in URIBL.com black";
         one_shot = true;
         groups = ["uribl"];
     }
     "URIBL_RED" {
         weight = 3.5;
-        description = "uribl.com red url";
+        description = "A domain in the message is listed in URIBL.com red";
         one_shot = true;
         groups = ["uribl"];
     }
     "URIBL_GREY" {
         weight = 1.5;
-        description = "uribl.com grey url";
+        description = "A domain in the message is listed in URIBL.com grey";
         one_shot = true;
         groups = ["uribl"];
     }
+
     "SPAMHAUS_ZEN_URIBL" {
         ignore = true;
         weight = 0.0;
-        description = "Spamhaus ZEN URIBL: Filtered result";
+        description = "Unrecognised result from Spamhaus ZEN URIBL";
         one_shot = true;
         groups = ["spamhaus"];
     }
@@ -229,7 +231,7 @@ symbols = {
     "URIBL_SBL_CSS" {
         ignore = true;
         weight = 6.5;
-        description = "A domain in the message body resolves to an IP listed in Spamhaus SBL CSS";
+        description = "A domain in the message body resolves to an IP listed in Spamhaus CSS";
         one_shot = true;
         groups = ["spamhaus"];
     }
@@ -254,9 +256,4 @@ symbols = {
         one_shot = true;
         groups = ["spamhaus"];
     }
-    #"RBL_SARBL_BAD" {
-    #    weight = 2.5;
-    #    description = "A domain in the message body is blacklisted in SARBL";
-    #    one_shot = true;
-    #}
 }

debian/compat

@@ -1 +1 @@
-9
+10

debian/control

@@ -6,10 +6,7 @@ Build-Depends: cmake,
                debhelper (>= 10),
                libcurl4-openssl-dev,
                libglib2.0-dev,
-               libhyperscan-dev [amd64 i386],
                libicu-dev,
-               libjemalloc-dev,
-               libluajit-5.1-dev [amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64 ppc64el] | liblua5.1-dev,
                libmagic-dev,
                libpcre2-dev,
                libsodium-dev,
@@ -26,15 +23,35 @@ Vcs-Browser: https://github.com/vstakhov/rspamd
 
 Package: rspamd
 Architecture: any
+Conflicts: rspamd-asan
+Provides: rspamd
 Depends: adduser, lsb-base, ca-certificates, ${misc:Depends}, ${shlibs:Depends}
 Description: Rapid spam filtering system
  Rspamd is a rapid, modular and lightweight spam filter. It is designed to work
  with big amount of mail and can be easily extended with own filters written in
  Lua.
 
+Package: rspamd-asan
+Architecture: any
+Conflicts: rspamd
+Provides: rspamd
+Depends: adduser, lsb-base, ca-certificates, ${misc:Depends}, ${shlibs:Depends}
+Description: Rapid spam filtering system
+ Rspamd is a rapid, modular and lightweight spam filter. It is designed to work
+ with big amount of mail and can be easily extended with own filters written in
+ Lua. This package is build with no compile optimisations and with ASAN library
+ to be used when there is a need to debug Rspamd.
+
 Package: rspamd-dbg
 Architecture: any
 Section: debug
 Depends: rspamd (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends}
 Description: debugging symbols for rspamd
  This package contains the debugging symbols for rspamd.
+
+Package: rspamd-dbg-asan
+Architecture: any
+Section: debug
+Depends: rspamd-asan (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends}
+Description: debugging symbols for rspamd-asan
+ This package contains the debugging symbols for rspamd-asan.

debian/rules

@@ -5,26 +5,39 @@ export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
 export ASAN_OPTIONS=detect_leaks=0
 
 DEB_HOST_ARCH ?= $(shell dpkg-architecture -qDEB_HOST_ARCH)
+builddir = debian/build/flavor-
+FLAVORS = release asan
+CONFIG_asan = -DCMAKE_BUILD_TYPE=Debug -DSANITIZE=address -DENABLE_LTO=OFF
+CONFIG_release = -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_LTO=ON
 
 ifneq ($(filter $(DEB_HOST_ARCH),amd64 armel armhf i386 mips mips64el mipsel ppc64 ppc64el),)
-	ENABLE_LUAJIT := ON
+	ENABLE_LUAJIT := -DENABLE_LUAJIT=ON -DLUA_ROOT=/luajit-build
 else
-	ENABLE_LUAJIT := OFF
+	ENABLE_LUAJIT := -DENABLE_LUAJIT=OFF
 endif
 
 # Can be filled by a build system
 EXTRA_ARGS :=
+ENABLE_JEMALLOC := -DENABLE_JEMALLOC=OFF
 
-ifneq ($(filter $(DEB_HOST_ARCH),amd64 i386),)
+ifneq ($(filter $(DEB_HOST_ARCH),amd64 i386 arm64),)
+ifeq ($(filter $(DEB_HOST_ARCH),arm64),)
+	ENABLE_JEMALLOC := -DENABLE_JEMALLOC=ON
+	ENABLE_HYPERSCAN := -DENABLE_HYPERSCAN=ON -DHYPERSCAN_ROOT_DIR=/vectorscan
+else
 	ENABLE_HYPERSCAN := -DENABLE_HYPERSCAN=ON
+endif
 else
-	ENABLE_HYPERSCAN :=
+	ENABLE_HYPERSCAN := -DENABLE_HYPERSCAN=OFF
 endif
+
 %:
 	dh $@ --with systemd --parallel
 
-override_dh_auto_configure:
-	dh_auto_configure -- -DCONFDIR=/etc/rspamd \
+override_dh_auto_configure: $(patsubst %,configure_%,$(FLAVORS))
+configure_%:
+	mkdir -p $(builddir)$*
+	cd $(builddir)$* && cmake ../../../  -DCONFDIR=/etc/rspamd -DCMAKE_C_COMPILER=clang-16 -DCMAKE_CXX_COMPILER=clang++-16 \
 		-DMANDIR=/usr/share/man \
 		-DRUNDIR=/run/rspamd \
 		-DDBDIR=/var/lib/rspamd \
@@ -33,34 +46,55 @@ override_dh_auto_configure:
 		-DLIBDIR=/usr/lib/rspamd \
 		-DINCLUDEDIR=/usr/include \
 		-DSYSTEMDDIR=/lib/systemd/system \
-		-DCMAKE_BUILD_TYPE=None \
+		-DCMAKE_INSTALL_PREFIX:PATH=/usr \
 		-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-		-DENABLE_LTO=ON \
 		-DDEBIAN_BUILD=1 \
-		-DENABLE_JEMALLOC=ON \
 		-DENABLE_PCRE2=ON \
 		-DENABLE_LUAJIT=$(ENABLE_LUAJIT) \
 		-DENABLE_LIBUNWIND=ON \
 		-DWANT_SYSTEMD_UNITS=ON \
 		-DENABLE_STATIC_LIBCXX=OFF \
-		-DENABLE_LIBCXX=OFF \
+		-DENABLE_LIBCXX=AUTO \
 		$(ENABLE_HYPERSCAN) \
+		$(ENABLE_LUAJIT) \
+		$(ENABLE_JEMALLOC) \
+		$(CONFIG_$*) \
 		$(EXTRA_ARGS)
 
-override_dh_strip:
-	dh_strip --dbg-package=rspamd-dbg
+override_dh_auto_build: $(patsubst %,build_%,$(FLAVORS))
+build_%:
+	dh_auto_build --sourcedirectory=$(builddir)$*
+
+override_dh_strip: $(patsubst %,strip_%,$(FLAVORS))
+strip_%:
+	if [ "$*" = "asan" ]; then \
+		dh_strip -prspamd-asan --dbg-package=rspamd-dbg-asan ; \
+	else \
+		dh_strip -prspamd --dbg-package=rspamd-dbg ; \
+	fi
 
 override_dh_makeshlibs:
 	dh_makeshlibs -Xrspamd-actrie
 
-override_dh_auto_install:
-	dh_auto_install --destdir=debian/rspamd
-
-override_dh_auto_clean:
-	dh_auto_clean
-	rm -f src/modules.c
+override_dh_auto_install: $(patsubst %,install_%,$(FLAVORS))
+install_%:
+	if [ "$*" = "asan" ]; then \
+		cd $(builddir)$* && make install DESTDIR=../../../debian/rspamd-asan ; \
+		cd ../../../debian/rspamd-asan &&  mkdir -p var/lib/rspamd \
+																								 var/log/rspamd \
+																								 etc/rspamd/local.d \
+																								 etc/rspamd/override.d  ; \
+	else \
+		cd $(builddir)$* && make install DESTDIR=../../../debian/rspamd ; \
+		cd ../../../debian/rspamd &&  mkdir -p var/lib/rspamd \
+																								 var/log/rspamd \
+																								 etc/rspamd/local.d \
+																								 etc/rspamd/override.d  ; \
+	fi
 
 override_dh_fixperms:
 	dh_fixperms
 	chmod 750 $(CURDIR)/debian/rspamd/var/lib/rspamd
 	chmod 750 $(CURDIR)/debian/rspamd/var/log/rspamd
+	chmod 750 $(CURDIR)/debian/rspamd-asan/var/lib/rspamd
+	chmod 750 $(CURDIR)/debian/rspamd-asan/var/log/rspamd

rules/content.lua

@@ -113,4 +113,4 @@ rspamd_config:register_symbol{
   name = 'PDF_TIMEOUT',
   parent = id,
   groups = {"content", "pdf"},
-}
+}

rules/forwarding.lua

@@ -153,4 +153,3 @@ rspamd_config.FORWARDED = {
   description = "Message was forwarded",
   group = "forwarding"
 }
-

rules/mid.lua

@@ -125,4 +125,3 @@ rspamd_config:register_symbol {
     end
   end
 }
-

rules/regexp/compromised_hosts.lua

@@ -211,4 +211,3 @@ reconf['WWW_DOT_DOMAIN'] = {
   score = 0.5,
   group = "compromised_hosts"
 }
-

rules/regexp/headers.lua

@@ -439,7 +439,7 @@ reconf['FORGED_MUA_OPERA_MSGID'] = {
 -- Detect forged Mozilla Mail/Thunderbird/Seamonkey/Postbox headers
 -- Mozilla based X-Mailer
 local user_agent_mozilla5	= 'User-Agent=/^\\s*Mozilla\\/5\\.0/H'
-local user_agent_thunderbird	= 'User-Agent=/^\\s*(Thunderbird|Mozilla Thunderbird|Mozilla\\/.*Gecko\\/.*(Thunderbird|Icedove)\\/)/H'
+local user_agent_thunderbird	= 'User-Agent=/^\\s*(Thunderbird|Mozilla Thunderbird|Mozilla\\/.*Gecko\\/.*(Thunderbird|Betterbird|Icedove)\\/)/H'
 local user_agent_seamonkey	= 'User-Agent=/^\\s*Mozilla\\/5\\.0\\s.+\\sSeaMonkey\\/\\d+\\.\\d+/H'
 local user_agent_postbox	= [[User-Agent=/^\s*Mozilla\/5\.0\s\([^)]+\)\sGecko\/\d+\sPostboxApp\/\d+(?:\.\d+){2,3}$/H]]
 local user_agent_mozilla	= string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla5, user_agent_thunderbird, user_agent_seamonkey, user_agent_postbox)
@@ -908,14 +908,21 @@ reconf['HAS_LIST_UNSUB'] = {
 
 reconf['HAS_GUC_PROXY_URI'] = {
   re = '/\\.googleusercontent\\.com\\/proxy/{url}i',
-  description = 'Has googleusercontent.com proxy URI',
-  score = 0.01,
-  group = 'experimental'
+  description = 'Has googleusercontent.com proxy URL',
+  score = 1.0,
+  group = 'url'
 }
 
 reconf['HAS_GOOGLE_REDIR'] = {
   re = '/\\.google\\.com\\/url\\?/{url}i',
   description = 'Has google.com/url redirection',
+  score = 1.0,
+  group = 'url'
+}
+
+reconf['HAS_GOOGLE_FIREBASE_URL'] = {
+  re = '/\\.firebasestorage\\.googleapis\\.com\\//{url}i',
+  description = 'Contains firebasestorage.googleapis.com URL',
   score = 0.01,
   group = 'experimental'
 }

rules/regexp/misc.lua

@@ -99,4 +99,4 @@ reconf['LEAKED_PASSWORD_SCAM_RE'] = {
   group = 'scams'
 }
 
-rspamd_config:register_dependency('LEAKED_PASSWORD_SCAM', 'BITCOIN_ADDR')
+rspamd_config:register_dependency('LEAKED_PASSWORD_SCAM', 'BITCOIN_ADDR')

rules/subject_checks.lua

@@ -67,4 +67,4 @@ rspamd_config.LONG_SUBJ = {
   group = 'subject',
   type = 'mime',
   description = 'Subject is too long'
-}
+}