Merge pull request #10 from rspamd/master

merge upstream into local master

.luacheckrc

@@ -33,6 +33,7 @@ globals = {
   'loadstring',
   'rspamadm_ev_base',
   'rspamadm_session',
+  'jit'
 }
 
 ignore = {
@@ -72,4 +73,4 @@ files['/**/rules/'].ignore = {'631'}
 files['/**/test/functional/'].ignore = {'631'}
 
 max_string_line_length = 500
-max_comment_line_length = 500
+max_comment_line_length = 500

conf/composites.conf

@@ -16,118 +16,125 @@
 
 composites {
 
-    FORGED_RECIPIENTS_MAILLIST {
-        expression = "FORGED_RECIPIENTS & -MAILLIST";
-    }
-    FORGED_SENDER_MAILLIST {
-        expression = "FORGED_SENDER & -MAILLIST";
-    }
-    FORGED_SENDER_FORWARDING {
-        expression = "FORGED_SENDER & g:forwarding";
-        description = "Forged sender, but message is forwarded";
-        policy = "remove_weight";
-    }
-    SPF_FAIL_FORWARDING {
-        expression = "g:forwarding & (R_SPF_SOFTFAIL | R_SPF_FAIL)";
-        policy = "remove_weight";
-    }
-    DMARC_POLICY_ALLOW_WITH_FAILURES {
-        expression = "DMARC_POLICY_ALLOW & (R_SPF_SOFTFAIL | R_SPF_FAIL | R_DKIM_REJECT)";
-        policy = "remove_weight";
-    }
-    FORGED_RECIPIENTS_FORWARDING {
-        expression = "FORGED_RECIPIENTS & g:forwarding";
-        policy = "remove_weight";
-    }
-    FORGED_SENDER_VERP_SRS {
-        expression = "FORGED_SENDER & (ENVFROM_PRVS | ENVFROM_VERP)";
-    }
-    FORGED_MUA_MAILLIST {
-        expression = "g:mua & -MAILLIST";
-    }
-    RBL_SPAMHAUS_XBL_ANY {
-        expression = "RBL_SPAMHAUS_XBL & RECEIVED_SPAMHAUS_XBL";
-        description = "From and Received address are listed in Spamhaus XBL";
-    }
-    AUTH_NA {
-        expression = "R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA";
-        score = 1.0;
-        policy = "remove_weight";
-        description = "Authenticating message via SPF/DKIM/DMARC/ARC not possible";
-    }
-    DKIM_MIXED {
-        expression = "-R_DKIM_ALLOW & (R_DKIM_DNSFAIL | R_DKIM_PERMFAIL | R_DKIM_REJECT)"
-        policy = "remove_weight";
-    }
-    MAIL_RU_MAILER_BASE64 {
-        expression = "MAIL_RU_MAILER & (FROM_EXCESS_BASE64 | MIME_BASE64_TEXT | REPLYTO_EXCESS_BASE64 | SUBJ_EXCESS_BASE64 | TO_EXCESS_BASE64)";
-    }
-    YANDEX_RU_MAILER_CTYPE_MIXED_BOGUS {
-        expression = "YANDEX_RU_MAILER & -HAS_ATTACHMENT & CTYPE_MIXED_BOGUS";
-    }
-    MAILER_1C_8_BASE64 {
-        expression = "MAILER_1C_8 & (FROM_EXCESS_BASE64 | MIME_BASE64_TEXT | SUBJ_EXCESS_BASE64 | TO_EXCESS_BASE64)";
-        description = "Message was sent by '1C:Enterprise 8' and uses base64 encoded data";
-    }
-    HACKED_WP_PHISHING {
-        expression = "(HAS_X_POS | HAS_PHPMAILER_SIG) & HAS_WP_URI & (PHISHING | DBL_PHISH | PHISHED_OPENPHISH | PHISHED_PHISHTANK)";
-        description = "Phish message sent by hacked Wordpress instance";
-        policy = "leave";
-    }
-    COMPROMISED_ACCT_BULK {
-        expression = "(HAS_XOIP | RCVD_FROM_SMTP_AUTH) & DCC_BULK";
-        description = "Likely to be from a compromised account";
-        score = 3.0;
-        policy = "leave";
-    }
-    UNDISC_RCPTS_BULK {
-        expression = "DCC_BULK & (MISSING_TO | R_UNDISC_RCPT)";
-        description = "Missing or undisclosed recipients with a bulk signature";
-        score = 3.0;
-        policy = "leave";
-    }
-    RCVD_UNAUTH_PBL {
-        expression = "RECEIVED_PBL & -RCVD_VIA_SMTP_AUTH";
-        description = "Relayed through ZEN PBL IP without sufficient authentication (possible indicating an open relay)";
-        score = 2.0;
-        policy = "leave";
-    }
-    RCVD_DKIM_ARC_DNSWL_MED {
-        expression = "(R_DKIM_ALLOW | ARC_ALLOW) & RCVD_IN_DNSWL_MED";
-        description = "Sufficiently DKIM/ARC signed and received from IP with medium trust at DNSWL";
-        score = -0.5;
-        policy = "leave";
-    }
-    RCVD_DKIM_ARC_DNSWL_HI {
-        expression = "(R_DKIM_ALLOW | ARC_ALLOW) & RCVD_IN_DNSWL_HI";
-        description = "Sufficiently DKIM/ARC signed and received from IP with high trust at DNSWL";
-        score = -1.0;
-        policy = "leave";
-    }
-    AUTOGEN_PHP_SPAMMY {
-        expression = "(HAS_X_POS | HAS_PHPMAILER_SIG | HAS_X_PHP_SCRIPT) & (SUBJECT_ENDS_QUESTION | SUBJECT_ENDS_EXCLAIM | MANY_INVISIBLE_PARTS)";
-        description = "Message was generated by PHP script and contains some spam indicators";
-        score = 1.0;
-        policy = "leave";
-    }
-    PHISH_EMOTION {
-        expression = "(PHISHING | DBL_PHISH | PHISHED_OPENPHISH | PHISHED_PHISHTANK) & (SUBJECT_ENDS_QUESTION | SUBJECT_ENDS_EXCLAIM)";
-        description = "Phish message with subject trying to address users emotion";
-        score = 1.0;
-        policy = "leave";
-    }
-    HAS_ANON_DOMAIN {
-        expression = "HAS_GUC_PROXY_URI | URIBL_RED | DBL_ABUSE_REDIR | HAS_ONION_URI";
-        description = "Contains one or more domains trying to disguise owner/destination";
-        score = 0.1;
-        policy = "leave";
-    }
-    BAD_REP_POLICIES {
-      description = "Contains valid policies but are also marked by fuzzy/bayes/surbl/rbl";
-      expression = "(~g-:policies) & (-g+:fuzzy | -g+:bayes | -g+:surbl | -g+:rbl)";
-      score = 0.1;
-    }
+  FORGED_RECIPIENTS_MAILLIST {
+    expression = "FORGED_RECIPIENTS & -MAILLIST";
+  }
+  FORGED_SENDER_MAILLIST {
+    expression = "FORGED_SENDER & -MAILLIST";
+  }
+  FORGED_SENDER_FORWARDING {
+    expression = "FORGED_SENDER & g:forwarding";
+    description = "Forged sender, but message is forwarded";
+    policy = "remove_weight";
+  }
+  SPF_FAIL_FORWARDING {
+    expression = "g:forwarding & (R_SPF_SOFTFAIL | R_SPF_FAIL)";
+    policy = "remove_weight";
+  }
+  DMARC_POLICY_ALLOW_WITH_FAILURES {
+    expression = "DMARC_POLICY_ALLOW & (R_SPF_SOFTFAIL | R_SPF_FAIL | R_DKIM_REJECT)";
+    policy = "remove_weight";
+  }
+  FORGED_RECIPIENTS_FORWARDING {
+    expression = "FORGED_RECIPIENTS & g:forwarding";
+    policy = "remove_weight";
+  }
+  FORGED_SENDER_VERP_SRS {
+    expression = "FORGED_SENDER & (ENVFROM_PRVS | ENVFROM_VERP)";
+  }
+  FORGED_MUA_MAILLIST {
+    expression = "g:mua & -MAILLIST";
+  }
+  RBL_SPAMHAUS_XBL_ANY {
+    expression = "RBL_SPAMHAUS_XBL & RECEIVED_SPAMHAUS_XBL";
+    description = "From and Received address are listed in Spamhaus XBL";
+  }
+  AUTH_NA {
+    expression = "R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA";
+    score = 1.0;
+    policy = "remove_weight";
+    description = "Authenticating message via SPF/DKIM/DMARC/ARC not possible";
+  }
+  DKIM_MIXED {
+    expression = "-R_DKIM_ALLOW & (R_DKIM_DNSFAIL | R_DKIM_PERMFAIL | R_DKIM_REJECT)"
+    policy = "remove_weight";
+  }
+  MAIL_RU_MAILER_BASE64 {
+    expression = "MAIL_RU_MAILER & (FROM_EXCESS_BASE64 | MIME_BASE64_TEXT | REPLYTO_EXCESS_BASE64 | SUBJ_EXCESS_BASE64 | TO_EXCESS_BASE64)";
+  }
+  YANDEX_RU_MAILER_CTYPE_MIXED_BOGUS {
+    expression = "YANDEX_RU_MAILER & -HAS_ATTACHMENT & CTYPE_MIXED_BOGUS";
+  }
+  MAILER_1C_8_BASE64 {
+    expression = "MAILER_1C_8 & (FROM_EXCESS_BASE64 | MIME_BASE64_TEXT | SUBJ_EXCESS_BASE64 | TO_EXCESS_BASE64)";
+    description = "Message was sent by '1C:Enterprise 8' and uses base64 encoded data";
+  }
+  HACKED_WP_PHISHING {
+    expression = "(HAS_X_POS | HAS_PHPMAILER_SIG) & HAS_WP_URI & (PHISHING | DBL_PHISH | PHISHED_OPENPHISH | PHISHED_PHISHTANK)";
+    description = "Phish message sent by hacked Wordpress instance";
+    policy = "leave";
+  }
+  COMPROMISED_ACCT_BULK {
+    expression = "(HAS_XOIP | RCVD_FROM_SMTP_AUTH) & DCC_BULK";
+    description = "Likely to be from a compromised account";
+    score = 3.0;
+    policy = "leave";
+  }
+  UNDISC_RCPTS_BULK {
+    expression = "DCC_BULK & (MISSING_TO | R_UNDISC_RCPT)";
+    description = "Missing or undisclosed recipients with a bulk signature";
+    score = 3.0;
+    policy = "leave";
+  }
+  RCVD_UNAUTH_PBL {
+    expression = "RECEIVED_PBL & -RCVD_VIA_SMTP_AUTH";
+    description = "Relayed through ZEN PBL IP without sufficient authentication (possible indicating an open relay)";
+    score = 2.0;
+    policy = "leave";
+  }
+  RCVD_DKIM_ARC_DNSWL_MED {
+    expression = "(R_DKIM_ALLOW | ARC_ALLOW) & RCVD_IN_DNSWL_MED";
+    description = "Sufficiently DKIM/ARC signed and received from IP with medium trust at DNSWL";
+    score = -0.5;
+    policy = "leave";
+  }
+  RCVD_DKIM_ARC_DNSWL_HI {
+    expression = "(R_DKIM_ALLOW | ARC_ALLOW) & RCVD_IN_DNSWL_HI";
+    description = "Sufficiently DKIM/ARC signed and received from IP with high trust at DNSWL";
+    score = -1.0;
+    policy = "leave";
+  }
+  AUTOGEN_PHP_SPAMMY {
+    expression = "(HAS_X_POS | HAS_PHPMAILER_SIG | HAS_X_PHP_SCRIPT) & (SUBJECT_ENDS_QUESTION | SUBJECT_ENDS_EXCLAIM | MANY_INVISIBLE_PARTS)";
+    description = "Message was generated by PHP script and contains some spam indicators";
+    score = 1.0;
+    policy = "leave";
+  }
+  PHISH_EMOTION {
+    expression = "(PHISHING | DBL_PHISH | PHISHED_OPENPHISH | PHISHED_PHISHTANK) & (SUBJECT_ENDS_QUESTION | SUBJECT_ENDS_EXCLAIM)";
+    description = "Phish message with subject trying to address users emotion";
+    score = 1.0;
+    policy = "leave";
+  }
+  HAS_ANON_DOMAIN {
+    expression = "HAS_GUC_PROXY_URI | URIBL_RED | DBL_ABUSE_REDIR | HAS_ONION_URI";
+    description = "Contains one or more domains trying to disguise owner/destination";
+    score = 0.1;
+    policy = "leave";
+  }
+  BAD_REP_POLICIES {
+    description = "Contains valid policies but are also marked by fuzzy/bayes/surbl/rbl";
+    expression = "(~g-:policies) & (-g+:fuzzy | -g+:bayes | -g+:surbl | -g+:rbl)";
+    score = 0.1;
+  }
 
-    .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/composites.conf"
-    .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/composites.conf"
+  VIOLATED_DIRECT_SPF {
+    description = "Has no Received (or no trusted received relays) and SPF policy fails or soft fails";
+    expression = "(R_SPF_FAIL | R_SPF_SOFTFAIL) & (RCVD_COUNT_ZERO | RCVD_NO_TLS_LAST)";
+    policy = "leave";
+    score = 3.5;
+  }
+
+  .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/composites.conf"
+  .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/composites.conf"
 }

conf/groups.conf

@@ -107,5 +107,15 @@ group "neural" {
     .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/neural_group.conf"
 }
 
+group "antivirus" {
+    .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/antivirus_group.conf"
+    .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/antivirus_group.conf"
+}
+
+group "external_services" {
+    .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/external_services_group.conf"
+    .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/external_services_group.conf"
+}
+
 .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/groups.conf"
-.include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/groups.conf"
+.include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/groups.conf"

conf/modules.d/antivirus.conf

@@ -45,6 +45,10 @@ antivirus {
       # symbol_name = "pattern";
       JUST_EICAR = '^Eicar-Test-Signature$';
     }
+    patterns_fail {
+      # symbol_name = "pattern";
+      #CLAM_PROTOCOL_ERROR = '^unhandled response';
+    }
     # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
     whitelist = "/etc/rspamd/antivirus.wl";
   }

conf/modules.d/dkim_signing.conf

@@ -58,10 +58,17 @@ dkim_signing {
   # Domain specific settings
   #domain {
   #  example.com {
-  #    # Private key path
-  #    path = "/var/lib/rspamd/dkim/example.key";
-  #    # Selector
-  #    selector = "ds";
+  #    selectors [
+  #      { # Private key path
+  #        path = "/var/lib/rspamd/dkim/example.key";
+  #        # Selector
+  #        selector = "ds";
+  #      },
+  #      { # multiple dkim signature
+  #        path = "/var/lib/rspamd/dkim/eddsa.key";
+  #        selector = "eddsa";
+  #      }
+  #    ]
   #  }
   #}
 

conf/modules.d/external_services.conf

@@ -0,0 +1,95 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+external_services {
+  oletools {
+    # If set force this action if any virus is found (default unset: no action is forced)
+    # action = "reject";
+    # If set, then rejection message is set to this value (mention single quotes)
+    # If `max_size` is set, messages > n bytes in size are not scanned
+    # max_size = 20000000;
+    # log_clean = true;
+    # servers = "127.0.0.1:10050";
+    # cache_expire = 86400;
+    # scan_mime_parts = true;
+    # extended = false;
+    # if `patterns` is specified virus name will be matched against provided regexes and the related
+    # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+    patterns {
+      # symbol_name = "pattern";
+      JUST_EICAR = "^Eicar-Test-Signature$";
+    }
+    # mime-part regex matching in content-type or filename
+    mime_parts_filter_regex {
+      #GEN1 = "application\/octet-stream";
+      DOC2 = "application\/msword";
+      DOC3 = "application\/vnd\.ms-word.*";
+      XLS = "application\/vnd\.ms-excel.*";
+      PPT = "application\/vnd\.ms-powerpoint.*";
+      GEN2 = "application\/vnd\.openxmlformats-officedocument.*";
+    }
+    # Mime-Part filename extension matching (no regex)
+    mime_parts_filter_ext {
+      doc = "doc";
+      dot = "dot";
+      docx = "docx";
+      dotx = "dotx";
+      docm = "docm";
+      dotm = "dotm";
+      xls = "xls";
+      xlt = "xlt";
+      xla = "xla";
+      xlsx = "xlsx";
+      xltx = "xltx";
+      xlsm = "xlsm";
+      xltm = "xltm";
+      xlam = "xlam";
+      xlsb = "xlsb";
+      ppt = "ppt";
+      pot = "pot";
+      pps = "pps";
+      ppa = "ppa";
+      pptx = "pptx";
+      potx = "potx";
+      ppsx = "ppsx";
+      ppam = "ppam";
+      pptm = "pptm";
+      potm = "potm";
+      ppsm = "ppsm";
+    }
+    # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+    whitelist = "/etc/rspamd/antivirus.wl";
+  }
+  dcc {
+    # If set force this action if any virus is found (default unset: no action is forced)
+    # action = "reject";
+    # If set, then rejection message is set to this value (mention single quotes)
+    # If `max_size` is set, messages > n bytes in size are not scanned
+    max_size = 20000000;
+    #servers = "127.0.0.1:10045";
+    # if `patterns` is specified virus name will be matched against provided regexes and the related
+    # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+    patterns {
+      # symbol_name = "pattern";
+      JUST_EICAR = "^Eicar-Test-Signature$";
+    }
+    # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+    whitelist = "/etc/rspamd/antivirus.wl";
+  }
+
+  .include(try=true,priority=5) "${DBDIR}/dynamic/external_services.conf"
+  .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/external_services.conf"
+  .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/external_services.conf"
+}

conf/modules.d/rbl.conf

@@ -80,11 +80,6 @@ rbl {
             rbl = "bl.score.senderscore.com";
         }
 
-        abusech {
-            symbol = "RBL_ABUSECH";
-            rbl = "spam.abuse.ch";
-        }
-
         sem {
             symbol = "RBL_SEM";
             rbl = "bl.spameatingmonkey.net";

conf/scores.d/mime_types_group.conf

@@ -56,4 +56,9 @@ symbols = {
         description = "Bad extension";
         one_shot = true;
     }
+    "MIME_BAD_UNICODE" {
+        weight = 8.0;
+        description = "Filename with known obscured unicode characters";
+        one_shot = true;
+    }
 }

conf/scores.d/rbl_group.conf

@@ -125,10 +125,6 @@ symbols = {
         weight = 2.0;
         description = "From address is listed in senderscore.com BL";
     }
-    "RBL_ABUSECH" {
-        weight = 1.0;
-        description = "From address is listed in ABUSE.CH BL";
-    }
     "MAILSPIKE" {
         weight = 0.0;
         description = "Unrecognised result from Mailspike";

config.h.in

@@ -80,7 +80,7 @@
 #cmakedefine HAVE_SA_SIGINFO     1
 #cmakedefine HAVE_SANE_SHMEM     1
 #cmakedefine HAVE_SANE_TZSET     1
-#cmakedefine HAVE_SCHED_YEILD    1
+#cmakedefine HAVE_SCHED_YIELD    1
 #cmakedefine HAVE_SC_NPROCESSORS_ONLN 1
 #cmakedefine HAVE_SEARCH_H       1
 #cmakedefine HAVE_SENDFILE       1

contrib/librdns/resolver.c

@@ -605,6 +605,33 @@ rdns_make_request_full (
 				return NULL;
 			}
 
+			if (cur_name[0] == '.') {
+				/* Skip dots at the begin */
+				unsigned int ndots = strspn (cur_name, ".");
+
+				cur_name += ndots;
+				clen -= ndots;
+
+				if (clen == 0) {
+					rdns_warn ("got empty name to resolve");
+					rdns_request_free (req);
+					return NULL;
+				}
+			}
+
+			if (cur_name[clen - 1] == '.') {
+				/* Skip trailing dots */
+				while (clen >= 1 && cur_name[clen - 1] == '.') {
+					clen --;
+				}
+
+				if (clen == 0) {
+					rdns_warn ("got empty name to resolve");
+					rdns_request_free (req);
+					return NULL;
+				}
+			}
+
 			if (last_name == NULL && queries == 1 && clen < MAX_FAKE_NAME) {
 				/* We allocate structure in the static space */
 				idx = (struct rdns_fake_reply_idx *)align_ptr (fake_buf, 16);

doc/Makefile

@@ -21,7 +21,7 @@ lua-dirs:
 lua-doc: lua-dirs rspamd_regexp rspamd_ip rspamd_config rspamd_task ucl rspamd_http rspamd_trie \
 	rspamd_resolver rspamd_redis rspamd_upstream_list rspamd_expression rspamd_mimepart rspamd_logger rspamd_url \
 	rspamd_tcp rspamd_mempool rspamd_html rspamd_util rspamd_fann rspamd_sqlite3 rspamd_cryptobox rspamd_map \
-	lua_redis lua_util lua_maps lua_clickhouse lua_selectors
+	lua_redis lua_util lua_maps lua_clickhouse lua_selectors rspamd_udp
 
 lua_redis:
 	$(LLUADOC) < ../lualib/lua_redis.lua > markdown/lua/lua_redis.md
@@ -81,4 +81,6 @@ rspamd_sqlite3: ../src/lua/lua_sqlite3.c
 rspamd_cryptobox: ../src/lua/lua_cryptobox.c
 	$(LUADOC) < ../src/lua/lua_cryptobox.c > markdown/lua/rspamd_cryptobox.md
 rspamd_map: ../src/lua/lua_map.c
-	$(LUADOC) < ../src/lua/lua_map.c > markdown/lua/rspamd_map.md
+	$(LUADOC) < ../src/lua/lua_map.c > markdown/lua/rspamd_map.md
+rspamd_udp: ../src/lua/lua_udp.c
+	$(LUADOC) < ../src/lua/lua_udp.c > markdown/lua/rspamd_udp.md

interface/js/app/upload.js

@@ -74,9 +74,9 @@ define(["jquery"],
                         rspamd.alertMessage("alert-success", "Data successfully scanned");
                         var action = "";
 
-                        if (json.action === "clean" || "no action") {
+                        if (json.action === "clean" || json.action === "no action") {
                             action = "label-success";
-                        } else if (json.action === "rewrite subject" || "add header" || "probable spam") {
+                        } else if (json.action === "rewrite subject" || json.action === "add header" || json.action === "probable spam") {
                             action = "label-warning";
                         } else if (json.action === "spam") {
                             action = "label-danger";

lualib/lua_cfg_transform.lua

@@ -16,6 +16,7 @@ limitations under the License.
 
 local logger = require "rspamd_logger"
 local lua_util = require "lua_util"
+local rspamd_util = require "rspamd_util"
 
 local function is_implicit(t)
   local mt = getmetatable(t)
@@ -216,6 +217,21 @@ local function merge_groups(groups)
   return ret
 end
 
+-- Checks configuration files for statistics
+local function check_statistics_sanity()
+  local local_conf = rspamd_paths['LOCAL_CONFDIR']
+  local local_stat = string.format('%s/local.d/%s', local_conf,
+      'statistic.conf')
+  local local_bayes = string.format('%s/local.d/%s', local_conf,
+      'classifier-bayes.conf')
+
+  if rspamd_util.file_exists(local_stat) and
+      rspamd_util.file_exists(local_bayes) then
+    logger.warnx(rspamd_config, 'conflicting files %s and %s are found: '..
+        'Rspamd classifier configuration might be broken!', local_stat, local_bayes)
+  end
+end
+
 return function(cfg)
   local ret = false
 
@@ -226,6 +242,8 @@ return function(cfg)
     ret = true
   end
 
+  check_statistics_sanity()
+
   if not cfg.actions then
     logger.errx('no actions defined')
   else

lualib/lua_dkim_tools.lua

@@ -22,21 +22,57 @@ local lua_util = require "lua_util"
 local rspamd_util = require "rspamd_util"
 local logger = require "rspamd_logger"
 
-local function check_violation(N, task, domain, selector)
+local function check_violation(N, task, domain)
   -- Check for DKIM_REJECT
   local sym_check = 'R_DKIM_REJECT'
 
   if N == 'arc' then sym_check = 'ARC_REJECT' end
   if task:has_symbol(sym_check) then
     local sym = task:get_symbol(sym_check)
-    logger.infox(task, 'skip signing for %s:%s: violation %s found: %s',
-        domain, selector, sym_check, sym.options)
+    logger.infox(task, 'skip signing for %s: violation %s found: %s',
+        domain, sym_check, sym.options)
     return false
   end
 
   return true
 end
 
+local function insert_or_update_prop(N, task, p, prop, origin, data)
+  if #p == 0 then
+    local k = {}
+    k[prop] = data
+    table.insert(p, k)
+    lua_util.debugm(N, task, 'add %s "%s" using %s', prop, data, origin)
+  else
+    for _, k in ipairs(p) do
+      if not k[prop] then
+        k[prop] = data
+        lua_util.debugm(N, task, 'set %s to "%s" using %s', prop, data, origin)
+      end
+    end
+  end
+end
+
+local function get_mempool_selectors(N, task)
+  local p = {}
+  local key_var = "dkim_key"
+  local selector_var = "dkim_selector"
+  if N == "arc" then
+    key_var = "arc_key"
+    selector_var = "arc_selector"
+  end
+
+  p.key = task:get_mempool():get_variable(key_var)
+  p.selector = task:get_mempool():get_variable(selector_var)
+
+  if (not p.key or not p.selector) then
+    return false, {}
+  end
+
+  lua_util.debugm(N, task, 'override selector and key to %s:%s', p.key, p.selector)
+  return true, p
+end
+
 local function parse_dkim_http_headers(N, task, settings)
   -- Configure headers
   local headers = {
@@ -66,11 +102,14 @@ local function parse_dkim_http_headers(N, task, settings)
       end
     end
 
-    return true,{
-      rawkey = tostring(key),
+    local p = {}
+    local k = {
       domain = tostring(domain),
-      selector = tostring(selector)
+      rawkey = tostring(key),
+      selector = tostring(selector),
     }
+    table.insert(p, k)
+    return true, p
   end
 
   lua_util.debugm(N, task, 'no sign header %s', headers.sign_header)
@@ -214,76 +253,71 @@ local function prepare_dkim_signing(N, task, settings)
   local p = {}
 
   if settings.domain[dkim_domain] then
-    p.selector = settings.domain[dkim_domain].selector
-    p.key = settings.domain[dkim_domain].path
-  end
-
-  if not p.key and p.selector then
-    local key_var = "dkim_key"
-    local selector_var = "dkim_selector"
-    if N == "arc" then
-      key_var = "arc_key"
-      selector_var = "arc_selector"
+    -- support old style selector/paths
+    if settings.domain[dkim_domain].selector or
+       settings.domain[dkim_domain].path then
+      local k = {}
+      k.selector = settings.domain[dkim_domain].selector
+      k.key = settings.domain[dkim_domain].path
+      table.insert(p, k)
     end
-
-    p.key = task:get_mempool():get_variable(key_var)
-    local selector_override = task:get_mempool():get_variable(selector_var)
-
-    if selector_override then
-      p.selector = selector_override
+    for _, s in ipairs((settings.domain[dkim_domain].selectors or {})) do
+      lua_util.debugm(N, task, 'adding selector: %1', s)
+      local k = {}
+      k.selector = s.selector
+      k.key = s.path
+      table.insert(p, k)
     end
+  end
 
-    if (not p.key or not p.selector) and (not (settings.try_fallback or
-        settings.use_redis or settings.selector_map
-        or settings.path_map)) then
-      lua_util.debugm(N, task, 'dkim unconfigured and fallback disabled')
-      return false,{}
+  if #p == 0 then
+    local ret, k = get_mempool_selectors(N, task)
+    if ret then
+      table.insert(p, k)
+      lua_util.debugm(N, task, 'using mempool selector %s with key %s',
+                      k.selector, k.key)
     end
-
-    lua_util.debugm(N, task, 'override selector and key to %s:%s', p.key, p.selector)
   end
 
-  if not p.selector and settings.selector_map then
+  if settings.selector_map then
     local data = settings.selector_map:get_key(dkim_domain)
     if data then
-      p.selector = data
-      lua_util.debugm(N, task, 'override selector to "%s" using selector_map', p.selector)
-    elseif not settings.try_fallback then
-      lua_util.debugm(N, task, 'no selector for %s', dkim_domain)
-      return false,{}
+      insert_or_update_prop(N, task, p, 'selector', 'selector_map', data)
+    else
+      lua_util.debugm(N, task, 'no selector in map for %s', dkim_domain)
     end
   end
 
-  if not p.key and settings.path_map then
+  if settings.path_map then
     local data = settings.path_map:get_key(dkim_domain)
     if data then
-      p.key = data
-      lua_util.debugm(N, task, 'override key to "%s" using path_map', p.key)
-    elseif not settings.try_fallback then
-      lua_util.debugm(N, task, 'no key for %s', dkim_domain)
-      return false,{}
+      insert_or_update_prop(N, task, p, 'key', 'path_map', data)
+    else
+      lua_util.debugm(N, task, 'no key in map for %s', dkim_domain)
     end
   end
 
-  if not p.key then
-    if not settings.use_redis then
-      p.key = settings.path
-      lua_util.debugm(N, task, 'use default key "%s" from path', p.key)
-    end
+  if #p == 0 and not settings.try_fallback then
+    lua_util.debugm(N, task, 'dkim unconfigured and fallback disabled')
+    return false,{}
   end
 
-  if not p.selector then
-    p.selector = settings.selector
-    lua_util.debugm(N, task, 'use default selector "%s"', p.selector)
+  if not settings.use_redis then
+    insert_or_update_prop(N, task, p, 'key',
+        'default path', settings.path)
   end
 
+  insert_or_update_prop(N, task, p, 'selector',
+        'default selector', settings.selector)
+
   if settings.check_violation then
-    if not check_violation(N, task, p.domain, p.selector) then
+    if not check_violation(N, task, p.domain) then
       return false,{}
     end
   end
 
-  p.domain = dkim_domain
+  insert_or_update_prop(N, task, p, 'domain', 'dkim_domain',
+    dkim_domain)
 
   return true,p
 end

lualib/lua_ffi/common.lua

@@ -0,0 +1,34 @@
+--[[
+Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_ffi/common
+-- Common ffi definitions
+--]]
+
+local ffi = require 'ffi'
+
+ffi.cdef[[
+struct GString {
+  char  *str;
+  size_t len;
+  size_t allocated_len;
+};
+
+void g_string_free (struct GString *st, int free_data);
+]]
+
+return {}

lualib/lua_ffi/dkim.lua

@@ -0,0 +1,140 @@
+--[[
+Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_ffi/dkim
+-- This module contains ffi interfaces to DKIM
+--]]
+
+local ffi = require 'ffi'
+
+ffi.cdef[[
+struct rspamd_dkim_sign_context_s;
+struct rspamd_dkim_key_s;
+struct rspamd_task;
+enum rspamd_dkim_key_format {
+  RSPAMD_DKIM_KEY_FILE = 0,
+  RSPAMD_DKIM_KEY_PEM,
+  RSPAMD_DKIM_KEY_BASE64,
+  RSPAMD_DKIM_KEY_RAW,
+};
+enum rspamd_dkim_type {
+	RSPAMD_DKIM_NORMAL,
+	RSPAMD_DKIM_ARC_SIG,
+	RSPAMD_DKIM_ARC_SEAL
+};
+struct rspamd_dkim_sign_context_s*
+rspamd_create_dkim_sign_context (struct rspamd_task *task,
+    struct rspamd_dkim_key_s *priv_key,
+    int headers_canon,
+    int body_canon,
+    const char *dkim_headers,
+    enum rspamd_dkim_type type,
+    void *unused);
+struct rspamd_dkim_key_s* rspamd_dkim_sign_key_load (const char *what, size_t len,
+    enum rspamd_dkim_key_format,
+    void *err);
+void rspamd_dkim_key_unref (struct rspamd_dkim_key_s *k);
+struct GString *rspamd_dkim_sign (struct rspamd_task *task,
+    const char *selector,
+    const char *domain,
+    unsigned long expire,
+    size_t len,
+    unsigned int idx,
+    const char *arc_cv,
+    struct rspamd_dkim_sign_context_s *ctx);
+]]
+
+local function load_sign_key(what, format)
+  if not format then
+    format = ffi.C.RSPAMD_DKIM_KEY_PEM
+  else
+    if format == 'file' then
+      format = ffi.C.RSPAMD_DKIM_KEY_FILE
+    elseif format == 'base64' then
+      format = ffi.C.RSPAMD_DKIM_KEY_BASE64
+    elseif format == 'raw' then
+      format = ffi.C.RSPAMD_DKIM_KEY_RAW
+    else
+      return nil,'unknown key format'
+    end
+  end
+
+  return ffi.C.rspamd_dkim_sign_key_load(what, #what, format, nil)
+end
+
+local default_dkim_headers =
+"(o)from:(o)sender:(o)reply-to:(o)subject:(o)date:(o)message-id:" ..
+"(o)to:(o)cc:(o)mime-version:(o)content-type:(o)content-transfer-encoding:" ..
+"resent-to:resent-cc:resent-from:resent-sender:resent-message-id:" ..
+"(o)in-reply-to:(o)references:list-id:list-owner:list-unsubscribe:" ..
+"list-subscribe:list-post"
+
+local function create_sign_context(task, privkey, dkim_headers, sign_type)
+  if not task or not privkey then
+    return nil,'invalid arguments'
+  end
+
+  if not dkim_headers then
+    dkim_headers = default_dkim_headers
+  end
+
+  if not sign_type then
+    sign_type = 'dkim'
+  end
+
+  if sign_type == 'dkim' then
+    sign_type = ffi.C.RSPAMD_DKIM_NORMAL
+  elseif sign_type == 'arc-sig' then
+    sign_type = ffi.C.RSPAMD_DKIM_ARC_SIG
+  elseif sign_type == 'arc-seal' then
+    sign_type = ffi.C.RSPAMD_DKIM_ARC_SEAL
+  else
+    return nil,'invalid sign type'
+  end
+
+
+  return ffi.C.rspamd_create_dkim_sign_context(task:topointer(), privkey,
+      1, 1, dkim_headers, sign_type, nil)
+end
+
+local function do_sign(task, sign_context, selector, domain,
+                       expire, len, arc_idx)
+  if not task or not sign_context or not selector or not domain then
+    return nil,'invalid arguments'
+  end
+
+  if not expire then expire = 0 end
+  if not len then len = 0 end
+  if not arc_idx then arc_idx = 0 end
+
+  local gstring = ffi.C.rspamd_dkim_sign(task:topointer(), selector, domain, expire, len, arc_idx, nil, sign_context)
+
+  if not gstring then
+    return nil,'cannot sign'
+  end
+
+  local ret = ffi.string(gstring.str, gstring.len)
+  ffi.C.g_string_free(gstring, true)
+
+  return ret
+end
+
+return {
+  load_sign_key = load_sign_key,
+  create_sign_context = create_sign_context,
+  do_sign = do_sign
+}

lualib/lua_ffi/init.lua

@@ -0,0 +1,52 @@
+--[[
+Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_ffi
+-- This module contains ffi interfaces (requires luajit or lua-ffi)
+--]]
+
+local ffi
+
+local exports = {}
+
+if type(jit) == 'table' then
+  ffi = require "ffi"
+  local NULL = ffi.new 'void*'
+
+  exports.is_null = function(o)
+    return o ~= NULL
+  end
+else
+  local ret,result_or_err = pcall(require, 'ffi')
+
+  if not ret then
+    io.stderr:write('FFI support is required: please use LuaJIT or install lua-ffi')
+    os.exit(1)
+  end
+
+  ffi = result_or_err
+  -- Lua ffi
+  local NULL = ffi.NULL or ffi.C.NULL
+  exports.is_null = function(o)
+    return o ~= NULL
+  end
+end
+
+exports.common = require "lua_ffi/common"
+exports.dkim = require "lua_ffi/dkim"
+
+return exports

lualib/lua_scanners/clamav.lua

@@ -32,9 +32,10 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"'
 
 local function clamav_config(opts)
   local clamav_conf = {
-    scan_mime_parts = true;
-    scan_text_mime = false;
-    scan_image_mime = false;
+    name = N,
+    scan_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
     default_port = 3310,
     log_clean = false,
     timeout = 5.0, -- FIXME: this will break task_timeout!
@@ -44,12 +45,18 @@ local function clamav_config(opts)
     message = default_message,
   }
 
-  for k,v in pairs(opts) do
-    clamav_conf[k] = v
-  end
+  clamav_conf = lua_util.override_defaults(clamav_conf, opts)
 
   if not clamav_conf.prefix then
-    clamav_conf.prefix = 'rs_cl'
+    clamav_conf.prefix = 'rs_' .. clamav_conf.name .. '_'
+  end
+
+  if not clamav_conf.log_prefix then
+    if clamav_conf.name:lower() == clamav_conf.type:lower() then
+      clamav_conf.log_prefix = clamav_conf.name
+    else
+      clamav_conf.log_prefix = clamav_conf.name .. ' (' .. clamav_conf.type .. ')'
+    end
   end
 
   if not clamav_conf['servers'] then
@@ -63,7 +70,7 @@ local function clamav_config(opts)
       clamav_conf.default_port)
 
   if clamav_conf['upstreams'] then
-    lua_util.add_debug_alias('antivirus', N)
+    lua_util.add_debug_alias('antivirus', clamav_conf.name)
     return clamav_conf
   end
 
@@ -96,7 +103,8 @@ local function clamav_check(task, content, digest, rule)
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+          lua_util.debugm(rule.name, task, '%s: retry IP: %s',
+              rule.log_prefix, addr)
 
           tcp.request({
             task = task,
@@ -108,34 +116,42 @@ local function clamav_check(task, content, digest, rule)
             stop_pattern = '\0'
           })
         else
-          rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', rule.log_prefix)
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
 
       else
         upstream:ok()
         data = tostring(data)
         local cached
-        lua_util.debugm(N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data)
+        lua_util.debugm(rule.name, task, '%s: got reply: %s',
+            rule.log_prefix, data)
         if data == 'stream: OK' then
           cached = 'OK'
           if rule['log_clean'] then
-            rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+            rspamd_logger.infox(task, '%s: message or mime_part is clean',
+                rule.log_prefix)
           else
-            lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+            lua_util.debugm(rule.name, task, '%s: message or mime_part is clean', rule.log_prefix)
           end
         else
           local vname = string.match(data, 'stream: (.+) FOUND')
-          if vname then
-            common.yield_result(task, rule, vname, N)
+          if string.find(vname, '^Heuristics%.Encrypted') then
+            rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix)
+            common.yield_result(task, rule, 'File is encrypted: '.. vname, 0.0, 'fail')
+          elseif string.find(vname, '^Heuristics%.Limits%.Exceeded') then
+            rspamd_logger.errx(task, '%s: ClamAV Limits Exceeded', rule.log_prefix)
+            common.yield_result(task, rule, 'Limits Exceeded: '.. vname, 0.0, 'fail')
+          elseif vname then
+            common.yield_result(task, rule, vname)
             cached = vname
           else
-            rspamd_logger.errx(task, 'unhandled response: %s', data)
-            task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response')
+            rspamd_logger.errx(task, '%s: unhandled response: %s', rule.log_prefix, data)
+            common.yield_result(task, rule, 'unhandled response:' .. vname, 0.0, 'fail')
           end
         end
         if cached then
-          common.save_av_cache(task, digest, rule, cached, N)
+          common.save_av_cache(task, digest, rule, cached)
         end
       end
     end
@@ -151,8 +167,8 @@ local function clamav_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule, N) then
-    if common.check_av_cache(task, digest, rule, clamav_check_uncached, N) then
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, clamav_check_uncached) then
       return
     else
       clamav_check_uncached()
@@ -165,5 +181,5 @@ return {
   description = 'clamav antivirus',
   configure = clamav_config,
   check = clamav_check,
-  name = 'clamav'
-}
+  name = N
+}

lualib/lua_scanners/common.lua

@@ -1,5 +1,6 @@
 --[[
 Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,99 +21,131 @@ limitations under the License.
 --]]
 
 local rspamd_logger = require "rspamd_logger"
+local rspamd_regexp = require "rspamd_regexp"
 local lua_util = require "lua_util"
 local lua_redis = require "lua_redis"
 local fun = require "fun"
 
 local exports = {}
 
-local function match_patterns(default_sym, found, patterns)
-  if type(patterns) ~= 'table' then return default_sym end
+local function log_clean(task, rule, msg)
+
+  msg = msg or 'message or mime_part is clean'
+
+  if rule.log_clean then
+    rspamd_logger.infox(task, '%s: %s', rule.log_prefix, msg)
+  else
+    lua_util.debugm(rule.name, task, '%s: %s', rule.log_prefix, msg)
+  end
+
+end
+
+local function match_patterns(default_sym, found, patterns, dyn_weight)
+  if type(patterns) ~= 'table' then return default_sym, dyn_weight end
   if not patterns[1] then
     for sym, pat in pairs(patterns) do
       if pat:match(found) then
-        return sym
+        return sym, '1'
       end
     end
-    return default_sym
+    return default_sym, dyn_weight
   else
     for _, p in ipairs(patterns) do
       for sym, pat in pairs(p) do
         if pat:match(found) then
-          return sym
+          return sym, '1'
         end
       end
     end
-    return default_sym
+    return default_sym, dyn_weight
   end
 end
 
-local function yield_result(task, rule, vname, N, dyn_weight)
+local function yield_result(task, rule, vname, dyn_weight, is_fail)
   local all_whitelisted = true
-  if not dyn_weight then dyn_weight = 1.0 end
+  local patterns
+  local symbol
+
+  -- This should be more generic
+  if not is_fail then
+    patterns = rule.patterns
+    symbol = rule.symbol
+    if not dyn_weight then dyn_weight = 1.0 end
+  elseif is_fail == 'fail' then
+    patterns = rule.patterns_fail
+    symbol = rule.symbol_fail
+    dyn_weight = 0.0
+  end
+
   if type(vname) == 'string' then
-    local symname = match_patterns(rule.symbol, vname, rule.patterns)
+    local symname, symscore = match_patterns(symbol,
+        vname,
+        patterns,
+        dyn_weight)
     if rule.whitelist and rule.whitelist:get_key(vname) then
-      rspamd_logger.infox(task, '%s: "%s" is in whitelist', N, vname)
+      rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vname)
       return
     end
-    task:insert_result(symname, 1.0, vname)
-    rspamd_logger.infox(task, '%s: %s found: "%s"', N, rule.detection_category, vname)
+    task:insert_result(symname, symscore, vname)
+    rspamd_logger.infox(task, '%s: %s found: "%s - score: %s"',
+        rule.log_prefix, rule.detection_category, vname, symscore)
   elseif type(vname) == 'table' then
     for _, vn in ipairs(vname) do
-      local symname = match_patterns(rule.symbol, vn, rule.patterns)
+      local symname, symscore = match_patterns(symbol, vn, patterns, dyn_weight)
       if rule.whitelist and rule.whitelist:get_key(vn) then
-        rspamd_logger.infox(task, '%s: "%s" is in whitelist', N, vn)
+        rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vn)
       else
         all_whitelisted = false
-        task:insert_result(symname, dyn_weight, vn)
-        rspamd_logger.infox(task, '%s: %s found: "%s"',
-            N, rule.detection_category, vn)
+        task:insert_result(symname, symscore, vn)
+        rspamd_logger.infox(task, '%s: %s found: "%s - score: %s"',
+            rule.log_prefix, rule.detection_category, vn, symscore)
       end
     end
   end
-  if rule.action then
+  if rule.action and is_fail ~= 'fail' then
     if type(vname) == 'table' then
       if all_whitelisted then return end
       vname = table.concat(vname, '; ')
     end
-    task:set_pre_result(rule['action'],
+    task:set_pre_result(rule.action,
         lua_util.template(rule.message or 'Rejected', {
-          SCANNER = N,
+          SCANNER = rule.name,
           VIRUS = vname,
-        }), N)
+        }), rule.name)
   end
 end
 
-local function message_not_too_large(task, content, rule, N)
+local function message_not_too_large(task, content, rule)
   local max_size = tonumber(rule.max_size)
   if not max_size then return true end
   if #content > max_size then
     rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)",
-        N, #content, max_size)
+        rule.log_prefix, #content, max_size)
     return false
   end
   return true
 end
 
-local function need_av_check(task, content, rule, N)
-  return message_not_too_large(task, content, rule, N)
+local function need_av_check(task, content, rule)
+  return message_not_too_large(task, content, rule)
 end
 
-local function check_av_cache(task, digest, rule, fn, N)
+local function check_av_cache(task, digest, rule, fn)
   local key = digest
 
   local function redis_av_cb(err, data)
     if data and type(data) == 'string' then
       -- Cached
-      if data ~= 'OK' then
-        lua_util.debugm(N, task, 'got cached result for %s: %s',
-            key, data)
-        data = lua_util.str_split(data, '\v')
-        yield_result(task, rule, data, N)
+      data = rspamd_str_split(data, '\t')
+      local threat_string = rspamd_str_split(data[1], '\v')
+      local score = data[2] or rule.default_score
+      if threat_string[1] ~= 'OK' then
+        lua_util.debugm(rule.name, task, '%s: got cached threat result for %s: %s - score: %s',
+          rule.log_prefix, key, threat_string[1], score)
+        yield_result(task, rule, threat_string, score)
       else
-        lua_util.debugm(N, task, 'got cached result for %s: %s',
-            key, data)
+        lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s',
+          rule.log_prefix, key, threat_string[1])
       end
     else
       if err then
@@ -124,7 +157,7 @@ local function check_av_cache(task, digest, rule, fn, N)
 
   if rule.redis_params then
 
-    key = rule['prefix'] .. key
+    key = rule.prefix .. key
 
     if lua_redis.redis_make_request(task,
         rule.redis_params, -- connect params
@@ -141,8 +174,9 @@ local function check_av_cache(task, digest, rule, fn, N)
   return false
 end
 
-local function save_av_cache(task, digest, rule, to_save, N)
+local function save_av_cache(task, digest, rule, to_save, dyn_weight)
   local key = digest
+  if not dyn_weight then dyn_weight = 1.0 end
 
   local function redis_set_cb(err)
     -- Do nothing
@@ -150,8 +184,8 @@ local function save_av_cache(task, digest, rule, to_save, N)
       rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s',
           rule.detection_category, to_save, key, err)
     else
-      lua_util.debugm(N, task, 'saved cached result for %s: %s',
-          key, to_save)
+      lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s',
+        rule.log_prefix, key, to_save, dyn_weight)
     end
   end
 
@@ -159,6 +193,8 @@ local function save_av_cache(task, digest, rule, to_save, N)
     to_save = table.concat(to_save, '\v')
   end
 
+  local value = table.concat({to_save, dyn_weight}, '\t')
+
   if rule.redis_params and rule.prefix then
     key = rule.prefix .. key
 
@@ -168,29 +204,150 @@ local function save_av_cache(task, digest, rule, to_save, N)
         true, -- is write
         redis_set_cb, --callback
         'SETEX', -- command
-        { key, rule.cache_expire or 0, to_save }
+        { key, rule.cache_expire or 0, value }
     )
   end
 
   return false
 end
 
-local function text_parts_min_words(task, min_words)
-  local filter_func = function(p)
-    return p:get_words_count() >= min_words
+local function create_regex_table(patterns)
+  local regex_table = {}
+  if patterns[1] then
+    for i, p in ipairs(patterns) do
+      if type(p) == 'table' then
+        local new_set = {}
+        for k, v in pairs(p) do
+          new_set[k] = rspamd_regexp.create_cached(v)
+        end
+        regex_table[i] = new_set
+      else
+        regex_table[i] = {}
+      end
+    end
+  else
+    for k, v in pairs(patterns) do
+      regex_table[k] = rspamd_regexp.create_cached(v)
+    end
   end
+  return regex_table
+end
 
-  return fun.any(filter_func, task:get_text_parts())
+local function match_filter(task, found, patterns)
+  if type(patterns) ~= 'table' then return false end
+  if not patterns[1] then
+    for _, pat in pairs(patterns) do
+      if pat:match(found) then
+        return true
+      end
+    end
+    return false
+  else
+    for _, p in ipairs(patterns) do
+      for _, pat in ipairs(p) do
+        if pat:match(found) then
+          return true
+        end
+      end
+    end
+    return false
+  end
+end
 
+-- borrowed from mime_types.lua
+-- ext is the last extension, LOWERCASED
+-- ext2 is the one before last extension LOWERCASED
+local function gen_extension(fname)
+  local filename_parts = rspamd_str_split(fname, '.')
+
+  local ext = {}
+  for n = 1, 2 do
+      ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil
+  end
+  return ext[1],ext[2],filename_parts
 end
 
+local function check_parts_match(task, rule)
+
+  local filter_func = function(p)
+    local mtype,msubtype = p:get_type()
+    local dmtype,dmsubtype = p:get_detected_type()
+    local fname = p:get_filename()
+    local ext, ext2
+    local extension_check = false
+    local content_type_check = false
+    local text_part_min_words_check = true
+
+    if rule.scan_all_mime_parts == false then
+    -- check file extension and filename regex matching
+      if fname ~= nil then
+        ext,ext2 = gen_extension(fname)
+        if match_filter(task, ext, rule.mime_parts_filter_ext)
+          or match_filter(task, ext2, rule.mime_parts_filter_ext) then
+          lua_util.debugm(rule.name, task, '%s: extension matched: %s', rule.log_prefix, ext)
+          extension_check = true
+        end
+        if match_filter(task, fname, rule.mime_parts_filter_regex) then
+          content_type_check = true
+        end
+      end
+      -- check content type string regex matching
+      if mtype ~= nil and msubtype ~= nil then
+        local ct = string.format('%s/%s', mtype, msubtype):lower()
+        if match_filter(task, ct, rule.mime_parts_filter_regex) then
+          lua_util.debugm(rule.name, task, '%s: regex content-type: %s', rule.log_prefix, ct)
+          content_type_check = true
+        end
+      end
+      -- check detected content type (libmagic) regex matching
+      if dmtype ~= nil and dmsubtype ~= nil then
+        local ct = string.format('%s/%s', mtype, msubtype):lower()
+        if match_filter(task, ct, rule.mime_parts_filter_regex) then
+          lua_util.debugm(rule.name, task, '%s: regex detected libmagic content-type: %s', rule.log_prefix, ct)
+          content_type_check = true
+        end
+      end
+      -- check filenames in archives
+      if p:is_archive() then
+        local arch = p:get_archive()
+        local filelist = arch:get_files_full()
+        for _,f in ipairs(filelist) do
+          ext,ext2 = gen_extension(f.name)
+          if match_filter(task, ext, rule.mime_parts_filter_ext)
+            or match_filter(task, ext2, rule.mime_parts_filter_ext) then
+            lua_util.debugm(rule.name, task, '%s: extension matched in archive: %s', rule.log_prefix, ext)
+            extension_check = true
+          end
+          if match_filter(task, f.name, rule.mime_parts_filter_regex) then
+            content_type_check = true
+          end
+        end
+      end
+    end
+
+    -- check text_part has more words than text_part_min_words_check
+    if rule.text_part_min_words and p:is_text() then
+      text_part_min_words_check = p:get_words_count() >= tonumber(rule.text_part_min_words)
+    end
+
+    return (rule.scan_image_mime and p:is_image())
+        or (rule.scan_text_mime and text_part_min_words_check)
+        or (p:is_attachment() and rule.scan_all_mime_parts ~= false)
+        or extension_check
+        or content_type_check
+  end
+
+  return fun.filter(filter_func, task:get_parts())
+end
 
+exports.log_clean = log_clean
 exports.yield_result = yield_result
 exports.match_patterns = match_patterns
 exports.need_av_check = need_av_check
 exports.check_av_cache = check_av_cache
 exports.save_av_cache = save_av_cache
-exports.text_parts_min_words = text_parts_min_words
+exports.create_regex_table = create_regex_table
+exports.check_parts_match = check_parts_match
 
 setmetatable(exports, {
   __call = function(t, override)
@@ -210,4 +367,4 @@ setmetatable(exports, {
   end,
 })
 
-return exports
+return exports

lualib/lua_scanners/dcc.lua

@@ -29,7 +29,7 @@ local fun = require "fun"
 
 local N = 'dcc'
 
-local function dcc_check(task, content, _, rule)
+local function dcc_check(task, content, digest, rule)
   local function dcc_check_uncached ()
     local upstream = rule.upstreams:get_upstream_round_robin()
     local addr = upstream:get_addr()
@@ -81,8 +81,7 @@ local function dcc_check(task, content, _, rule)
 
     local function dcc_callback(err, data, conn)
 
-      if err then
-
+      local function dcc_requery()
         -- set current upstream to fail because an error occurred
         upstream:fail()
 
@@ -91,11 +90,15 @@ local function dcc_check(task, content, _, rule)
 
           retransmits = retransmits - 1
 
+          lua_util.debugm(rule.name, task, '%s: Request Error: %s - retries left: %s',
+            rule.log_prefix, err, retransmits)
+
           -- Select a different upstream!
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s: retry IP: %s', rule.log_prefix, addr)
+          lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+            rule.log_prefix, addr, addr:get_port())
 
           tcp.request({
             task = task,
@@ -110,35 +113,34 @@ local function dcc_check(task, content, _, rule)
             fuz2_max = 999999,
           })
         else
-          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+            'exceed', rule.log_prefix)
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
+      end
+
+      if err then
+
+        dcc_requery()
+
       else
         -- Parse the response
         if upstream then upstream:ok() end
         local _,_,result,disposition,header = tostring(data):find("(.-)\n(.-)\n(.-)\n")
-        lua_util.debugm(N, task, 'DCC result=%1 disposition=%2 header="%3"',
+        lua_util.debugm(rule.name, task, 'DCC result=%1 disposition=%2 header="%3"',
             result, disposition, header)
 
         if header then
           local _,_,info = header:find("; (.-)$")
-
           if (result == 'R') then
             -- Reject
             common.yield_result(task, rule, info, rule.default_score)
+            common.save_av_cache(task, digest, rule, info, rule.default_score)
           elseif (result == 'T') then
             -- Temporary failure
             rspamd_logger.warnx(task, 'DCC returned a temporary failure result: %s', result)
-            task:insert_result(rule.symbol_fail,
-                0.0,
-                'tempfail:' .. result)
+            dcc_requery()
           elseif result == 'A' then
-            if rule.log_clean then
-              rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
-                  rule.log_prefix, info)
-            else
-              lua_util.debugm(N, task, '%s: returned result A - info: %s',
-                  rule.log_prefix, info)
 
               local opts = {}
               local score = 0.0
@@ -188,28 +190,37 @@ local function dcc_check(task, content, _, rule)
                 task:insert_result(rule.symbol_bulk,
                     score,
                     opts)
+                common.save_av_cache(task, digest, rule, opts, score)
+              else
+                common.save_av_cache(task, digest, rule, 'OK')
+                if rule.log_clean then
+                  rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
+                      rule.log_prefix, info)
+                else
+                  lua_util.debugm(rule.name, task, '%s: returned result A - info: %s',
+                      rule.log_prefix, info)
               end
             end
           elseif result == 'G' then
             -- do nothing
+            common.save_av_cache(task, digest, rule, 'OK')
             if rule.log_clean then
               rspamd_logger.infox(task, '%s: clean, returned result G - info: %s', rule.log_prefix, info)
             else
-              lua_util.debugm(N, task, '%s: returned result G - info: %s', rule.log_prefix, info)
+              lua_util.debugm(rule.name, task, '%s: returned result G - info: %s', rule.log_prefix, info)
             end
           elseif result == 'S' then
             -- do nothing
+            common.save_av_cache(task, digest, rule, 'OK')
             if rule.log_clean then
               rspamd_logger.infox(task, '%s: clean, returned result S - info: %s', rule.log_prefix, info)
             else
-              lua_util.debugm(N, task, '%s: returned result S - info: %s', rule.log_prefix, info)
+              lua_util.debugm(rule.name, task, '%s: returned result S - info: %s', rule.log_prefix, info)
             end
           else
             -- Unknown result
-            rspamd_logger.warnx(task, 'DCC result error: %1', result);
-            task:insert_result(rule.symbol_fail,
-                0.0,
-                'error: ' .. result)
+            rspamd_logger.warnx(task, '%s: result error: %1', rule.log_prefix, result);
+            common.yield_result(task, rule, 'error: ' .. result, 0.0, 'fail')
           end
         end
       end
@@ -222,21 +233,30 @@ local function dcc_check(task, content, _, rule)
       timeout = rule.timeout or 2.0,
       shutdown = true,
       data = request_data,
-      callback = dcc_callback
+      callback = dcc_callback,
+      body_max = 999999,
+      fuz1_max = 999999,
+      fuz2_max = 999999,
     })
   end
-  if common.need_av_check(task, content, rule, N) then
-    dcc_check_uncached()
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, dcc_check_uncached) then
+      return
+    else
+      dcc_check_uncached()
+    end
   end
 end
 
 local function dcc_config(opts)
 
   local dcc_conf = {
+    name = N,
     default_port = 10045,
     timeout = 5.0,
     log_clean = false,
     retransmits = 2,
+    cache_expire = 7200, -- expire redis in 2h
     message = '${SCANNER}: bulk message found: "${VIRUS}"',
     detection_category = "hash",
     default_score = 1,
@@ -252,8 +272,12 @@ local function dcc_config(opts)
 
   dcc_conf = lua_util.override_defaults(dcc_conf, opts)
 
+  if not dcc_conf.prefix then
+    dcc_conf.prefix = 'rs_' .. dcc_conf.name .. '_'
+  end
+
   if not dcc_conf.log_prefix then
-    dcc_conf.log_prefix = N
+    dcc_conf.log_prefix = dcc_conf.name
   end
 
   if not dcc_conf.servers and dcc_conf.socket then
@@ -271,7 +295,7 @@ local function dcc_config(opts)
       dcc_conf.default_port)
 
   if dcc_conf.upstreams then
-    lua_util.add_debug_alias('external_services', N)
+    lua_util.add_debug_alias('external_services', dcc_conf.name)
     return dcc_conf
   end
 
@@ -285,5 +309,5 @@ return {
   description = 'dcc bulk scanner',
   configure = dcc_config,
   check = dcc_check,
-  name = 'dcc'
-}
+  name = N
+}

lualib/lua_scanners/fprot.lua

@@ -31,9 +31,10 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"'
 
 local function fprot_config(opts)
   local fprot_conf = {
-    scan_mime_parts = true;
-    scan_text_mime = false;
-    scan_image_mime = false;
+    name = N,
+    scan_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
     default_port = 10200,
     timeout = 5.0, -- FIXME: this will break task_timeout!
     log_clean = false,
@@ -43,12 +44,18 @@ local function fprot_config(opts)
     message = default_message,
   }
 
-  for k,v in pairs(opts) do
-    fprot_conf[k] = v
-  end
+  fprot_conf = lua_util.override_defaults(fprot_conf, opts)
 
   if not fprot_conf.prefix then
-    fprot_conf.prefix = 'rs_fp'
+    fprot_conf.prefix = 'rs_' .. fprot_conf.name .. '_'
+  end
+
+  if not fprot_conf.log_prefix then
+    if fprot_conf.name:lower() == fprot_conf.type:lower() then
+      fprot_conf.log_prefix = fprot_conf.name
+    else
+      fprot_conf.log_prefix = fprot_conf.name .. ' (' .. fprot_conf.type .. ')'
+    end
   end
 
   if not fprot_conf['servers'] then
@@ -62,7 +69,7 @@ local function fprot_config(opts)
       fprot_conf.default_port)
 
   if fprot_conf['upstreams'] then
-    lua_util.add_debug_alias('antivirus', N)
+    lua_util.add_debug_alias('antivirus', fprot_conf.name)
     return fprot_conf
   end
 
@@ -96,7 +103,7 @@ local function fprot_check(task, content, digest, rule)
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+          lua_util.debugm(rule.name, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
 
           tcp.request({
             task = task,
@@ -111,8 +118,7 @@ local function fprot_check(task, content, digest, rule)
           rspamd_logger.errx(task,
               '%s [%s]: failed to scan, maximum retransmits exceed',
               rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0,
-              'failed to scan and retransmits exceed')
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
       else
         upstream:ok()
@@ -133,12 +139,12 @@ local function fprot_check(task, content, digest, rule)
           if not vname then
             rspamd_logger.errx(task, 'Unhandled response: %s', data)
           else
-            common.yield_result(task, rule, vname, N)
+            common.yield_result(task, rule, vname)
             cached = vname
           end
         end
         if cached then
-          common.save_av_cache(task, digest, rule, cached, N)
+          common.save_av_cache(task, digest, rule, cached)
         end
       end
     end
@@ -154,8 +160,8 @@ local function fprot_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule, N) then
-    if common.check_av_cache(task, digest, rule, fprot_check_uncached, N) then
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, fprot_check_uncached) then
       return
     else
       fprot_check_uncached()
@@ -168,5 +174,5 @@ return {
   description = 'fprot antivirus',
   configure = fprot_config,
   check = fprot_check,
-  name = 'fprot'
-}
+  name = N
+}

lualib/lua_scanners/icap.lua

@@ -0,0 +1,330 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module icap
+-- This module contains icap access functions.
+-- Currently tested with Symantec, Sophos Savdi, ClamAV/c-icap
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'icap'
+
+local function icap_check(task, content, digest, rule)
+  local function icap_check_uncached ()
+    local upstream = rule.upstreams:get_upstream_round_robin()
+    local addr = upstream:get_addr()
+    local retransmits = rule.retransmits
+    local respond_headers = {}
+
+    -- Build the icap queries
+    local options_request = {
+      "OPTIONS icap://" .. addr:to_string() .. ":" .. addr:get_port() .. "/" .. rule.scheme .. " ICAP/1.0\r\n",
+      "Host:" .. addr:to_string() .. "\r\n",
+      "User-Agent: Rspamd\r\n",
+      "Encapsulated: null-body=0\r\n\r\n",
+    }
+    local size = string.format("%x", tonumber(#content))
+
+    local function get_respond_query()
+      table.insert(respond_headers, 1,
+          'RESPMOD icap://' .. addr:to_string() .. ':' .. addr:get_port() .. '/'
+        .. rule.scheme .. ' ICAP/1.0\r\n')
+      table.insert(respond_headers, 'Encapsulated: res-body=0\r\n')
+      table.insert(respond_headers, '\r\n')
+      table.insert(respond_headers, size .. '\r\n')
+      table.insert(respond_headers, content)
+      table.insert(respond_headers, '\r\n0\r\n\r\n')
+      return respond_headers
+    end
+
+    local function add_respond_header(name, value)
+      table.insert(respond_headers, name .. ': ' .. value .. '\r\n' )
+    end
+
+    local function icap_result_header_table(result)
+      local icap_headers = {}
+      for s in result:gmatch("[^\r\n]+") do
+        if string.find(s, '^ICAP') then
+          icap_headers['icap'] = s
+        end
+        if string.find(s, '[%a%d-+]-:') then
+          local _,_,key,value = tostring(s):find("([%a%d-+]-):%s?(.+)")
+          if key ~= nil then
+            icap_headers[key] = value
+          end
+        end
+      end
+      lua_util.debugm(rule.name, task, '%s: icap_headers: %s',
+          rule.log_prefix, icap_headers)
+      return icap_headers
+    end
+
+    local function icap_parse_result(icap_headers)
+
+      local threat_string = {}
+
+      --[[
+      @ToDo: handle type in response
+
+      Generic Strings:
+        X-Infection-Found: Type=0; Resolution=2; Threat=Troj/DocDl-OYC;
+        X-Infection-Found: Type=0; Resolution=2; Threat=W97M.Downloader;
+      Symantec String:
+        X-Infection-Found: Type=2; Resolution=2; Threat=Container size violation
+        X-Infection-Found: Type=2; Resolution=2; Threat=Encrypted container violation;
+      Sophos Strings:
+        X-Virus-ID: Troj/DocDl-OYC
+      Kaspersky Strings:
+        X-Virus-ID: HEUR:Backdoor.Java.QRat.gen
+        X-Response-Info: blocked
+
+        X-Virus-ID: no threats
+        X-Response-Info: blocked
+
+        X-Response-Info: passed
+      ]] --
+
+      if icap_headers['X-Infection-Found'] ~= nil then
+        local _,_,icap_type,_,icap_threat =
+          icap_headers['X-Infection-Found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$")
+
+        if not icap_type or icap_type == 2 then
+          -- error returned
+          lua_util.debugm(rule.name, task,
+              '%s: icap error X-Infection-Found: %s', rule.log_prefix, icap_threat)
+          common.yield_result(task, rule, icap_threat, 0, 'fail')
+        else
+          lua_util.debugm(rule.name, task,
+              '%s: icap X-Infection-Found: %s', rule.log_prefix, icap_threat)
+          table.insert(threat_string, icap_threat)
+        end
+
+      elseif icap_headers['X-Virus-ID'] ~= nil and icap_headers['X-Virus-ID'] ~= "no threats" then
+        lua_util.debugm(rule.name, task,
+            '%s: icap X-Virus-ID: %s', rule.log_prefix, icap_headers['X-Virus-ID'])
+
+        if string.find(icap_headers['X-Virus-ID'], ', ') then
+          local vnames = rspamd_str_split(string.gsub(icap_headers['X-Virus-ID'], "%s", ""), ',') or {}
+
+          for _,v in ipairs(vnames) do
+            table.insert(threat_string, v)
+          end
+        else
+          table.insert(threat_string, icap_headers['X-Virus-ID'])
+        end
+      end
+
+      if #threat_string > 0 then
+        common.yield_result(task, rule, threat_string, rule.default_score)
+        common.save_av_cache(task, digest, rule, threat_string, rule.default_score)
+      else
+        common.save_av_cache(task, digest, rule, 'OK', 0)
+        common.log_clean(task, rule)
+      end
+    end
+
+    local function icap_r_respond_cb(err, data, conn)
+      local result = tostring(data)
+      conn:close()
+
+      local icap_headers = icap_result_header_table(result)
+      -- Find ICAP/1.x 2xx response
+      if string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then
+        icap_parse_result(icap_headers)
+      elseif string.find(icap_headers.icap, 'ICAP%/1%.. [45]%d%d') then
+        -- Find ICAP/1.x 5/4xx response
+        --[[
+        Symantec String:
+          ICAP/1.0 539 Aborted - No AV scanning license
+        SquidClamAV/C-ICAP:
+          ICAP/1.0 500 Server error
+        ]]--
+        rspamd_logger.errx(task, '%s: ICAP ERROR: %s', rule.log_prefix, icap_headers.icap)
+        common.yield_result(task, rule, icap_headers.icap, 0.0, 'fail')
+        return false
+      else
+        rspamd_logger.errx(task, '%s: unhandled response |%s|',
+          rule.log_prefix, string.gsub(result, "\r\n", ", "))
+        common.yield_result(task, rule, 'unhandled icap response: ' .. icap_headers.icap, 0.0, 'fail')
+      end
+    end
+
+    local function icap_w_respond_cb(err, conn)
+      conn:add_read(icap_r_respond_cb, '\r\n\r\n')
+    end
+
+    local function icap_r_options_cb(err, data, conn)
+      local icap_headers = icap_result_header_table(tostring(data))
+
+      if string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then
+        if icap_headers['Methods'] ~= nil and string.find(icap_headers['Methods'], 'RESPMOD') then
+          if icap_headers['Allow'] ~= nil and string.find(icap_headers['Allow'], '204') then
+            add_respond_header('Allow', '204')
+          end
+          conn:add_write(icap_w_respond_cb, get_respond_query())
+        else
+          rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s',
+            rule.log_prefix, icap_headers['Methods'])
+          common.yield_result(task, rule, 'NO RESPMOD', 0.0, 'fail')
+        end
+      else
+        rspamd_logger.errx(task, '%s: OPTIONS query failed: %s',
+          rule.log_prefix, icap_headers.icap)
+        common.yield_result(task, rule, 'OPTIONS query failed', 0.0, 'fail')
+      end
+    end
+
+    local function icap_callback(err, conn)
+
+      local function icap_requery(error)
+        -- set current upstream to fail because an error occurred
+        upstream:fail()
+
+        -- retry with another upstream until retransmits exceeds
+        if retransmits > 0 then
+
+          retransmits = retransmits - 1
+
+          lua_util.debugm(rule.name, task,
+              '%s: Request Error: %s - retries left: %s',
+              rule.log_prefix, error, retransmits)
+
+          -- Select a different upstream!
+          upstream = rule.upstreams:get_upstream_round_robin()
+          addr = upstream:get_addr()
+
+          lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+            rule.log_prefix, addr, addr:get_port())
+
+          tcp.request({
+            task = task,
+            host = addr:to_string(),
+            port = addr:get_port(),
+            timeout = rule.timeout,
+            stop_pattern = '\r\n',
+            data = options_request,
+            read = false,
+            callback = icap_callback,
+          })
+        else
+          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+            'exceed - err: %s', rule.log_prefix, error)
+          common.yield_result(task, rule, 'failed - err: ' .. error, 0.0, 'fail')
+        end
+      end
+
+      if err then
+        icap_requery(err)
+      else
+        -- set upstream ok
+        if upstream then upstream:ok() end
+        conn:add_read(icap_r_options_cb, '\r\n\r\n')
+      end
+    end
+
+    tcp.request({
+      task = task,
+      host = addr:to_string(),
+      port = addr:get_port(),
+      timeout = rule.timeout,
+      stop_pattern = '\r\n',
+      data = options_request,
+      read = false,
+      callback = icap_callback,
+    })
+  end
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, icap_check_uncached) then
+      return
+    else
+      icap_check_uncached()
+    end
+  end
+end
+
+
+local function icap_config(opts)
+
+  local icap_conf = {
+    name = N,
+    scan_mime_parts = true,
+    scan_all_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
+    scheme = "scan",
+    default_port = 4020,
+    timeout = 10.0,
+    log_clean = false,
+    retransmits = 2,
+    cache_expire = 7200, -- expire redis in one hour
+    message = '${SCANNER}: threat found with icap scanner: "${VIRUS}"',
+    detection_category = "virus",
+    default_score = 1,
+    action = false,
+  }
+
+  icap_conf = lua_util.override_defaults(icap_conf, opts)
+
+  if not icap_conf.prefix then
+    icap_conf.prefix = 'rs_' .. icap_conf.name .. '_'
+  end
+
+  if not icap_conf.log_prefix then
+    icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')'
+  end
+
+  if not icap_conf.log_prefix then
+    if icap_conf.name:lower() == icap_conf.type:lower() then
+      icap_conf.log_prefix = icap_conf.name
+    else
+      icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')'
+    end
+  end
+
+  if not icap_conf.servers then
+    rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+    return nil
+  end
+
+  icap_conf.upstreams = upstream_list.create(rspamd_config,
+    icap_conf.servers,
+    icap_conf.default_port)
+
+  if icap_conf.upstreams then
+    lua_util.add_debug_alias('external_services', icap_conf.name)
+    return icap_conf
+  end
+
+  rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+    icap_conf.servers)
+  return nil
+end
+
+return {
+  type = {N, 'virus', 'virus', 'scanner'},
+  description = 'generic icap antivirus',
+  configure = icap_config,
+  check = icap_check,
+  name = N
+}

lualib/lua_scanners/init.lua

@@ -39,6 +39,9 @@ require_scanner('sophos')
 
 -- Other scanners
 require_scanner('dcc')
+require_scanner('oletools')
+require_scanner('icap')
+require_scanner('vadesecure')
 
 exports.add_scanner = function(name, t, conf_func, check_func)
   assert(type(conf_func) == 'function' and type(check_func) == 'function',
@@ -59,4 +62,4 @@ exports.filter = function(t)
   end, exports))
 end
 
-return exports
+return exports

lualib/lua_scanners/kaspersky_av.lua

@@ -32,9 +32,10 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"'
 
 local function kaspersky_config(opts)
   local kaspersky_conf = {
-    scan_mime_parts = true;
-    scan_text_mime = false;
-    scan_image_mime = false;
+    name = N,
+    scan_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
     product_id = 0,
     log_clean = false,
     timeout = 5.0,
@@ -43,11 +44,22 @@ local function kaspersky_config(opts)
     message = default_message,
     detection_category = "virus",
     tmpdir = '/tmp',
-    prefix = 'rs_ak',
   }
 
   kaspersky_conf = lua_util.override_defaults(kaspersky_conf, opts)
 
+  if not kaspersky_conf.prefix then
+    kaspersky_conf.prefix = 'rs_' .. kaspersky_conf.name .. '_'
+  end
+
+  if not kaspersky_conf.log_prefix then
+    if kaspersky_conf.name:lower() == kaspersky_conf.type:lower() then
+      kaspersky_conf.log_prefix = kaspersky_conf.name
+    else
+      kaspersky_conf.log_prefix = kaspersky_conf.name .. ' (' .. kaspersky_conf.type .. ')'
+    end
+  end
+
   if not kaspersky_conf['servers'] then
     rspamd_logger.errx(rspamd_config, 'no servers defined')
 
@@ -58,7 +70,7 @@ local function kaspersky_config(opts)
       kaspersky_conf['servers'], 0)
 
   if kaspersky_conf['upstreams'] then
-    lua_util.add_debug_alias('antivirus', N)
+    lua_util.add_debug_alias('antivirus', kaspersky_conf.name)
     return kaspersky_conf
   end
 
@@ -110,7 +122,7 @@ local function kaspersky_check(task, content, digest, rule)
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task,
+          lua_util.debugm(rule.name, task,
               '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
 
           tcp.request({
@@ -126,37 +138,31 @@ local function kaspersky_check(task, content, digest, rule)
           rspamd_logger.errx(task,
               '%s [%s]: failed to scan, maximum retransmits exceed',
               rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0,
-              'failed to scan and retransmits exceed')
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
 
       else
         upstream:ok()
         data = tostring(data)
         local cached
-        lua_util.debugm(N, task, '%s [%s]: got reply: %s',
+        lua_util.debugm(rule.name, task,
+            '%s [%s]: got reply: %s',
             rule['symbol'], rule['type'], data)
-        if data == 'stream: OK' then
+        if data == 'stream: OK' or data == fname .. ': OK' then
           cached = 'OK'
-          if rule['log_clean'] then
-            rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean',
-                rule['symbol'], rule['type'])
-          else
-            lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean',
-                rule['symbol'], rule['type'])
-          end
+          common.log_clean(task, rule)
         else
           local vname = string.match(data, ': (.+) FOUND')
           if vname then
-            common.yield_result(task, rule, vname, N)
+            common.yield_result(task, rule, vname)
             cached = vname
           else
             rspamd_logger.errx(task, 'unhandled response: %s', data)
-            task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response')
+            common.yield_result(task, rule, 'unhandled response', 0.0, 'fail')
           end
         end
         if cached then
-          common.save_av_cache(task, digest, rule, cached, N)
+          common.save_av_cache(task, digest, rule, cached)
         end
       end
     end
@@ -172,8 +178,8 @@ local function kaspersky_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule, N) then
-    if common.check_av_cache(task, digest, rule, kaspersky_check_uncached, N) then
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, kaspersky_check_uncached) then
       return
     else
       kaspersky_check_uncached()
@@ -186,5 +192,5 @@ return {
   description = 'kaspersky antivirus',
   configure = kaspersky_config,
   check = kaspersky_check,
-  name = 'kaspersky'
-}
+  name = N
+}

lualib/lua_scanners/oletools.lua

@@ -0,0 +1,308 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module oletools
+-- This module contains oletools access functions.
+-- Olefy is needed: https://github.com/HeinleinSupport/olefy
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local common = require "lua_scanners/common"
+
+local N = 'oletools'
+
+local function oletools_check(task, content, digest, rule)
+  local function oletools_check_uncached ()
+    local upstream = rule.upstreams:get_upstream_round_robin()
+    local addr = upstream:get_addr()
+    local retransmits = rule.retransmits
+    local protocol = 'OLEFY/1.0\nMethod: oletools\nRspamd-ID: ' .. task:get_uid() .. '\n\n'
+
+    local function oletools_callback(err, data)
+
+      local function oletools_requery(error)
+        -- set current upstream to fail because an error occurred
+        upstream:fail()
+
+        -- retry with another upstream until retransmits exceeds
+        if retransmits > 0 then
+
+          retransmits = retransmits - 1
+
+          lua_util.debugm(rule.name, task,
+              '%s: Request Error: %s - retries left: %s',
+              rule.log_prefix, error, retransmits)
+
+          -- Select a different upstream!
+          upstream = rule.upstreams:get_upstream_round_robin()
+          addr = upstream:get_addr()
+
+          lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+              rule.log_prefix, addr, addr:get_port())
+
+          tcp.request({
+            task = task,
+            host = addr:to_string(),
+            port = addr:get_port(),
+            timeout = rule.timeout,
+            shutdown = true,
+            data = { protocol, content },
+            callback = oletools_callback,
+          })
+        else
+          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+              'exceed - err: %s', rule.log_prefix, error)
+          common.yield_result(task, rule, 'failed to scan, maximum retransmits exceed - err: ' .. error, 0.0, 'fail')
+        end
+      end
+
+      if err then
+
+        oletools_requery(err)
+
+      else
+        -- Parse the response
+        if upstream then upstream:ok() end
+
+        data = tostring(data)
+
+        local ucl_parser = ucl.parser()
+        local ok, ucl_err = ucl_parser:parse_string(tostring(data))
+        if not ok then
+          rspamd_logger.errx(task, "%s: error parsing json response: %s",
+              rule.log_prefix, ucl_err)
+          return
+        end
+
+        local result = ucl_parser:get_object()
+
+        local oletools_rc = {
+          [0] = 'RETURN_OK',
+          [1] = 'RETURN_WARNINGS',
+          [2] = 'RETURN_WRONG_ARGS',
+          [3] = 'RETURN_FILE_NOT_FOUND',
+          [4] = 'RETURN_XGLOB_ERR',
+          [5] = 'RETURN_OPEN_ERROR',
+          [6] = 'RETURN_PARSE_ERROR',
+          [7] = 'RETURN_SEVERAL_ERRS',
+          [8] = 'RETURN_UNEXPECTED',
+          [9] = 'RETURN_ENCRYPTED',
+        }
+
+        if result[1].error ~= nil then
+          rspamd_logger.errx(task, '%s: ERROR found: %s', rule.log_prefix,
+              result[1].error)
+          if result[1].error == 'File too small' then
+            common.save_av_cache(task, digest, rule, 'OK')
+            common.log_clean(task, rule, 'File too small to be scanned for macros')
+          else
+            oletools_requery(result[1].error)
+          end
+        elseif result[3]['return_code'] == 9 then
+          rspamd_logger.warnx(task, '%s: File is encrypted.', rule.log_prefix)
+          common.yield_result(task, rule, 'failed - err: ' .. oletools_rc[result[3]['return_code']], 0.0, 'fail')
+        elseif result[3]['return_code'] > 6 then
+          rspamd_logger.errx(task, '%s: Error Returned: %s',
+              rule.log_prefix, oletools_rc[result[3]['return_code']])
+          rspamd_logger.errx(task, '%s: Error message: %s',
+              rule.log_prefix, result[2]['message'])
+          common.yield_result(task, rule, 'failed - err: ' .. oletools_rc[result[3]['return_code']], 0.0, 'fail')
+        elseif result[3]['return_code'] > 1 then
+          rspamd_logger.errx(task, '%s: Error message: %s',
+              rule.log_prefix, result[2]['message'])
+          oletools_requery(oletools_rc[result[3]['return_code']])
+        elseif #result[2]['analysis'] == 0 and #result[2]['macros'] == 0 then
+          rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix)
+          common.yield_result(task, rule, 'oletools unhandled error', 0.0, 'fail')
+        elseif result[2]['analysis'] == 'null' and #result[2]['macros'] == 0 then
+          common.save_av_cache(task, digest, rule, 'OK')
+          common.log_clean(task, rule, 'No macro found')
+        elseif #result[2]['macros'] > 0 then
+          -- M=Macros, A=Auto-executable, S=Suspicious keywords, I=IOCs,
+          -- H=Hex strings, B=Base64 strings, D=Dridex strings, V=VBA strings
+          local m_exist = 'M'
+          local m_autoexec = '-'
+          local m_suspicious = '-'
+          local m_iocs = '-'
+          local m_hex = '-'
+          local m_base64 = '-'
+          local m_dridex = '-'
+          local m_vba = '-'
+
+          lua_util.debugm(rule.name, task,
+              '%s: filename: %s', rule.log_prefix, result[2]['file'])
+          lua_util.debugm(rule.name, task,
+              '%s: type: %s', rule.log_prefix, result[2]['type'])
+
+          for _,m in ipairs(result[2]['macros']) do
+            lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, '..
+                'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename)
+          end
+
+          local analysis_keyword_table = {}
+
+          for _,a in ipairs(result[2]['analysis']) do
+            lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, '..
+                'description: %s', rule.log_prefix, a.type, a.keyword, a.description)
+            if a.type == 'AutoExec' then
+              m_autoexec = 'A'
+              table.insert(analysis_keyword_table, a.keyword)
+            elseif a.type == 'Suspicious' then
+              if rule.extended == true or
+                (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings')
+              then
+                m_suspicious = 'S'
+                table.insert(analysis_keyword_table, a.keyword)
+              end
+            elseif a.type == 'IOCs' then
+              m_iocs = 'I'
+            elseif a.type == 'Hex strings' then
+              m_hex = 'H'
+            elseif a.type == 'Base64 strings' then
+              m_base64 = 'B'
+            elseif a.type == 'Dridex strings' then
+              m_dridex = 'D'
+            elseif a.type == 'VBA strings' then
+              m_vba = 'V'
+            end
+          end
+
+          --lua_util.debugm(N, task, '%s: analysis_keyword_table: %s', rule.log_prefix, analysis_keyword_table)
+
+          if rule.extended == false and m_autoexec == 'A' and m_suspicious == 'S' then
+            -- use single string as virus name
+            local threat = 'AutoExec + Suspicious (' .. table.concat(analysis_keyword_table, ',') .. ')'
+            lua_util.debugm(rule.name, task, '%s: threat result: %s', rule.log_prefix, threat)
+            common.yield_result(task, rule, threat, rule.default_score)
+            common.save_av_cache(task, digest, rule, threat, rule.default_score)
+
+          elseif rule.extended == true and #analysis_keyword_table > 0 then
+            -- report any flags (types) and any most keywords as individual virus name
+
+            local flags = m_exist ..
+                m_autoexec ..
+                m_suspicious ..
+                m_iocs ..
+                m_hex ..
+                m_base64 ..
+                m_dridex ..
+                m_vba
+            table.insert(analysis_keyword_table, 1, flags)
+
+            lua_util.debugm(rule.name, task, '%s: extended threat result: %s',
+                rule.log_prefix, table.concat(analysis_keyword_table, ','))
+
+            common.yield_result(task, rule, analysis_keyword_table, rule.default_score)
+            common.save_av_cache(task, digest, rule, analysis_keyword_table, rule.default_score)
+          else
+            common.save_av_cache(task, digest, rule, 'OK')
+            common.log_clean(task, rule, 'Scanned Macro is OK')
+          end
+
+        else
+          rspamd_logger.warnx(task, '%s: unhandled response', rule.log_prefix)
+          common.yield_result(task, rule, 'unhandled error', 0.0, 'fail')
+        end
+      end
+    end
+
+    tcp.request({
+      task = task,
+      host = addr:to_string(),
+      port = addr:get_port(),
+      timeout = rule.timeout,
+      shutdown = true,
+      data = { protocol, content },
+      callback = oletools_callback,
+    })
+
+  end
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, oletools_check_uncached) then
+      return
+    else
+      oletools_check_uncached()
+    end
+  end
+end
+
+local function oletools_config(opts)
+
+  local oletools_conf = {
+    name = N,
+    scan_mime_parts = false,
+    scan_text_mime = false,
+    scan_image_mime = false,
+    default_port = 10050,
+    timeout = 15.0,
+    log_clean = false,
+    retransmits = 2,
+    cache_expire = 86400, -- expire redis in 1d
+    symbol = "OLETOOLS",
+    message = '${SCANNER}: Oletools threat message found: "${VIRUS}"',
+    detection_category = "office macro",
+    default_score = 1,
+    action = false,
+    extended = false,
+  }
+
+  oletools_conf = lua_util.override_defaults(oletools_conf, opts)
+
+  if not oletools_conf.prefix then
+    oletools_conf.prefix = 'rs_' .. oletools_conf.name .. '_'
+  end
+
+  if not oletools_conf.log_prefix then
+    if oletools_conf.name:lower() == oletools_conf.type:lower() then
+      oletools_conf.log_prefix = oletools_conf.name
+    else
+      oletools_conf.log_prefix = oletools_conf.name .. ' (' .. oletools_conf.type .. ')'
+    end
+  end
+
+  if not oletools_conf.servers then
+    rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+    return nil
+  end
+
+  oletools_conf.upstreams = upstream_list.create(rspamd_config,
+      oletools_conf.servers,
+      oletools_conf.default_port)
+
+  if oletools_conf.upstreams then
+    lua_util.add_debug_alias('external_services', oletools_conf.name)
+    return oletools_conf
+  end
+
+  rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+      oletools_conf.servers)
+  return nil
+end
+
+return {
+  type = {N, 'attachment scanner', 'hash', 'scanner'},
+  description = 'oletools office macro scanner',
+  configure = oletools_config,
+  check = oletools_check,
+  name = N
+}

lualib/lua_scanners/savapi.lua

@@ -32,9 +32,10 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"'
 
 local function savapi_config(opts)
   local savapi_conf = {
-    scan_mime_parts = true;
-    scan_text_mime = false;
-    scan_image_mime = false;
+    name = N,
+    scan_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
     default_port = 4444, -- note: You must set ListenAddress in savapi.conf
     product_id = 0,
     log_clean = false,
@@ -46,12 +47,18 @@ local function savapi_config(opts)
     tmpdir = '/tmp',
   }
 
-  for k,v in pairs(opts) do
-    savapi_conf[k] = v
-  end
+  savapi_conf = lua_util.override_defaults(savapi_conf, opts)
 
   if not savapi_conf.prefix then
-    savapi_conf.prefix = 'rs_ap'
+    savapi_conf.prefix = 'rs_' .. savapi_conf.name .. '_'
+  end
+
+  if not savapi_conf.log_prefix then
+    if savapi_conf.name:lower() == savapi_conf.type:lower() then
+      savapi_conf.log_prefix = savapi_conf.name
+    else
+      savapi_conf.log_prefix = savapi_conf.name .. ' (' .. savapi_conf.type .. ')'
+    end
   end
 
   if not savapi_conf['servers'] then
@@ -65,7 +72,7 @@ local function savapi_config(opts)
       savapi_conf.default_port)
 
   if savapi_conf['upstreams'] then
-    lua_util.add_debug_alias('antivirus', N)
+    lua_util.add_debug_alias('antivirus', savapi_conf.name)
     return savapi_conf
   end
 
@@ -112,15 +119,15 @@ local function savapi_check(task, content, digest, rule)
       for virus,_ in pairs(vnames) do
         table.insert(vnames_reordered, virus)
       end
-      lua_util.debugm(N, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered)
+      lua_util.debugm(rule.name, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered)
       if #vnames_reordered > 0 then
         local vname = {}
         for _,virus in ipairs(vnames_reordered) do
           table.insert(vname, virus)
         end
 
-        common.yield_result(task, rule, vname, N)
-        common.save_av_cache(task, digest, rule, vname, N)
+        common.yield_result(task, rule, vname)
+        common.save_av_cache(task, digest, rule, vname)
       end
       if conn then
         conn:close()
@@ -129,15 +136,15 @@ local function savapi_check(task, content, digest, rule)
 
     local function savapi_scan2_cb(err, data, conn)
       local result = tostring(data)
-      lua_util.debugm(N, task, "%s: got reply: %s",
-          rule['type'], result)
+      lua_util.debugm(rule.name, task, "%s: got reply: %s",
+          rule.type, result)
 
       -- Terminal response - clean
       if string.find(result, '200') or string.find(result, '210') then
         if rule['log_clean'] then
           rspamd_logger.infox(task, '%s: message or mime_part is clean', rule['type'])
         end
-        common.save_av_cache(task, digest, rule, 'OK', N)
+        common.save_av_cache(task, digest, rule, 'OK')
         conn:add_write(savapi_fin_cb, 'QUIT\n')
 
         -- Terminal response - infected
@@ -153,6 +160,7 @@ local function savapi_check(task, content, digest, rule)
           if not virus then
             rspamd_logger.errx(task, "%s: virus result unparseable: %s",
                 rule['type'], result)
+            common.yield_result(task, rule, 'virus result unparseable: ' .. result, 0.0, 'fail')
             return
           end
         end
@@ -171,13 +179,14 @@ local function savapi_check(task, content, digest, rule)
     local function savapi_greet2_cb(err, data, conn)
       local result = tostring(data)
       if string.find(result, '100 PRODUCT') then
-        lua_util.debugm(N, task, "%s: scanning file: %s",
+        lua_util.debugm(rule.name, task, "%s: scanning file: %s",
             rule['type'], fname)
         conn:add_write(savapi_scan1_cb, {string.format('SCAN %s\n',
             fname)})
       else
         rspamd_logger.errx(task, '%s: invalid product id %s', rule['type'],
             rule['product_id'])
+        common.yield_result(task, rule, 'invalid product id: ' .. result, 0.0, 'fail')
         conn:add_write(savapi_fin_cb, 'QUIT\n')
       end
     end
@@ -201,7 +210,9 @@ local function savapi_check(task, content, digest, rule)
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+          lua_util.debugm(rule.name, task,
+              '%s [%s]: retry IP: %s', rule['symbol'],
+              rule['type'], addr)
 
           tcp.request({
             task = task,
@@ -213,7 +224,7 @@ local function savapi_check(task, content, digest, rule)
           })
         else
           rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
       else
         upstream:ok()
@@ -236,8 +247,8 @@ local function savapi_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule, N) then
-    if common.check_av_cache(task, digest, rule, savapi_check_uncached, N) then
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, savapi_check_uncached) then
       return
     else
       savapi_check_uncached()
@@ -250,5 +261,5 @@ return {
   description = 'savapi avira antivirus',
   configure = savapi_config,
   check = savapi_check,
-  name = 'savapi'
-}
+  name = N
+}

lualib/lua_scanners/sophos.lua

@@ -31,9 +31,10 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"'
 
 local function sophos_config(opts)
   local sophos_conf = {
-    scan_mime_parts = true;
-    scan_text_mime = false;
-    scan_image_mime = false;
+    name = N,
+    scan_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
     default_port = 4010,
     timeout = 15.0,
     log_clean = false,
@@ -45,12 +46,18 @@ local function sophos_config(opts)
     savdi_report_oversize = false,
   }
 
-  for k,v in pairs(opts) do
-    sophos_conf[k] = v
-  end
+  sophos_conf = lua_util.override_defaults(sophos_conf, opts)
 
   if not sophos_conf.prefix then
-    sophos_conf.prefix = 'rs_sp'
+    sophos_conf.prefix = 'rs_' .. sophos_conf.name .. '_'
+  end
+
+  if not sophos_conf.log_prefix then
+    if sophos_conf.name:lower() == sophos_conf.type:lower() then
+      sophos_conf.log_prefix = sophos_conf.name
+    else
+      sophos_conf.log_prefix = sophos_conf.name .. ' (' .. sophos_conf.type .. ')'
+    end
   end
 
   if not sophos_conf['servers'] then
@@ -64,7 +71,7 @@ local function sophos_config(opts)
       sophos_conf.default_port)
 
   if sophos_conf['upstreams'] then
-    lua_util.add_debug_alias('antivirus', N)
+    lua_util.add_debug_alias('antivirus', sophos_conf.name)
     return sophos_conf
   end
 
@@ -97,7 +104,8 @@ local function sophos_check(task, content, digest, rule)
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+          lua_util.debugm(rule.name, task,
+              '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
 
           tcp.request({
             task = task,
@@ -109,51 +117,42 @@ local function sophos_check(task, content, digest, rule)
           })
         else
           rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
       else
         upstream:ok()
         data = tostring(data)
-        lua_util.debugm(N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data)
+        lua_util.debugm(rule.name, task,
+            '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data)
         local vname = string.match(data, 'VIRUS (%S+) ')
         if vname then
-          common.yield_result(task, rule, vname, N)
-          common.save_av_cache(task, digest, rule, vname, N)
+          common.yield_result(task, rule, vname)
+          common.save_av_cache(task, digest, rule, vname)
         else
           if string.find(data, 'DONE OK') then
             if rule['log_clean'] then
-              rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+              rspamd_logger.infox(task, '%s: message or mime_part is clean', rule.log_prefix)
             else
-              lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+              lua_util.debugm(rule.name, task,
+                  '%s: message or mime_part is clean', rule.log_prefix)
             end
-            common.save_av_cache(task, digest, rule, 'OK', N)
+            common.save_av_cache(task, digest, rule, 'OK')
             -- not finished - continue
           elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then
             conn:add_read(sophos_callback)
-            -- set pseudo virus if configured, else do nothing since it's no fatal
           elseif string.find(data, 'FAIL 0212') then
-            rspamd_logger.infox(task, 'Message is ENCRYPTED (0212 SOPHOS_SAVI_ERROR_FILE_ENCRYPTED): %s', data)
-            if rule['savdi_report_encrypted'] then
-              common.yield_result(task, rule, "SAVDI_FILE_ENCRYPTED", N)
-              common.save_av_cache(task, digest, rule, "SAVDI_FILE_ENCRYPTED", N)
-            end
-            -- set pseudo virus if configured, else set fail since part was not scanned
+            rspamd_logger.warnx(task, 'Message is encrypted (FAIL 0212): %s', data)
+            common.yield_result(task, rule, 'SAVDI: Message is encrypted (FAIL 0212)', 0.0, 'fail')
           elseif string.find(data, 'REJ 4') then
-            if rule['savdi_report_oversize'] then
-              rspamd_logger.infox(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data)
-              common.yield_result(task, rule, "SAVDI_FILE_OVERSIZED", N)
-              common.save_av_cache(task, digest, rule, "SAVDI_FILE_OVERSIZED", N)
-            else
-              rspamd_logger.errx(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data)
-              task:insert_result(rule['symbol_fail'], 0.0, 'Message is OVERSIZED (SSSP reject code 4):' .. data)
-            end
+            rspamd_logger.warnx(task, 'Message is oversized (REJ 4): %s', data)
+            common.yield_result(task, rule, 'SAVDI: Message oversized (REJ 4)', 0.0, 'fail')
             -- excplicitly set REJ1 message when SAVDIreports a protocol error
           elseif string.find(data, 'REJ 1') then
             rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data)
-            task:insert_result(rule['symbol_fail'], 0.0, 'SAVDI (Protocol error (REJ 1)):' .. data)
+            common.yield_result(task, rule, 'SAVDI: Protocol error (REJ 1)', 0.0, 'fail')
           else
             rspamd_logger.errx(task, 'unhandled response: %s', data)
-            task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response')
+            common.yield_result(task, rule, 'unhandled response: ' .. data, 0.0, 'fail')
           end
 
         end
@@ -170,8 +169,8 @@ local function sophos_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule, N) then
-    if common.check_av_cache(task, digest, rule, sophos_check_uncached, N) then
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, sophos_check_uncached) then
       return
     else
       sophos_check_uncached()
@@ -184,5 +183,5 @@ return {
   description = 'sophos antivirus',
   configure = sophos_config,
   check = sophos_check,
-  name = 'sophos'
-}
+  name = N
+}

lualib/lua_scanners/spamassassin.lua

@@ -0,0 +1,217 @@
+--[[
+Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module spamassassin
+-- This module contains spamd access functions.
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'spamassassin'
+
+local function spamassassin_check(task, content, digest, rule)
+  local function spamassassin_check_uncached ()
+    local upstream = rule.upstreams:get_upstream_round_robin()
+    local addr = upstream:get_addr()
+    local retransmits = rule.retransmits
+
+    -- Build the spamd query
+    -- https://svn.apache.org/repos/asf/spamassassin/trunk/spamd/PROTOCOL
+    local request_data = {
+      "HEADERS SPAMC/1.5\r\n",
+      "User: root\r\n",
+      "Content-length: ".. #content .. "\r\n",
+      "\r\n",
+      content,
+    }
+
+    local function spamassassin_callback(err, data, conn)
+
+      local function spamassassin_requery(error)
+        -- set current upstream to fail because an error occurred
+        upstream:fail()
+
+        -- retry with another upstream until retransmits exceeds
+        if retransmits > 0 then
+
+          retransmits = retransmits - 1
+
+          lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s',
+            rule.log_prefix, error, retransmits)
+
+          -- Select a different upstream!
+          upstream = rule.upstreams:get_upstream_round_robin()
+          addr = upstream:get_addr()
+
+          lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s',
+            rule.log_prefix, addr, addr:get_port())
+
+          tcp.request({
+            task = task,
+            host = addr:to_string(),
+            port = addr:get_port(),
+            timeout = rule['timeout'],
+            data = request_data,
+            callback = spamassassin_callback,
+          })
+        else
+          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+            'exceed - err: %s', rule.log_prefix, error)
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed: ' .. error, 0.0, 'fail')
+        end
+      end
+
+      if err then
+
+        spamassassin_requery(err)
+
+      else
+        -- Parse the response
+        if upstream then upstream:ok() end
+
+        --lua_util.debugm(rule.N, task, '%s: returned result: %s', rule.log_prefix, data)
+
+        --[[
+        patterns tested against Spamassassin 3.4.2
+
+        Spam: False ; 1.1 / 5.0
+
+        X-Spam-Status: No, score=1.1 required=5.0 tests=HTML_MESSAGE,MIME_HTML_ONLY,
+          TVD_RCVD_SPACE_BRACKET,UNPARSEABLE_RELAY autolearn=no
+          autolearn_force=no version=3.4.2
+        ]] --
+        local header = string.gsub(tostring(data), "[\r\n]+[\t ]", " ")
+        --lua_util.debugm(rule.N, task, '%s: returned header: %s', rule.log_prefix, header)
+
+        local symbols
+        local spam_score
+        for s in header:gmatch("[^\r\n]+") do
+            if string.find(s, 'Spam: .* / 5.0') then
+              local pattern_symbols = "(Spam:.*; )(%-?%d?%d%.%d)( / 5%.0)"
+              spam_score = string.gsub(s, pattern_symbols, "%2")
+              lua_util.debugm(rule.N, task, '%s: spamd Spam line: %s', rule.log_prefix, spam_score)
+            end
+            if string.find(s, 'X%-Spam%-Status') then
+              local pattern_symbols = "(.*X%-Spam%-Status.*tests%=)(.*)(autolearn%=.*version%=%d%.%d%.%d.*)"
+              symbols = string.gsub(s, pattern_symbols, "%2")
+              symbols = string.gsub(symbols, "%s", "")
+            end
+        end
+
+        if tonumber(spam_score) > 0 and #symbols > 0 and symbols ~= "none" then
+
+          if rule.extended == false then
+            common.yield_result(task, rule, symbols, spam_score)
+            common.save_av_cache(task, digest, rule, symbols, spam_score)
+          else
+            local symbols_table = {}
+            symbols_table = rspamd_str_split(symbols, ",")
+            lua_util.debugm(rule.N, task, '%s: returned symbols as table: %s', rule.log_prefix, symbols_table)
+
+            common.yield_result(task, rule, symbols_table, spam_score)
+            common.save_av_cache(task, digest, rule, symbols_table, spam_score)
+          end
+        else
+          common.log_clean(task, rule, 'no spam detected - spam score: ' .. spam_score .. ', symbols: ' .. symbols)
+        end
+      end
+    end
+
+    tcp.request({
+      task = task,
+      host = addr:to_string(),
+      port = addr:get_port(),
+      timeout = rule['timeout'],
+      data = request_data,
+      callback = spamassassin_callback,
+    })
+  end
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, spamassassin_check_uncached) then
+      return
+    else
+      spamassassin_check_uncached()
+    end
+  end
+end
+
+local function spamassassin_config(opts)
+
+  local spamassassin_conf = {
+    N = N,
+    scan_mime_parts = false,
+    scan_text_mime = false,
+    scan_image_mime = false,
+    default_port = 783,
+    timeout = 15.0,
+    log_clean = false,
+    retransmits = 2,
+    cache_expire = 3600, -- expire redis in one hour
+    symbol = "SPAMD",
+    message = '${SCANNER}: Spamassassin bulk message found: "${VIRUS}"',
+    detection_category = "spam",
+    default_score = 1,
+    action = false,
+    extended = false,
+  }
+
+  spamassassin_conf = lua_util.override_defaults(spamassassin_conf, opts)
+
+  if not spamassassin_conf.prefix then
+    spamassassin_conf.prefix = 'rs_' .. spamassassin_conf.name .. '_'
+  end
+
+  if not spamassassin_conf.log_prefix then
+    if spamassassin_conf.name:lower() == spamassassin_conf.type:lower() then
+      spamassassin_conf.log_prefix = spamassassin_conf.name
+    else
+      spamassassin_conf.log_prefix = spamassassin_conf.name .. ' (' .. spamassassin_conf.type .. ')'
+    end
+  end
+
+  if not spamassassin_conf.servers then
+    rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+    return nil
+  end
+
+  spamassassin_conf.upstreams = upstream_list.create(rspamd_config,
+    spamassassin_conf.servers,
+    spamassassin_conf.default_port)
+
+  if spamassassin_conf.upstreams then
+    lua_util.add_debug_alias('external_services', spamassassin_conf.N)
+    return spamassassin_conf
+  end
+
+  rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+    spamassassin_conf.servers)
+  return nil
+end
+
+return {
+  type = {N,'spam', 'scanner'},
+  description = 'spamassassin spam scanner',
+  configure = spamassassin_config,
+  check = spamassassin_check,
+  name = N
+}

lualib/lua_scanners/vadesecure.lua

@@ -0,0 +1,318 @@
+--[[
+Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module vadesecure
+-- This module contains Vadesecure Filterd interface
+--]]
+
+local lua_util = require "lua_util"
+local http = require "rspamd_http"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+
+local N = 'vadesecure'
+
+local function vade_check(task, content, digest, rule)
+  local function vade_url(addr)
+    local url
+    if rule.use_https then
+      url = string.format('https://%s:%d%s', tostring(addr),
+          rule.default_port, rule.url)
+    else
+      url = string.format('http://%s:%d%s', tostring(addr),
+          rule.default_port, rule.url)
+    end
+
+    return url
+  end
+
+  local upstream = rule.upstreams:get_upstream_round_robin()
+  local addr = upstream:get_addr()
+  local retransmits = rule.retransmits
+
+  local url = vade_url(addr)
+  local hdrs = {}
+
+  local helo = task:get_helo()
+  if helo then
+    hdrs['X-Helo'] = helo
+  end
+  local mail_from = task:get_from('smtp') or {}
+  if mail_from[1] and #mail_from[1].addr > 1 then
+    hdrs['X-Mailfrom'] = mail_from[1].addr
+  end
+
+  local rcpt_to = task:get_recipients('smtp')
+  if rcpt_to then
+    hdrs['X-Rcptto'] = {}
+    for _, r in ipairs(rcpt_to) do
+      table.insert(hdrs['X-Rcptto'], r.addr)
+    end
+  end
+
+  local fip = task:get_from_ip()
+  if fip and fip:is_valid() then
+    hdrs['X-Inet'] = tostring(fip)
+  end
+
+  local request_data = {
+    task = task,
+    url = url,
+    body = task:get_content(),
+    headers = hdrs,
+    timeout = rule.timeout,
+  }
+
+  local function vade_callback(http_err, code, body, headers)
+
+    local function vade_requery()
+      -- set current upstream to fail because an error occurred
+      upstream:fail()
+
+      -- retry with another upstream until retransmits exceeds
+      if retransmits > 0 then
+
+        retransmits = retransmits - 1
+
+        lua_util.debugm(rule.name, task,
+            '%s: Request Error: %s - retries left: %s',
+            rule.log_prefix, http_err, retransmits)
+
+        -- Select a different upstream!
+        upstream = rule.upstreams:get_upstream_round_robin()
+        addr = upstream:get_addr()
+        url = vade_url(addr)
+
+        lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
+            rule.log_prefix, addr, addr:get_port())
+        request_data.url = url
+
+        http.request(request_data)
+      else
+        rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+            'exceed', rule.log_prefix)
+        task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '..
+            'retransmits exceed')
+      end
+    end
+
+    if http_err then
+      vade_requery()
+    else
+      -- Parse the response
+      if upstream then upstream:ok() end
+      if code ~= 200 then
+        rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
+        task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
+        return
+      end
+      local parser = ucl.parser()
+      local ret, err = parser:parse_string(body)
+      if not ret then
+        rspamd_logger.errx(task, 'vade: bad response body (raw): %s', body)
+        task:insert_result(rule.symbol_fail, 1.0, 'Parser error: ' .. err)
+        return
+      end
+      local obj = parser:get_object()
+      local verdict = obj.verdict
+      if not verdict then
+        rspamd_logger.errx(task, 'vade: bad response JSON (no verdict): %s', obj)
+        task:insert_result(rule.symbol_fail, 1.0, 'No verdict/unknown verdict')
+        return
+      end
+      local vparts = lua_util.rspamd_str_split(verdict, ":")
+      verdict = table.remove(vparts, 1) or verdict
+
+      local sym = rule.symbols[verdict]
+      if not sym then
+        sym = rule.symbols.other
+      end
+
+      if not sym.symbol then
+        -- Subcategory match
+        local lvl = 'low'
+        if vparts and vparts[1] then
+          lvl = vparts[1]
+        end
+
+        if sym[lvl] then
+          sym = sym[lvl]
+        else
+          sym = rule.symbols.other
+        end
+      end
+
+      local opts = {}
+      if obj.score then
+        table.insert(opts, 'score=' .. obj.score)
+      end
+      if obj.elapsed then
+        table.insert(opts, 'elapsed=' .. obj.elapsed)
+      end
+
+      if rule.log_spamcause and obj.spamcause then
+        rspamd_logger.infox(task, 'vadesecure returned verdict="%s", score=%s, spamcause="%s"',
+            verdict, obj.score, obj.spamcause)
+      else
+        lua_util.debugm(rule.name, task, 'vadesecure returned verdict="%s", score=%s, spamcause="%s"',
+            verdict, obj.score, obj.spamcause)
+      end
+
+      if #vparts > 0 then
+        table.insert(opts, 'verdict=' .. verdict .. ';' .. table.concat(vparts, ':'))
+      end
+
+      task:insert_result(sym.symbol, 1.0, opts)
+    end
+  end
+
+  request_data.callback = vade_callback
+  http.request(request_data)
+end
+
+
+local function vade_config(opts)
+
+  local vade_conf = {
+    name = N,
+    default_port = 23808,
+    url = '/api/v1/scan',
+    use_https = false,
+    timeout = 5.0,
+    log_clean = false,
+    retransmits = 1,
+    cache_expire = 7200, -- expire redis in 2h
+    message = '${SCANNER}: spam message found: "${VIRUS}"',
+    detection_category = "hash",
+    default_score = 1,
+    action = false,
+    log_spamcause = true,
+    symbol_fail = 'VADE_FAIL',
+    symbol = 'VADE_CHECK',
+    symbols = {
+      clean = {
+        symbol = 'VADE_CLEAN',
+        score = -0.5,
+        description = 'VadeSecure decided message to be clean'
+      },
+      spam = {
+        high = {
+          symbol = 'VADE_SPAM_HIGH',
+          score = 8.0,
+          description = 'VadeSecure decided message to be clearly spam'
+        },
+        medium = {
+          symbol = 'VADE_SPAM_MEDIUM',
+          score = 5.0,
+          description = 'VadeSecure decided message to be highly likely spam'
+        },
+        low = {
+          symbol = 'VADE_SPAM_LOW',
+          score = 2.0,
+          description = 'VadeSecure decided message to be likely spam'
+        },
+      },
+      malware = {
+        symbol = 'VADE_MALWARE',
+        score = 8.0,
+        description = 'VadeSecure decided message to be malware'
+      },
+      scam = {
+        symbol = 'VADE_SCAM',
+        score = 7.0,
+        description = 'VadeSecure decided message to be scam'
+      },
+      phishing = {
+        symbol = 'VADE_PHISHING',
+        score = 8.0,
+        description = 'VadeSecure decided message to be phishing'
+      },
+      commercial =  {
+        symbol = 'VADE_COMMERCIAL',
+        score = 0.0,
+        description = 'VadeSecure decided message to be commercial message'
+      },
+      community =  {
+        symbol = 'VADE_COMMUNITY',
+        score = 0.0,
+        description = 'VadeSecure decided message to be community message'
+      },
+      transactional =  {
+        symbol = 'VADE_TRANSACTIONAL',
+        score = 0.0,
+        description = 'VadeSecure decided message to be transactional message'
+      },
+      suspect = {
+        symbol = 'VADE_SUSPECT',
+        score = 3.0,
+        description = 'VadeSecure decided message to be suspicious message'
+      },
+      bounce = {
+        symbol = 'VADE_BOUNCE',
+        score = 0.0,
+        description = 'VadeSecure decided message to be bounce message'
+      },
+      other = 'VADE_OTHER',
+    }
+  }
+
+  vade_conf = lua_util.override_defaults(vade_conf, opts)
+
+  if not vade_conf.prefix then
+    vade_conf.prefix = 'rs_' .. vade_conf.name .. '_'
+  end
+
+  if not vade_conf.log_prefix then
+    if vade_conf.name:lower() == vade_conf.type:lower() then
+      vade_conf.log_prefix = vade_conf.name
+    else
+      vade_conf.log_prefix = vade_conf.name .. ' (' .. vade_conf.type .. ')'
+    end
+  end
+
+  if not vade_conf.servers and vade_conf.socket then
+    vade_conf.servers = vade_conf.socket
+  end
+
+  if not vade_conf.servers then
+    rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+    return nil
+  end
+
+  vade_conf.upstreams = upstream_list.create(rspamd_config,
+      vade_conf.servers,
+      vade_conf.default_port)
+
+  if vade_conf.upstreams then
+    lua_util.add_debug_alias('external_services', vade_conf.name)
+    return vade_conf
+  end
+
+  rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+      vade_conf['servers'])
+  return nil
+end
+
+return {
+  type = {'vadesecure', 'scanner'},
+  description = 'VadeSecure Filterd interface',
+  configure = vade_config,
+  check = vade_check,
+  name = N
+}

lualib/lua_squeeze_rules.lua

@@ -322,12 +322,13 @@ exports.handle_settings = function(task, settings)
   local symbols_disabled = {}
   local symbols_enabled = {}
   local found = false
+  local disabled = false
 
   if settings.default then settings = settings.default end
 
   local function disable_all()
-    for k,_ in pairs(squeezed_symbols) do
-      if not symbols_enabled[k] then
+    for k,sym in pairs(squeezed_symbols) do
+      if not symbols_enabled[k] and not (sym.flags and sym.flags.explicit_disable) then
         symbols_disabled[k] = true
       end
     end
@@ -336,6 +337,7 @@ exports.handle_settings = function(task, settings)
   if settings.symbols_enabled then
     disable_all()
     found = true
+    disabled = true
     for _,s in ipairs(settings.symbols_enabled) do
       if squeezed_symbols[s] then
         lua_util.debugm(SN, task, 'enable symbol %s as it is in `symbols_enabled`', s)
@@ -346,7 +348,9 @@ exports.handle_settings = function(task, settings)
   end
 
   if settings.groups_enabled then
-    disable_all()
+    if not disabled then
+      disable_all()
+    end
     found = true
     for _,gr in ipairs(settings.groups_enabled) do
       if squeezed_groups[gr] then

lualib/lua_util.lua

@@ -593,7 +593,8 @@ exports.extract_specific_urls = function(params_or_task, lim, need_emails, filte
   if params.prefix then
     cache_key = params.prefix
   else
-    cache_key = string.format('sp_urls_%d%s', params.limit, params.need_emails)
+    cache_key = string.format('sp_urls_%d%s', params.limit,
+        tostring(params.need_emails))
   end
 
 
@@ -875,5 +876,35 @@ exports.get_task_verdict = function(task)
   return 'uncertain'
 end
 
+---[[[
+-- @function lua_util.maybe_obfuscate_subject(subject, settings)
+-- Obfuscate subject if enabled in settings. Also checks utf8 validity.
+-- Supported settings:
+-- * subject_privacy = false - subject privacy is off
+-- * subject_privacy_alg = 'blake2' - default hash-algorithm to obfuscate subject
+-- * subject_privacy_prefix = 'obf' - prefix to show it's obfuscated
+-- * subject_privacy_length = 16 - cut the length of the hash
+-- @return obfuscated or validated subject
+--]]
+
+exports.maybe_obfuscate_subject = function(subject, settings)
+  local hash = require 'rspamd_cryptobox_hash'
+  if subject and not rspamd_util.is_valid_utf8(subject) then
+    subject = '???'
+  elseif settings.subject_privacy then
+    local hash_alg = settings.subject_privacy_alg or 'blake2'
+    local subject_hash = hash.create_specific(hash_alg, subject)
+
+    if settings.subject_privacy_length then
+      subject = (settings.subject_privacy_prefix or 'obf') .. ':' ..
+          subject_hash:hex():sub(1, settings.subject_privacy_length)
+    else
+      subject = (settings.subject_privacy_prefix or '') .. ':' ..
+          subject_hash:hex()
+    end
+  end
+
+  return subject
+end
 
 return exports

lualib/rspamadm/mime.lua

@@ -16,7 +16,7 @@ limitations under the License.
 
 local argparse = require "argparse"
 local ansicolors = require "ansicolors"
---local rspamd_util = require "rspamd_util"
+local rspamd_util = require "rspamd_util"
 local rspamd_task = require "rspamd_task"
 local rspamd_logger = require "rspamd_logger"
 local lua_meta = require "lua_meta"
@@ -153,6 +153,42 @@ modify:option "-H --html-footer"
       :description "Adds footer to text/html parts from a specific file"
       :argname "<file>"
 
+local sign = parser:command "sign"
+                     :description "Performs DKIM signing"
+sign:argument "file"
+      :description "File to process"
+      :argname "<file>"
+      :args "+"
+
+sign:option "-d --domain"
+    :description "Use specific domain"
+    :argname "<domain>"
+    :count "1"
+sign:option "-s --selector"
+    :description "Use specific selector"
+    :argname "<selector>"
+    :count "1"
+sign:option "-k --key"
+    :description "Use specific key of file"
+    :argname "<key>"
+    :count "1"
+sign:option "-t type"
+    :description "ARC or DKIM signing"
+    :argname("<arc|dkim>")
+    :convert {
+      ['arc'] = 'arc',
+      ['dkim'] = 'dkim',
+    }
+    :default 'dkim'
+sign:option "-o --output"
+    :description "Output format"
+    :argname("<message|signature>")
+    :convert {
+      ['message'] = 'message',
+      ['signature'] = 'signature',
+    }
+    :default 'message'
+
 local function load_config(opts)
   local _r,err = rspamd_config:load_ucl(opts['config'])
 
@@ -170,7 +206,7 @@ end
 
 local function load_task(opts, fname)
   if not fname then
-    parser:error('no file specified')
+    fname = '-'
   end
 
   local res,task = rspamd_task.load_from_file(fname, rspamd_config)
@@ -583,22 +619,21 @@ local function urls_handler(opts)
   print_elts(out_elts, opts)
 end
 
-local function modify_handler(opts)
-  local rspamd_util = require "rspamd_util"
-  load_config(opts)
-  rspamd_url.init(rspamd_config:get_tld_path())
+local function newline(task)
+  local t = task:get_newlines_type()
 
-  local function newline(task)
-    local t = task:get_newlines_type()
+  if t == 'cr' then
+    return '\r'
+  elseif t == 'lf' then
+    return '\n'
+  end
 
-    if t == 'cr' then
-      return '\r'
-    elseif t == 'lf' then
-      return '\n'
-    end
+  return '\r\n'
+end
 
-    return '\r\n'
-  end
+local function modify_handler(opts)
+  load_config(opts)
+  rspamd_url.init(rspamd_config:get_tld_path())
 
   local function read_file(file)
     local f = assert(io.open(file, "rb"))
@@ -607,7 +642,7 @@ local function modify_handler(opts)
     return content
   end
 
-  local function do_append_footer(task, part, footer, is_multipart)
+  local function do_append_footer(task, part, footer, is_multipart, out)
     local newline_s = newline(task)
     local tp = part:get_text()
     local ct = 'text/plain'
@@ -622,10 +657,10 @@ local function modify_handler(opts)
     end
 
     if is_multipart then
-      io.write(string.format('Content-Type: %s; charset=utf-8%s'..
-          'Content-Transfer-Encoding: %s%s%s',
-          ct, newline_s, cte, newline_s, newline_s))
-      io.flush()
+      out[#out + 1] = string.format('Content-Type: %s; charset=utf-8%s'..
+          'Content-Transfer-Encoding: %s',
+          ct, newline_s, cte)
+      out[#out + 1] = ''
     end
 
     local content = tostring(tp:get_content('raw_utf') or '')
@@ -636,17 +671,16 @@ local function modify_handler(opts)
       content = string.format('%s%s',
           content:sub(-(#newline_s), #newline_s + 1), -- content without last newline
           footer)
-      rspamd_util.encode_qp(content,
-          80, task:get_newlines_type()):save_in_file(1)
-      io.write(double_nline)
+      out[#out + 1] = {rspamd_util.encode_qp(content,
+          80, task:get_newlines_type()), true}
+      out[#out + 1] = ''
     else
       content = content .. footer
-      rspamd_util.encode_qp(content,
-          80, task:get_newlines_type()):save_in_file(1)
-      io.write(double_nline)
+      out[#out + 1] = {rspamd_util.encode_qp(content,
+          80, task:get_newlines_type()), true}
+      out[#out + 1] = ''
     end
 
-
   end
 
   local text_footer, html_footer
@@ -665,6 +699,7 @@ local function modify_handler(opts)
     local need_rewrite_ct = false
     local parsed_ct
     local seen_cte = false
+    local out = {}
 
     local function process_headers_cb(name, hdr)
 
@@ -678,31 +713,31 @@ local function modify_handler(opts)
         local hname,hpattern = h:match('^([^=]+)=(.+)$')
         if hname == name then
           local new_value = string.format(hpattern, hdr.decoded)
-          new_value = string.format('%s:%s%s%s',
+          new_value = string.format('%s:%s%s',
               name, hdr.separator,
               rspamd_util.fold_header(name,
                   rspamd_util.mime_header_encode(new_value),
-                  task:get_newlines_type()), newline_s)
-          io.write(new_value)
+                  task:get_newlines_type()))
+          out[#out + 1] = new_value
           return
         end
       end
 
       if need_rewrite_ct then
         if name:lower() == 'content-type' then
-          local nct = string.format('%s: %s/%s; charset=utf-8%s',
-              'Content-Type', parsed_ct.type, parsed_ct.subtype, newline_s)
-          io.write(nct)
+          local nct = string.format('%s: %s/%s; charset=utf-8',
+              'Content-Type', parsed_ct.type, parsed_ct.subtype)
+          out[#out + 1] = nct
           return
         elseif name:lower() == 'content-transfer-encoding' then
           seen_cte = true
-          io.write(string.format('%s: %s%s',
-              'Content-Transfer-Encoding', 'quoted-printable', newline_s))
+          out[#out + 1] = string.format('%s: %s',
+              'Content-Transfer-Encoding', 'quoted-printable')
           return
         end
       end
 
-      io.write(hdr.raw)
+      out[#out + 1] = hdr.raw:gsub('\r?\n?$', '')
     end
 
     if html_footer or text_footer then
@@ -755,19 +790,19 @@ local function modify_handler(opts)
       local hname,hvalue = h:match('^([^=]+)=(.+)$')
 
       if hname and hvalue then
-        io.write(string.format('%s: %s%s', hname,
-            rspamd_util.fold_header(hname, hvalue, task:get_newlines_type()),
-            newline_s))
+        out[#out + 1] = string.format('%s: %s', hname,
+            rspamd_util.fold_header(hname, hvalue, task:get_newlines_type()))
       end
     end
 
     if not seen_cte and need_rewrite_ct then
-      io.write(string.format('%s: %s%s',
-          'Content-Transfer-Encoding', 'quoted-printable', newline_s))
+      out[#out + 1] = string.format('%s: %s',
+          'Content-Transfer-Encoding', 'quoted-printable')
     end
 
     -- End of headers
-    io.write(newline_s)
+    --local eoh_pos = #out
+    out[#out + 1] = ''
 
     local boundaries = {}
     local cur_boundary
@@ -776,36 +811,31 @@ local function modify_handler(opts)
       local boundary = part:get_boundary()
       if part:is_multipart() then
         if cur_boundary then
-          io.write(string.format('--%s%s',
-              boundaries[#boundaries], newline_s))
+          out[#out + 1] = string.format('--%s',
+              boundaries[#boundaries])
         end
 
         boundaries[#boundaries + 1] = boundary or '--XXX'
         cur_boundary = boundary
-        io.flush ()
 
         local rh = part:get_raw_headers()
         if #rh > 0 then
-          rh:save_in_file(1)
-          io.write(newline_s)
-          io.flush()
+          out[#out + 1] = {rh, true}
         end
       elseif part:is_message() then
         if boundary then
           if cur_boundary and boundary ~= cur_boundary then
             -- Need to close boundary
-            io.write(string.format('--%s--%s%s',
-                boundaries[#boundaries], newline_s, newline_s))
+            out[#out + 1] = string.format('--%s--%s',
+                boundaries[#boundaries], newline_s)
             table.remove(boundaries)
             cur_boundary = nil
           end
-          io.write(string.format('--%s%s',
-              boundary, newline_s))
+          out[#out + 1] = string.format('--%s',
+              boundary)
         end
 
-        io.flush()
-        part:get_raw_headers():save_in_file(1)
-        io.write(newline_s)
+        out[#out + 1] = {part:get_raw_headers(), true}
       else
         local append_footer = false
         local skip_footer = part:is_attachment()
@@ -838,25 +868,23 @@ local function modify_handler(opts)
         if boundary then
           if cur_boundary and boundary ~= cur_boundary then
             -- Need to close boundary
-            io.write(string.format('--%s--%s%s',
-                boundaries[#boundaries], newline_s, newline_s))
+            out[#out + 1] = string.format('--%s--%s',
+                boundaries[#boundaries], newline_s)
             table.remove(boundaries)
             cur_boundary = boundary
           end
-          io.write(string.format('--%s%s',
-              boundary, newline_s))
+          out[#out + 1] = string.format('--%s',
+              boundary)
         end
 
         io.flush()
 
         if append_footer and not skip_footer then
           do_append_footer(task, part, append_footer,
-              parent and parent:is_multipart())
+              parent and parent:is_multipart(), out)
         else
-          part:get_raw_headers():save_in_file(1)
-          io.write(newline_s)
-          io.flush()
-          part:get_raw_content():save_in_file(1)
+          out[#out + 1] = {part:get_raw_headers(), true}
+          out[#out + 1] = {part:get_raw_content(), false}
         end
       end
     end
@@ -864,13 +892,81 @@ local function modify_handler(opts)
     -- Close remaining
     local b = table.remove(boundaries)
     while b do
-      io.write(string.format('--%s--%s', b, newline_s))
+      out[#out + 1] = string.format('--%s--', b)
       if #boundaries > 0 then
-        io.write(newline_s)
+        out[#out + 1] = ''
       end
       b = table.remove(boundaries)
     end
 
+    for _,o in ipairs(out) do
+      if type(o) == 'string' then
+        io.write(o)
+        io.write(newline_s)
+      else
+        io.flush()
+        o[1]:save_in_file(1)
+
+        if o[2] then
+          io.write(newline_s)
+        end
+      end
+    end
+
+    task:destroy() -- No automatic dtor
+  end
+end
+
+local function sign_handler(opts)
+  load_config(opts)
+  rspamd_url.init(rspamd_config:get_tld_path())
+
+  local lua_dkim = require("lua_ffi").dkim
+
+  local sign_key
+  if rspamd_util.file_exists(opts.key) then
+    sign_key = lua_dkim.load_sign_key(opts.key, 'file')
+  else
+    sign_key = lua_dkim.load_sign_key(opts.key, 'base64')
+  end
+
+  if not sign_key then
+    io.stderr:write('Cannot load key: ' .. opts.key .. '\n')
+    os.exit(1)
+  end
+
+  for _,fname in ipairs(opts.file) do
+    local task = load_task(opts, fname)
+    local ctx = lua_dkim.create_sign_context(task, sign_key, nil, opts.algorithm)
+
+    if not ctx then
+      io.stderr:write('Cannot init signing\n')
+      os.exit(1)
+    end
+
+    local sig = lua_dkim.do_sign(task, ctx, opts.selector, opts.domain)
+
+    if not sig then
+      io.stderr:write('Cannot create signature\n')
+      os.exit(1)
+    end
+
+    if opts.output == 'signature' then
+      io.write(sig)
+      io.write('\n')
+      io.flush()
+    else
+      local dkim_hdr = string.format('%s: %s%s',
+          'DKIM-Signature',
+          rspamd_util.fold_header('DKIM-Signature',
+              rspamd_util.mime_header_encode(sig),
+              task:get_newlines_type()),
+          newline(task))
+      io.write(dkim_hdr)
+      io.flush()
+      task:get_content():save_in_file(1)
+    end
+
     task:destroy() -- No automatic dtor
   end
 end
@@ -894,6 +990,8 @@ local function handler(args)
     urls_handler(opts)
   elseif command == 'modify' then
     modify_handler(opts)
+  elseif command == 'sign' then
+    sign_handler(opts)
   else
     parser:error('command %s is not implemented', command)
   end

rules/misc.lua

@@ -101,25 +101,41 @@ rspamd_config.DATE_IN_PAST = {
   type = 'mime',
 }
 
-rspamd_config.R_SUSPICIOUS_URL = {
+local obscured_id = rspamd_config:register_symbol{
   callback = function(task)
     local urls = task:get_urls()
 
     if urls then
       for _,u in ipairs(urls) do
-        if u:is_obscured() then
+        local fl = u:get_flags()
+        if fl.obscured then
           task:insert_result('R_SUSPICIOUS_URL', 1.0, u:get_host())
         end
+        if fl.zw_spaces then
+          task:insert_result('ZERO_WIDTH_SPACE_URL', 1.0, u:get_host())
+        end
       end
     end
+
     return false
   end,
+  name = 'R_SUSPICIOUS_URL',
   score = 5.0,
   one_shot = true,
   description = 'Obfusicated or suspicious URL has been found in a message',
   group = 'url'
 }
 
+rspamd_config:register_symbol{
+  type = 'virtual',
+  name = 'ZERO_WIDTH_SPACE_URL',
+  score = 7.0,
+  one_shot = true,
+  description = 'Zero width space in url',
+  group = 'url',
+  parent = obscured_id,
+}
+
 
 rspamd_config.ENVFROM_PRVS = {
   callback = function (task)
@@ -198,7 +214,7 @@ local check_rcvd = rspamd_config:register_symbol{
   group = 'headers',
   callback = function (task)
     local rcvds = task:get_received_headers()
-    if not rcvds then return false end
+    if not rcvds or #rcvds == 0 then return false end
 
     local all_tls = fun.all(function(rc)
       return rc.flags and rc.flags['ssl']
@@ -357,26 +373,31 @@ rspamd_config.OMOGRAPH_URL = {
       local bad_omographs = 0
       local single_bad_omograps = 0
       local bad_urls = {}
+      local seen = {}
 
       fun.each(function(u)
         if u:is_phished() then
+
           local h1 = u:get_host()
           local h2 = u:get_phished():get_host()
           if h1 and h2 then
-            if util.is_utf_spoofed(h1, h2) then
-              table.insert(bad_urls, string.format('%s->%s', h1, h2))
+            local selt = string.format('%s->%s', h1, h2)
+            if not seen[selt] and util.is_utf_spoofed(h1, h2) then
+              bad_urls[#bad_urls + 1] = selt
               bad_omographs = bad_omographs + 1
             end
+            seen[selt] = true
           end
         end
         if not u:is_html_displayed() then
           local h = u:get_tld()
 
           if h then
-            if util.is_utf_spoofed(h) then
-              table.insert(bad_urls, string.format('%s', h))
+            if not seen[h] and util.is_utf_spoofed(h) then
+              bad_urls[#bad_urls + 1] = h
               single_bad_omograps = single_bad_omograps + 1
             end
+            seen[h] = true
           end
         end
       end, urls)

rules/regexp/misc.lua

@@ -61,15 +61,36 @@ reconf['HAS_ONION_URI'] = {
     group = 'experimental'
 }
 
+local my_victim = [[/(?:victim|prey)/{words}]]
+local your_webcam = [[/webcam/{words}]]
+local your_onan = [[/(?:mast[ur]{2}bati(?:on|ng)|onanism|solitary)/{words}]]
 local password_in_words = [[/^pass(?:(?:word)|(?:phrase))/i{words}]]
 local btc_wallet_address = [[/^[13][0-9a-zA-Z]{25,34}$/{words}]]
-local wallet_word = [[/^wallet$/i{words}]]
+local wallet_word = [[/^wallet$/{words}]]
 local broken_unicode = [[has_flag(bad_unicode)]]
 
 reconf['LEAKED_PASSWORD_SCAM'] = {
-  re = string.format('%s & (%s | %s | %s)',
-      btc_wallet_address, password_in_words, wallet_word, broken_unicode),
+  re = string.format('%s & (%s | %s | %s | %s | %s | %s | lua:check_data_images)',
+      btc_wallet_address, password_in_words, wallet_word,
+      my_victim, your_webcam, your_onan, broken_unicode),
   description = 'Contains password word and BTC wallet address',
+  functions = {
+    check_data_images = function(task)
+      local tp = task:get_text_parts() or {}
+
+      for _,p in ipairs(tp) do
+        if p:is_html() then
+          local hc = p:get_html()
+
+          if hc and hc:has_property('data_urls') then
+            return true
+          end
+        end
+      end
+
+      return false
+    end
+  },
   score = 7.0,
   group = 'scams'
 }

src/CMakeLists.txt

@@ -117,19 +117,13 @@ LIST(LENGTH PLUGINSSRC RSPAMD_MODULES_NUM)
 SET(RAGEL_DEPENDS "${CMAKE_SOURCE_DIR}/src/ragel/smtp_address.rl"
 	"${CMAKE_SOURCE_DIR}/src/ragel/smtp_date.rl"
 	"${CMAKE_SOURCE_DIR}/src/ragel/smtp_ip.rl"
-	"${CMAKE_SOURCE_DIR}/src/ragel/smtp_whitespace.rl"
-	"${CMAKE_SOURCE_DIR}/src/ragel/smtp_received.rl"
+	"${CMAKE_SOURCE_DIR}/src/ragel/smtp_base.rl"
 	"${CMAKE_SOURCE_DIR}/src/ragel/content_disposition.rl")
 RAGEL_TARGET(ragel_smtp_addr
 	INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_addr_parser.rl
 	DEPENDS ${RAGEL_DEPENDS}
 	COMPILE_FLAGS -T1
 	OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/smtp_addr_parser.rl.c)
-RAGEL_TARGET(ragel_smtp_received
-	INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_received_parser.rl
-	DEPENDS ${RAGEL_DEPENDS}
-	COMPILE_FLAGS -T1
-	OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/smtp_received_parser.rl.c)
 RAGEL_TARGET(ragel_content_disposition
 	INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/content_disposition_parser.rl
 	DEPENDS ${RAGEL_DEPENDS}
@@ -145,24 +139,49 @@ RAGEL_TARGET(ragel_smtp_date
 	DEPENDS ${RAGEL_DEPENDS}
 	COMPILE_FLAGS -G2
 	OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/date_parser.rl.c)
+RAGEL_TARGET(ragel_smtp_ip
+	INPUTS ${CMAKE_SOURCE_DIR}/src/ragel/smtp_ip_parser.rl
+	DEPENDS ${RAGEL_DEPENDS}
+	COMPILE_FLAGS -G2
+	OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ip_parser.rl.c)
 ######################### LINK SECTION ###############################
 
-ADD_LIBRARY(rspamd-server STATIC
-		${RSPAMD_CRYPTOBOX}
-		${RSPAMD_UTIL}
-		${RSPAMD_LUA}
-		${RSPAMD_SERVER}
-		${RSPAMD_STAT}
-		${RSPAMD_MIME}
-		${CMAKE_CURRENT_BINARY_DIR}/modules.c
-		${PLUGINSSRC}
-		"${RAGEL_ragel_smtp_addr_OUTPUTS}"
-		"${RAGEL_ragel_smtp_received_OUTPUTS}"
-		"${RAGEL_ragel_newlines_strip_OUTPUTS}"
-		"${RAGEL_ragel_content_type_OUTPUTS}"
-		"${RAGEL_ragel_content_disposition_OUTPUTS}"
-		"${RAGEL_ragel_rfc2047_OUTPUTS}"
-		"${RAGEL_ragel_smtp_date_OUTPUTS}")
+IF(ENABLE_STATIC MATCHES "ON")
+	ADD_LIBRARY(rspamd-server STATIC
+			${RSPAMD_CRYPTOBOX}
+			${RSPAMD_UTIL}
+			${RSPAMD_LUA}
+			${RSPAMD_SERVER}
+			${RSPAMD_STAT}
+			${RSPAMD_MIME}
+			${CMAKE_CURRENT_BINARY_DIR}/modules.c
+			${PLUGINSSRC}
+			"${RAGEL_ragel_smtp_addr_OUTPUTS}"
+			"${RAGEL_ragel_newlines_strip_OUTPUTS}"
+			"${RAGEL_ragel_content_type_OUTPUTS}"
+			"${RAGEL_ragel_content_disposition_OUTPUTS}"
+			"${RAGEL_ragel_rfc2047_OUTPUTS}"
+			"${RAGEL_ragel_smtp_date_OUTPUTS}"
+			"${RAGEL_ragel_smtp_ip_OUTPUTS}")
+ELSE()
+	ADD_LIBRARY(rspamd-server SHARED
+			${RSPAMD_CRYPTOBOX}
+			${RSPAMD_UTIL}
+			${RSPAMD_LUA}
+			${RSPAMD_SERVER}
+			${RSPAMD_STAT}
+			${RSPAMD_MIME}
+			${CMAKE_CURRENT_BINARY_DIR}/modules.c
+			${PLUGINSSRC}
+			"${RAGEL_ragel_smtp_addr_OUTPUTS}"
+			"${RAGEL_ragel_newlines_strip_OUTPUTS}"
+			"${RAGEL_ragel_content_type_OUTPUTS}"
+			"${RAGEL_ragel_content_disposition_OUTPUTS}"
+			"${RAGEL_ragel_rfc2047_OUTPUTS}"
+			"${RAGEL_ragel_smtp_date_OUTPUTS}"
+			"${RAGEL_ragel_smtp_ip_OUTPUTS}")
+ENDIF()
+
 TARGET_LINK_LIBRARIES(rspamd-server rspamd-http-parser)
 TARGET_LINK_LIBRARIES(rspamd-server rspamd-cdb)
 TARGET_LINK_LIBRARIES(rspamd-server rspamd-lpeg)
@@ -173,33 +192,36 @@ IF (ENABLE_CLANG_PLUGIN MATCHES "ON")
 	ADD_DEPENDENCIES(rspamd-server rspamd-clang)
 ENDIF()
 
-ADD_EXECUTABLE(rspamd ${RSPAMDSRC} ${CMAKE_CURRENT_BINARY_DIR}/workers.c)
-SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE C)
-SET_TARGET_PROPERTIES(rspamd PROPERTIES COMPILE_FLAGS "-DRSPAMD_MAIN")
-IF(NOT DEBIAN_BUILD)
-	SET_TARGET_PROPERTIES(rspamd PROPERTIES VERSION ${RSPAMD_VERSION})
-ENDIF(NOT DEBIAN_BUILD)
-
-TARGET_LINK_LIBRARIES(rspamd rspamd-server)
 IF (ENABLE_SNOWBALL MATCHES "ON")
-	TARGET_LINK_LIBRARIES(rspamd stemmer)
+	TARGET_LINK_LIBRARIES(rspamd-server stemmer)
 ENDIF()
-TARGET_LINK_LIBRARIES(rspamd rspamd-hiredis)
+TARGET_LINK_LIBRARIES(rspamd-server rspamd-hiredis)
 
 IF (ENABLE_FANN MATCHES "ON")
-	TARGET_LINK_LIBRARIES(rspamd fann)
+	TARGET_LINK_LIBRARIES(rspamd-server fann)
 ENDIF ()
 
 IF (ENABLE_HYPERSCAN MATCHES "ON")
-	TARGET_LINK_LIBRARIES(rspamd hs)
+	TARGET_LINK_LIBRARIES(rspamd-server hs)
 ENDIF()
 
-TARGET_LINK_LIBRARIES(rspamd rspamd-linenoise)
+TARGET_LINK_LIBRARIES(rspamd-server rspamd-linenoise)
 
 IF(USE_CXX_LINKER)
 	SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE CXX)
+	SET_TARGET_PROPERTIES(rspamd-server PROPERTIES LINKER_LANGUAGE CXX)
 ENDIF()
 
-TARGET_LINK_LIBRARIES(rspamd ${RSPAMD_REQUIRED_LIBRARIES})
+TARGET_LINK_LIBRARIES(rspamd-server ${RSPAMD_REQUIRED_LIBRARIES})
+
+ADD_EXECUTABLE(rspamd ${RSPAMDSRC} ${CMAKE_CURRENT_BINARY_DIR}/workers.c)
+SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE C)
+SET_TARGET_PROPERTIES(rspamd PROPERTIES COMPILE_FLAGS "-DRSPAMD_MAIN")
+IF(NOT DEBIAN_BUILD)
+	SET_TARGET_PROPERTIES(rspamd PROPERTIES VERSION ${RSPAMD_VERSION})
+ENDIF(NOT DEBIAN_BUILD)
+
+TARGET_LINK_LIBRARIES(rspamd rspamd-server)
 
 INSTALL(TARGETS rspamd RUNTIME DESTINATION bin)
+INSTALL(TARGETS rspamd-server LIBRARY DESTINATION ${RSPAMD_LIBDIR})

src/client/CMakeLists.txt

@@ -8,7 +8,6 @@ ADD_EXECUTABLE(rspamc ${RSPAMCSRC} ${LIBRSPAMDCLIENTSRC})
 SET_TARGET_PROPERTIES(rspamc PROPERTIES COMPILE_FLAGS "-I${CMAKE_SOURCE_DIR}/lib")
 TARGET_LINK_LIBRARIES(rspamc rspamd-server)
 TARGET_LINK_LIBRARIES(rspamc ${RSPAMD_REQUIRED_LIBRARIES})
-TARGET_LINK_LIBRARIES(rspamc rspamd-linenoise)
 IF(USE_CXX_LINKER)
 	SET_TARGET_PROPERTIES(rspamc PROPERTIES LINKER_LANGUAGE CXX)
 ENDIF()

src/client/rspamc.c

@@ -887,7 +887,15 @@ rspamc_symbols_output (FILE *out, ucl_object_t *obj)
 		}
 	}
 
-	PRINT_PROTOCOL_STRING ("dkim-signature", "DKIM-Signature");
+	elt = ucl_object_lookup (obj, "dkim-signature");
+	if (elt && elt->type == UCL_STRING) {
+		rspamd_fprintf (out, "DKIM-Signature: %s\n", ucl_object_tostring (elt));
+	} else if (elt && elt->type == UCL_ARRAY) {
+		mit = NULL;
+		while ((cmesg = ucl_object_iterate (elt, &mit, true)) != NULL) {
+			rspamd_fprintf (out, "DKIM-Signature: %s\n", ucl_object_tostring (cmesg));
+		}
+	}
 
 	elt = ucl_object_lookup (obj, "profile");
 
@@ -1372,11 +1380,16 @@ rspamc_mime_output (FILE *out, ucl_object_t *result, GString *input,
 		g_string_free (folded_symbuf, TRUE);
 		g_string_free (symbuf, TRUE);
 
-		if (ucl_object_lookup (result, "dkim-signature")) {
+		res = ucl_object_lookup (result, "dkim-signature");
+		if (res && res->type == UCL_STRING) {
 			rspamd_printf_gstring (added_headers, "DKIM-Signature: %s%s",
-					ucl_object_tostring (
-							ucl_object_lookup (result, "dkim-signature")),
-					line_end);
+					ucl_object_tostring (res), line_end);
+		} else if (res && res->type == UCL_ARRAY) {
+			it = NULL;
+			while ((cur = ucl_object_iterate (res, &it, true)) != NULL) {
+				rspamd_printf_gstring (added_headers, "DKIM-Signature: %s%s",
+					ucl_object_tostring (cur), line_end);
+			}
 		}
 
 		if (json || raw || compact) {

src/controller.c

@@ -15,6 +15,7 @@
  */
 #include "config.h"
 #include "libserver/dynamic_cfg.h"
+#include "libserver/cfg_file_private.h"
 #include "libutil/rrd.h"
 #include "libutil/map.h"
 #include "libutil/map_helpers.h"
@@ -864,8 +865,7 @@ rspamd_controller_handle_actions (struct rspamd_http_connection_entry *conn_ent,
 	struct rspamd_http_message *msg)
 {
 	struct rspamd_controller_session *session = conn_ent->ud;
-	struct rspamd_action *act;
-	gint i;
+	struct rspamd_action *act, *tmp;
 	ucl_object_t *obj, *top;
 
 	if (!rspamd_controller_check_password (conn_ent, session, msg, FALSE)) {
@@ -874,15 +874,14 @@ rspamd_controller_handle_actions (struct rspamd_http_connection_entry *conn_ent,
 
 	top = ucl_object_typed_new (UCL_ARRAY);
 
-	/* Get actions for default metric */
-	for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) {
-		act = &session->cfg->actions[i];
+	HASH_ITER (hh, session->cfg->actions, act, tmp) {
 		obj = ucl_object_typed_new (UCL_OBJECT);
 		ucl_object_insert_key (obj,
-				ucl_object_fromstring (rspamd_action_to_str (
-						act->action)), "action", 0, false);
-		ucl_object_insert_key (obj, ucl_object_fromdouble (
-				act->score), "value", 0, false);
+				ucl_object_fromstring (act->name),
+				"action", 0, false);
+		ucl_object_insert_key (obj,
+				ucl_object_fromdouble (act->threshold),
+				"value", 0, false);
 		ucl_array_append (top, obj);
 	}
 
@@ -2238,8 +2237,8 @@ rspamd_controller_handle_saveactions (
 			score = ucl_object_todouble (cur);
 		}
 
-		if ((isnan (session->cfg->actions[act].score) != isnan (score)) ||
-				(session->cfg->actions[act].score != score)) {
+		if ((isnan (session->cfg->actions[act].threshold) != isnan (score)) ||
+				(session->cfg->actions[act].threshold != score)) {
 			add_dynamic_action (ctx->cfg, DEFAULT_METRIC, act, score);
 			added ++;
 		}

src/libcryptobox/base64/base64.c

@@ -146,3 +146,28 @@ base64_test (bool generic, size_t niters, size_t len)
 
 	return cycles;
 }
+
+
+gboolean
+rspamd_cryptobox_base64_is_valid (const gchar *in, gsize inlen)
+{
+	const guchar *p, *end;
+
+	if (inlen == 0) {
+		return FALSE;
+	}
+
+	p = in;
+	end = in + inlen;
+
+	while (p < end && *p != '=') {
+		if (!g_ascii_isspace (*p)) {
+			if (base64_table_dec[*p] == 255) {
+				return FALSE;
+			}
+		}
+		p ++;
+	}
+
+	return TRUE;
+}

src/libcryptobox/cryptobox.h

@@ -399,4 +399,13 @@ guint64 rspamd_cryptobox_fast_hash_specific (
  */
 gboolean rspamd_cryptobox_base64_decode (const gchar *in, gsize inlen,
 		guchar *out, gsize *outlen);
+
+/**
+ * Returns TRUE if data looks like a valid base64 string
+ * @param in
+ * @param inlen
+ * @return
+ */
+gboolean rspamd_cryptobox_base64_is_valid (const gchar *in, gsize inlen);
+
 #endif /* CRYPTOBOX_H_ */

src/libcryptobox/curve25519/avx.c

@@ -61,7 +61,7 @@ fe_frombytes (fe h, const unsigned char *s)
 	guint64 h6 = load_3 (s + 20) << 7;
 	guint64 h7 = load_3 (s + 23) << 5;
 	guint64 h8 = load_3 (s + 26) << 4;
-	guint64 h9 = load_3 (s + 29) << 2;
+	guint64 h9 = (load_3(s + 29) & 8388607) << 2;
 	guint64 carry0;
 	guint64 carry1;
 	guint64 carry2;

src/libcryptobox/curve25519/constants.S

@@ -1,20 +1,4 @@
-SECTION_RODATA
-
-.globl v0_0
-.globl v1_0
-.globl v2_1
-.globl v19_19
-.globl v38_1
-.globl v38_38
-.globl v121666_121666
-.globl m25
-.globl m26
-.globl subc0
-.globl subc2
-.globl v9_0
-.globl v9_9
-.globl base64_table_dec
-.globl REDMASK51
+.data
 
 .p2align 4
 

src/libcryptobox/ed25519/ed25519.c

@@ -26,6 +26,7 @@ typedef struct ed25519_impl_s {
 	const char *desc;
 
 	void (*keypair) (unsigned char *pk, unsigned char *sk);
+	void (*seed_keypair) (unsigned char *pk, unsigned char *sk, unsigned char *seed);
 	void (*sign) (unsigned char *sig, size_t *siglen_p,
 			const unsigned char *m, size_t mlen,
 			const unsigned char *sk);
@@ -37,6 +38,7 @@ typedef struct ed25519_impl_s {
 
 #define ED25519_DECLARE(ext) \
     void ed_keypair_##ext(unsigned char *pk, unsigned char *sk); \
+    void ed_seed_keypair_##ext(unsigned char *pk, unsigned char *sk, unsigned char *seed); \
     void ed_sign_##ext(unsigned char *sig, size_t *siglen_p, \
         const unsigned char *m, size_t mlen, \
         const unsigned char *sk); \
@@ -46,7 +48,7 @@ typedef struct ed25519_impl_s {
         const unsigned char *pk)
 
 #define ED25519_IMPL(cpuflags, desc, ext) \
-    {(cpuflags), desc, ed_keypair_##ext, ed_sign_##ext, ed_verify_##ext}
+    {(cpuflags), desc, ed_keypair_##ext, ed_seed_keypair_##ext, ed_sign_##ext, ed_verify_##ext}
 
 ED25519_DECLARE(ref);
 #define ED25519_REF ED25519_IMPL(0, "ref", ref)
@@ -78,6 +80,12 @@ ed25519_load (void)
 	return ed25519_opt->desc;
 }
 
+void
+ed25519_seed_keypair (unsigned char *pk, unsigned char *sk, unsigned char *seed)
+{
+	ed25519_opt->seed_keypair (pk, sk, seed);
+}
+
 void
 ed25519_keypair (unsigned char *pk, unsigned char *sk)
 {

src/libcryptobox/ed25519/ed25519.h

@@ -22,6 +22,7 @@
 
 const char* ed25519_load (void);
 void ed25519_keypair (unsigned char *pk, unsigned char *sk);
+void ed25519_seed_keypair (unsigned char *pk, unsigned char *sk, unsigned char *seed);
 void ed25519_sign (unsigned char *sig, size_t *siglen_p,
 		const unsigned char *m, size_t mlen,
 		const unsigned char *sk);

src/libcryptobox/ed25519/ref.c

@@ -23,7 +23,7 @@
 #include "ottery.h"
 #include <openssl/evp.h> /* SHA512 */
 
-static int
+int
 ed_seed_keypair_ref (unsigned char *pk, unsigned char *sk,
 		const unsigned char *seed)
 {

src/libmime/archives.c

@@ -1757,9 +1757,19 @@ rspamd_archive_process_gzip (struct rspamd_task *task,
 				const gchar *fname_start = part->cd->filename.begin;
 
 				f = g_malloc0 (sizeof (*f));
-				f->fname = g_string_sized_new (dot_pos - slash_pos);
-				g_string_append_len (f->fname, fname_start,
-						dot_pos - fname_start);
+
+				if (memchr (fname_start, '.', part->cd->filename.len) != dot_pos) {
+					/* Double dots, something like foo.exe.gz */
+					f->fname = g_string_sized_new (dot_pos - fname_start);
+					g_string_append_len (f->fname, fname_start,
+							dot_pos - fname_start);
+				}
+				else {
+					/* Single dot, something like foo.gzz */
+					f->fname = g_string_sized_new (part->cd->filename.len);
+					g_string_append_len (f->fname, fname_start,
+							part->cd->filename.len);
+				}
 
 				msg_debug_archive ("fallback to gzip filename based on cd: %v",
 						f->fname);

src/libmime/content_type.c

@@ -17,72 +17,312 @@
 #include "libmime/content_type.h"
 #include "smtp_parsers.h"
 #include "utlist.h"
+#include "libserver/url.h"
+#include "libmime/mime_encoding.h"
 
-void
-rspamd_content_type_add_param (rspamd_mempool_t *pool,
-		struct rspamd_content_type *ct,
-		gchar *name_start, gchar *name_end,
+static gboolean
+rspamd_rfc2231_decode (rspamd_mempool_t *pool,
+		struct rspamd_content_type_param *param,
 		gchar *value_start, gchar *value_end)
 {
-	rspamd_ftok_t srch;
-	struct rspamd_content_type_param *found = NULL, *nparam;
+	gchar *quote_pos;
 
-	g_assert (ct != NULL);
+	quote_pos = memchr (value_start, '\'', value_end - value_start);
+
+	if (quote_pos == NULL) {
+		/* Plain percent encoding */
+		gsize r = rspamd_url_decode (value_start, value_start,
+				value_end - value_start);
+		param->value.begin = value_start;
+		param->value.len = r;
+	}
+	else {
+		/*
+		 * We can have encoding'language'data, or
+		 * encoding'data (in theory).
+		 * Try to handle both...
+		 */
+		const gchar *charset = NULL;
+		rspamd_ftok_t ctok;
 
+		ctok.begin = value_start;
+		ctok.len = quote_pos - value_start;
 
-	nparam = rspamd_mempool_alloc (pool, sizeof (*nparam));
-	nparam->name.begin = name_start;
-	nparam->name.len = name_end - name_start;
-	rspamd_str_lc (name_start, name_end - name_start);
+		charset = rspamd_mime_detect_charset (&ctok, pool);
+
+		if (charset == NULL) {
+			msg_warn_pool ("cannot convert parameter from charset %T", &ctok);
+
+			return FALSE;
+		}
+
+		/* Now, we can check for either next quote sign or, eh, ignore that */
+		value_start = quote_pos + 1;
+
+		quote_pos = memchr (value_start, '\'', value_end - value_start);
+
+		if (quote_pos) {
+			/* Ignore language */
+			value_start = quote_pos + 1;
+		}
+
+		/* Perform percent decoding */
+		gsize r = rspamd_url_decode (value_start, value_start,
+				value_end - value_start);
+		GError *err = NULL;
+
+		param->value.begin = rspamd_mime_text_to_utf8 (pool,
+				value_start, r,
+				charset, &param->value.len, &err);
+
+		if (param->value.begin == NULL) {
+			msg_warn_pool ("cannot convert parameter from charset %s: %e",
+					charset, err);
+
+			if (err) {
+				g_error_free (err);
+			}
+
+			return FALSE;
+		}
+	}
+
+	param->flags |= RSPAMD_CONTENT_PARAM_RFC2231;
+
+	return TRUE;
+}
+
+static gboolean
+rspamd_param_maybe_rfc2231_process (rspamd_mempool_t *pool,
+									struct rspamd_content_type_param *param,
+									gchar *name_start, gchar *name_end,
+									gchar *value_start, gchar *value_end)
+{
+	const gchar *star_pos;
+
+	star_pos = memchr (name_start, '*', name_end - name_start);
+
+	if (star_pos == NULL) {
+		return FALSE;
+	}
+
+	/* We have three possibilities here:
+	 * 1. name* (just name + 2231 encoding)
+	 * 2. name*(\d+) (piecewise stuff but no rfc2231 encoding)
+	 * 3. name*(\d+)* (piecewise stuff and rfc2231 encoding)
+	 */
+
+	if (star_pos == name_end - 1) {
+		/* First */
+		if (rspamd_rfc2231_decode (pool, param, value_start, value_end)) {
+			param->name.begin = name_start;
+			param->name.len = name_end - name_start - 1;
+		}
+	}
+	else if (*(name_end - 1) == '*') {
+		/* Third */
+		/* Check number */
+		gulong tmp;
+
+		if (!rspamd_strtoul (star_pos + 1, name_end - star_pos - 2, &tmp)) {
+			return FALSE;
+		}
+
+		param->flags |= RSPAMD_CONTENT_PARAM_PIECEWISE|RSPAMD_CONTENT_PARAM_RFC2231;
+		param->rfc2231_id = tmp;
+		param->name.begin = name_start;
+		param->name.len = star_pos - name_start;
+		param->value.begin = value_start;
+		param->value.len = value_end - value_start;
+
+		/* Deal with that later... */
+	}
+	else {
+		/* Second case */
+		gulong tmp;
+
+		if (!rspamd_strtoul (star_pos + 1, name_end - star_pos - 1, &tmp)) {
+			return FALSE;
+		}
+
+		param->flags |= RSPAMD_CONTENT_PARAM_PIECEWISE;
+		param->rfc2231_id = tmp;
+		param->name.begin = name_start;
+		param->name.len = star_pos - name_start;
+		param->value.begin = value_start;
+		param->value.len = value_end - value_start;
+	}
+
+	return TRUE;
+}
+
+static gint32
+rspamd_cmp_pieces (struct rspamd_content_type_param *p1, struct rspamd_content_type_param *p2)
+{
+	return p1->rfc2231_id - p2->rfc2231_id;
+}
+
+static void
+rspamd_postprocess_ct_attributes (rspamd_mempool_t *pool,
+								  GHashTable *htb,
+								  void (*proc)(rspamd_mempool_t *, struct rspamd_content_type_param *, gpointer ud),
+								  gpointer procd)
+{
+	GHashTableIter it;
+	gpointer k, v;
+	struct rspamd_content_type_param *param, *sorted, *cur;
+
+	if (htb == NULL) {
+		return;
+	}
+
+	g_hash_table_iter_init (&it, htb);
+
+	while (g_hash_table_iter_next (&it, &k, &v)) {
+		param = (struct rspamd_content_type_param *)v;
+
+		if (param->flags & RSPAMD_CONTENT_PARAM_PIECEWISE) {
+			/* Reconstruct param */
+			gsize tlen = 0;
+			gchar *ndata, *pos;
+
+			sorted = param;
+			DL_SORT (sorted, rspamd_cmp_pieces);
+
+			DL_FOREACH (sorted, cur) {
+				tlen += cur->value.len;
+			}
+
+			ndata = rspamd_mempool_alloc (pool, tlen);
+			pos = ndata;
+
+			DL_FOREACH (sorted, cur) {
+				memcpy (pos, cur->value.begin, cur->value.len);
+				pos += cur->value.len;
+			}
+
+			if (param->flags & RSPAMD_CONTENT_PARAM_RFC2231) {
+				if (!rspamd_rfc2231_decode (pool, param,
+						ndata, pos)) {
+					param->flags |= RSPAMD_CONTENT_PARAM_BROKEN;
+					param->value.begin = ndata;
+					param->value.len = tlen;
+				}
+			}
+			else {
+				param->value.begin = ndata;
+				param->value.len = tlen;
+			}
+
+			/* Detach from list */
+			param->next = NULL;
+			param->prev = param;
+		}
+
+		proc (pool, param, procd);
+	}
+}
+
+static void
+rspamd_content_type_postprocess (rspamd_mempool_t *pool,
+								 struct rspamd_content_type_param *param,
+								 gpointer ud)
+{
+	rspamd_ftok_t srch;
+	struct rspamd_content_type_param *found = NULL;
 
-	nparam->value.begin = value_start;
-	nparam->value.len = value_end - value_start;
+	struct rspamd_content_type *ct = (struct rspamd_content_type *)ud;
 
 	RSPAMD_FTOK_ASSIGN (&srch, "charset");
 
-	if (rspamd_ftok_cmp (&nparam->name, &srch) == 0) {
+	if (rspamd_ftok_cmp (&param->name, &srch) == 0) {
 		/* Adjust charset */
-		found = nparam;
-		ct->charset.begin = nparam->value.begin;
-		ct->charset.len = nparam->value.len;
+		found = param;
+		ct->charset.begin = param->value.begin;
+		ct->charset.len = param->value.len;
 	}
 
 	RSPAMD_FTOK_ASSIGN (&srch, "boundary");
 
-	if (rspamd_ftok_cmp (&nparam->name, &srch) == 0) {
-		found = nparam;
+	if (rspamd_ftok_cmp (&param->name, &srch) == 0) {
+		found = param;
 		gchar *lc_boundary;
 		/* Adjust boundary */
-		lc_boundary = rspamd_mempool_alloc (pool, nparam->value.len);
-		memcpy (lc_boundary, nparam->value.begin, nparam->value.len);
-		rspamd_str_lc (lc_boundary, nparam->value.len);
+		lc_boundary = rspamd_mempool_alloc (pool, param->value.len);
+		memcpy (lc_boundary, param->value.begin, param->value.len);
+		rspamd_str_lc (lc_boundary, param->value.len);
 		ct->boundary.begin = lc_boundary;
-		ct->boundary.len = nparam->value.len;
+		ct->boundary.len = param->value.len;
 		/* Preserve original (case sensitive) boundary */
-		ct->orig_boundary.begin = nparam->value.begin;
-		ct->orig_boundary.len = nparam->value.len;
+		ct->orig_boundary.begin = param->value.begin;
+		ct->orig_boundary.len = param->value.len;
 	}
 
 	if (!found) {
-		srch.begin = nparam->name.begin;
-		srch.len = nparam->name.len;
+		/* Just lowercase */
+		rspamd_str_lc ((gchar *)param->value.begin, param->value.len);
+	}
+}
 
-		rspamd_str_lc (value_start, value_end - value_start);
+static void
+rspamd_content_disposition_postprocess (rspamd_mempool_t *pool,
+										struct rspamd_content_type_param *param,
+										gpointer ud)
+{
+	rspamd_ftok_t srch;
+	struct rspamd_content_disposition *cd = (struct rspamd_content_disposition *)ud;
 
-		if (ct->attrs) {
-			found = g_hash_table_lookup (ct->attrs, &srch);
-		} else {
-			ct->attrs = g_hash_table_new (rspamd_ftok_icase_hash,
-					rspamd_ftok_icase_equal);
-		}
+	srch.begin = "filename";
+	srch.len = 8;
 
-		if (!found) {
-			DL_APPEND (found, nparam);
-			g_hash_table_insert (ct->attrs, &nparam->name, nparam);
-		}
-		else {
-			DL_APPEND (found, nparam);
-		}
+	if (rspamd_ftok_cmp (&param->name, &srch) == 0) {
+		/* Adjust filename */
+		cd->filename.begin = param->value.begin;
+		cd->filename.len = param->value.len;
+	}
+}
+
+void
+rspamd_content_type_add_param (rspamd_mempool_t *pool,
+		struct rspamd_content_type *ct,
+		gchar *name_start, gchar *name_end,
+		gchar *value_start, gchar *value_end)
+{
+	struct rspamd_content_type_param *nparam;
+	rspamd_ftok_t srch;
+	struct rspamd_content_type_param *found = NULL;
+
+	g_assert (ct != NULL);
+
+	nparam = rspamd_mempool_alloc0 (pool, sizeof (*nparam));
+	rspamd_str_lc (name_start, name_end - name_start);
+
+	if (!rspamd_param_maybe_rfc2231_process (pool, nparam, name_start,
+			name_end, value_start, value_end)) {
+		nparam->name.begin = name_start;
+		nparam->name.len = name_end - name_start;
+		nparam->value.begin = value_start;
+		nparam->value.len = value_end - value_start;
+	}
+
+	RSPAMD_FTOK_ASSIGN (&srch, "charset");
+
+	srch.begin = nparam->name.begin;
+	srch.len = nparam->name.len;
+
+	if (ct->attrs) {
+		found = g_hash_table_lookup (ct->attrs, &srch);
+	} else {
+		ct->attrs = g_hash_table_new (rspamd_ftok_icase_hash,
+				rspamd_ftok_icase_equal);
+	}
+
+	if (!found) {
+		DL_APPEND (found, nparam);
+		g_hash_table_insert (ct->attrs, &nparam->name, nparam);
+	}
+	else {
+		DL_APPEND (found, nparam);
 	}
 }
 
@@ -361,9 +601,6 @@ rspamd_content_type_parser (gchar *in, gsize len, rspamd_mempool_t *pool)
 	if (val.type.len > 0) {
 		res = rspamd_mempool_alloc (pool, sizeof (val));
 		memcpy (res, &val, sizeof (val));
-
-		/* Lowercase common thingies */
-
 	}
 
 	return res;
@@ -384,6 +621,9 @@ rspamd_content_type_parse (const gchar *in,
 		if (res->attrs) {
 			rspamd_mempool_add_destructor (pool,
 					(rspamd_mempool_destruct_t)g_hash_table_unref, res->attrs);
+
+			rspamd_postprocess_ct_attributes (pool, res->attrs,
+					rspamd_content_type_postprocess, res);
 		}
 
 		/* Now do some hacks to work with broken content types */
@@ -491,7 +731,7 @@ rspamd_content_disposition_add_param (rspamd_mempool_t *pool,
 				(rspamd_mempool_destruct_t)g_hash_table_unref, cd->attrs);
 	}
 
-	nparam = rspamd_mempool_alloc (pool, sizeof (*nparam));
+	nparam = rspamd_mempool_alloc0 (pool, sizeof (*nparam));
 	nparam->name.begin = name_start;
 	nparam->name.len = name_end - name_start;
 	decoded = rspamd_mime_header_decode (pool, value_start,
@@ -503,15 +743,6 @@ rspamd_content_disposition_add_param (rspamd_mempool_t *pool,
 	}
 
 	DL_APPEND (found, nparam);
-
-	srch.begin = "filename";
-	srch.len = 8;
-
-	if (rspamd_ftok_cmp (&nparam->name, &srch) == 0) {
-		/* Adjust filename */
-		cd->filename.begin = nparam->value.begin;
-		cd->filename.len = nparam->value.len;
-	}
 }
 
 struct rspamd_content_disposition *
@@ -526,6 +757,8 @@ rspamd_content_disposition_parse (const gchar *in,
 		res->lc_data = rspamd_mempool_alloc (pool, len + 1);
 		rspamd_strlcpy (res->lc_data, in, len + 1);
 		rspamd_str_lc (res->lc_data, len);
+		rspamd_postprocess_ct_attributes (pool, res->attrs,
+				rspamd_content_disposition_postprocess, res);
 	}
 	else {
 		msg_warn_pool ("cannot parse content disposition: %*s",

src/libmime/content_type.h

@@ -34,9 +34,18 @@ enum rspamd_content_type_flags {
 #define IS_CT_TEXT(ct) ((ct) && ((ct)->flags & RSPAMD_CONTENT_TYPE_TEXT))
 #define IS_CT_MESSAGE(ct) ((ct) &&((ct)->flags & RSPAMD_CONTENT_TYPE_MESSAGE))
 
+enum rspamd_content_param_flags {
+	RSPAMD_CONTENT_PARAM_NORMAL = 0,
+	RSPAMD_CONTENT_PARAM_RFC2231 = (1 << 0),
+	RSPAMD_CONTENT_PARAM_PIECEWISE = (1 << 1),
+	RSPAMD_CONTENT_PARAM_BROKEN = (1 << 2),
+};
+
 struct rspamd_content_type_param {
 	rspamd_ftok_t name;
 	rspamd_ftok_t value;
+	guint rfc2231_id;
+	enum rspamd_content_param_flags flags;
 	struct rspamd_content_type_param *prev, *next;
 };
 

src/libmime/email_addr.c

@@ -87,15 +87,17 @@ rspamd_email_address_from_smtp (const gchar *str, guint len)
 void
 rspamd_email_address_free (struct rspamd_email_address *addr)
 {
-	if (addr->flags & RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED) {
-		g_free ((void *)addr->addr);
-	}
+	if (addr) {
+		if (addr->flags & RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED) {
+			g_free ((void *) addr->addr);
+		}
 
-	if (addr->flags & RSPAMD_EMAIL_ADDR_USER_ALLOCATED) {
-		g_free ((void *)addr->user);
-	}
+		if (addr->flags & RSPAMD_EMAIL_ADDR_USER_ALLOCATED) {
+			g_free ((void *) addr->user);
+		}
 
-	g_free (addr);
+		g_free (addr);
+	}
 }
 
 static inline void
@@ -137,6 +139,7 @@ rspamd_email_address_add (rspamd_mempool_t *pool,
 	}
 
 	if (name->len > 0) {
+		rspamd_gstring_strip (name, " \t\v");
 		elt->name = rspamd_mime_header_decode (pool, name->str, name->len, NULL);
 	}
 
@@ -233,13 +236,12 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 	gboolean seen_at = FALSE;
 
 	const gchar *p = hdr, *end = hdr + len, *c = hdr, *t;
-	GString *ns;
+	GString *ns, *cpy;
 	gint obraces, ebraces;
 	enum {
 		parse_name = 0,
 		parse_quoted,
 		parse_addr,
-		skip_comment,
 		skip_spaces
 	} state = parse_name, next_state = parse_name;
 
@@ -249,7 +251,70 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 				res);
 	}
 
-	ns = g_string_sized_new (127);
+	ns = g_string_sized_new (len);
+	cpy = g_string_sized_new (len);
+
+	rspamd_mempool_add_destructor (pool, rspamd_gstring_free_hard, cpy);
+
+	/* First, we need to remove all comments as they are terrible */
+	obraces = 0;
+	ebraces = 0;
+
+	while (p < end) {
+		if (state == parse_name) {
+			if (*p == '\\') {
+				if (obraces == 0) {
+					g_string_append_c (cpy, *p);
+				}
+
+				p++;
+			}
+			else {
+				if (*p == '"') {
+					state = parse_quoted;
+				}
+				else if (*p == '(') {
+					obraces ++;
+				}
+				else if (*p == ')') {
+					ebraces ++;
+				}
+
+				if (obraces == ebraces) {
+					obraces = 0;
+					ebraces = 0;
+				}
+			}
+
+			if (p < end && obraces == 0) {
+				g_string_append_c (cpy, *p);
+			}
+		}
+		else {
+			/* Quoted elt */
+			if (*p == '\\') {
+				g_string_append_c (cpy, *p);
+				p++;
+			}
+			else {
+				if (*p == '"') {
+					state = parse_name;
+				}
+			}
+
+			if (p < end) {
+				g_string_append_c (cpy, *p);
+			}
+		}
+
+		p++;
+	}
+
+	state = parse_name;
+
+	p = cpy->str;
+	c = p;
+	end = p + cpy->len;
 
 	while (p < end) {
 		switch (state) {
@@ -257,13 +322,20 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 			if (*p == '"') {
 				/* We need to strip last spaces and update `ns` */
 				if (p > c) {
+					guint nspaces = 0;
+
 					t = p - 1;
 
 					while (t > c && g_ascii_isspace (*t)) {
 						t --;
+						nspaces ++;
 					}
 
 					g_string_append_len (ns, c, t - c + 1);
+
+					if (nspaces > 0) {
+						g_string_append_c (ns, ' ');
+					}
 				}
 
 				state = parse_quoted;
@@ -311,33 +383,7 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 			else if (*p == '@') {
 				seen_at = TRUE;
 			}
-			else if (*p == '(') {
-				if (p > c) {
-					t = p - 1;
 
-					while (t > c && g_ascii_isspace (*t)) {
-						t --;
-					}
-
-					g_string_append_len (ns, c, t - c + 1);
-
-					if (seen_at) {
-						if (!rspamd_email_address_check_and_add (c, t - c + 1,
-								res, pool, ns)) {
-							rspamd_email_address_add (pool, res, NULL, ns);
-						}
-
-						g_string_set_size (ns, 0);
-						seen_at = FALSE;
-					}
-				}
-
-				c = p;
-				obraces = 1;
-				ebraces = 0;
-				state = skip_comment;
-				next_state = parse_name;
-			}
 			p ++;
 			break;
 		case parse_quoted:
@@ -346,6 +392,10 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 					g_string_append_len (ns, c, p - c);
 				}
 
+				if (p + 1 < end && g_ascii_isspace (p[1])) {
+					g_string_append_c (ns, ' ');
+				}
+
 				state = skip_spaces;
 				next_state = parse_name;
 			}
@@ -367,12 +417,6 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 			else if (*p == '@') {
 				seen_at = TRUE;
 			}
-			else if (*p == '(') {
-				obraces = 1;
-				ebraces = 0;
-				state = skip_comment;
-				next_state = parse_addr;
-			}
 			p ++;
 			break;
 		case skip_spaces:
@@ -384,36 +428,6 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 				p ++;
 			}
 			break;
-		case skip_comment:
-			if (*p == '(') {
-				obraces ++;
-			}
-			else if (*p == ')') {
-				ebraces ++;
-			}
-
-			if (obraces == ebraces) {
-				if (next_state == parse_name) {
-					if (ns->len > 0) {
-						/* Include comment in name if it has been seen */
-						if (p > c) {
-							t = p - 1;
-
-							while (t > c && g_ascii_isspace (*t)) {
-								t --;
-							}
-
-							g_string_append_len (ns, c, t - c + 1);
-						}
-					}
-
-					c = p + 1;
-				}
-
-				state = next_state;
-			}
-			p ++;
-			break;
 		}
 	}
 
@@ -460,7 +474,6 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 		}
 		break;
 	case parse_quoted:
-	case skip_comment:
 		/* Unfinished quoted string or a comment */
 		break;
 	default:
@@ -485,30 +498,4 @@ rspamd_email_address_list_destroy (gpointer ptr)
 	}
 
 	g_ptr_array_free (ar, TRUE);
-}
-
-void rspamd_smtp_maybe_process_smtp_comment (struct rspamd_task *task,
-											 const char *data, size_t len,
-											 struct received_header *rh)
-{
-	if (!rh->by_hostname) {
-		/* Heuristic to detect IP addresses like in Exim received:
-		 * [xxx]:port or [xxx]
-		 */
-
-		if (*data == '[' && len > 2) {
-			const gchar *p = data + 1;
-			gsize iplen = rspamd_memcspn (p, "]", len - 1);
-
-			if (iplen > 0) {
-				guchar tbuf[sizeof(struct in6_addr) + sizeof(guint32)];
-
-				if (rspamd_parse_inet_address_ip4 (p, iplen, tbuf) ||
-						rspamd_parse_inet_address_ip6 (p, iplen, tbuf)) {
-					rh->comment_ip = rspamd_mempool_alloc (task->task_pool, iplen + 1);
-					rspamd_strlcpy (rh->comment_ip, p, iplen + 1);
-				}
-			}
-		}
-	}
 }

src/libmime/email_addr.h

@@ -53,16 +53,6 @@ struct rspamd_email_address {
 
 struct received_header;
 struct rspamd_task;
-/**
- * Try to parse SMTP comment to process stupid Exim received headers
- * @param task
- * @param data
- * @param len
- * @param rh
- */
-void rspamd_smtp_maybe_process_smtp_comment (struct rspamd_task *task,
-											 const char *data, size_t len,
-											 struct received_header *rh);
 
 /**
  * Create email address from a single rfc822 address (e.g. from mail from:)

src/libmime/filter.c

@@ -19,6 +19,8 @@
 #include "rspamd.h"
 #include "message.h"
 #include "lua/lua_common.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/filter_private.h"
 #include <math.h>
 #include "contrib/uthash/utlist.h"
 
@@ -70,14 +72,22 @@ rspamd_create_metric_result (struct rspamd_task *task)
 	}
 
 	if (task->cfg) {
-		for (i = 0; i < METRIC_ACTION_MAX; i++) {
-			metric_res->actions_limits[i] = task->cfg->actions[i].score;
-		}
-	}
-	else {
-		for (i = 0; i < METRIC_ACTION_MAX; i++) {
-			metric_res->actions_limits[i] = NAN;
+		struct rspamd_action *act, *tmp;
+
+		metric_res->actions_limits = rspamd_mempool_alloc0 (task->task_pool,
+			sizeof (struct rspamd_action_result) * HASH_COUNT (task->cfg->actions));
+		i = 0;
+
+		HASH_ITER (hh, task->cfg->actions, act, tmp) {
+			if (!(act->flags & RSPAMD_ACTION_NO_THRESHOLD)) {
+				metric_res->actions_limits[i].cur_limit = act->threshold;
+			}
+			metric_res->actions_limits[i].action = act;
+
+			i ++;
 		}
+
+		metric_res->nactions = i;
 	}
 
 	rspamd_mempool_add_destructor (task->task_pool,
@@ -96,7 +106,7 @@ rspamd_pr_sort (const struct rspamd_passthrough_result *pra,
 
 void
 rspamd_add_passthrough_result (struct rspamd_task *task,
-									enum rspamd_action_type action,
+									struct rspamd_action *action,
 									guint priority,
 									double target_score,
 									const gchar *message,
@@ -119,13 +129,13 @@ rspamd_add_passthrough_result (struct rspamd_task *task,
 
 	if (!isnan (target_score)) {
 		msg_info_task ("<%s>: set pre-result to %s (%.2f): '%s' from %s(%d)",
-				task->message_id, rspamd_action_to_str (action), target_score,
+				task->message_id, action->name, target_score,
 				message, module, priority);
 	}
 
 	else {
 		msg_info_task ("<%s>: set pre-result to %s (no score): '%s' from %s(%d)",
-				task->message_id, rspamd_action_to_str (action),
+				task->message_id, action->name,
 				message, module, priority);
 	}
 }
@@ -475,43 +485,50 @@ rspamd_task_add_result_option (struct rspamd_task *task,
 	return ret;
 }
 
-enum rspamd_action_type
-rspamd_check_action_metric (struct rspamd_task *task, struct rspamd_metric_result *mres)
+struct rspamd_action*
+rspamd_check_action_metric (struct rspamd_task *task)
 {
-	struct rspamd_action *action, *selected_action = NULL;
+	struct rspamd_action_result *action_lim,
+			*noaction = NULL;
+	struct rspamd_action *selected_action = NULL;
 	struct rspamd_passthrough_result *pr;
 	double max_score = -(G_MAXDOUBLE), sc;
 	int i;
-	gboolean set_action = FALSE;
+	struct rspamd_metric_result *mres = task->result;
 
 	/* We are not certain about the results during processing */
-	if (task->result->passthrough_result == NULL) {
-		for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) {
-			action = &task->cfg->actions[i];
-			sc = mres->actions_limits[i];
+	if (mres->passthrough_result == NULL) {
+		for (i = mres->nactions - 1; i >= 0; i--) {
+			action_lim = &mres->actions_limits[i];
+			sc = action_lim->cur_limit;
+
+			if (action_lim->action->action_type == METRIC_ACTION_NOACTION) {
+				noaction = action_lim;
+			}
 
-			if (isnan (sc)) {
+			if (isnan (sc) ||
+			 	(action_lim->action->flags & (RSPAMD_ACTION_NO_THRESHOLD|RSPAMD_ACTION_HAM))) {
 				continue;
 			}
 
 			if (mres->score >= sc && sc > max_score) {
-				selected_action = action;
+				selected_action = action_lim->action;
 				max_score = sc;
 			}
 		}
 
-		if (set_action && selected_action == NULL) {
-			selected_action = &task->cfg->actions[METRIC_ACTION_NOACTION];
+		if (selected_action == NULL) {
+			selected_action = noaction->action;
 		}
 	}
 	else {
 		/* Peek the highest priority result */
-		pr = task->result->passthrough_result;
+		pr = mres->passthrough_result;
 		sc = pr->target_score;
-		selected_action = &task->cfg->actions[pr->action];
+		selected_action = pr->action;
 
 		if (!isnan (sc)) {
-			if (pr->action == METRIC_ACTION_NOACTION) {
+			if (pr->action->action_type == METRIC_ACTION_NOACTION) {
 				mres->score = MIN (sc, mres->score);
 			}
 			else {
@@ -521,10 +538,10 @@ rspamd_check_action_metric (struct rspamd_task *task, struct rspamd_metric_resul
 	}
 
 	if (selected_action) {
-		return selected_action->action;
+		return selected_action;
 	}
 
-	return METRIC_ACTION_NOACTION;
+	return noaction ? noaction->action : NULL;
 }
 
 struct rspamd_symbol_result*

src/libmime/filter.h

@@ -9,7 +9,6 @@
 #include "config.h"
 #include "rspamd_symcache.h"
 #include "task.h"
-#include "khash.h"
 
 struct rspamd_task;
 struct rspamd_settings;
@@ -25,14 +24,14 @@ enum rspamd_symbol_result_flags {
 	RSPAMD_SYMBOL_RESULT_IGNORED = (1 << 0)
 };
 
+struct kh_rspamd_options_hash_s;
+
 /**
  * Rspamd symbol
  */
-
-KHASH_MAP_INIT_STR (rspamd_options_hash, struct rspamd_symbol_option *);
 struct rspamd_symbol_result {
 	double score;                                  /**< symbol's score							*/
-	khash_t(rspamd_options_hash) *options;         /**< list of symbol's options				*/
+	struct kh_rspamd_options_hash_s *options;         /**< list of symbol's options				*/
 	struct rspamd_symbol_option *opts_head;        /**< head of linked list of options			*/
 	const gchar *name;
 	struct rspamd_symbol *sym;                     /**< symbol configuration					*/
@@ -40,24 +39,6 @@ struct rspamd_symbol_result {
 	enum rspamd_symbol_result_flags flags;
 };
 
-/**
- * Result of metric processing
- */
-KHASH_MAP_INIT_STR (rspamd_symbols_hash, struct rspamd_symbol_result);
-#if UINTPTR_MAX <= UINT_MAX
-/* 32 bit */
-#define rspamd_ptr_hash_func(key) (khint32_t)(((uintptr_t)(key))>>1)
-#else
-/* likely 64 bit */
-#define rspamd_ptr_hash_func(key) (khint32_t)(((uintptr_t)(key))>>3)
-#endif
-#define rspamd_ptr_equal_func(a, b) ((a) == (b))
-KHASH_INIT (rspamd_symbols_group_hash,
-		void *,
-		double,
-		1,
-		rspamd_ptr_hash_func,
-		rspamd_ptr_equal_func);
 
 #define RSPAMD_PASSTHROUGH_NORMAL 1
 #define RSPAMD_PASSTHROUGH_LOW 0
@@ -65,7 +46,7 @@ KHASH_INIT (rspamd_symbols_group_hash,
 #define RSPAMD_PASSTHROUGH_CRITICAL 3
 
 struct rspamd_passthrough_result {
-	enum rspamd_action_type action;
+	struct rspamd_action *action;
 	guint priority;
 	double target_score;
 	const gchar *message;
@@ -73,6 +54,15 @@ struct rspamd_passthrough_result {
 	struct rspamd_passthrough_result *prev, *next;
 };
 
+struct rspamd_action_result {
+	gdouble cur_limit;
+	struct rspamd_action *action;
+};
+
+struct kh_rspamd_symbols_hash_s;
+struct kh_rspamd_symbols_group_hash_s;
+
+
 struct rspamd_metric_result {
 	double score;									/**< total score							*/
 	double grow_factor;								/**< current grow factor					*/
@@ -81,9 +71,10 @@ struct rspamd_metric_result {
 	guint nnegative;
 	double positive_score;
 	double negative_score;
-	khash_t(rspamd_symbols_hash) *symbols;			/**< symbols of metric						*/
-	khash_t(rspamd_symbols_group_hash) *sym_groups; /**< groups of symbols						*/
-	gdouble actions_limits[METRIC_ACTION_MAX];		/**< set of actions for this metric			*/
+	struct kh_rspamd_symbols_hash_s *symbols;			/**< symbols of metric						*/
+	struct kh_rspamd_symbols_group_hash_s *sym_groups; /**< groups of symbols						*/
+	struct rspamd_action_result *actions_limits;
+	guint nactions;
 };
 
 /**
@@ -103,7 +94,7 @@ struct rspamd_metric_result * rspamd_create_metric_result (struct rspamd_task *t
  * @param module
  */
 void rspamd_add_passthrough_result (struct rspamd_task *task,
-									enum rspamd_action_type action,
+									struct rspamd_action *action,
 									guint priority,
 									double target_score,
 									const gchar *message,
@@ -175,10 +166,11 @@ double rspamd_factor_consolidation_func (struct rspamd_task *task,
 	const gchar *unused);
 
 
-/*
- * Get action for specific metric
+/**
+ * Check thresholds and return action for a task
+ * @param task
+ * @return
  */
-enum rspamd_action_type rspamd_check_action_metric (struct rspamd_task *task,
-	struct rspamd_metric_result *mres);
+struct rspamd_action* rspamd_check_action_metric (struct rspamd_task *task);
 
 #endif

src/libmime/filter_private.h

@@ -0,0 +1,31 @@
+//
+// Created by Vsevolod Stakhov on 2019-01-14.
+//
+
+#ifndef RSPAMD_FILTER_PRIVATE_H
+#define RSPAMD_FILTER_PRIVATE_H
+
+#include "filter.h"
+#include "contrib/libucl/khash.h"
+
+KHASH_MAP_INIT_STR (rspamd_options_hash, struct rspamd_symbol_option *);
+/**
+ * Result of metric processing
+ */
+KHASH_MAP_INIT_STR (rspamd_symbols_hash, struct rspamd_symbol_result);
+#if UINTPTR_MAX <= UINT_MAX
+/* 32 bit */
+#define rspamd_ptr_hash_func(key) (khint32_t)(((uintptr_t)(key))>>1)
+#else
+/* likely 64 bit */
+#define rspamd_ptr_hash_func(key) (khint32_t)(((uintptr_t)(key))>>3)
+#endif
+#define rspamd_ptr_equal_func(a, b) ((a) == (b))
+KHASH_INIT (rspamd_symbols_group_hash,
+		void *,
+		double,
+		1,
+		rspamd_ptr_hash_func,
+		rspamd_ptr_equal_func);
+
+#endif //RSPAMD_FILTER_PRIVATE_H

src/libmime/images.c

@@ -92,14 +92,14 @@ detect_image_type (rspamd_ftok_t *data)
 
 
 static struct rspamd_image *
-process_png_image (struct rspamd_task *task, rspamd_ftok_t *data)
+process_png_image (rspamd_mempool_t *pool, rspamd_ftok_t *data)
 {
 	struct rspamd_image *img;
 	guint32 t;
 	const guint8 *p;
 
 	if (data->len < 24) {
-		msg_info_task ("bad png detected (maybe striped)");
+		msg_info_pool ("bad png detected (maybe striped)");
 		return NULL;
 	}
 
@@ -107,11 +107,11 @@ process_png_image (struct rspamd_task *task, rspamd_ftok_t *data)
 	/* Skip signature and read header section */
 	p = data->begin + 12;
 	if (memcmp (p, "IHDR", 4) != 0) {
-		msg_info_task ("png doesn't begins with IHDR section");
+		msg_info_pool ("png doesn't begins with IHDR section");
 		return NULL;
 	}
 
-	img = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct rspamd_image));
+	img = rspamd_mempool_alloc0 (pool, sizeof (struct rspamd_image));
 	img->type = IMAGE_TYPE_PNG;
 	img->data = data;
 
@@ -126,13 +126,13 @@ process_png_image (struct rspamd_task *task, rspamd_ftok_t *data)
 }
 
 static struct rspamd_image *
-process_jpg_image (struct rspamd_task *task, rspamd_ftok_t *data)
+process_jpg_image (rspamd_mempool_t *pool, rspamd_ftok_t *data)
 {
 	const guint8 *p, *end;
 	guint16 h, w;
 	struct rspamd_image *img;
 
-	img = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct rspamd_image));
+	img = rspamd_mempool_alloc0 (pool, sizeof (struct rspamd_image));
 	img->type = IMAGE_TYPE_JPG;
 	img->data = data;
 
@@ -169,18 +169,18 @@ process_jpg_image (struct rspamd_task *task, rspamd_ftok_t *data)
 }
 
 static struct rspamd_image *
-process_gif_image (struct rspamd_task *task, rspamd_ftok_t *data)
+process_gif_image (rspamd_mempool_t *pool, rspamd_ftok_t *data)
 {
 	struct rspamd_image *img;
 	const guint8 *p;
 	guint16 t;
 
 	if (data->len < 10) {
-		msg_info_task ("bad gif detected (maybe striped)");
+		msg_info_pool ("bad gif detected (maybe striped)");
 		return NULL;
 	}
 
-	img = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct rspamd_image));
+	img = rspamd_mempool_alloc0 (pool, sizeof (struct rspamd_image));
 	img->type = IMAGE_TYPE_GIF;
 	img->data = data;
 
@@ -194,18 +194,18 @@ process_gif_image (struct rspamd_task *task, rspamd_ftok_t *data)
 }
 
 static struct rspamd_image *
-process_bmp_image (struct rspamd_task *task, rspamd_ftok_t *data)
+process_bmp_image (rspamd_mempool_t *pool, rspamd_ftok_t *data)
 {
 	struct rspamd_image *img;
 	gint32 t;
 	const guint8 *p;
 
 	if (data->len < 28) {
-		msg_info_task ("bad bmp detected (maybe striped)");
+		msg_info_pool ("bad bmp detected (maybe striped)");
 		return NULL;
 	}
 
-	img = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct rspamd_image));
+	img = rspamd_mempool_alloc0 (pool, sizeof (struct rspamd_image));
 	img->type = IMAGE_TYPE_BMP;
 	img->data = data;
 	p = data->begin + 18;
@@ -558,31 +558,26 @@ rspamd_image_normalize (struct rspamd_task *task, struct rspamd_image *img)
 #endif
 }
 
-static void
-process_image (struct rspamd_task *task, struct rspamd_mime_part *part)
+struct rspamd_image*
+rspamd_maybe_process_image (rspamd_mempool_t *pool,
+							rspamd_ftok_t *data)
 {
 	enum rspamd_image_type type;
 	struct rspamd_image *img = NULL;
-	struct rspamd_mime_header *rh;
-	struct rspamd_mime_text_part *tp;
-	struct html_image *himg;
-	const gchar *cid, *html_cid;
-	guint cid_len, i, j;
-	GPtrArray *ar;
 
-	if ((type = detect_image_type (&part->parsed_data)) != IMAGE_TYPE_UNKNOWN) {
+	if ((type = detect_image_type (data)) != IMAGE_TYPE_UNKNOWN) {
 		switch (type) {
 		case IMAGE_TYPE_PNG:
-			img = process_png_image (task, &part->parsed_data);
+			img = process_png_image (pool, data);
 			break;
 		case IMAGE_TYPE_JPG:
-			img = process_jpg_image (task, &part->parsed_data);
+			img = process_jpg_image (pool, data);
 			break;
 		case IMAGE_TYPE_GIF:
-			img = process_gif_image (task, &part->parsed_data);
+			img = process_gif_image (pool, data);
 			break;
 		case IMAGE_TYPE_BMP:
-			img = process_bmp_image (task, &part->parsed_data);
+			img = process_bmp_image (pool, data);
 			break;
 		default:
 			img = NULL;
@@ -590,6 +585,23 @@ process_image (struct rspamd_task *task, struct rspamd_mime_part *part)
 		}
 	}
 
+	return img;
+}
+
+static void
+process_image (struct rspamd_task *task, struct rspamd_mime_part *part)
+{
+	struct rspamd_mime_header *rh;
+	struct rspamd_mime_text_part *tp;
+	struct html_image *himg;
+	const gchar *cid, *html_cid;
+	guint cid_len, i, j;
+	GPtrArray *ar;
+	struct rspamd_image *img;
+
+
+	img = rspamd_maybe_process_image (task->task_pool, &part->parsed_data);
+
 	if (img != NULL) {
 		debug_task ("detected %s image of size %ud x %ud in message <%s>",
 			rspamd_image_type_str (img->type),
@@ -643,6 +655,7 @@ process_image (struct rspamd_task *task, struct rspamd_mime_part *part)
 								if (strlen (html_cid) == cid_len &&
 										memcmp (html_cid, cid, cid_len) == 0) {
 									img->html_image = himg;
+									himg->embedded_image = img;
 
 									debug_task ("found linked image by cid: <%s>",
 											cid);

src/libmime/images.h

@@ -35,6 +35,15 @@ struct rspamd_image {
  */
 void rspamd_images_process (struct rspamd_task *task);
 
+/**
+ * Processes image in raw data
+ * @param task
+ * @param data
+ * @return
+ */
+struct rspamd_image* rspamd_maybe_process_image (rspamd_mempool_t *pool,
+		rspamd_ftok_t *data);
+
 /*
  * Get textual representation of an image's type
  */

src/libmime/message.c

@@ -34,6 +34,7 @@
 
 #include <math.h>
 #include <unicode/uchar.h>
+#include <src/libserver/cfg_file_private.h>
 
 #define GTUBE_SYMBOL "GTUBE"
 
@@ -238,7 +239,7 @@ rspamd_strip_newlines_parse (struct rspamd_task *task,
 
 			if (uc != -1) {
 				while (p < pe) {
-					if (uc == 0x200b) {
+					if (IS_ZERO_WIDTH_SPACE (uc)) {
 						/* Invisible space ! */
 						task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE;
 						part->spaces ++;
@@ -252,7 +253,7 @@ rspamd_strip_newlines_parse (struct rspamd_task *task,
 
 						U8_NEXT (begin, off, pe - begin, uc);
 
-						if (uc != 0x200b) {
+						if (!IS_ZERO_WIDTH_SPACE (uc)) {
 							break;
 						}
 
@@ -561,8 +562,8 @@ rspamd_words_levenshtein_distance (struct rspamd_task *task,
 	s2len = w2->len;
 
 	if (s1len + s2len > max_words) {
-		msg_err_task ("cannot compare parts with more than %ud words: %ud",
-				max_words, s1len);
+		msg_err_task ("cannot compare parts with more than %ud words: (%ud + %ud)",
+				max_words, s1len, s2len);
 		return 0;
 	}
 
@@ -883,23 +884,23 @@ rspamd_message_process_text_part_maybe (struct rspamd_task *task,
 
 	act = rspamd_check_gtube (task, text_part);
 	if (act != METRIC_ACTION_NOACTION) {
-		struct rspamd_metric_result *mres = task->result;
+		struct rspamd_action *action;
 		gdouble score = NAN;
 
-		if (act == METRIC_ACTION_REJECT) {
-			score = rspamd_task_get_required_score (task, mres);
-		}
-		else {
-			score = mres->actions_limits[act];
-		}
+		action = rspamd_config_get_action_by_type (task->cfg, act);
 
-		rspamd_add_passthrough_result (task, act, RSPAMD_PASSTHROUGH_CRITICAL,
-				score, "Gtube pattern", "GTUBE");
+		if (action) {
+			score = action->threshold;
 
-		if (ucl_object_lookup (task->messages, "smtp_message") == NULL) {
-			ucl_object_replace_key (task->messages,
-					ucl_object_fromstring ("Gtube pattern"), "smtp_message", 0,
-					false);
+			rspamd_add_passthrough_result (task, action,
+					RSPAMD_PASSTHROUGH_CRITICAL,
+					score, "Gtube pattern", "GTUBE");
+
+			if (ucl_object_lookup (task->messages, "smtp_message") == NULL) {
+				ucl_object_replace_key (task->messages,
+						ucl_object_fromstring ("Gtube pattern"),
+						"smtp_message", 0, false);
+			}
 		}
 
 		rspamd_task_insert_result (task, GTUBE_SYMBOL, 0, NULL);

src/libmime/message.h

@@ -125,36 +125,6 @@ struct rspamd_mime_text_part {
 	guint unicode_scripts;
 };
 
-enum rspamd_received_type {
-	RSPAMD_RECEIVED_SMTP = 0,
-	RSPAMD_RECEIVED_ESMTP,
-	RSPAMD_RECEIVED_ESMTPA,
-	RSPAMD_RECEIVED_ESMTPS,
-	RSPAMD_RECEIVED_ESMTPSA,
-	RSPAMD_RECEIVED_LMTP,
-	RSPAMD_RECEIVED_IMAP,
-	RSPAMD_RECEIVED_UNKNOWN
-};
-
-#define RSPAMD_RECEIVED_FLAG_ARTIFICIAL (1 << 0)
-#define RSPAMD_RECEIVED_FLAG_SSL (1 << 1)
-#define RSPAMD_RECEIVED_FLAG_AUTHENTICATED (1 << 2)
-
-struct received_header {
-	gchar *from_hostname;
-	gchar *from_ip;
-	gchar *real_hostname;
-	gchar *real_ip;
-	gchar *by_hostname;
-	gchar *for_mbox;
-	gchar *comment_ip;
-	rspamd_inet_addr_t *addr;
-	struct rspamd_mime_header *hdr;
-	time_t timestamp;
-	enum rspamd_received_type type;
-	gint flags;
-};
-
 /**
  * Parse and pre-process mime message
  * @param task worker_task object

src/libmime/mime_encoding.c

@@ -180,6 +180,9 @@ rspamd_mime_get_converter_cached (const gchar *enc, UErrorCode *err)
 			conv = g_malloc0 (sizeof (*conv));
 			conv->is_internal = TRUE;
 			conv->d.cnv_table = iso_8859_16_map;
+			conv->canon_name = g_strdup (canon_name);
+
+			rspamd_lru_hash_insert (cache, conv->canon_name, conv, 0, 0);
 		}
 	}
 

src/libmime/mime_expressions.c

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#include <contrib/libucl/ucl.h>
 #include "config.h"
 #include "util.h"
 #include "cfg_file.h"
@@ -117,7 +118,8 @@ struct rspamd_function_atom {
 enum rspamd_mime_atom_type {
 	MIME_ATOM_REGEXP = 0,
 	MIME_ATOM_INTERNAL_FUNCTION,
-	MIME_ATOM_LUA_FUNCTION
+	MIME_ATOM_LUA_FUNCTION,
+	MIME_ATOM_LOCAL_LUA_FUNCTION, /* New style */
 };
 
 struct rspamd_mime_atom {
@@ -126,6 +128,7 @@ struct rspamd_mime_atom {
 		struct rspamd_regexp_atom *re;
 		struct rspamd_function_atom *func;
 		const gchar *lua_function;
+		gint lua_cbref;
 	} d;
 	enum rspamd_mime_atom_type type;
 };
@@ -637,8 +640,9 @@ rspamd_mime_expr_parse (const gchar *line, gsize len,
 {
 	rspamd_expression_atom_t *a = NULL;
 	struct rspamd_mime_atom *mime_atom = NULL;
-	const gchar *p, *end;
-	struct rspamd_config *cfg = ud;
+	const gchar *p, *end, *c = NULL;
+	struct rspamd_mime_expr_ud *real_ud = (struct rspamd_mime_expr_ud *)ud;
+	struct rspamd_config *cfg;
 	rspamd_regexp_t *own_re;
 	gchar t;
 	gint type = MIME_ATOM_REGEXP, obraces = 0, ebraces = 0;
@@ -652,6 +656,7 @@ rspamd_mime_expr_parse (const gchar *line, gsize len,
 		in_flags_brace,
 		got_obrace,
 		in_function,
+		in_local_function,
 		got_ebrace,
 		end_atom,
 		bad_atom
@@ -659,6 +664,7 @@ rspamd_mime_expr_parse (const gchar *line, gsize len,
 
 	p = line;
 	end = p + len;
+	cfg = real_ud->cfg;
 
 	while (p < end) {
 		t = *p;
@@ -674,11 +680,20 @@ rspamd_mime_expr_parse (const gchar *line, gsize len,
 				state = got_obrace;
 			}
 			else if (!g_ascii_isalnum (t) && t != '_' && t != '-' && t != '=') {
-				/* Likely lua function, identified by just a string */
-				type = MIME_ATOM_LUA_FUNCTION;
-				state = end_atom;
-				/* Do not increase p */
-				continue;
+				if (t == ':') {
+					if (p - line == 3 && memcmp (line, "lua", 3) == 0) {
+						type = MIME_ATOM_LOCAL_LUA_FUNCTION;
+						state = in_local_function;
+						c = p + 1;
+					}
+				}
+				else {
+					/* Likely lua function, identified by just a string */
+					type = MIME_ATOM_LUA_FUNCTION;
+					state = end_atom;
+					/* Do not increase p */
+					continue;
+				}
 			}
 			else if (g_ascii_isspace (t)) {
 				state = bad_atom;
@@ -744,6 +759,15 @@ rspamd_mime_expr_parse (const gchar *line, gsize len,
 			}
 			p ++;
 			break;
+		case in_local_function:
+			if (!(g_ascii_isalnum (t) || t == '-' || t == '_')) {
+				g_assert (c != NULL);
+				state = end_atom;
+			}
+			else {
+				p++;
+			}
+			break;
 		case got_ebrace:
 			state = end_atom;
 			break;
@@ -851,8 +875,59 @@ set:
 
 			goto err;
 		}
+
 		lua_pop (cfg->lua_state, 1);
 	}
+	else if (type == MIME_ATOM_LOCAL_LUA_FUNCTION) {
+		/* p pointer is set to the start of Lua function name */
+
+		if (real_ud->conf_obj == NULL) {
+			g_set_error (err, rspamd_mime_expr_quark(), 300,
+					"no config object for '%s'",
+					mime_atom->str);
+			goto err;
+		}
+
+		const ucl_object_t *functions = ucl_object_lookup (real_ud->conf_obj,
+				"functions");
+
+		if (functions == NULL) {
+			g_set_error (err, rspamd_mime_expr_quark(), 310,
+					"no functions defined for '%s'",
+					mime_atom->str);
+			goto err;
+		}
+
+		if (ucl_object_type (functions) != UCL_OBJECT) {
+			g_set_error (err, rspamd_mime_expr_quark(), 320,
+					"functions is not a table for '%s'",
+					mime_atom->str);
+			goto err;
+		}
+
+		const ucl_object_t *function_obj;
+
+		function_obj = ucl_object_lookup_len (functions, c,
+				p - c);
+
+		if (function_obj == NULL) {
+			g_set_error (err, rspamd_mime_expr_quark(), 320,
+					"function %*.s is not found for '%s'",
+					(int)(p - c), c, mime_atom->str);
+			goto err;
+		}
+
+		if (ucl_object_type (function_obj) != UCL_USERDATA) {
+			g_set_error (err, rspamd_mime_expr_quark(), 320,
+					"function %*.s has invalid type for '%s'",
+					(int)(p - c), c, mime_atom->str);
+			goto err;
+		}
+
+		struct ucl_lua_funcdata *fd = function_obj->value.ud;
+
+		mime_atom->d.lua_cbref = fd->idx;
+	}
 	else {
 		mime_atom->d.func = rspamd_mime_expr_parse_function_atom (pool,
 				mime_atom->str);
@@ -933,6 +1008,7 @@ rspamd_mime_expr_priority (rspamd_expression_atom_t *atom)
 		ret = 50;
 		break;
 	case MIME_ATOM_LUA_FUNCTION:
+	case MIME_ATOM_LOCAL_LUA_FUNCTION:
 		ret = 50;
 		break;
 	case MIME_ATOM_REGEXP:
@@ -1036,6 +1112,41 @@ rspamd_mime_expr_process (struct rspamd_expr_process_data *process_data, rspamd_
 			lua_pop (L, 1);
 		}
 	}
+	else if (mime_atom->type == MIME_ATOM_LOCAL_LUA_FUNCTION) {
+		gint err_idx;
+		GString *tb;
+
+		L = task->cfg->lua_state;
+		lua_pushcfunction (L, &rspamd_lua_traceback);
+		err_idx = lua_gettop (L);
+
+		lua_rawgeti (L, LUA_REGISTRYINDEX, mime_atom->d.lua_cbref);
+		rspamd_lua_task_push (L, task);
+
+		if (lua_pcall (L, 1, 1, err_idx) != 0) {
+			tb = lua_touserdata (L, -1);
+			msg_info_task ("lua call to local function for atom '%s' failed: %v",
+					mime_atom->str,
+					tb);
+			if (tb) {
+				g_string_free (tb, TRUE);
+			}
+		}
+		else {
+			if (lua_type (L, -1) == LUA_TBOOLEAN) {
+				ret = lua_toboolean (L, -1);
+			}
+			else if (lua_type (L, -1) == LUA_TNUMBER) {
+				ret = lua_tonumber (L, 1);
+			}
+			else {
+				msg_err_task ("%s returned wrong return type: %s",
+						mime_atom->str, lua_typename (L, lua_type (L, -1)));
+			}
+		}
+
+		lua_settop (L, 0);
+	}
 	else {
 		ret = rspamd_mime_expr_process_function (mime_atom->d.func, task,
 				task->cfg->lua_state);
@@ -1391,7 +1502,7 @@ rspamd_is_html_balanced (struct rspamd_task * task, GArray * args, void *unused)
 	for (i = 0; i < task->text_parts->len; i ++) {
 
 		p = g_ptr_array_index (task->text_parts, i);
-		if (!IS_PART_EMPTY (p) && IS_PART_HTML (p)) {
+		if (IS_PART_HTML (p)) {
 			if (p->flags & RSPAMD_MIME_TEXT_PART_FLAG_BALANCED) {
 				res = TRUE;
 			}
@@ -1428,7 +1539,7 @@ rspamd_has_html_tag (struct rspamd_task * task, GArray * args, void *unused)
 	for (i = 0; i < task->text_parts->len; i ++) {
 		p = g_ptr_array_index (task->text_parts, i);
 
-		if (!IS_PART_EMPTY (p) && IS_PART_HTML (p) && p->html) {
+		if (IS_PART_HTML (p) && p->html) {
 			res = rspamd_html_tag_seen (p->html, arg->data);
 		}
 
@@ -1451,7 +1562,7 @@ rspamd_has_fake_html (struct rspamd_task * task, GArray * args, void *unused)
 	for (i = 0; i < task->text_parts->len; i ++) {
 		p = g_ptr_array_index (task->text_parts, i);
 
-		if (!IS_PART_EMPTY (p) && IS_PART_HTML (p) && p->html->html_tags == NULL) {
+		if (IS_PART_HTML (p) && (p->html == NULL || p->html->html_tags == NULL)) {
 			res = TRUE;
 		}
 

src/libmime/mime_expressions.h

@@ -8,8 +8,15 @@
 
 #include "config.h"
 #include "expression.h"
+#include "contrib/libucl/ucl.h"
 
 struct rspamd_task;
+struct rspamd_config;
+
+struct rspamd_mime_expr_ud {
+	struct rspamd_config *cfg;
+	const ucl_object_t *conf_obj;
+};
 
 extern const struct rspamd_atom_subr mime_expr_subr;
 

src/libmime/mime_headers.c

@@ -17,7 +17,9 @@
 #include "mime_headers.h"
 #include "smtp_parsers.h"
 #include "mime_encoding.h"
+#include "contrib/uthash/utlist.h"
 #include "libserver/mempool_vars_internal.h"
+#include "libserver/url.h"
 #include <unicode/utf8.h>
 
 static void
@@ -36,19 +38,12 @@ rspamd_mime_header_check_special (struct rspamd_task *task,
 		recv = rspamd_mempool_alloc0 (task->task_pool,
 				sizeof (struct received_header));
 		recv->hdr = rh;
-		rspamd_smtp_received_parse (task, rh->decoded,
-				strlen (rh->decoded), recv);
-		/* Set flags */
-		if (recv->type == RSPAMD_RECEIVED_ESMTPA ||
-				recv->type == RSPAMD_RECEIVED_ESMTPSA) {
-			recv->flags |= RSPAMD_RECEIVED_FLAG_AUTHENTICATED;
-		}
-		if (recv->type == RSPAMD_RECEIVED_ESMTPS ||
-				recv->type == RSPAMD_RECEIVED_ESMTPSA) {
-			recv->flags |= RSPAMD_RECEIVED_FLAG_SSL;
+
+		if (rspamd_smtp_received_parse (task, rh->decoded,
+				strlen (rh->decoded), recv) != -1) {
+			g_ptr_array_add (task->received, recv);
 		}
 
-		g_ptr_array_add (task->received, recv);
 		rh->type = RSPAMD_HEADER_RECEIVED;
 		break;
 	case 0x76F31A09F4352521ULL:	/* to */
@@ -164,6 +159,7 @@ rspamd_mime_header_add (struct rspamd_task *task,
 	}
 }
 
+
 /* Convert raw headers to a list of struct raw_header * */
 void
 rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
@@ -207,7 +203,7 @@ rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
 						sizeof (struct rspamd_mime_header));
 				l = p - c;
 				tmp = rspamd_mempool_alloc (task->task_pool, l + 1);
-				rspamd_strlcpy (tmp, c, l + 1);
+				rspamd_null_safe_copy (c, l, tmp, l + 1);
 				nh->name = tmp;
 				nh->empty_separator = TRUE;
 				nh->raw_value = c;
@@ -256,7 +252,7 @@ rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
 				l = p - c;
 				if (l > 0) {
 					tmp = rspamd_mempool_alloc (task->task_pool, l + 1);
-					rspamd_strlcpy (tmp, c, l + 1);
+					rspamd_null_safe_copy (c, l, tmp, l + 1);
 					nh->separator = tmp;
 				}
 				next_state = 3;
@@ -268,7 +264,7 @@ rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
 				l = p - c;
 				if (l >= 0) {
 					tmp = rspamd_mempool_alloc (task->task_pool, l + 1);
-					rspamd_strlcpy (tmp, c, l + 1);
+					rspamd_null_safe_copy (c, l, tmp, l + 1);
 					nh->separator = tmp;
 				}
 				c = p;
@@ -302,6 +298,12 @@ rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
 			break;
 		case 4:
 			/* Copy header's value */
+
+			/*
+			 * XXX:
+			 * The original decision to use here null terminated
+			 * strings was extremely poor!
+			 */
 			l = p - c;
 			tmp = rspamd_mempool_alloc (task->task_pool, l + 1);
 			tp = tmp;
@@ -315,7 +317,12 @@ rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
 						*tp++ = ' ';
 					}
 					else {
-						*tp++ = *c++;
+						if (*c != '\0') {
+							*tp++ = *c++;
+						}
+						else {
+							c++;
+						}
 					}
 				}
 				else if (t_state == 1) {
@@ -325,7 +332,12 @@ rspamd_mime_headers_process (struct rspamd_task *task, GHashTable *target,
 					}
 					else {
 						t_state = 0;
-						*tp++ = *c++;
+						if (*c != '\0') {
+							*tp++ = *c++;
+						}
+						else {
+							c++;
+						}
 					}
 				}
 			}
@@ -848,3 +860,727 @@ rspamd_mime_message_id_generate (const gchar *fqdn)
 
 	return g_string_free (out, FALSE);
 }
+
+enum rspamd_received_part_type {
+	RSPAMD_RECEIVED_PART_FROM,
+	RSPAMD_RECEIVED_PART_BY,
+	RSPAMD_RECEIVED_PART_FOR,
+	RSPAMD_RECEIVED_PART_WITH,
+	RSPAMD_RECEIVED_PART_UNKNOWN,
+};
+
+struct rspamd_received_comment {
+	gchar *data;
+	gsize dlen;
+	struct rspamd_received_comment *prev;
+};
+
+struct rspamd_received_part {
+	enum rspamd_received_part_type type;
+	gchar *data;
+	gsize dlen;
+	struct rspamd_received_comment *tail_comment;
+	struct rspamd_received_comment *head_comment;
+	struct rspamd_received_part *prev, *next;
+};
+
+static void
+rspamd_smtp_received_part_set_or_append (struct rspamd_task *task,
+										 const gchar *begin,
+										 gsize len,
+										 gchar **dest,
+										 gsize *destlen)
+{
+	if (len == 0) {
+		return;
+	}
+
+	if (*dest) {
+		/* Append */
+		gsize total_len = *destlen + len;
+		gchar *new_dest;
+
+		new_dest = rspamd_mempool_alloc (task->task_pool, total_len);
+		memcpy (new_dest, *dest, *destlen);
+		memcpy (new_dest + *destlen, begin, len);
+		rspamd_str_lc (new_dest + *destlen, len);
+		*dest = new_dest;
+		*destlen = total_len;
+	}
+	else {
+		/* Set */
+		*dest = rspamd_mempool_alloc (task->task_pool, len);
+		memcpy (*dest, begin, len);
+		rspamd_str_lc (*dest, len);
+		*dest = (gchar *)rspamd_string_len_strip (*dest, &len, " \t");
+		*destlen = len;
+	}
+}
+
+static struct rspamd_received_part *
+rspamd_smtp_received_process_part (struct rspamd_task *task,
+								   const char *data,
+								   size_t len,
+								   enum rspamd_received_part_type type,
+								   goffset *last)
+{
+	struct rspamd_received_part *npart;
+	const guchar *p, *c, *end;
+	guint obraces = 0, ebraces = 0;
+	gboolean seen_tcpinfo = FALSE;
+	enum _parse_state {
+		skip_spaces,
+		in_comment,
+		read_data,
+		read_tcpinfo,
+		all_done
+	} state, next_state;
+
+	npart = rspamd_mempool_alloc0 (task->task_pool, sizeof (*npart));
+	npart->type = type;
+
+	/* In this function, we just process comments and data separately */
+	p = data;
+	end = data + len;
+	c = data;
+	state = skip_spaces;
+	next_state = read_data;
+
+	while (p < end) {
+		switch (state) {
+		case skip_spaces:
+			if (!g_ascii_isspace (*p)) {
+				c = p;
+				state = next_state;
+			}
+			else {
+				p ++;
+			}
+			break;
+		case in_comment:
+			if (*p == '(') {
+				obraces ++;
+			}
+			else if (*p == ')') {
+				ebraces ++;
+
+				if (ebraces >= obraces) {
+					if (type != RSPAMD_RECEIVED_PART_UNKNOWN) {
+						if (p > c) {
+							struct rspamd_received_comment *comment;
+
+
+							comment = rspamd_mempool_alloc0 (task->task_pool,
+									sizeof (*comment));
+							rspamd_smtp_received_part_set_or_append (task,
+									c, p - c,
+									&comment->data, &comment->dlen);
+
+							if (!npart->head_comment) {
+								comment->prev = NULL;
+								npart->head_comment = comment;
+								npart->tail_comment = comment;
+							}
+							else {
+								comment->prev = npart->tail_comment;
+								npart->tail_comment = comment;
+							}
+						}
+					}
+
+					p ++;
+					c = p;
+					state = skip_spaces;
+					next_state = read_data;
+
+					continue;
+				}
+			}
+
+			p ++;
+			break;
+		case read_data:
+			if (*p == '(') {
+				if (p > c) {
+					if (type != RSPAMD_RECEIVED_PART_UNKNOWN) {
+						rspamd_smtp_received_part_set_or_append (task,
+								c, p - c,
+								&npart->data, &npart->dlen);
+					}
+				}
+
+				state = in_comment;
+				obraces = 1;
+				ebraces = 0;
+				p ++;
+				c = p;
+			}
+			else if (g_ascii_isspace (*p)) {
+				if (p > c) {
+					if (type != RSPAMD_RECEIVED_PART_UNKNOWN) {
+						rspamd_smtp_received_part_set_or_append (task,
+								c, p - c,
+								&npart->data, &npart->dlen);
+					}
+				}
+
+				state = skip_spaces;
+				next_state = read_data;
+				c = p;
+			}
+			else if (*p == ';') {
+				/* It is actually delimiter of date part if not in the comments */
+				if (p > c) {
+					if (type != RSPAMD_RECEIVED_PART_UNKNOWN) {
+						rspamd_smtp_received_part_set_or_append (task,
+								c, p - c,
+								&npart->data, &npart->dlen);
+					}
+				}
+
+				state = all_done;
+				continue;
+			}
+			else if (npart->dlen > 0) {
+				/* We have already received data and find something with no ( */
+				if (!seen_tcpinfo && type == RSPAMD_RECEIVED_PART_FROM) {
+					/* Check if we have something special here, such as TCPinfo */
+					if (*c == '[') {
+						state = read_tcpinfo;
+						p ++;
+					}
+					else {
+						state = all_done;
+						continue;
+					}
+				}
+				else {
+					state = all_done;
+					continue;
+				}
+			}
+			else {
+				p ++;
+			}
+			break;
+		case read_tcpinfo:
+			if (*p == ']') {
+				rspamd_smtp_received_part_set_or_append (task,
+						c, p - c + 1,
+						&npart->data, &npart->dlen);
+				seen_tcpinfo = TRUE;
+				state = skip_spaces;
+				next_state = read_data;
+				c = p;
+			}
+			p ++;
+			break;
+		case all_done:
+			*last = p - (const guchar *)data;
+			return npart;
+			break;
+		}
+	}
+
+	/* Leftover */
+	switch (state) {
+	case read_data:
+		if (p > c) {
+			if (type != RSPAMD_RECEIVED_PART_UNKNOWN) {
+				rspamd_smtp_received_part_set_or_append (task,
+						c, p - c,
+						&npart->data, &npart->dlen);
+			}
+
+			*last = p - (const guchar *)data;
+
+			return npart;
+		}
+		break;
+	case skip_spaces:
+		*last = p - (const guchar *)data;
+
+		return npart;
+	default:
+		break;
+	}
+
+	return NULL;
+}
+
+static struct rspamd_received_part *
+rspamd_smtp_received_spill (struct rspamd_task *task,
+							const char *data,
+							size_t len,
+							goffset *date_pos)
+{
+	const guchar *p, *end;
+	struct rspamd_received_part *cur_part, *head = NULL;
+	goffset pos = 0;
+
+	p = data;
+	end = data + len;
+
+	while (p < end && g_ascii_isspace (*p)) {
+		p ++;
+	}
+
+	len = end - p;
+
+	/* Ignore all received but those started from from part */
+	if (len <= 4 || (lc_map[p[0]] != 'f' &&
+					 lc_map[p[1]] != 'r' &&
+					 lc_map[p[2]] != 'o' &&
+					 lc_map[p[3]] != 'm')) {
+		return NULL;
+	}
+
+	p += sizeof ("from") - 1;
+
+	/* We can now store from part */
+	cur_part = rspamd_smtp_received_process_part (task, p, end - p,
+			RSPAMD_RECEIVED_PART_FROM, &pos);
+
+	if (!cur_part) {
+		return NULL;
+	}
+
+	g_assert (pos != 0);
+	p += pos;
+	len = end > p ? end - p : 0;
+	DL_APPEND (head, cur_part);
+
+
+	if (len > 2 && (lc_map[p[0]] == 'b' &&
+					lc_map[p[1]] == 'y')) {
+		p += sizeof ("by") - 1;
+
+		cur_part = rspamd_smtp_received_process_part (task, p, end - p,
+				RSPAMD_RECEIVED_PART_BY, &pos);
+
+		if (!cur_part) {
+			return NULL;
+		}
+
+		g_assert (pos != 0);
+		p += pos;
+		len = end > p ? end - p : 0;
+		DL_APPEND (head, cur_part);
+	}
+
+	while (p < end) {
+		if (*p == ';') {
+			/* We are at the date separator, stop here */
+			*date_pos = p - (const guchar *)data + 1;
+			break;
+		}
+		else {
+			if (len > sizeof ("with") && (lc_map[p[0]] == 'w' &&
+										  lc_map[p[1]] == 'i' &&
+										  lc_map[p[2]] == 't' &&
+										  lc_map[p[3]] == 'h')) {
+				p += sizeof ("with") - 1;
+
+				cur_part = rspamd_smtp_received_process_part (task, p, end - p,
+						RSPAMD_RECEIVED_PART_WITH, &pos);
+			}
+			else if (len > sizeof ("for") && (lc_map[p[0]] == 'f' &&
+											  lc_map[p[1]] == 'o' &&
+											  lc_map[p[2]] == 'r')) {
+				p += sizeof ("for") - 1;
+				cur_part = rspamd_smtp_received_process_part (task, p, end - p,
+						RSPAMD_RECEIVED_PART_FOR, &pos);
+			}
+			else {
+				while (p < end) {
+					if (!(g_ascii_isspace (*p) || *p == '(' || *p == ';')) {
+						p ++;
+					}
+					else {
+						break;
+					}
+				}
+
+				if (p == end) {
+					return NULL;
+				}
+				else if (*p == ';') {
+					*date_pos = p - (const guchar *)data + 1;
+					break;
+				}
+				else {
+					cur_part = rspamd_smtp_received_process_part (task, p, end - p,
+							RSPAMD_RECEIVED_PART_UNKNOWN, &pos);
+				}
+			}
+
+			if (!cur_part) {
+				return NULL;
+			}
+			else {
+				g_assert (pos != 0);
+				p += pos;
+				len = end > p ? end - p : 0;
+				DL_APPEND (head, cur_part);
+			}
+		}
+	}
+
+	return head;
+}
+
+static gboolean
+rspamd_smtp_received_process_rdns (struct rspamd_task *task,
+								   const gchar *begin,
+								   gsize len,
+								   const gchar **pdest)
+{
+	const gchar *p, *end;
+	gsize hlen = 0;
+	gboolean seen_dot = FALSE;
+
+	p = begin;
+	end = begin + len;
+
+	while (p < end) {
+		if (!g_ascii_isspace (*p) && rspamd_url_is_domain (*p)) {
+			if (*p == '.') {
+				seen_dot = TRUE;
+			}
+
+			hlen ++;
+		}
+		else {
+			break;
+		}
+
+		p ++;
+	}
+
+	if (hlen > 0) {
+		if (p == end) {
+			/* All data looks like a hostname */
+			gchar *dest;
+
+			dest = rspamd_mempool_alloc (task->task_pool,
+					hlen + 1);
+			rspamd_strlcpy (dest, begin, hlen + 1);
+			*pdest = dest;
+
+			return TRUE;
+		}
+		else if (seen_dot && (g_ascii_isspace (*p) || *p == '[' || *p == '(')) {
+			gchar *dest;
+
+			dest = rspamd_mempool_alloc (task->task_pool,
+					hlen + 1);
+			rspamd_strlcpy (dest, begin, hlen + 1);
+			*pdest = dest;
+
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+static gboolean
+rspamd_smtp_received_process_host_tcpinfo (struct rspamd_task *task,
+										   struct received_header *rh,
+										   const gchar *data,
+										   gsize len)
+{
+	rspamd_inet_addr_t *addr = NULL;
+	gboolean ret = FALSE;
+
+	if (data[0] == '[') {
+		/* Likely Exim version */
+
+		const gchar *brace_pos = memchr (data, ']', len);
+
+		if (brace_pos) {
+			addr = rspamd_parse_inet_address_pool (data + 1,
+					brace_pos - data - 1,
+					task->task_pool);
+
+			if (addr) {
+				rh->addr = addr;
+				rh->real_ip = rspamd_mempool_strdup (task->task_pool,
+						rspamd_inet_address_to_string (addr));
+				rh->from_ip = rh->real_ip;
+			}
+		}
+	}
+	else {
+		if (g_ascii_isxdigit (data[0])) {
+			/* Try to parse IP address */
+			addr = rspamd_parse_inet_address_pool (data,
+					len, task->task_pool);
+			if (addr) {
+				rh->addr = addr;
+				rh->real_ip = rspamd_mempool_strdup (task->task_pool,
+						rspamd_inet_address_to_string (addr));
+				rh->from_ip = rh->real_ip;
+			}
+		}
+
+		if (!addr) {
+			/* Try canonical Postfix version: rdns [ip] */
+			const gchar *obrace_pos = memchr (data, '[', len),
+					*ebrace_pos, *dend;
+
+			if (obrace_pos) {
+				dend = data + len;
+				ebrace_pos = memchr (obrace_pos, ']', dend - obrace_pos);
+
+				if (ebrace_pos) {
+					addr = rspamd_parse_inet_address_pool (obrace_pos + 1,
+							ebrace_pos - obrace_pos - 1,
+							task->task_pool);
+
+					if (addr) {
+						rh->addr = addr;
+						rh->real_ip = rspamd_mempool_strdup (task->task_pool,
+								rspamd_inet_address_to_string (addr));
+						rh->from_ip = rh->real_ip;
+
+						/* Process with rDNS */
+						if (rspamd_smtp_received_process_rdns (task,
+								data,
+								obrace_pos - data,
+								&rh->real_hostname)) {
+							ret = TRUE;
+						}
+					}
+				}
+			}
+			else {
+				/* Hostname or some crap, sigh... */
+				if (rspamd_smtp_received_process_rdns (task,
+						data,
+						len,
+						&rh->real_hostname)) {
+					ret = TRUE;
+				}
+			}
+		}
+	}
+
+	return ret;
+}
+
+static void
+rspamd_smtp_received_process_from (struct rspamd_task *task,
+								   struct rspamd_received_part *rpart,
+								   struct received_header *rh)
+{
+	if (rpart->dlen > 0) {
+		/* We have seen multiple cases:
+		 * - [ip] (hostname/unknown [real_ip])
+		 * - helo (hostname/unknown [real_ip])
+		 * - [ip]
+		 * - hostname
+		 * - hostname ([ip]:port helo=xxx)
+		 * Maybe more...
+		 */
+		gboolean seen_ip_in_data = FALSE, seen_rdns_in_comment = FALSE;
+
+		if (rpart->head_comment && rpart->head_comment->dlen > 0) {
+			/* We can have info within comment as part of RFC */
+			seen_rdns_in_comment = rspamd_smtp_received_process_host_tcpinfo (
+					task, rh,
+					rpart->head_comment->data, rpart->head_comment->dlen);
+		}
+
+		if (!rh->real_ip) {
+			if (rpart->data[0] == '[') {
+				/* No comment, just something that looks like SMTP IP */
+				const gchar *brace_pos = memchr (rpart->data, ']', rpart->dlen);
+				rspamd_inet_addr_t *addr;
+
+				if (brace_pos) {
+					addr = rspamd_parse_inet_address_pool (rpart->data + 1,
+							brace_pos - rpart->data - 1,
+							task->task_pool);
+
+					if (addr) {
+						seen_ip_in_data = TRUE;
+						rh->addr = addr;
+						rh->real_ip = rspamd_mempool_strdup (task->task_pool,
+								rspamd_inet_address_to_string (addr));
+						rh->from_ip = rh->real_ip;
+					}
+				}
+			} else if (g_ascii_isxdigit (rpart->data[0])) {
+				/* Try to parse IP address */
+				rspamd_inet_addr_t *addr;
+				addr = rspamd_parse_inet_address_pool (rpart->data,
+						rpart->dlen, task->task_pool);
+				if (addr) {
+					seen_ip_in_data = TRUE;
+					rh->addr = addr;
+					rh->real_ip = rspamd_mempool_strdup (task->task_pool,
+							rspamd_inet_address_to_string (addr));
+					rh->from_ip = rh->real_ip;
+				}
+			}
+		}
+
+		if (!seen_ip_in_data) {
+			if (rh->real_ip) {
+				/* Get anounced hostname (usually helo) */
+				rspamd_smtp_received_process_rdns (task,
+						rpart->data,
+						rpart->dlen,
+						&rh->from_hostname);
+			}
+			else {
+				rspamd_smtp_received_process_host_tcpinfo (task,
+						rh, rpart->data, rpart->dlen);
+			}
+		}
+	}
+	else {
+		/* rpart->dlen = 0 */
+
+		if (rpart->head_comment && rpart->head_comment->dlen > 0) {
+			rspamd_smtp_received_process_host_tcpinfo (task,
+					rh,
+					rpart->head_comment->data,
+					rpart->head_comment->dlen);
+		}
+	}
+}
+
+int
+rspamd_smtp_received_parse (struct rspamd_task *task,
+							const char *data,
+							size_t len,
+							struct received_header *rh)
+{
+	goffset date_pos = -1;
+	struct rspamd_received_part *head, *cur;
+	rspamd_ftok_t t1, t2;
+
+	head = rspamd_smtp_received_spill (task, data, len, &date_pos);
+
+	if (head == NULL) {
+		return -1;
+	}
+
+	rh->type = RSPAMD_RECEIVED_UNKNOWN;
+
+	DL_FOREACH (head, cur) {
+		switch (cur->type) {
+		case RSPAMD_RECEIVED_PART_FROM:
+			rspamd_smtp_received_process_from (task, cur, rh);
+			break;
+		case RSPAMD_RECEIVED_PART_BY:
+			rspamd_smtp_received_process_rdns (task,
+					cur->data,
+					cur->dlen,
+					&rh->by_hostname);
+			break;
+		case RSPAMD_RECEIVED_PART_WITH:
+			t1.begin = cur->data;
+			t1.len = cur->dlen;
+
+			if (t1.len > 0) {
+				RSPAMD_FTOK_ASSIGN (&t2, "smtp");
+
+				if (rspamd_ftok_cmp (&t1, &t2) == 0) {
+					rh->type = RSPAMD_RECEIVED_SMTP;
+				}
+
+				RSPAMD_FTOK_ASSIGN (&t2, "esmtp");
+
+				if (rspamd_ftok_starts_with (&t1, &t2)) {
+					/*
+					 * esmtp, esmtps, esmtpsa
+					 */
+					if (t1.len == t2.len + 1) {
+						if (t1.begin[t2.len] == 'a') {
+							rh->type = RSPAMD_RECEIVED_ESMTPA;
+							rh->flags |= RSPAMD_RECEIVED_FLAG_AUTHENTICATED;
+						}
+						else if (t1.begin[t2.len] == 's') {
+							rh->type = RSPAMD_RECEIVED_ESMTPS;
+							rh->flags |= RSPAMD_RECEIVED_FLAG_SSL;
+						}
+						continue;
+					}
+					else if (t1.len == t2.len + 2) {
+						if (t1.begin[t2.len] == 's' &&
+								t1.begin[t2.len + 1] == 'a') {
+							rh->type = RSPAMD_RECEIVED_ESMTPSA;
+							rh->flags |= RSPAMD_RECEIVED_FLAG_AUTHENTICATED;
+							rh->flags |= RSPAMD_RECEIVED_FLAG_SSL;
+						}
+						continue;
+					}
+					else if (t1.len == t2.len) {
+						rh->type = RSPAMD_RECEIVED_ESMTP;
+						continue;
+					}
+				}
+
+				RSPAMD_FTOK_ASSIGN (&t2, "lmtp");
+
+				if (rspamd_ftok_cmp (&t1, &t2) == 0) {
+					rh->type = RSPAMD_RECEIVED_LMTP;
+					continue;
+				}
+
+				RSPAMD_FTOK_ASSIGN (&t2, "imap");
+
+				if (rspamd_ftok_cmp (&t1, &t2) == 0) {
+					rh->type = RSPAMD_RECEIVED_IMAP;
+					continue;
+				}
+
+				RSPAMD_FTOK_ASSIGN (&t2, "local");
+
+				if (rspamd_ftok_cmp (&t1, &t2) == 0) {
+					rh->type = RSPAMD_RECEIVED_LOCAL;
+					continue;
+				}
+
+				RSPAMD_FTOK_ASSIGN (&t2, "http");
+
+				if (rspamd_ftok_starts_with (&t1, &t2)) {
+					if (t1.len == t2.len + 1) {
+						if (t1.begin[t2.len] == 's') {
+							rh->type = RSPAMD_RECEIVED_HTTP;
+							rh->flags |= RSPAMD_RECEIVED_FLAG_SSL;
+						}
+					}
+					else if (t1.len == t2.len) {
+						rh->type = RSPAMD_RECEIVED_HTTP;
+					}
+
+					continue;
+				}
+			}
+
+			break;
+		default:
+			/* Do nothing */
+			break;
+		}
+	}
+
+	if (rh->real_ip && !rh->from_ip) {
+		rh->from_ip = rh->real_ip;
+	}
+
+	if (rh->real_hostname && !rh->from_hostname) {
+		rh->from_hostname = rh->real_hostname;
+	}
+
+	if (date_pos > 0 && date_pos < len) {
+		rh->timestamp = rspamd_parse_smtp_date (data + date_pos,
+				len - date_pos);
+	}
+
+	return 0;
+}

src/libmime/mime_headers.h

@@ -18,6 +18,7 @@
 
 #include "config.h"
 #include "libutil/mem_pool.h"
+#include "libutil/addr.h"
 
 struct rspamd_task;
 
@@ -55,6 +56,38 @@ struct rspamd_mime_header {
 	gchar *decoded;
 };
 
+enum rspamd_received_type {
+	RSPAMD_RECEIVED_SMTP = 0,
+	RSPAMD_RECEIVED_ESMTP,
+	RSPAMD_RECEIVED_ESMTPA,
+	RSPAMD_RECEIVED_ESMTPS,
+	RSPAMD_RECEIVED_ESMTPSA,
+	RSPAMD_RECEIVED_LMTP,
+	RSPAMD_RECEIVED_IMAP,
+	RSPAMD_RECEIVED_LOCAL,
+	RSPAMD_RECEIVED_HTTP,
+	RSPAMD_RECEIVED_MAPI,
+	RSPAMD_RECEIVED_UNKNOWN
+};
+
+#define RSPAMD_RECEIVED_FLAG_ARTIFICIAL (1 << 0)
+#define RSPAMD_RECEIVED_FLAG_SSL (1 << 1)
+#define RSPAMD_RECEIVED_FLAG_AUTHENTICATED (1 << 2)
+
+struct received_header {
+	const gchar *from_hostname;
+	const gchar *from_ip;
+	const gchar *real_hostname;
+	const gchar *real_ip;
+	const gchar *by_hostname;
+	const gchar *for_mbox;
+	rspamd_inet_addr_t *addr;
+	struct rspamd_mime_header *hdr;
+	time_t timestamp;
+	enum rspamd_received_type type;
+	gint flags;
+};
+
 /**
  * Process headers and store them in `target`
  * @param task

src/libmime/mime_parser.c

@@ -376,9 +376,6 @@ rspamd_mime_part_get_cd (struct rspamd_task *task, struct rspamd_mime_part *part
 					task->task_pool);
 
 			if (cd) {
-				msg_debug_mime ("processed content disposition: %s",
-						cd->lc_data);
-
 				/* We still need to check filename */
 				if (cd->filename.len == 0) {
 					if (part->ct && part->ct->attrs) {
@@ -397,6 +394,9 @@ rspamd_mime_part_get_cd (struct rspamd_task *task, struct rspamd_mime_part *part
 						}
 					}
 				}
+
+				msg_debug_mime ("processed content disposition: %s, file: \"%T\"",
+						cd->lc_data, &cd->filename);
 				break;
 			}
 		}
@@ -680,16 +680,17 @@ rspamd_mime_parse_multipart_cb (struct rspamd_task *task,
 			/* We should have seen some boundary */
 			g_assert (cb->cur_boundary != NULL);
 
-
 			if ((ret = rspamd_mime_process_multipart_node (task, cb->st,
 					cb->multipart, cb->part_start, pos, cb->err))
 					!= RSPAMD_MIME_PARSE_OK) {
 				return ret;
 			}
 
-			/* Go towards the next part */
-			cb->part_start = st->start + b->start;
-			cb->st->pos = cb->part_start;
+			if (b->start > 0) {
+				/* Go towards the next part */
+				cb->part_start = st->start + b->start;
+				cb->st->pos = cb->part_start;
+			}
 		}
 		else {
 			/* We have an empty boundary, do nothing */
@@ -795,6 +796,7 @@ rspamd_multipart_boundaries_filter (struct rspamd_task *task,
 		struct rspamd_mime_boundary fb;
 
 		fb.boundary = last_offset;
+		fb.start = -1;
 
 		if ((ret = rspamd_mime_parse_multipart_cb (task, multipart, st,
 				cb, &fb)) != RSPAMD_MIME_PARSE_OK) {
@@ -941,15 +943,18 @@ rspamd_mime_preprocess_cb (struct rspamd_multipattern *mp,
 
 			rspamd_cryptobox_siphash ((guchar *)&b.hash, lc_copy, blen,
 					lib_ctx->hkey);
-			msg_debug_mime ("normal hash: %*s -> %L", (gint)blen, lc_copy, b.hash);
+			msg_debug_mime ("normal hash: %*s -> %L, %d boffset, %d data offset",
+					(gint)blen, lc_copy, b.hash, (int)b.boundary, (int)b.start);
 
 			if (closing) {
 				b.flags = RSPAMD_MIME_BOUNDARY_FLAG_CLOSED;
 				rspamd_cryptobox_siphash ((guchar *)&b.closed_hash, lc_copy,
 						blen + 2,
 						lib_ctx->hkey);
-				msg_debug_mime ("closing hash: %*s -> %L", (gint)blen + 2, lc_copy,
-						b.closed_hash);
+				msg_debug_mime ("closing hash: %*s -> %L, %d boffset, %d data offset",
+						(gint)blen + 2, lc_copy,
+						b.closed_hash,
+						(int)b.boundary, (int)b.start);
 			}
 			else {
 				b.flags = 0;
@@ -1183,8 +1188,6 @@ rspamd_mime_parse_message (struct rspamd_task *task,
 		 */
 		nst = g_malloc0 (sizeof (*st));
 		nst->stack = g_ptr_array_sized_new (4);
-		nst->pos = task->raw_headers_content.body_start;
-		nst->end = task->msg.begin + task->msg.len;
 		nst->boundaries = g_array_sized_new (FALSE, FALSE,
 				sizeof (struct rspamd_mime_boundary), 8);
 		nst->start = part->parsed_data.begin;
@@ -1214,14 +1217,17 @@ rspamd_mime_parse_message (struct rspamd_task *task,
 						npart->raw_headers_len,
 						FALSE);
 			}
+
+			hdrs = rspamd_message_get_header_from_hash (npart->raw_headers,
+					task->task_pool,
+					"Content-Type", FALSE);
+		}
+		else {
+			body_pos = 0;
 		}
 
 		pbegin = part->parsed_data.begin + body_pos;
 		plen = part->parsed_data.len - body_pos;
-
-		hdrs = rspamd_message_get_header_from_hash (npart->raw_headers,
-				task->task_pool,
-				"Content-Type", FALSE);
 	}
 
 	npart->raw_data.begin = pbegin;

src/libmime/smtp_parsers.h

@@ -34,6 +34,9 @@ rspamd_rfc2047_parser (const gchar *in, gsize len, gint *pencoding,
 		const gchar **charset, gsize *charset_len,
 		const gchar **encoded, gsize *encoded_len);
 
+rspamd_inet_addr_t* rspamd_parse_smtp_ip (const char *data, size_t len,
+		rspamd_mempool_t *pool);
+
 guint64 rspamd_parse_smtp_date (const char *data, size_t len);
 
 #endif /* SRC_LIBMIME_SMTP_PARSERS_H_ */

src/libserver/cfg_file.h

@@ -115,7 +115,7 @@ struct rspamd_symbols_group {
 #define RSPAMD_SYMBOL_FLAG_UNGROUPPED (1 << 3)
 
 /**
- * Symbol definition
+ * Symbol config definition
  */
 struct rspamd_symbol {
 	gchar *name;
@@ -258,6 +258,9 @@ struct rspamd_log_format {
 	struct rspamd_log_format *prev, *next;
 };
 
+/**
+ * Standard actions
+ */
 enum rspamd_action_type {
 	METRIC_ACTION_REJECT = 0,
 	METRIC_ACTION_SOFT_REJECT,
@@ -265,15 +268,23 @@ enum rspamd_action_type {
 	METRIC_ACTION_ADD_HEADER,
 	METRIC_ACTION_GREYLIST,
 	METRIC_ACTION_NOACTION,
-	METRIC_ACTION_MAX
+	METRIC_ACTION_MAX,
+	METRIC_ACTION_CUSTOM = 999,
+	METRIC_ACTION_DISCARD,
+	METRIC_ACTION_QUARANTINE
 };
 
-struct rspamd_action {
-	enum rspamd_action_type action;
-	gdouble score;
-	guint priority;
+enum rspamd_action_flags {
+	RSPAMD_ACTION_NORMAL = 0,
+	RSPAMD_ACTION_NO_THRESHOLD = (1u << 0),
+	RSPAMD_ACTION_THRESHOLD_ONLY = (1u << 1),
+	RSPAMD_ACTION_HAM = (1u << 2),
+	RSPAMD_ACTION_MILTER = (1u << 3),
 };
 
+
+struct rspamd_action;
+
 struct rspamd_config_post_load_script {
 	gint cbref;
 	struct rspamd_config_post_load_script *prev, *next;
@@ -300,8 +311,8 @@ struct rspamd_config {
 	gdouble grow_factor;                            /**< grow factor for metric							*/
 	GHashTable *symbols;                            /**< weights of symbols in metric					*/
 	const gchar *subject;                           /**< subject rewrite string							*/
-	GHashTable * groups; 		                    /**< groups of symbols								*/
-	struct rspamd_action actions[METRIC_ACTION_MAX]; /**< all actions of the metric						*/
+	GHashTable * groups;                            /**< groups of symbols								*/
+	struct rspamd_action *actions;                  /**< all actions of the metric						*/
 
 	gboolean raw_mode;                              /**< work in raw mode instead of utf one				*/
 	gboolean one_shot_mode;                         /**< rules add only one symbol							*/
@@ -626,14 +637,23 @@ gboolean rspamd_config_add_symbol_group (struct rspamd_config *cfg,
  * @param cfg config file
  * @param metric metric name (or NULL for default metric)
  * @param action_name symbolic name of action
- * @param score score limit
- * @param priority priority for action
+ * @param obj data to set for action
  * @return TRUE if symbol has been inserted or FALSE if action already exists with higher priority
  */
 gboolean rspamd_config_set_action_score (struct rspamd_config *cfg,
 		const gchar *action_name,
-		gdouble score,
-		guint priority);
+		const ucl_object_t *obj);
+
+/**
+ * Check priority and maybe disable action completely
+ * @param cfg
+ * @param action_name
+ * @param priority
+ * @return
+ */
+gboolean rspamd_config_maybe_disable_action (struct rspamd_config *cfg,
+											 const gchar *action_name,
+											 guint priority);
 
 /**
  * Checks if a specified C or lua module is enabled or disabled in the config.
@@ -662,6 +682,11 @@ gboolean rspamd_action_from_str (const gchar *data, gint *result);
 const gchar * rspamd_action_to_str (enum rspamd_action_type action);
 const gchar * rspamd_action_to_str_alt (enum rspamd_action_type action);
 
+/*
+ * Resort all actions (needed to operate with thresholds)
+ */
+void rspamd_actions_sort (struct rspamd_config *cfg);
+
 /**
  * Parse radix tree or radix map from ucl object
  * @param cfg configuration object
@@ -677,6 +702,18 @@ gboolean rspamd_config_radix_from_ucl (struct rspamd_config *cfg,
 		struct rspamd_radix_map_helper **target,
 		GError **err);
 
+/**
+ * Returns action object by name
+ * @param cfg
+ * @param name
+ * @return
+ */
+struct rspamd_action * rspamd_config_get_action (struct rspamd_config *cfg,
+												 const gchar *name);
+
+struct rspamd_action * rspamd_config_get_action_by_type (struct rspamd_config *cfg,
+												 enum rspamd_action_type type);
+
 #define msg_err_config(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
         cfg->cfg_pool->tag.tagname, cfg->checksum, \
         G_STRFUNC, \

src/libserver/cfg_file_private.h

@@ -0,0 +1,38 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_CFG_FILE_PRIVATE_H
+#define RSPAMD_CFG_FILE_PRIVATE_H
+
+#include "cfg_file.h"
+#include "../../contrib/mumhash/mum.h"
+#define HASH_CASELESS
+#include "uthash_strcase.h"
+
+/**
+ * Action config definition
+ */
+struct rspamd_action {
+	enum rspamd_action_type action_type;
+	enum rspamd_action_flags flags;
+	guint priority;
+	gint lua_handler_ref; /* If special handling is needed */
+	gdouble threshold;
+	gchar *name;
+	struct UT_hash_handle hh; /* Index by name */
+};
+
+#endif

src/libserver/cfg_rcl.c

@@ -15,9 +15,7 @@
  */
 #include "cfg_rcl.h"
 #include "rspamd.h"
-#include "../../contrib/mumhash/mum.h"
-#define HASH_CASELESS
-#include "uthash_strcase.h"
+#include "cfg_file_private.h"
 #include "utlist.h"
 #include "cfg_file.h"
 #include "lua/lua_common.h"
@@ -541,7 +539,8 @@ rspamd_rcl_actions_handler (rspamd_mempool_t *pool, const ucl_object_t *obj,
 		}
 		else {
 			if (ucl_object_type (cur) == UCL_NULL) {
-				action_score = NAN;
+				rspamd_config_maybe_disable_action (cfg, ucl_object_key (cur),
+						ucl_object_get_priority (cur));
 			}
 			else {
 				if (!ucl_object_todouble_safe (cur, &action_score)) {
@@ -558,8 +557,7 @@ rspamd_rcl_actions_handler (rspamd_mempool_t *pool, const ucl_object_t *obj,
 
 			rspamd_config_set_action_score (cfg,
 					ucl_object_key (cur),
-					action_score,
-					ucl_object_get_priority (cur));
+					cur);
 		}
 	}
 

src/libserver/cfg_utils.c

@@ -17,7 +17,7 @@
 
 #include "cfg_file.h"
 #include "rspamd.h"
-#include "uthash_strcase.h"
+#include "cfg_file_private.h"
 #include "filter.h"
 #include "lua/lua_common.h"
 #include "lua/lua_thread_pool.h"
@@ -133,6 +133,30 @@ rspamd_config_new (enum rspamd_config_init_flags flags)
 	/* 16 sockets per DNS server */
 	cfg->dns_io_per_server = 16;
 
+	/* Add all internal actions to keep compatibility */
+	for (int i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i ++) {
+		struct rspamd_action *action;
+
+		action = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*action));
+		action->threshold = NAN;
+		action->name = rspamd_mempool_strdup (cfg->cfg_pool,
+				rspamd_action_to_str (i));
+		action->action_type = i;
+
+		if (i == METRIC_ACTION_SOFT_REJECT) {
+			action->flags |= RSPAMD_ACTION_NO_THRESHOLD|RSPAMD_ACTION_HAM;
+		}
+		else if (i == METRIC_ACTION_GREYLIST) {
+			action->flags |= RSPAMD_ACTION_THRESHOLD_ONLY|RSPAMD_ACTION_HAM;
+		}
+		else if (i == METRIC_ACTION_NOACTION) {
+			action->flags |= RSPAMD_ACTION_HAM;
+		}
+
+		HASH_ADD_KEYPTR (hh, cfg->actions,
+				action->name, strlen (action->name), action);
+	}
+
 	/* Disable timeout */
 	cfg->task_timeout = DEFAULT_TASK_TIMEOUT;
 
@@ -265,6 +289,8 @@ rspamd_config_free (struct rspamd_config *cfg)
 		rspamd_monitored_ctx_destroy (cfg->monitored_ctx);
 	}
 
+	HASH_CLEAR (hh, cfg->actions);
+
 	rspamd_mempool_delete (cfg->cfg_pool);
 
 	if (cfg->checksum) {
@@ -821,29 +847,10 @@ rspamd_config_post_load (struct rspamd_config *cfg,
 	/* Validate cache */
 	if (opts & RSPAMD_CONFIG_INIT_VALIDATE) {
 		/* Check for actions sanity */
-		int i, prev_act = 0;
-		gdouble prev_score = NAN;
 		gboolean seen_controller = FALSE;
 		GList *cur;
 		struct rspamd_worker_conf *wcf;
 
-		for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i ++) {
-			if (!isnan (prev_score) && !isnan (cfg->actions[i].score)) {
-				if (prev_score <= isnan (cfg->actions[i].score)) {
-					msg_warn_config ("incorrect metrics scores: action %s"
-							" has lower score: %.2f than action %s: %.2f",
-							rspamd_action_to_str (prev_act), prev_score,
-							rspamd_action_to_str (i), cfg->actions[i].score);
-					ret = FALSE;
-				}
-			}
-
-			if (!isnan (cfg->actions[i].score)) {
-				prev_score = cfg->actions[i].score;
-				prev_act = i;
-			}
-		}
-
 		cur = cfg->workers;
 		while (cur) {
 			wcf = cur->data;
@@ -1007,18 +1014,10 @@ rspamd_config_new_statfile (struct rspamd_config *cfg,
 void
 rspamd_config_init_metric (struct rspamd_config *cfg)
 {
-	int i;
-
 	cfg->grow_factor = 1.0;
 	cfg->symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
 	cfg->groups = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal);
 
-	for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) {
-		cfg->actions[i].score = NAN;
-		cfg->actions[i].action = i;
-		cfg->actions[i].priority = 0;
-	}
-
 	cfg->subject = SPAM_SUBJECT;
 	rspamd_mempool_add_destructor (cfg->cfg_pool,
 			(rspamd_mempool_destruct_t) g_hash_table_unref,
@@ -1279,7 +1278,7 @@ gboolean
 rspamd_config_check_statfiles (struct rspamd_classifier_config *cf)
 {
 	struct rspamd_statfile_config *st;
-	gboolean has_other = FALSE, res = FALSE, cur_class;
+	gboolean has_other = FALSE, res = FALSE, cur_class = FALSE;
 	GList *cur;
 
 	/* First check classes directly */
@@ -1887,58 +1886,231 @@ rspamd_config_is_module_enabled (struct rspamd_config *cfg,
 	return TRUE;
 }
 
+static gboolean
+rspamd_config_action_from_ucl (struct rspamd_config *cfg,
+							   struct rspamd_action *act,
+							   const ucl_object_t *obj,
+							   guint priority)
+{
+	const ucl_object_t *elt;
+	gdouble threshold = NAN;
+	guint flags = 0, std_act, obj_type;
+
+	obj_type = ucl_object_type (obj);
+
+	if (obj_type == UCL_OBJECT) {
+		obj_type = ucl_object_type (obj);
+
+		elt = ucl_object_lookup_any (obj, "score", "threshold", NULL);
+
+		if (elt) {
+			threshold = ucl_object_todouble (elt);
+		}
+
+		elt = ucl_object_lookup (obj, "flags");
+
+		if (elt && ucl_object_type (elt) == UCL_ARRAY) {
+			const ucl_object_t *cur;
+			ucl_object_iter_t it = NULL;
+
+			while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) {
+				if (ucl_object_type (cur) == UCL_STRING) {
+					const gchar *fl_str = ucl_object_tostring (cur);
+
+					if (g_ascii_strcasecmp (fl_str, "no_threshold") == 0) {
+						flags |= RSPAMD_ACTION_NO_THRESHOLD;
+					} else if (g_ascii_strcasecmp (fl_str, "threshold_only") == 0) {
+						flags |= RSPAMD_ACTION_THRESHOLD_ONLY;
+					} else if (g_ascii_strcasecmp (fl_str, "ham") == 0) {
+						flags |= RSPAMD_ACTION_HAM;
+					} else {
+						msg_warn_config ("unknown action flag: %s", fl_str);
+					}
+				}
+			}
+		}
+
+		elt = ucl_object_lookup (obj, "milter");
+
+		if (elt) {
+			const gchar *milter_action = ucl_object_tostring (elt);
+
+			if (strcmp (milter_action, "discard") == 0) {
+				flags |= RSPAMD_ACTION_MILTER;
+				act->action_type = METRIC_ACTION_DISCARD;
+			}
+			else if (strcmp (milter_action, "quarantine") == 0) {
+				flags |= RSPAMD_ACTION_MILTER;
+				act->action_type = METRIC_ACTION_QUARANTINE;
+			}
+			else {
+				msg_warn_config ("unknown milter action: %s", milter_action);
+			}
+		}
+	}
+	else if (obj_type == UCL_FLOAT || obj_type == UCL_INT) {
+		threshold = ucl_object_todouble (obj);
+	}
+
+	/* TODO: add lua references support */
+
+	if (isnan (threshold) && !(flags & RSPAMD_ACTION_NO_THRESHOLD)) {
+		msg_err_config ("action %s has no threshold being set and it is not"
+				  " a no threshold action", act->name);
+
+		return FALSE;
+	}
+
+	act->threshold = threshold;
+	act->flags = flags;
+
+	if (!(flags & RSPAMD_ACTION_MILTER)) {
+		if (rspamd_action_from_str (act->name, &std_act)) {
+			act->action_type = std_act;
+		} else {
+			act->action_type = METRIC_ACTION_CUSTOM;
+		}
+	}
+
+	return TRUE;
+}
+
 gboolean
 rspamd_config_set_action_score (struct rspamd_config *cfg,
 		const gchar *action_name,
-		gdouble score,
-		guint priority)
+		const ucl_object_t *obj)
 {
 	struct rspamd_action *act;
-	gint act_num;
+	enum rspamd_action_type std_act;
+	const ucl_object_t *elt;
+	guint priority = ucl_object_get_priority (obj), obj_type;
 
 	g_assert (cfg != NULL);
 	g_assert (action_name != NULL);
 
-	if (!rspamd_action_from_str (action_name, &act_num)) {
-		msg_err_config ("invalid action name: %s", action_name);
-		return FALSE;
-	}
+	obj_type = ucl_object_type (obj);
 
-	g_assert (act_num >= METRIC_ACTION_REJECT && act_num < METRIC_ACTION_MAX);
+	if (obj_type == UCL_OBJECT) {
+		elt = ucl_object_lookup (obj, "priority");
 
-	act = &cfg->actions[act_num];
+		if (elt) {
+			priority = ucl_object_toint (elt);
+		}
+	}
+
+	/* Here are dragons:
+	 * We have `canonical` name for actions, such as `soft reject` and
+	 * configuration names for actions (used to be more convenient), such
+	 * as `soft_reject`. Unfortunately, we must have heuristic for this
+	 * variance of names.
+	 */
 
-	if (isnan (act->score)) {
-		act->score = score;
-		act->priority = priority;
+	if (rspamd_action_from_str (action_name, (gint *)&std_act)) {
+		action_name = rspamd_action_to_str (std_act);
 	}
-	else {
-		if (act->priority > priority) {
+
+	HASH_FIND_STR (cfg->actions, action_name, act);
+
+	if (act) {
+		/* Existing element */
+		if (act->priority <= priority) {
+			/* We can replace data */
 			msg_info_config ("action %s has been already registered with "
-					"priority %ud, do not override (new priority: %ud)",
+							 "priority %ud, override it with new priority: %ud, "
+							 "old score: %.2f",
 					action_name,
 					act->priority,
-					priority);
-			return FALSE;
+					priority,
+					act->threshold);
+			if (rspamd_config_action_from_ucl (cfg, act, obj, priority)) {
+				rspamd_actions_sort (cfg);
+			}
+			else {
+				return FALSE;
+			}
 		}
 		else {
 			msg_info_config ("action %s has been already registered with "
-					"priority %ud, override it with new priority: %ud, "
-					"old score: %.2f, new score: %.2f",
+							 "priority %ud, do not override (new priority: %ud)",
 					action_name,
 					act->priority,
-					priority,
-					act->score,
-					score);
+					priority);
+		}
+	}
+	else {
+		/* Add new element */
+		act = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*act));
+		act->name = rspamd_mempool_strdup (cfg->cfg_pool, action_name);
 
-			act->score = score;
-			act->priority = priority;
+		if (rspamd_config_action_from_ucl (cfg, act, obj, priority)) {
+			HASH_ADD_KEYPTR (hh, cfg->actions,
+					act->name, strlen (act->name), act);
+			rspamd_actions_sort (cfg);
+		}
+		else {
+			return FALSE;
 		}
 	}
 
 	return TRUE;
 }
 
+gboolean
+rspamd_config_maybe_disable_action (struct rspamd_config *cfg,
+											 const gchar *action_name,
+											 guint priority)
+{
+	struct rspamd_action *act;
+
+	HASH_FIND_STR (cfg->actions, action_name, act);
+
+	if (act) {
+		if (priority >= act->priority) {
+			msg_info_config ("disable action %s; old priority: %ud, new priority: %ud",
+					action_name,
+					act->priority,
+					priority);
+
+			HASH_DEL (cfg->actions, act);
+
+			return TRUE;
+		}
+		else {
+			msg_info_config ("action %s has been already registered with "
+							 "priority %ud, cannot disable it with new priority: %ud",
+					action_name,
+					act->priority,
+					priority);
+		}
+	}
+
+	return FALSE;
+}
+
+struct rspamd_action *
+rspamd_config_get_action (struct rspamd_config *cfg, const gchar *name)
+{
+	struct rspamd_action *res = NULL;
+
+	HASH_FIND_STR (cfg->actions, name, res);
+
+	return res;
+}
+
+struct rspamd_action *
+rspamd_config_get_action_by_type (struct rspamd_config *cfg,
+								  enum rspamd_action_type type)
+{
+	struct rspamd_action *cur, *tmp;
+
+	HASH_ITER (hh, cfg->actions, cur, tmp) {
+		if (cur->action_type == type) {
+			return cur;
+		}
+	}
+
+	return NULL;
+}
 
 gboolean
 rspamd_config_radix_from_ucl (struct rspamd_config *cfg,
@@ -2067,6 +2239,12 @@ rspamd_action_from_str (const gchar *data, gint *result)
 	case 0x167C0DF4BAA9BCECULL: /* accept */
 		*result = METRIC_ACTION_NOACTION;
 		break;
+	case 0x4E9666ECCD3FC314ULL: /* quarantine */
+		*result = METRIC_ACTION_QUARANTINE;
+		break;
+	case 0x93B346242F7F69B3ULL: /* discard */
+		*result = METRIC_ACTION_DISCARD;
+		break;
 	default:
 		return FALSE;
 	}
@@ -2092,6 +2270,12 @@ rspamd_action_to_str (enum rspamd_action_type action)
 		return "no action";
 	case METRIC_ACTION_MAX:
 		return "invalid max action";
+	case METRIC_ACTION_CUSTOM:
+		return "custom";
+	case METRIC_ACTION_DISCARD:
+		return "discard";
+	case METRIC_ACTION_QUARANTINE:
+		return "quarantine";
 	}
 
 	return "unknown action";
@@ -2115,7 +2299,44 @@ rspamd_action_to_str_alt (enum rspamd_action_type action)
 		return "no action";
 	case METRIC_ACTION_MAX:
 		return "invalid max action";
+	case METRIC_ACTION_CUSTOM:
+		return "custom";
+	case METRIC_ACTION_DISCARD:
+		return "discard";
+	case METRIC_ACTION_QUARANTINE:
+		return "quarantine";
 	}
 
 	return "unknown action";
 }
+
+static int
+rspamd_actions_cmp (const struct rspamd_action *a1, const struct rspamd_action *a2)
+{
+	if (!isnan (a1->threshold) && !isnan (a2->threshold)) {
+		if (a1->threshold < a2->threshold) {
+			return -1;
+		}
+		else if (a1->threshold > a2->threshold) {
+			return 1;
+		}
+
+		return 0;
+	}
+
+	if (isnan (a1->threshold) && isnan (a2->threshold)) {
+		return 0;
+	}
+	else if (isnan (a1->threshold)) {
+		return 1;
+	}
+	else {
+		return -1;
+	}
+}
+
+void
+rspamd_actions_sort (struct rspamd_config *cfg)
+{
+	HASH_SORT (cfg->actions, rspamd_actions_cmp);
+}

src/libserver/dkim.c

@@ -21,6 +21,7 @@
 #include "utlist.h"
 #include "unix-std.h"
 #include "mempool_vars_internal.h"
+#include "libcryptobox/ed25519/ed25519.h"
 
 #include <openssl/evp.h>
 #include <openssl/rsa.h>
@@ -29,6 +30,10 @@
 /* special DNS tokens */
 #define DKIM_DNSKEYNAME     "_domainkey"
 
+/* ed25519 key lengths */
+#define ED25519_B64_BYTES	45
+#define ED25519_BYTES		32
+
 /* Canonization methods */
 #define DKIM_CANON_UNKNOWN  (-1)    /* unknown method */
 #define DKIM_CANON_SIMPLE   0   /* as specified in DKIM spec */
@@ -78,6 +83,10 @@ enum rspamd_dkim_param_type {
         rspamd_dkim_log_id, "dkim", ctx->pool->tag.uid, \
         G_STRFUNC, \
         __VA_ARGS__)
+#define msg_debug_dkim_taskless(...)  rspamd_conditional_debug_fast (NULL, NULL, \
+        rspamd_dkim_log_id, "dkim", "", \
+        G_STRFUNC, \
+        __VA_ARGS__)
 
 INIT_LOG_MODULE(dkim)
 
@@ -135,7 +144,7 @@ struct rspamd_dkim_context_s {
 
 struct rspamd_dkim_key_s {
 	guint8 *keydata;
-	guint keylen;
+	gsize keylen;
 	gsize decoded_len;
 	guint ttl;
 	union {
@@ -146,6 +155,7 @@ struct rspamd_dkim_key_s {
 	enum rspamd_dkim_key_type type;
 	BIO *key_bio;
 	EVP_PKEY *key_evp;
+	time_t mtime;
 	ref_entry_t ref;
 };
 
@@ -154,18 +164,6 @@ struct rspamd_dkim_sign_context_s {
 	rspamd_dkim_sign_key_t *key;
 };
 
-struct rspamd_dkim_sign_key_s {
-	enum rspamd_dkim_sign_key_type type;
-	guint8 *keydata;
-	gsize keylen;
-	RSA *key_rsa;
-	BIO *key_bio;
-	EVP_PKEY *key_evp;
-	time_t mtime;
-	ref_entry_t ref;
-};
-
-
 struct rspamd_dkim_header {
 	const gchar *name;
 	guint count;
@@ -1342,21 +1340,18 @@ rspamd_dkim_sign_key_free (rspamd_dkim_sign_key_t *key)
 	if (key->key_evp) {
 		EVP_PKEY_free (key->key_evp);
 	}
-	if (key->key_rsa) {
-		RSA_free (key->key_rsa);
+	if (key->type == RSPAMD_DKIM_KEY_RSA) {
+		if (key->key.key_rsa) {
+			RSA_free (key->key.key_rsa);
+		}
 	}
 	if (key->key_bio) {
 		BIO_free (key->key_bio);
 	}
 
-	if (key->keydata && key->keylen > 0) {
-
-		if (key->type == RSPAMD_DKIM_SIGN_KEY_FILE) {
-			munmap (key->keydata, key->keylen);
-		}
-		else {
-			g_free (key->keydata);
-		}
+	if (key->type == RSPAMD_DKIM_KEY_EDDSA) {
+		rspamd_explicit_memzero (key->key.key_eddsa, key->keylen);
+		g_free (key->keydata);
 	}
 
 	g_free (key);
@@ -2609,145 +2604,156 @@ rspamd_dkim_get_dns_key (rspamd_dkim_context_t *ctx)
 	return NULL;
 }
 
+#define PEM_SIG "-----BEGIN"
+
 rspamd_dkim_sign_key_t*
-rspamd_dkim_sign_key_load (const gchar *what, gsize len,
-		enum rspamd_dkim_sign_key_type type,
+rspamd_dkim_sign_key_load (const gchar *key, gsize len,
+		enum rspamd_dkim_key_format type,
 		GError **err)
 {
-	gpointer map;
-	gsize map_len = 0;
+	guchar *map = NULL, *tmp = NULL;
+	gsize maplen;
 	rspamd_dkim_sign_key_t *nkey;
-	struct stat st;
-	time_t mtime = 0;
+	time_t mtime = time (NULL);
 
-	if (type == RSPAMD_DKIM_SIGN_KEY_FILE) {
-		gchar fpath[PATH_MAX];
+	if (type < 0 || type > RSPAMD_DKIM_KEY_UNKNOWN || len == 0 || key == NULL) {
+		g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
+				"invalid key type to load: %d", type);
+		return NULL;
+	}
+
+	nkey = g_malloc0 (sizeof (*nkey));
+	nkey->mtime = mtime;
 
-		rspamd_snprintf (fpath, sizeof (fpath), "%*s", (gint)len, what);
+	msg_debug_dkim_taskless ("got public key with length %d and type %d",
+			len, type);
 
-		if (stat (fpath, &st) == -1) {
+	/* Load key file if needed */
+	if (type == RSPAMD_DKIM_KEY_FILE) {
+		struct stat st;
+
+		if (stat (key, &st) != 0) {
 			g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
-					"cannot stat private key %s: %s",
-					fpath, strerror (errno));
+					"cannot stat key file: '%s' %s", key, strerror (errno));
 
 			return NULL;
 		}
 
-		mtime = st.st_mtime;
-		map = rspamd_file_xmap (fpath, PROT_READ, &map_len, TRUE);
+		nkey->mtime = st.st_mtime;
+		map = rspamd_file_xmap (key, PROT_READ, &maplen, TRUE);
 
 		if (map == NULL) {
 			g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
-					"cannot map private key %s: %s",
-					fpath, strerror (errno));
+					"cannot map key file: '%s' %s", key, strerror (errno));
 
 			return NULL;
 		}
-	}
 
-	nkey = g_malloc0 (sizeof (*nkey));
-	nkey->type = type;
-	nkey->mtime = mtime;
+		key = map;
+		len = maplen;
 
-	switch (type) {
-	case RSPAMD_DKIM_SIGN_KEY_FILE:
-		(void)mlock (map, len);
-		nkey->keydata = map;
-		nkey->keylen = map_len;
-		break;
-	case RSPAMD_DKIM_SIGN_KEY_BASE64:
-		nkey->keydata = g_malloc (len);
-		nkey->keylen = len;
-		rspamd_cryptobox_base64_decode (what, len, nkey->keydata,
-				&nkey->keylen);
-		break;
-	case RSPAMD_DKIM_SIGN_KEY_DER:
-	case RSPAMD_DKIM_SIGN_KEY_PEM:
-		nkey->keydata = g_malloc (len);
-		memcpy (nkey->keydata, what, len);
-		nkey->keylen = len;
+		if (maplen > sizeof (PEM_SIG) &&
+			strncmp (map, PEM_SIG, sizeof (PEM_SIG) - 1) == 0) {
+			type = RSPAMD_DKIM_KEY_PEM;
+		}
+		else if (rspamd_cryptobox_base64_is_valid (map, maplen)) {
+			type = RSPAMD_DKIM_KEY_BASE64;
+		}
+		else {
+			type = RSPAMD_DKIM_KEY_RAW;
+		}
 	}
 
-	(void)mlock (nkey->keydata, nkey->keylen);
-	nkey->key_bio = BIO_new_mem_buf (nkey->keydata, nkey->keylen);
-
-	if (type == RSPAMD_DKIM_SIGN_KEY_DER || type == RSPAMD_DKIM_SIGN_KEY_BASE64) {
-		if (d2i_PrivateKey_bio (nkey->key_bio, &nkey->key_evp) == NULL) {
-			if (type == RSPAMD_DKIM_SIGN_KEY_FILE) {
-				g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
-						"cannot read private key from %*s: %s",
-						(gint)len, what,
-						ERR_error_string (ERR_get_error (), NULL));
-			}
-			else {
-				g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
-						"cannot read private key from string: %s",
-						ERR_error_string (ERR_get_error (), NULL));
-			}
+	if (type == RSPAMD_DKIM_KEY_UNKNOWN) {
+		if (len > sizeof (PEM_SIG) &&
+			memcmp (key, PEM_SIG, sizeof (PEM_SIG) - 1) == 0) {
+			type = RSPAMD_DKIM_KEY_PEM;
+		}
+		else {
+			type = RSPAMD_DKIM_KEY_RAW;
+		}
+	}
 
-			rspamd_dkim_sign_key_free (nkey);
+	if (type == RSPAMD_DKIM_KEY_BASE64) {
+		type = RSPAMD_DKIM_KEY_RAW;
+		tmp = g_malloc (len);
+		rspamd_cryptobox_base64_decode (key, len, tmp, &len);
+		key = tmp;
+	}
 
-			return NULL;
-		}
+	if (type == RSPAMD_DKIM_KEY_RAW && len == 32) {
+		unsigned char pk[32];
+		nkey->type = RSPAMD_DKIM_KEY_EDDSA;
+		nkey->key.key_eddsa = g_malloc (
+				rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519));
+		ed25519_seed_keypair (pk, nkey->key.key_eddsa, (char *)key);
+		nkey->keylen = rspamd_cryptobox_sk_sig_bytes (RSPAMD_CRYPTOBOX_MODE_25519);
 	}
 	else {
-		if (!PEM_read_bio_PrivateKey (nkey->key_bio, &nkey->key_evp, NULL, NULL)) {
-			if (type == RSPAMD_DKIM_SIGN_KEY_FILE) {
+		nkey->key_bio = BIO_new_mem_buf (key, len);
+
+		if (type == RSPAMD_DKIM_KEY_RAW) {
+			if (d2i_PrivateKey_bio (nkey->key_bio, &nkey->key_evp) == NULL) {
 				g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
-						"cannot read private key from %*s: %s",
-						(gint)len, what,
+						"cannot parse raw private key: %s",
 						ERR_error_string (ERR_get_error (), NULL));
+
+				rspamd_dkim_sign_key_free (nkey);
+				nkey = NULL;
+
+				goto end;
 			}
-			else {
+		}
+		else {
+			if (!PEM_read_bio_PrivateKey (nkey->key_bio, &nkey->key_evp, NULL, NULL)) {
 				g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
-						"cannot read private key from string: %s",
+						"cannot parse pem private key: %s",
 						ERR_error_string (ERR_get_error (), NULL));
+				rspamd_dkim_sign_key_free (nkey);
+				nkey = NULL;
+
+				goto end;
 			}
 
+		}
+		nkey->key.key_rsa = EVP_PKEY_get1_RSA (nkey->key_evp);
+		if (nkey->key.key_rsa == NULL) {
+			g_set_error (err,
+					DKIM_ERROR,
+					DKIM_SIGERROR_KEYFAIL,
+					"cannot extract rsa key from evp key");
 			rspamd_dkim_sign_key_free (nkey);
+			nkey = NULL;
 
-			return NULL;
+			goto end;
 		}
+		nkey->type = RSPAMD_DKIM_KEY_RSA;
 	}
 
-	nkey->key_rsa = EVP_PKEY_get1_RSA (nkey->key_evp);
-	if (nkey->key_rsa == NULL) {
-		g_set_error (err,
-				DKIM_ERROR,
-				DKIM_SIGERROR_KEYFAIL,
-				"cannot extract rsa key from evp key");
-		rspamd_dkim_sign_key_free (nkey);
+	REF_INIT_RETAIN (nkey, rspamd_dkim_sign_key_free);
 
-		return NULL;
+end:
+
+	if (map != NULL) {
+		munmap (map, maplen);
 	}
 
-	REF_INIT_RETAIN (nkey, rspamd_dkim_sign_key_free);
+	if (tmp != NULL) {
+		rspamd_explicit_memzero (tmp, len);
+		g_free (tmp);
+	}
 
 	return nkey;
 }
 
+#undef PEM_SIG
+
 gboolean
-rspamd_dkim_sign_key_maybe_invalidate (rspamd_dkim_sign_key_t *key,
-		enum rspamd_dkim_sign_key_type type,
-		const gchar *what, gsize len)
+rspamd_dkim_sign_key_maybe_invalidate (rspamd_dkim_sign_key_t *key, time_t mtime)
 {
-	struct stat st;
-
-	if (type == RSPAMD_DKIM_SIGN_KEY_FILE) {
-		gchar fpath[PATH_MAX];
-
-		rspamd_snprintf (fpath, sizeof (fpath), "%*s", (gint) len, what);
-
-		if (stat (fpath, &st) == -1) {
-			/* Prefer to use cached key since it is absent on FS */
-			return FALSE;
-		}
-
-		if (st.st_mtime > key->mtime) {
+	if (mtime > key->mtime) {
 			return TRUE;
-		}
 	}
-
 	return FALSE;
 }
 
@@ -2779,7 +2785,7 @@ rspamd_create_dkim_sign_context (struct rspamd_task *task,
 		return NULL;
 	}
 
-	if (!priv_key || !priv_key->key_rsa) {
+	if (!priv_key || (!priv_key->key.key_rsa && !priv_key->key.key_eddsa)) {
 		g_set_error (err,
 				DKIM_ERROR,
 				DKIM_SIGERROR_KEYFAIL,
@@ -2847,8 +2853,8 @@ rspamd_dkim_sign (struct rspamd_task *task, const gchar *selector,
 	gsize dlen = 0;
 	guint i, j;
 	gchar *b64_data;
-	guchar *rsa_buf;
-	guint rsa_len;
+	guchar *sig_buf;
+	guint sig_len;
 	guint headers_len = 0, cur_len = 0;
 	union rspamd_dkim_header_stat hstat;
 
@@ -2884,7 +2890,9 @@ rspamd_dkim_sign (struct rspamd_task *task, const gchar *selector,
 	hdr = g_string_sized_new (255);
 
 	if (ctx->common.type == RSPAMD_DKIM_NORMAL) {
-		rspamd_printf_gstring (hdr, "v=1; a=rsa-sha256; c=%s/%s; d=%s; s=%s; ",
+		rspamd_printf_gstring (hdr, "v=1; a=%s; c=%s/%s; d=%s; s=%s; ",
+				ctx->key->type == RSPAMD_DKIM_KEY_RSA ?
+						"rsa-sha256" : "ed25519-sha256",
 				ctx->common.header_canon_type == DKIM_CANON_RELAXED ?
 						"relaxed" : "simple",
 				ctx->common.body_canon_type == DKIM_CANON_RELAXED ?
@@ -2892,8 +2900,10 @@ rspamd_dkim_sign (struct rspamd_task *task, const gchar *selector,
 				domain, selector);
 	}
 	else if (ctx->common.type == RSPAMD_DKIM_ARC_SIG) {
-		rspamd_printf_gstring (hdr, "i=%d; a=rsa-sha256; c=%s/%s; d=%s; s=%s; ",
+		rspamd_printf_gstring (hdr, "i=%d; a=%s; c=%s/%s; d=%s; s=%s; ",
 				idx,
+				ctx->key->type == RSPAMD_DKIM_KEY_RSA ?
+						"rsa-sha256" : "ed25519-sha256",
 				ctx->common.header_canon_type == DKIM_CANON_RELAXED ?
 						"relaxed" : "simple",
 				ctx->common.body_canon_type == DKIM_CANON_RELAXED ?
@@ -2902,8 +2912,10 @@ rspamd_dkim_sign (struct rspamd_task *task, const gchar *selector,
 	}
 	else {
 		g_assert (arc_cv != NULL);
-		rspamd_printf_gstring (hdr, "i=%d; a=rsa-sha256; c=%s/%s; d=%s; s=%s; cv=%s; ",
+		rspamd_printf_gstring (hdr, "i=%d; a=%s; c=%s/%s; d=%s; s=%s; cv=%s; ",
 				arc_cv,
+				ctx->key->type == RSPAMD_DKIM_KEY_RSA ?
+						"rsa-sha256" : "ed25519-sha256",
 				idx,
 				domain, selector);
 	}
@@ -3035,24 +3047,37 @@ rspamd_dkim_sign (struct rspamd_task *task, const gchar *selector,
 
 	dlen = EVP_MD_CTX_size (ctx->common.headers_hash);
 	EVP_DigestFinal_ex (ctx->common.headers_hash, raw_digest, NULL);
-	rsa_len = RSA_size (ctx->key->key_rsa);
-	rsa_buf = g_alloca (rsa_len);
+	if (ctx->key->type == RSPAMD_DKIM_KEY_RSA) {
+		sig_len = RSA_size (ctx->key->key.key_rsa);
+		sig_buf = g_alloca (sig_len);
 
-	if (RSA_sign (NID_sha256, raw_digest, dlen, rsa_buf, &rsa_len,
-			ctx->key->key_rsa) != 1) {
+		if (RSA_sign (NID_sha256, raw_digest, dlen, sig_buf, &sig_len,
+				ctx->key->key.key_rsa) != 1) {
+			g_string_free (hdr, TRUE);
+			msg_err_task ("rsa sign error: %s",
+					ERR_error_string (ERR_get_error (), NULL));
+
+			return NULL;
+		}
+	} else if (ctx->key->type == RSPAMD_DKIM_KEY_EDDSA) {
+		sig_len = rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519);
+		sig_buf = g_alloca (sig_len);
+
+		rspamd_cryptobox_sign (sig_buf, NULL, raw_digest, dlen,
+				ctx->key->key.key_eddsa, RSPAMD_CRYPTOBOX_MODE_25519);
+	} else {
 		g_string_free (hdr, TRUE);
-		msg_err_task ("rsa sign error: %s",
-				ERR_error_string (ERR_get_error (), NULL));
+		msg_err_task ("unsupported key type for signing");
 
 		return NULL;
 	}
 
 	if (task->flags & RSPAMD_TASK_FLAG_MILTER) {
-		b64_data = rspamd_encode_base64_fold (rsa_buf, rsa_len, 70, NULL,
+		b64_data = rspamd_encode_base64_fold (sig_buf, sig_len, 70, NULL,
 				RSPAMD_TASK_NEWLINES_LF);
 	}
 	else {
-		b64_data = rspamd_encode_base64_fold (rsa_buf, rsa_len, 70, NULL,
+		b64_data = rspamd_encode_base64_fold (sig_buf, sig_len, 70, NULL,
 				task->nlines_type);
 	}
 
@@ -3067,33 +3092,30 @@ rspamd_dkim_match_keys (rspamd_dkim_key_t *pk,
 								 rspamd_dkim_sign_key_t *sk,
 								 GError **err)
 {
-	const BIGNUM *n1, *n2;
-
 	if (pk == NULL || sk == NULL) {
 		g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
 				"missing public or private key");
 		return FALSE;
 	}
-
-	if (pk->type != RSPAMD_DKIM_KEY_RSA) {
+	if (pk->type != sk->type) {
 		g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYFAIL,
-				"pubkey is not RSA key");
+				"public and private key types do not match");
 		return FALSE;
 	}
 
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
-	RSA_get0_key (pk->key.key_rsa, &n1, NULL, NULL);
-	RSA_get0_key (sk->key_rsa, &n2, NULL, NULL);
-#else
-	n1 = pk->key.key_rsa->n;
-	n2 = sk->key_rsa->n;
-#endif
+	if (pk->type == RSPAMD_DKIM_KEY_EDDSA) {
+		if (memcmp(sk->key.key_eddsa + 32, pk->key.key_eddsa, 32) != 0) {
+			g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYHASHMISMATCH,
+					"pubkey does not match private key");
+			return FALSE;
+		}
 
-	if (BN_cmp (n1, n2) != 0) {
+	}
+	else if (EVP_PKEY_cmp (pk->key_evp, sk->key_evp) != 1) {
 		g_set_error (err, dkim_error_quark (), DKIM_SIGERROR_KEYHASHMISMATCH,
 				"pubkey does not match private key");
 		return FALSE;
 	}
 
 	return TRUE;
-}
+}

src/libserver/dkim.h

@@ -101,17 +101,16 @@ typedef struct rspamd_dkim_sign_context_s rspamd_dkim_sign_context_t;
 
 struct rspamd_dkim_key_s;
 typedef struct rspamd_dkim_key_s rspamd_dkim_key_t;
-
-struct rspamd_dkim_sign_key_s;
-typedef struct rspamd_dkim_sign_key_s rspamd_dkim_sign_key_t;
+typedef struct rspamd_dkim_key_s rspamd_dkim_sign_key_t;
 
 struct rspamd_task;
 
-enum rspamd_dkim_sign_key_type {
-	RSPAMD_DKIM_SIGN_KEY_FILE = 0,
-	RSPAMD_DKIM_SIGN_KEY_PEM,
-	RSPAMD_DKIM_SIGN_KEY_BASE64,
-	RSPAMD_DKIM_SIGN_KEY_DER
+enum rspamd_dkim_key_format {
+	RSPAMD_DKIM_KEY_FILE = 0,
+	RSPAMD_DKIM_KEY_PEM,
+	RSPAMD_DKIM_KEY_BASE64,
+	RSPAMD_DKIM_KEY_RAW,
+	RSPAMD_DKIM_KEY_UNKNOWN
 };
 
 enum rspamd_dkim_type {
@@ -188,17 +187,16 @@ rspamd_dkim_sign_context_t * rspamd_create_dkim_sign_context (struct rspamd_task
  * @return
  */
 rspamd_dkim_sign_key_t* rspamd_dkim_sign_key_load (const gchar *what, gsize len,
-		enum rspamd_dkim_sign_key_type type,
+		enum rspamd_dkim_key_format type,
 		GError **err);
 
 /**
  * Invalidate modified sign key
  * @param key
  * @return
- */
+*/
 gboolean rspamd_dkim_sign_key_maybe_invalidate (rspamd_dkim_sign_key_t *key,
-		enum rspamd_dkim_sign_key_type type,
-		const gchar *what, gsize len);
+		time_t mtime);
 
 /**
  * Make DNS request for specified context and obtain and parse key

src/libserver/dns.c

@@ -20,7 +20,6 @@
 #include "dns.h"
 #include "rspamd.h"
 #include "utlist.h"
-#include "uthash.h"
 #include "rdns_event.h"
 #include "unix-std.h"
 

src/libserver/dynamic_cfg.c

@@ -118,7 +118,13 @@ apply_dynamic_conf (const ucl_object_t *top, struct rspamd_config *cfg)
 						nscore = ucl_object_todouble (v);
 					}
 
-					rspamd_config_set_action_score (cfg, name, nscore, priority);
+					ucl_object_t *obj_tbl = ucl_object_typed_new (UCL_OBJECT);
+					ucl_object_insert_key (obj_tbl, ucl_object_fromdouble (nscore),
+							"score", 0, false);
+					ucl_object_insert_key (obj_tbl, ucl_object_fromdouble (priority),
+							"priority", 0, false);
+					rspamd_config_set_action_score (cfg, name, obj_tbl);
+					ucl_object_unref (obj_tbl);
 				}
 				else {
 					msg_info (

src/libserver/events.c

@@ -16,6 +16,7 @@
 #include "config.h"
 #include "rspamd.h"
 #include "contrib/uthash/utlist.h"
+#include "contrib/libucl/khash.h"
 #include "events.h"
 #include "cryptobox.h"
 

src/libserver/html.c

@@ -22,6 +22,9 @@
 #include "html_colors.h"
 #include "html_entities.h"
 #include "url.h"
+#include "contrib/libucl/khash.h"
+#include "libmime/images.h"
+
 #include <unicode/uversion.h>
 #include <unicode/ucnv.h>
 #if U_ICU_VERSION_MAJOR_NUM >= 46
@@ -334,13 +337,14 @@ rspamd_html_tag_by_id (gint id)
 
 /* Decode HTML entitles in text */
 guint
-rspamd_html_decode_entitles_inplace (gchar *s, guint len)
+rspamd_html_decode_entitles_inplace (gchar *s, gsize len)
 {
-	guint l, rep_len;
+	goffset l, rep_len;
 	gchar *t = s, *h = s, *e = s, *end_ptr;
 	const gchar *end;
 	const gchar *entity;
-	gint state = 0, val, base;
+	gint state = 0, base;
+	UChar32 uc;
 	khiter_t k;
 
 	if (len == 0) {
@@ -352,7 +356,7 @@ rspamd_html_decode_entitles_inplace (gchar *s, guint len)
 
 	end = s + l;
 
-	while (h - s < (gint)l) {
+	while (h - s < l) {
 		switch (state) {
 		/* Out of entity */
 		case 0:
@@ -374,9 +378,11 @@ rspamd_html_decode_entitles_inplace (gchar *s, guint len)
 				/* First find in entities table */
 				*h = '\0';
 				entity = e + 1;
+				uc = 0;
 
 				if (*entity != '#') {
 					k = kh_get (entity_by_name, html_entity_by_name, entity);
+					*h = ';';
 
 					if (k != kh_end (html_entity_by_name)) {
 						if (kh_val (html_entity_by_name, k)) {
@@ -388,12 +394,18 @@ rspamd_html_decode_entitles_inplace (gchar *s, guint len)
 								t += rep_len;
 							}
 						} else {
-							if (end - t >= h - e) {
-								memmove (t, e, h - e);
-								t += h - e;
+							if (end - t > h - e + 1) {
+								memmove (t, e, h - e + 1);
+								t += h - e + 1;
 							}
 						}
 					}
+					else {
+						if (end - t > h - e + 1) {
+							memmove (t, e, h - e + 1);
+							t += h - e + 1;
+						}
+					}
 				}
 				else if (e + 2 < h) {
 					if (*(e + 2) == 'x' || *(e + 2) == 'X') {
@@ -405,23 +417,27 @@ rspamd_html_decode_entitles_inplace (gchar *s, guint len)
 					else {
 						base = 10;
 					}
+
 					if (base == 10) {
-						val = strtoul ((e + 2), &end_ptr, base);
+						uc = strtoul ((e + 2), &end_ptr, base);
 					}
 					else {
-						val = strtoul ((e + 3), &end_ptr, base);
+						uc = strtoul ((e + 3), &end_ptr, base);
 					}
 
 					if (end_ptr != NULL && *end_ptr != '\0') {
 						/* Skip undecoded */
-						if (end - t >= h - e) {
-							memmove (t, e, h - e);
-							t += h - e;
+						*h = ';';
+
+						if (end - t > h - e + 1) {
+							memmove (t, e, h - e + 1);
+							t += h - e + 1;
 						}
 					}
 					else {
 						/* Search for a replacement */
-						k = kh_get (entity_by_number, html_entity_by_number, val);
+						*h = ';';
+						k = kh_get (entity_by_number, html_entity_by_number, uc);
 
 						if (k != kh_end (html_entity_by_number)) {
 							if (kh_val (html_entity_by_number, k)) {
@@ -433,33 +449,67 @@ rspamd_html_decode_entitles_inplace (gchar *s, guint len)
 									t += rep_len;
 								}
 							} else {
-								if (end - t >= h - e) {
-									memmove (t, e, h - e);
-									t += h - e;
+								if (end - t > h - e + 1) {
+									memmove (t, e, h - e + 1);
+									t += h - e + 1;
 								}
 							}
 						}
 						else {
 							/* Unicode point */
-							if (g_unichar_isgraph (val)) {
-								t += g_unichar_to_utf8 (val, t);
+							goffset off = t - s;
+							UBool is_error = 0;
+
+							if (uc > 0) {
+								U8_APPEND (s, off, len, uc, is_error);
+								if (!is_error) {
+									t = s + off;
+								}
+								else {
+									/* Leave invalid entities as is */
+									if (end - t > h - e + 1) {
+										memmove (t, e, h - e + 1);
+										t += h - e + 1;
+									}
+								}
 							}
-							else {
-								/* Remove unknown entities */
+							else if (end - t > h - e + 1) {
+								memmove (t, e, h - e + 1);
+								t += h - e + 1;
 							}
 						}
 					}
 				}
 
-				*h = ';';
 				state = 0;
 			}
+			else if (*h == '&') {
+				/* Previous `&` was bogus */
+				state = 1;
+
+				if (end - t > h - e) {
+					memmove (t, e, h - e);
+					t += h - e;
+				}
+
+				e = h;
+			}
+
 			h++;
 
 			break;
 		}
 	}
 
+	/* Leftover */
+	if (state == 1 && h > e) {
+		/* Unfinished entity, copy as is */
+		if (end - t > h - e) {
+			memmove (t, e, h - e);
+			t += h - e;
+		}
+	}
+
 	return (t - s);
 }
 
@@ -568,7 +618,8 @@ rspamd_html_url_is_phished (rspamd_mempool_t *pool,
 			}
 		}
 		text_url = rspamd_mempool_alloc0 (pool, sizeof (struct rspamd_url));
-		rc = rspamd_url_parse (text_url, url_str, strlen (url_str), pool);
+		rc = rspamd_url_parse (text_url, url_str, strlen (url_str), pool,
+				RSPAMD_URL_PARSE_TEXT);
 
 		if (rc == URI_ERRNO_OK) {
 			disp_tok.len = text_url->hostlen;
@@ -894,7 +945,7 @@ rspamd_html_parse_tag_component (rspamd_mempool_t *pool,
 	return ret;
 }
 
-static void
+static inline void
 rspamd_html_parse_tag_content (rspamd_mempool_t *pool,
 		struct html_content *hc, struct html_tag *tag, const guchar *in,
 		gint *statep, guchar const **savep)
@@ -988,23 +1039,61 @@ rspamd_html_parse_tag_content (rspamd_mempool_t *pool,
 			state = ignore_bad_tag;
 		}
 		else {
+			const guchar *attr_name_end = in;
+
 			if (*in == '=') {
 				state = parse_equal;
 			}
+			else if (*in == '"') {
+				/* No equal or something sane but we have quote character */
+				state = parse_start_dquote;
+				attr_name_end = in - 1;
+
+				while (attr_name_end > *savep) {
+					if (!g_ascii_isalnum (*attr_name_end)) {
+						attr_name_end --;
+					}
+					else {
+						break;
+					}
+				}
+
+				/* One character forward to obtain length */
+				attr_name_end ++;
+			}
 			else if (g_ascii_isspace (*in)) {
 				state = spaces_before_eq;
 			}
 			else if (*in == '/') {
 				tag->flags |= FL_CLOSED;
 			}
+			else if (!g_ascii_isgraph (*in)) {
+				state = parse_value;
+				attr_name_end = in - 1;
+
+				while (attr_name_end > *savep) {
+					if (!g_ascii_isalnum (*attr_name_end)) {
+						attr_name_end --;
+					}
+					else {
+						break;
+					}
+				}
+
+				/* One character forward to obtain length */
+				attr_name_end ++;
+			}
 			else {
 				return;
 			}
 
-			if (!rspamd_html_parse_tag_component (pool, *savep, in, tag)) {
+			if (!rspamd_html_parse_tag_component (pool, *savep, attr_name_end, tag)) {
 				/* Ignore unknown params */
 				*savep = NULL;
 			}
+			else if (state == parse_value) {
+				*savep = in + 1;
+			}
 		}
 
 		break;
@@ -1106,6 +1195,7 @@ rspamd_html_parse_tag_content (rspamd_mempool_t *pool,
 			store = TRUE;
 			state = parse_end_dquote;
 		}
+
 		if (store) {
 			if (*savep != NULL) {
 				gchar *s;
@@ -1150,7 +1240,7 @@ rspamd_html_parse_tag_content (rspamd_mempool_t *pool,
 			tag->flags |= FL_CLOSED;
 			store = TRUE;
 		}
-		else if (g_ascii_isspace (*in) || *in == '>') {
+		else if (g_ascii_isspace (*in) || *in == '>' || *in == '"') {
 			store = TRUE;
 			state = spaces_after_param;
 		}
@@ -1207,6 +1297,7 @@ rspamd_html_process_url (rspamd_mempool_t *pool, const gchar *start, guint len,
 		struct html_tag_component *comp)
 {
 	struct rspamd_url *url;
+	guint saved_flags = 0;
 	gchar *decoded;
 	gint rc;
 	gsize decoded_len;
@@ -1276,9 +1367,12 @@ rspamd_html_process_url (rspamd_mempool_t *pool, const gchar *start, guint len,
 		}
 	}
 
-	/* We also need to remove all internal newlines and encode unsafe characters */
+	/*
+	 * We also need to remove all internal newlines, spaces
+	 * and encode unsafe characters
+	 */
 	for (i = 0; i < len; i ++) {
-		if (G_UNLIKELY (s[i] == '\r' || s[i] == '\n')) {
+		if (G_UNLIKELY (g_ascii_isspace (s[i]))) {
 			continue;
 		}
 		else if (G_UNLIKELY (((guint)s[i]) < 0x80 && !g_ascii_isgraph (s[i]))) {
@@ -1298,13 +1392,27 @@ rspamd_html_process_url (rspamd_mempool_t *pool, const gchar *start, guint len,
 
 	url = rspamd_mempool_alloc0 (pool, sizeof (*url));
 
-	if (rspamd_normalise_unicode_inplace (pool, decoded, &dlen)) {
-		url->flags |= RSPAMD_URL_FLAG_UNNORMALISED;
+	enum rspamd_normalise_result norm_res;
+
+	norm_res = rspamd_normalise_unicode_inplace (pool, decoded, &dlen);
+
+	if (norm_res & RSPAMD_UNICODE_NORM_UNNORMAL) {
+		saved_flags |= RSPAMD_URL_FLAG_UNNORMALISED;
 	}
 
-	rc = rspamd_url_parse (url, decoded, dlen, pool);
+	if (norm_res & (RSPAMD_UNICODE_NORM_ZERO_SPACES|RSPAMD_UNICODE_NORM_ERROR)) {
+		saved_flags |= RSPAMD_URL_FLAG_OBSCURED;
+
+		if (norm_res & RSPAMD_UNICODE_NORM_ZERO_SPACES) {
+			saved_flags |= RSPAMD_URL_FLAG_ZW_SPACES;
+		}
+	}
+
+	rc = rspamd_url_parse (url, decoded, dlen, pool, RSPAMD_URL_PARSE_HREF);
 
 	if (rc == URI_ERRNO_OK) {
+		url->flags |= saved_flags;
+
 		if (has_bad_chars) {
 			url->flags |= RSPAMD_URL_FLAG_OBSCURED;
 		}
@@ -1428,7 +1536,7 @@ rspamd_process_html_url (rspamd_mempool_t *pool, struct rspamd_url *url,
 
 	if (url->querylen > 0) {
 
-		if (rspamd_url_find (pool, url->query, url->querylen, &url_str, TRUE,
+		if (rspamd_url_find (pool, url->query, url->querylen, &url_str, FALSE,
 				NULL, &prefix_added)) {
 			query_url = rspamd_mempool_alloc0 (pool,
 					sizeof (struct rspamd_url));
@@ -1436,7 +1544,8 @@ rspamd_process_html_url (rspamd_mempool_t *pool, struct rspamd_url *url,
 			rc = rspamd_url_parse (query_url,
 					url_str,
 					strlen (url_str),
-					pool);
+					pool,
+					RSPAMD_URL_PARSE_TEXT);
 
 			if (rc == URI_ERRNO_OK &&
 					query_url->hostlen > 0) {
@@ -1480,6 +1589,58 @@ rspamd_process_html_url (rspamd_mempool_t *pool, struct rspamd_url *url,
 	}
 }
 
+static void
+rspamd_html_process_data_image (rspamd_mempool_t *pool,
+								struct html_image *img,
+								struct html_tag_component *src)
+{
+	/*
+	 * Here, we do very basic processing of the data:
+	 * detect if we have something like: `data:image/xxx;base64,yyyzzz==`
+	 * We only parse base64 encoded data.
+	 * We ignore content type so far
+	 */
+	struct rspamd_image *parsed_image;
+	const gchar *semicolon_pos = NULL, *end = src->start + src->len;
+
+	semicolon_pos = src->start;
+
+	while ((semicolon_pos = memchr (semicolon_pos, ';', end - semicolon_pos)) != NULL) {
+		if (end - semicolon_pos > sizeof ("base64,")) {
+			if (memcmp (semicolon_pos + 1, "base64,", sizeof ("base64,") - 1) == 0) {
+				const gchar *data_pos = semicolon_pos + sizeof ("base64,");
+				gchar *decoded;
+				gsize encoded_len = end - data_pos, decoded_len;
+				rspamd_ftok_t inp;
+
+				decoded_len = (encoded_len / 4 * 3) + 12;
+				decoded = rspamd_mempool_alloc (pool, decoded_len);
+				rspamd_cryptobox_base64_decode (data_pos, encoded_len,
+						decoded, &decoded_len);
+				inp.begin = decoded;
+				inp.len = decoded_len;
+
+				parsed_image = rspamd_maybe_process_image (pool, &inp);
+
+				if (parsed_image) {
+					msg_debug_html ("detected %s image of size %ud x %ud in data url",
+							rspamd_image_type_str (parsed_image->type),
+							parsed_image->width, parsed_image->height);
+					img->embedded_image = parsed_image;
+				}
+			}
+
+			break;
+		}
+		else {
+			/* Nothing useful */
+			return;
+		}
+
+		semicolon_pos ++;
+	}
+}
+
 static void
 rspamd_html_process_img_tag (rspamd_mempool_t *pool, struct html_tag *tag,
 		struct html_content *hc)
@@ -1510,6 +1671,14 @@ rspamd_html_process_img_tag (rspamd_mempool_t *pool, struct html_tag *tag,
 				/* We have an embedded image */
 				img->flags |= RSPAMD_HTML_FLAG_IMAGE_EMBEDDED;
 			}
+			if (comp->len > sizeof ("data:") - 1 && memcmp (comp->start,
+					"data:", sizeof ("data:") - 1) == 0) {
+				/* We have an embedded image in HTML tag */
+				img->flags |=
+						(RSPAMD_HTML_FLAG_IMAGE_EMBEDDED|RSPAMD_HTML_FLAG_IMAGE_DATA);
+				rspamd_html_process_data_image (pool, img, comp);
+				hc->flags |= RSPAMD_HTML_FLAG_HAS_DATA_URLS;
+			}
 			else {
 				img->flags |= RSPAMD_HTML_FLAG_IMAGE_EXTERNAL;
 				if (img->src) {
@@ -1584,6 +1753,15 @@ rspamd_html_process_img_tag (rspamd_mempool_t *pool, struct html_tag *tag,
 				hc->images);
 	}
 
+	if (img->embedded_image) {
+		if (!seen_height) {
+			img->height = img->embedded_image->height;
+		}
+		if (!seen_width) {
+			img->width = img->embedded_image->width;
+		}
+	}
+
 	g_ptr_array_add (hc->images, img);
 	tag->extra = img;
 }
@@ -2411,8 +2589,7 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 				/* Empty tag */
 				hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
 				state = tag_end;
-				p ++;
-				break;
+				continue;
 			default:
 				state = tag_content;
 				substate = 0;
@@ -2462,6 +2639,7 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 		case xml_tag_end:
 			if (t == '>') {
 				state = tag_end;
+				continue;
 			}
 			else {
 				hc->flags |= RSPAMD_HTML_FLAG_BAD_ELEMENTS;
@@ -2478,6 +2656,7 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 			}
 			else if (t == '>' && obrace == ebrace) {
 				state = tag_end;
+				continue;
 			}
 			p ++;
 			break;
@@ -2495,7 +2674,7 @@ rspamd_html_process_part_full (rspamd_mempool_t *pool, struct html_content *hc,
 			if (t == '-') {
 				ebrace ++;
 			}
-			else if (t == '>' && ebrace == 2) {
+			else if (t == '>' && ebrace >= 2) {
 				state = tag_end;
 				continue;
 			}

src/libserver/html.h

@@ -18,12 +18,14 @@
 #define RSPAMD_HTML_FLAG_UNKNOWN_ELEMENTS (1 << 4)
 #define RSPAMD_HTML_FLAG_DUPLICATE_ELEMENTS (1 << 5)
 #define RSPAMD_HTML_FLAG_TOO_MANY_TAGS (1 << 6)
+#define RSPAMD_HTML_FLAG_HAS_DATA_URLS (1 << 7)
 
 /*
  * Image flags
  */
 #define RSPAMD_HTML_FLAG_IMAGE_EMBEDDED (1 << 0)
 #define RSPAMD_HTML_FLAG_IMAGE_EXTERNAL (1 << 1)
+#define RSPAMD_HTML_FLAG_IMAGE_DATA (1 << 2)
 
 enum html_component_type {
 	RSPAMD_HTML_COMPONENT_NAME = 0,
@@ -43,12 +45,16 @@ struct html_tag_component {
 	const guchar *start;
 };
 
+
+struct rspamd_image;
+
 struct html_image {
 	guint height;
 	guint width;
 	guint flags;
 	gchar *src;
 	struct rspamd_url *url;
+	struct rspamd_image *embedded_image;
 	struct html_tag *tag;
 };
 
@@ -121,7 +127,7 @@ struct html_content {
 /*
  * Decode HTML entitles in text. Text is modified in place.
  */
-guint rspamd_html_decode_entitles_inplace (gchar *s, guint len);
+guint rspamd_html_decode_entitles_inplace (gchar *s, gsize len);
 
 GByteArray* rspamd_html_process_part (rspamd_mempool_t *pool,
 		struct html_content *hc,

src/libserver/milter.c

@@ -25,6 +25,7 @@
 #include "libutil/http.h"
 #include "libutil/http_private.h"
 #include "libserver/protocol_internal.h"
+#include "libserver/cfg_file_private.h"
 #include "libmime/filter.h"
 #include "libserver/worker_util.h"
 #include "utlist.h"
@@ -1557,7 +1558,7 @@ rspamd_milter_remove_header_safe (struct rspamd_milter_session *session,
  */
 static gboolean
 rspamd_milter_process_milter_block (struct rspamd_milter_session *session,
-		const ucl_object_t *obj, gint action)
+		const ucl_object_t *obj, struct rspamd_action *action)
 {
 	const ucl_object_t *elt, *cur, *cur_elt;
 	ucl_object_iter_t it;
@@ -1731,7 +1732,7 @@ rspamd_milter_process_milter_block (struct rspamd_milter_session *session,
 		}
 	}
 
-	if (action == METRIC_ACTION_ADD_HEADER) {
+	if (action->action_type == METRIC_ACTION_ADD_HEADER) {
 		elt = ucl_object_lookup (obj, "spam_header");
 
 		if (elt) {
@@ -1783,7 +1784,7 @@ rspamd_milter_send_task_results (struct rspamd_milter_session *session,
 	const ucl_object_t *elt;
 	struct rspamd_milter_private *priv = session->priv;
 	const gchar *str_action;
-	gint action = METRIC_ACTION_REJECT;
+	struct rspamd_action *action;
 	rspamd_fstring_t *xcode = NULL, *rcode = NULL, *reply = NULL;
 	GString *hname, *hvalue;
 	gboolean processed = FALSE;
@@ -1805,7 +1806,14 @@ rspamd_milter_send_task_results (struct rspamd_milter_session *session,
 	}
 
 	str_action = ucl_object_tostring (elt);
-	rspamd_action_from_str (str_action, &action);
+	action = rspamd_config_get_action (milter_ctx->cfg, str_action);
+
+	if (action == NULL) {
+		msg_err_milter ("action %s has not been registered", str_action);
+		rspamd_milter_send_action (session, RSPAMD_MILTER_TEMPFAIL);
+
+		goto cleanup;
+	}
 
 	elt = ucl_object_lookup (results, "messages");
 	if (elt) {
@@ -1833,12 +1841,35 @@ rspamd_milter_send_task_results (struct rspamd_milter_session *session,
 
 	if (elt) {
 		hname = g_string_new (RSPAMD_MILTER_DKIM_HEADER);
-		hvalue = g_string_new (ucl_object_tostring (elt));
 
-		rspamd_milter_send_action (session, RSPAMD_MILTER_INSHEADER,
-				1, hname, hvalue);
+		if (ucl_object_type (elt) == UCL_STRING) {
+			hvalue = g_string_new (ucl_object_tostring (elt));
+
+			rspamd_milter_send_action (session, RSPAMD_MILTER_INSHEADER,
+					1, hname, hvalue);
+
+			g_string_free (hvalue, TRUE);
+		}
+		else {
+			ucl_object_iter_t it;
+			const ucl_object_t *cur;
+			int i = 1;
+
+			it = ucl_object_iterate_new (elt);
+
+			while ((cur = ucl_object_iterate_safe (it, true)) != NULL) {
+				hvalue = g_string_new (ucl_object_tostring (cur));
+
+				rspamd_milter_send_action (session, RSPAMD_MILTER_INSHEADER,
+						i++, hname, hvalue);
+
+				g_string_free (hvalue, TRUE);
+			}
+
+			ucl_object_iterate_free (it);
+		}
+
 		g_string_free (hname, TRUE);
-		g_string_free (hvalue, TRUE);
 	}
 
 	if (processed) {
@@ -1860,7 +1891,7 @@ rspamd_milter_send_task_results (struct rspamd_milter_session *session,
 		goto cleanup;
 	}
 
-	switch (action) {
+	switch (action->action_type) {
 	case METRIC_ACTION_REJECT:
 		if (priv->discard_on_reject) {
 			rspamd_milter_send_action (session, RSPAMD_MILTER_DISCARD);
@@ -1939,6 +1970,17 @@ rspamd_milter_send_task_results (struct rspamd_milter_session *session,
 		rspamd_milter_send_action (session, RSPAMD_MILTER_ACCEPT);
 		break;
 
+	case METRIC_ACTION_QUARANTINE:
+		/* TODO: be more flexible about SMTP messages */
+		rspamd_milter_send_action (session, RSPAMD_MILTER_QUARANTINE,
+				RSPAMD_MILTER_QUARANTINE_MESSAGE);
+
+		/* Quarantine also requires accept action, all hail Sendmail */
+		rspamd_milter_send_action (session, RSPAMD_MILTER_ACCEPT);
+		break;
+	case METRIC_ACTION_DISCARD:
+		rspamd_milter_send_action (session, RSPAMD_MILTER_DISCARD);
+		break;
 	case METRIC_ACTION_GREYLIST:
 	case METRIC_ACTION_NOACTION:
 	default:

src/libserver/milter.h

@@ -43,12 +43,14 @@ enum rspamd_milter_reply {
 struct rspamd_email_address;
 struct event_base;
 struct rspamd_http_message;
+struct rspamd_config;
 
 struct rspamd_milter_context {
 	const gchar *spam_header;
 	const gchar *client_ca_name;
 	const gchar *reject_message;
 	void *sessions_cache;
+	struct rspamd_config *cfg;
 	gboolean discard_on_reject;
 	gboolean quarantine_on_reject;
 };

src/libserver/protocol.c

@@ -19,6 +19,8 @@
 #include "utlist.h"
 #include "http_private.h"
 #include "worker_private.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/filter_private.h"
 #include "contrib/zstd/zstd.h"
 #include "lua/lua_common.h"
 #include "unix-std.h"
@@ -433,14 +435,17 @@ rspamd_protocol_handle_headers (struct rspamd_task *task,
 					}
 				}
 				IF_HEADER (URLS_HEADER) {
+					msg_debug_protocol ("read urls header, value: %V", hv);
+
 					srch.begin = "extended";
 					srch.len = 8;
 
-					msg_debug_protocol ("read urls header, value: %V", hv);
 					if (rspamd_ftok_casecmp (hv_tok, &srch) == 0) {
 						task->flags |= RSPAMD_TASK_FLAG_EXT_URLS;
 						msg_debug_protocol ("extended urls information");
 					}
+
+					/* TODO: add more formats there */
 				}
 				IF_HEADER (USER_AGENT_HEADER) {
 					msg_debug_protocol ("read user-agent header, value: %V", hv);
@@ -663,6 +668,7 @@ rspamd_protocol_handle_request (struct rspamd_task *task,
 /* Structure for writing tree data */
 struct tree_cb_data {
 	ucl_object_t *top;
+	GHashTable *seen;
 	struct rspamd_task *task;
 };
 
@@ -713,17 +719,37 @@ urls_protocol_cb (gpointer key, gpointer value, gpointer ud)
 	struct rspamd_url *url = value;
 	ucl_object_t *obj;
 	struct rspamd_task *task = cb->task;
-	const gchar *user_field = "unknown", *encoded;
+	const gchar *user_field = "unknown", *encoded = NULL;
 	gboolean has_user = FALSE;
 	guint len = 0;
-	gsize enclen;
-
-	encoded = rspamd_url_encode (url, &enclen, task->task_pool);
+	gsize enclen = 0;
 
 	if (!(task->flags & RSPAMD_TASK_FLAG_EXT_URLS)) {
-		obj = ucl_object_fromlstring (encoded, enclen);
+		if (url->hostlen > 0) {
+			if (g_hash_table_lookup (cb->seen, url)) {
+				return;
+			}
+
+			const gchar *end = NULL;
+
+			if (g_utf8_validate (url->host, url->hostlen, &end)) {
+				obj = ucl_object_fromlstring (url->host, url->hostlen);
+			}
+			else if (end - url->host > 0) {
+				obj = ucl_object_fromlstring (url->host, end - url->host);
+			}
+			else {
+				return;
+			}
+		}
+		else {
+			return;
+		}
+
+		g_hash_table_insert (cb->seen, url, url);
 	}
 	else {
+		encoded = rspamd_url_encode (url, &enclen, task->task_pool);
 		obj = rspamd_protocol_extended_url (task, url, encoded, enclen);
 	}
 
@@ -740,6 +766,10 @@ urls_protocol_cb (gpointer key, gpointer value, gpointer ud)
 			len = task->from_envelope->addr_len;
 		}
 
+		if (!encoded) {
+			encoded = rspamd_url_encode (url, &enclen, task->task_pool);
+		}
+
 		msg_notice_task_encrypted ("<%s> %s: %*s; ip: %s; URL: %*s",
 			task->message_id,
 			has_user ? "user" : "from",
@@ -758,9 +788,12 @@ rspamd_urls_tree_ucl (GHashTable *input, struct rspamd_task *task)
 	obj = ucl_object_typed_new (UCL_ARRAY);
 	cb.top = obj;
 	cb.task = task;
+	cb.seen = g_hash_table_new (rspamd_url_host_hash, rspamd_urls_host_cmp);
 
 	g_hash_table_foreach (input, urls_protocol_cb, &cb);
 
+	g_hash_table_unref (cb.seen);
+
 	return obj;
 }
 
@@ -922,12 +955,12 @@ rspamd_metric_result_ucl (struct rspamd_task *task,
 {
 	struct rspamd_symbol_result *sym;
 	gboolean is_spam;
-	enum rspamd_action_type action = METRIC_ACTION_NOACTION;
+	struct rspamd_action *action;
 	ucl_object_t *obj = NULL, *sobj;
 	const gchar *subject;
 
-	action = rspamd_check_action_metric (task, mres);
-	is_spam = (action < METRIC_ACTION_GREYLIST);
+	action = rspamd_check_action_metric (task);
+	is_spam = !(action->flags & RSPAMD_ACTION_HAM);
 
 	if (task->cmd != CMD_CHECK_V2) {
 		obj = ucl_object_typed_new (UCL_OBJECT);
@@ -955,10 +988,10 @@ rspamd_metric_result_ucl (struct rspamd_task *task,
 			ucl_object_fromdouble (rspamd_task_get_required_score (task, mres)),
 			"required_score", 0, false);
 	ucl_object_insert_key (obj,
-			ucl_object_fromstring (rspamd_action_to_str (action)),
+			ucl_object_fromstring (action->name),
 			"action", 0, false);
 
-	if (action == METRIC_ACTION_REWRITE_SUBJECT) {
+	if (action->action_type == METRIC_ACTION_REWRITE_SUBJECT) {
 		subject = rspamd_protocol_rewrite_subject (task);
 
 		if (subject) {
@@ -966,6 +999,17 @@ rspamd_metric_result_ucl (struct rspamd_task *task,
 				"subject", 0, false);
 		}
 	}
+	if (action->flags & RSPAMD_ACTION_MILTER) {
+		/* Treat milter action specially */
+		if (action->action_type == METRIC_ACTION_DISCARD) {
+			ucl_object_insert_key (obj, ucl_object_fromstring ("discard"),
+					"reject", 0, false);
+		}
+		else if (action->action_type == METRIC_ACTION_QUARANTINE) {
+			ucl_object_insert_key (obj, ucl_object_fromstring ("quarantine"),
+					"reject", 0, false);
+		}
+	}
 
 	/* Now handle symbols */
 	if (task->cmd == CMD_CHECK_V2) {
@@ -1119,6 +1163,7 @@ rspamd_protocol_write_ucl (struct rspamd_task *task,
 {
 	ucl_object_t *top = NULL;
 	GString *dkim_sig;
+	GList *dkim_sigs;
 	const ucl_object_t *milter_reply;
 
 	rspamd_task_set_finish_time (task);
@@ -1154,18 +1199,16 @@ rspamd_protocol_write_ucl (struct rspamd_task *task,
 	}
 
 	if (flags & RSPAMD_PROTOCOL_URLS) {
-		if (task->flags & RSPAMD_TASK_FLAG_EXT_URLS) {
-			if (g_hash_table_size (task->urls) > 0) {
-				ucl_object_insert_key (top,
-						rspamd_urls_tree_ucl (task->urls, task),
-						"urls", 0, false);
-			}
+		if (g_hash_table_size (task->urls) > 0) {
+			ucl_object_insert_key (top,
+					rspamd_urls_tree_ucl (task->urls, task),
+					"urls", 0, false);
+		}
 
-			if (g_hash_table_size (task->emails) > 0) {
-				ucl_object_insert_key (top,
-						rspamd_emails_tree_ucl (task->emails, task),
-						"emails", 0, false);
-			}
+		if (g_hash_table_size (task->emails) > 0) {
+			ucl_object_insert_key (top,
+					rspamd_emails_tree_ucl (task->emails, task),
+					"emails", 0, false);
 		}
 	}
 
@@ -1187,11 +1230,12 @@ rspamd_protocol_write_ucl (struct rspamd_task *task,
 	}
 
 	if (flags & RSPAMD_PROTOCOL_DKIM) {
-		dkim_sig = rspamd_mempool_get_variable (task->task_pool,
+		dkim_sigs = rspamd_mempool_get_variable (task->task_pool,
 				RSPAMD_MEMPOOL_DKIM_SIGNATURE);
 
-		if (dkim_sig) {
+		for (; dkim_sigs != NULL; dkim_sigs = dkim_sigs->next) {
 			GString *folded_header;
+			dkim_sig = (GString *) dkim_sigs->data;
 
 			if (task->flags & RSPAMD_TASK_FLAG_MILTER) {
 				folded_header = rspamd_header_value_fold ("DKIM-Signature",
@@ -1253,7 +1297,8 @@ rspamd_protocol_http_reply (struct rspamd_http_message *msg,
 	gpointer h, v;
 	ucl_object_t *top = NULL;
 	rspamd_fstring_t *reply;
-	gint action, flags = RSPAMD_PROTOCOL_DEFAULT;
+	gint flags = RSPAMD_PROTOCOL_DEFAULT;
+	struct rspamd_action *action;
 
 	/* Write custom headers */
 	g_hash_table_iter_init (&hiter, task->reply_headers);
@@ -1263,9 +1308,7 @@ rspamd_protocol_http_reply (struct rspamd_http_message *msg,
 		rspamd_http_message_add_header (msg, hn->begin, hv->begin);
 	}
 
-	if (task->cfg->log_urls || (task->flags & RSPAMD_TASK_FLAG_EXT_URLS)) {
-		flags |= RSPAMD_PROTOCOL_URLS;
-	}
+	flags |= RSPAMD_PROTOCOL_URLS;
 
 	top = rspamd_protocol_write_ucl (task, flags);
 
@@ -1376,19 +1419,24 @@ end:
 
 		if (metric_res != NULL) {
 
-			action = rspamd_check_action_metric (task, metric_res);
+			action = rspamd_check_action_metric (task);
 
-			if (action == METRIC_ACTION_SOFT_REJECT &&
+			/* TODO: handle custom actions in stats */
+			if (action->action_type == METRIC_ACTION_SOFT_REJECT &&
 					(task->flags & RSPAMD_TASK_FLAG_GREYLISTED)) {
 				/* Set stat action to greylist to display greylisted messages */
-				action = METRIC_ACTION_GREYLIST;
+#ifndef HAVE_ATOMIC_BUILTINS
+				task->worker->srv->stat->actions_stat[METRIC_ACTION_GREYLIST]++;
+#else
+				__atomic_add_fetch (&task->worker->srv->stat->actions_stat[METRIC_ACTION_GREYLIST],
+						1, __ATOMIC_RELEASE);
+#endif
 			}
-
-			if (action < METRIC_ACTION_MAX) {
+			else if (action->action_type < METRIC_ACTION_MAX) {
 #ifndef HAVE_ATOMIC_BUILTINS
-				task->worker->srv->stat->actions_stat[action]++;
+				task->worker->srv->stat->actions_stat[action->action_type]++;
 #else
-				__atomic_add_fetch (&task->worker->srv->stat->actions_stat[action],
+				__atomic_add_fetch (&task->worker->srv->stat->actions_stat[action->action_type],
 						1, __ATOMIC_RELEASE);
 #endif
 			}

src/libserver/roll_history.c

@@ -17,6 +17,7 @@
 #include "rspamd.h"
 #include "lua/lua_common.h"
 #include "unix-std.h"
+#include "cfg_file_private.h"
 
 static const gchar rspamd_history_magic_old[] = {'r', 's', 'h', '1'};
 
@@ -101,6 +102,7 @@ rspamd_roll_history_update (struct roll_history *history,
 	struct roll_history_row *row;
 	struct rspamd_metric_result *metric_res;
 	struct history_metric_callback_data cbdata;
+	struct rspamd_action *action;
 
 	if (history->disabled) {
 		return;
@@ -155,7 +157,8 @@ rspamd_roll_history_update (struct roll_history *history,
 	}
 	else {
 		row->score = metric_res->score;
-		row->action = rspamd_check_action_metric (task, metric_res);
+		action = rspamd_check_action_metric (task);
+		row->action = action->action_type;
 		row->required_score = rspamd_task_get_required_score (task, metric_res);
 		cbdata.pos = row->symbols;
 		cbdata.remain = sizeof (row->symbols);

src/libserver/rspamd_symcache.c

@@ -235,8 +235,6 @@ static void rspamd_symcache_disable_symbol_checkpoint (struct rspamd_task *task,
 		struct rspamd_symcache *cache, const gchar *symbol);
 static void rspamd_symcache_enable_symbol_checkpoint (struct rspamd_task *task,
 		struct rspamd_symcache *cache, const gchar *symbol);
-static void rspamd_symcache_disable_all_symbols (struct rspamd_task *task,
-		struct rspamd_symcache *cache);
 
 static void
 rspamd_symcache_order_dtor (gpointer p)
@@ -1598,7 +1596,8 @@ rspamd_symcache_process_settings (struct rspamd_task *task,
 
 	if (enabled) {
 		/* Disable all symbols but selected */
-		rspamd_symcache_disable_all_symbols (task, cache);
+		rspamd_symcache_disable_all_symbols (task, cache,
+				SYMBOL_TYPE_EXPLICIT_DISABLE);
 		already_disabled = TRUE;
 		it = NULL;
 
@@ -1615,7 +1614,8 @@ rspamd_symcache_process_settings (struct rspamd_task *task,
 		it = NULL;
 
 		if (!already_disabled) {
-			rspamd_symcache_disable_all_symbols (task, cache);
+			rspamd_symcache_disable_all_symbols (task, cache,
+					SYMBOL_TYPE_EXPLICIT_DISABLE);
 		}
 
 		while ((cur = ucl_iterate_object (enabled, &it, true)) != NULL) {
@@ -2274,9 +2274,10 @@ rspamd_symcache_stats_symbols_count (struct rspamd_symcache *cache)
 }
 
 
-static void
+void
 rspamd_symcache_disable_all_symbols (struct rspamd_task *task,
-		struct rspamd_symcache *cache)
+									 struct rspamd_symcache *cache,
+									 guint skip_mask)
 {
 	struct cache_savepoint *checkpoint;
 	guint i;
@@ -2295,7 +2296,7 @@ rspamd_symcache_disable_all_symbols (struct rspamd_task *task,
 	PTR_ARRAY_FOREACH (cache->items_by_id, i, item) {
 		dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
 
-		if (!(item->type & SYMBOL_TYPE_SQUEEZED)) {
+		if (!(item->type & (SYMBOL_TYPE_SQUEEZED|skip_mask))) {
 			SET_FINISH_BIT (checkpoint, dyn_item);
 			SET_START_BIT (checkpoint, dyn_item);
 		}
@@ -2426,7 +2427,7 @@ rspamd_symcache_disable_symbol_perm (struct rspamd_symcache *cache,
 	g_assert (cache != NULL);
 	g_assert (symbol != NULL);
 
-	item = g_hash_table_lookup (cache->items_by_symbol, symbol);
+	item = rspamd_symcache_find_filter (cache, symbol);
 
 	if (item) {
 		item->enabled = FALSE;
@@ -2442,7 +2443,7 @@ rspamd_symcache_enable_symbol_perm (struct rspamd_symcache *cache,
 	g_assert (cache != NULL);
 	g_assert (symbol != NULL);
 
-	item = g_hash_table_lookup (cache->items_by_symbol, symbol);
+	item = rspamd_symcache_find_filter (cache, symbol);
 
 	if (item) {
 		item->enabled = TRUE;
@@ -2773,4 +2774,64 @@ rspamd_symcache_item_async_dec_check_full (struct rspamd_task *task,
 	}
 
 	return FALSE;
+}
+
+gboolean
+rspamd_symcache_add_symbol_flags (struct rspamd_symcache *cache,
+										   const gchar *symbol,
+										   guint flags)
+{
+	struct rspamd_symcache_item *item;
+
+	g_assert (cache != NULL);
+	g_assert (symbol != NULL);
+
+	item = rspamd_symcache_find_filter (cache, symbol);
+
+	if (item) {
+		item->type |= flags;
+
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+gboolean
+rspamd_symcache_set_symbol_flags (struct rspamd_symcache *cache,
+										   const gchar *symbol,
+										   guint flags)
+{
+	struct rspamd_symcache_item *item;
+
+	g_assert (cache != NULL);
+	g_assert (symbol != NULL);
+
+	item = rspamd_symcache_find_filter (cache, symbol);
+
+	if (item) {
+		item->type = flags;
+
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+guint
+rspamd_symcache_get_symbol_flags (struct rspamd_symcache *cache,
+										const gchar *symbol)
+{
+	struct rspamd_symcache_item *item;
+
+	g_assert (cache != NULL);
+	g_assert (symbol != NULL);
+
+	item = rspamd_symcache_find_filter (cache, symbol);
+
+	if (item) {
+		return item->type;
+	}
+
+	return 0;
 }

src/libserver/rspamd_symcache.h

@@ -48,6 +48,8 @@ enum rspamd_symbol_type {
 	SYMBOL_TYPE_SQUEEZED = (1 << 13), /* Symbol is squeezed inside Lua */
 	SYMBOL_TYPE_TRIVIAL = (1 << 14), /* Symbol is trivial */
 	SYMBOL_TYPE_MIME_ONLY = (1 << 15), /* Symbol is mime only */
+	SYMBOL_TYPE_EXPLICIT_DISABLE = (1 << 16), /* Symbol should be disabled explicitly only */
+	SYMBOL_TYPE_IGNORE_PASSTHROUGH = (1 << 17), /* Symbol ignores passthrough result */
 };
 
 /**
@@ -247,7 +249,23 @@ void rspamd_symcache_enable_symbol_perm (struct rspamd_symcache *cache,
 struct rspamd_abstract_callback_data* rspamd_symcache_get_cbdata (
 		struct rspamd_symcache *cache, const gchar *symbol);
 
+/**
+ * Adds flags to a symbol
+ * @param cache
+ * @param symbol
+ * @param flags
+ * @return
+ */
+gboolean rspamd_symcache_add_symbol_flags (struct rspamd_symcache *cache,
+										  const gchar *symbol,
+										  guint flags);
+
+gboolean rspamd_symcache_set_symbol_flags (struct rspamd_symcache *cache,
+										   const gchar *symbol,
+										   guint flags);
 
+guint rspamd_symcache_get_symbol_flags (struct rspamd_symcache *cache,
+										   const gchar *symbol);
 /**
  * Process settings for task
  * @param task
@@ -374,4 +392,14 @@ gboolean rspamd_symcache_item_async_dec_check_full (struct rspamd_task *task,
 											   const gchar *loc);
 #define rspamd_symcache_item_async_dec_check(task, item, subsystem) \
 	rspamd_symcache_item_async_dec_check_full(task, item, subsystem, G_STRLOC)
+
+/**
+ * Disables execution of all symbols, excluding those specified in `skip_mask`
+ * @param task
+ * @param cache
+ * @param skip_mask
+ */
+void rspamd_symcache_disable_all_symbols (struct rspamd_task *task,
+										  struct rspamd_symcache *cache,
+										  guint skip_mask);
 #endif

src/libserver/task.c

@@ -26,7 +26,10 @@
 #include "utlist.h"
 #include "contrib/zstd/zstd.h"
 #include "libserver/mempool_vars_internal.h"
+#include "libserver/cfg_file_private.h"
 #include "libmime/lang_detection.h"
+#include "libmime/filter_private.h"
+
 #include <math.h>
 
 /*
@@ -1072,11 +1075,11 @@ rspamd_task_log_metric_res (struct rspamd_task *task,
 	rspamd_fstring_t *symbuf;
 	struct rspamd_symbol_result *sym;
 	GPtrArray *sorted_symbols;
-	enum rspamd_action_type act;
+	struct rspamd_action *act;
 	guint i, j;
 
 	mres = task->result;
-	act = rspamd_check_action_metric (task, mres);
+	act = rspamd_check_action_metric (task);
 
 	if (mres != NULL) {
 		switch (lf->type) {
@@ -1084,7 +1087,7 @@ rspamd_task_log_metric_res (struct rspamd_task *task,
 			if (RSPAMD_TASK_IS_SKIPPED (task)) {
 				res.begin = "S";
 			}
-			else if (act == METRIC_ACTION_REJECT) {
+			else if (!(act->flags & RSPAMD_ACTION_HAM)) {
 				res.begin = "T";
 			}
 			else {
@@ -1094,7 +1097,7 @@ rspamd_task_log_metric_res (struct rspamd_task *task,
 			res.len = 1;
 			break;
 		case RSPAMD_LOG_ACTION:
-			res.begin = rspamd_action_to_str (act);
+			res.begin = act->name;
 			res.len = strlen (res.begin);
 			break;
 		case RSPAMD_LOG_SCORES:
@@ -1441,14 +1444,17 @@ rspamd_task_log_variable (struct rspamd_task *task,
 			if (!isnan (pr->target_score)) {
 				var.len = rspamd_snprintf (numbuf, sizeof (numbuf),
 						"%s \"%s\"; score=%.2f (set by %s)",
-						rspamd_action_to_str (pr->action),
-						pr->message, pr->target_score, pr->module);
+						pr->action->name,
+						pr->message,
+						pr->target_score,
+						pr->module);
 			}
 			else {
 				var.len = rspamd_snprintf (numbuf, sizeof (numbuf),
 						"%s \"%s\"; score=nan (set by %s)",
-						rspamd_action_to_str (pr->action),
-						pr->message, pr->module);
+						pr->action->name,
+						pr->message,
+						pr->module);
 			}
 			var.begin = numbuf;
 		}
@@ -1536,7 +1542,7 @@ rspamd_task_write_log (struct rspamd_task *task)
 gdouble
 rspamd_task_get_required_score (struct rspamd_task *task, struct rspamd_metric_result *m)
 {
-	guint i;
+	gint i;
 
 	if (m == NULL) {
 		m = task->result;
@@ -1546,9 +1552,13 @@ rspamd_task_get_required_score (struct rspamd_task *task, struct rspamd_metric_r
 		}
 	}
 
-	for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_NOACTION; i ++) {
-		if (!isnan (m->actions_limits[i])) {
-			return m->actions_limits[i];
+	for (i = m->nactions - 1; i >= 0; i --) {
+		struct rspamd_action_result *action_lim = &m->actions_limits[i];
+
+
+		if (!isnan (action_lim->cur_limit) &&
+				!(action_lim->action->flags & (RSPAMD_ACTION_NO_THRESHOLD|RSPAMD_ACTION_HAM))) {
+			return m->actions_limits[i].cur_limit;
 		}
 	}
 

src/libserver/task.h

@@ -150,7 +150,7 @@ struct rspamd_task {
 	gchar *deliver_to;								/**< address to deliver								*/
 	gchar *user;									/**< user to deliver								*/
 	gchar *subject;									/**< subject (for non-mime)							*/
-	gchar *hostname;								/**< hostname reported by MTA						*/
+	const gchar *hostname;							/**< hostname reported by MTA						*/
 	GHashTable *request_headers;					/**< HTTP headers in a request						*/
 	GHashTable *reply_headers;						/**< Custom reply headers							*/
 	struct {

src/libserver/url.c

@@ -97,6 +97,11 @@ static const struct {
 				.name = "mailto",
 				.len = 6
 		},
+		{
+				.proto = PROTOCOL_TELEPHONE,
+				.name = "tel",
+				.len = 3
+		},
 		{
 				.proto = PROTOCOL_UNKNOWN,
 				.name = NULL,
@@ -120,36 +125,44 @@ struct url_matcher {
 };
 
 static gboolean url_file_start (struct url_callback_data *cb,
-		const gchar *pos,
-		url_match_t *match);
+								const gchar *pos,
+								url_match_t *match);
 
 static gboolean url_file_end (struct url_callback_data *cb,
-		const gchar *pos,
-		url_match_t *match);
+							  const gchar *pos,
+							  url_match_t *match);
 
 static gboolean url_web_start (struct url_callback_data *cb,
-		const gchar *pos,
-		url_match_t *match);
+							   const gchar *pos,
+							   url_match_t *match);
 
 static gboolean url_web_end (struct url_callback_data *cb,
-		const gchar *pos,
-		url_match_t *match);
+							 const gchar *pos,
+							 url_match_t *match);
 
 static gboolean url_tld_start (struct url_callback_data *cb,
-		const gchar *pos,
-		url_match_t *match);
+							   const gchar *pos,
+							   url_match_t *match);
 
 static gboolean url_tld_end (struct url_callback_data *cb,
-		const gchar *pos,
-		url_match_t *match);
+							 const gchar *pos,
+							 url_match_t *match);
 
 static gboolean url_email_start (struct url_callback_data *cb,
-		const gchar *pos,
-		url_match_t *match);
+								 const gchar *pos,
+								 url_match_t *match);
 
 static gboolean url_email_end (struct url_callback_data *cb,
-		const gchar *pos,
-		url_match_t *match);
+							   const gchar *pos,
+							   url_match_t *match);
+
+static gboolean url_tel_start (struct url_callback_data *cb,
+							   const gchar *pos,
+							   url_match_t *match);
+
+static gboolean url_tel_end (struct url_callback_data *cb,
+							 const gchar *pos,
+							 url_match_t *match);
 
 struct url_matcher static_matchers[] = {
 		/* Common prefixes */
@@ -173,6 +186,8 @@ struct url_matcher static_matchers[] = {
 				0, 0},
 		{"telnet://", "",          url_web_start,   url_web_end,
 				0, 0},
+		{"tel:", "",               url_tel_start,   url_tel_end,
+				0, 0},
 		{"webcal://", "",          url_web_start,   url_web_end,
 				0, 0},
 		{"mailto:",   "",          url_email_start, url_email_end,
@@ -564,8 +579,10 @@ is_url_end (gchar c)
 }
 
 static gint
-rspamd_mailto_parse (struct http_parser_url *u, const gchar *str, gsize len,
-		gchar const **end, gboolean strict, guint *flags)
+rspamd_mailto_parse (struct http_parser_url *u,
+					 const gchar *str, gsize len,
+					 gchar const **end,
+					 enum rspamd_url_parse_flags parse_flags, guint *flags)
 {
 	const gchar *p = str, *c = str, *last = str + len;
 	gchar t;
@@ -711,7 +728,7 @@ rspamd_mailto_parse (struct http_parser_url *u, const gchar *str, gsize len,
 		*end = p;
 	}
 
-	if (!strict) {
+	if ((parse_flags & RSPAMD_URL_PARSE_CHECK)) {
 		return 0;
 	}
 
@@ -720,7 +737,9 @@ rspamd_mailto_parse (struct http_parser_url *u, const gchar *str, gsize len,
 
 static gint
 rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
-		gchar const **end, gboolean strict, guint *flags)
+				  gchar const **end,
+				  enum rspamd_url_parse_flags parse_flags,
+				  guint *flags)
 {
 	const gchar *p = str, *c = str, *last = str + len, *slash = NULL,
 			*password_start = NULL, *user_start = NULL;
@@ -763,7 +782,7 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
 					SET_U (u, UF_SCHEMA);
 				}
 				else if (!g_ascii_isalnum (t) && t != '+' && t != '-') {
-					if (!strict && p > c) {
+					if ((parse_flags & RSPAMD_URL_PARSE_CHECK) && p > c) {
 						/* We might have some domain, but no protocol */
 						st = parse_domain;
 						p = c;
@@ -985,7 +1004,7 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
 					}
 					else if (*p != '.' && *p != '-' && *p != '_' && *p != '%') {
 						if (*p & 0x80) {
-							*flags |= RSPAMD_URL_FLAG_IDN;
+							(*flags) |= RSPAMD_URL_FLAG_IDN;
 							guint i = 0;
 
 							U8_NEXT (p, i, last - p, uc);
@@ -997,11 +1016,16 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
 
 							if (!u_isalnum (uc)) {
 								/* Bad symbol */
-								if (strict) {
-									goto out;
+								if (IS_ZERO_WIDTH_SPACE (uc)) {
+									(*flags) |= RSPAMD_URL_FLAG_OBSCURED;
 								}
 								else {
-									goto set;
+									if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+										goto out;
+									}
+									else {
+										goto set;
+									}
 								}
 							}
 
@@ -1011,11 +1035,18 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
 							p ++;
 						}
 						else {
-							if (strict) {
-								goto out;
+							if (parse_flags & RSPAMD_URL_PARSE_HREF) {
+								/* We have to use all shit we are given here */
+								p ++;
+								(*flags) |= RSPAMD_URL_FLAG_OBSCURED;
 							}
 							else {
-								goto set;
+								if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+									goto out;
+								}
+								else {
+									goto set;
+								}
 							}
 						}
 					}
@@ -1117,7 +1148,8 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
 					goto set;
 				}
 				else if (!g_ascii_isdigit (t)) {
-					if (strict || !g_ascii_isspace (t)) {
+					if (!(parse_flags & RSPAMD_URL_PARSE_CHECK) ||
+						!g_ascii_isspace (t)) {
 						goto out;
 					}
 					else {
@@ -1148,7 +1180,7 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
 					goto set;
 				}
 				else if (is_lwsp (t)) {
-					if (strict) {
+					if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
 						if (g_ascii_isspace (t)) {
 							goto set;
 						}
@@ -1172,7 +1204,7 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
 					goto set;
 				}
 				else if (is_lwsp (t)) {
-					if (strict) {
+					if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
 						if (g_ascii_isspace (t)) {
 							goto set;
 						}
@@ -1189,7 +1221,7 @@ rspamd_web_parse (struct http_parser_url *u, const gchar *str, gsize len,
 					goto set;
 				}
 				else if (is_lwsp (t)) {
-					if (strict) {
+					if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
 						if (g_ascii_isspace (t)) {
 							goto set;
 						}
@@ -1602,8 +1634,10 @@ rspamd_url_shift (struct rspamd_url *uri, gsize nlen,
 }
 
 enum uri_errno
-rspamd_url_parse (struct rspamd_url *uri, gchar *uristring, gsize len,
-		rspamd_mempool_t *pool)
+rspamd_url_parse (struct rspamd_url *uri,
+				  gchar *uristring, gsize len,
+				  rspamd_mempool_t *pool,
+				  enum rspamd_url_parse_flags parse_flags)
 {
 	struct http_parser_url u;
 	gchar *p, *comp;
@@ -1624,14 +1658,86 @@ rspamd_url_parse (struct rspamd_url *uri, gchar *uristring, gsize len,
 	if (len > sizeof ("mailto:") - 1) {
 		/* For mailto: urls we also need to add slashes to make it a valid URL */
 		if (g_ascii_strncasecmp (p, "mailto:", sizeof ("mailto:") - 1) == 0) {
-			ret = rspamd_mailto_parse (&u, uristring, len, &end, TRUE, &flags);
+			ret = rspamd_mailto_parse (&u, uristring, len, &end, parse_flags,
+					&flags);
+		}
+		else if (g_ascii_strncasecmp (p, "tel:", sizeof ("tel:") - 1) == 0) {
+			/* Telephone url */
+			gint nlen = 0;
+			gboolean has_plus = FALSE;
+			end = p + len;
+			gchar *t, *tend;
+			UChar32 uc;
+
+			uri->raw = p;
+			uri->rawlen = len;
+			uri->string = rspamd_mempool_alloc (pool, len + 1);
+			t = uri->string;
+			tend = t + len;
+			i = 4;
+
+			memcpy (t, "tel:", 4);
+			t += 4;
+			p += 4;
+			nlen = 4;
+
+			if (*p == '+') {
+				has_plus = TRUE;
+				*t++ = *p++;
+				nlen ++;
+				i ++;
+			}
+
+			while (t < tend && i < len) {
+				U8_NEXT (uristring, i, len, uc);
+
+				if (u_isdigit (uc)) {
+					if (g_ascii_isdigit (uc)) {
+						*t++ = uc;
+						nlen ++;
+					}
+					else {
+						/* Obfuscated number */
+						uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
+					}
+				}
+				else if (IS_OBSCURED_CHAR (uc)) {
+					uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
+				}
+			}
+
+			*t = '\0';
+
+			if (rspamd_normalise_unicode_inplace (pool, uri->string, &nlen)) {
+				uri->flags |= RSPAMD_URL_FLAG_UNNORMALISED;
+			}
+
+			uri->urllen = nlen;
+
+			uri->protocol = PROTOCOL_TELEPHONE;
+			uri->protocollen = 4;
+
+			uri->host = uri->string + 4;
+			uri->hostlen = nlen - 4;
+
+			if (has_plus) {
+				uri->tld = uri->string + 5;
+				uri->tldlen = nlen - 5;
+			}
+			else {
+				uri->tld = uri->string + 4;
+				uri->tldlen = nlen - 4;
+			}
+
+			return URI_ERRNO_OK;
 		}
 		else {
-			ret = rspamd_web_parse (&u, uristring, len, &end, TRUE, &flags);
+			ret = rspamd_web_parse (&u, uristring, len, &end, parse_flags,
+					&flags);
 		}
 	}
 	else {
-		ret = rspamd_web_parse (&u, uristring, len, &end, TRUE, &flags);
+		ret = rspamd_web_parse (&u, uristring, len, &end, parse_flags, &flags);
 	}
 
 	if (ret != 0) {
@@ -1715,9 +1821,11 @@ rspamd_url_parse (struct rspamd_url *uri, gchar *uristring, gsize len,
 			uri->protocollen);
 	rspamd_url_shift (uri, unquoted_len, UF_SCHEMA);
 	unquoted_len = rspamd_url_decode (uri->host, uri->host, uri->hostlen);
+
 	if (rspamd_normalise_unicode_inplace (pool, uri->host, &unquoted_len)) {
 		uri->flags |= RSPAMD_URL_FLAG_UNNORMALISED;
 	}
+
 	rspamd_url_shift (uri, unquoted_len, UF_HOST);
 
 	if (uri->datalen) {
@@ -1730,6 +1838,7 @@ rspamd_url_parse (struct rspamd_url *uri, gchar *uristring, gsize len,
 		rspamd_http_normalize_path_inplace (uri->data, uri->datalen, &unquoted_len);
 		rspamd_url_shift (uri, unquoted_len, UF_PATH);
 	}
+
 	if (uri->querylen) {
 		unquoted_len = rspamd_url_decode (uri->query,
 				uri->query,
@@ -1739,6 +1848,7 @@ rspamd_url_parse (struct rspamd_url *uri, gchar *uristring, gsize len,
 		}
 		rspamd_url_shift (uri, unquoted_len, UF_QUERY);
 	}
+
 	if (uri->fragmentlen) {
 		unquoted_len = rspamd_url_decode (uri->fragment,
 				uri->fragment,
@@ -1770,14 +1880,27 @@ rspamd_url_parse (struct rspamd_url *uri, gchar *uristring, gsize len,
 				rspamd_tld_trie_callback, uri, NULL);
 
 	if (uri->tldlen == 0) {
-		/* Ignore URL's without TLD if it is not a numeric URL */
-		if (!rspamd_url_is_ip (uri, pool)) {
-			return URI_ERRNO_TLD_MISSING;
+		if (!(parse_flags & RSPAMD_URL_PARSE_HREF)) {
+			/* Ignore URL's without TLD if it is not a numeric URL */
+			if (!rspamd_url_is_ip (uri, pool)) {
+				return URI_ERRNO_TLD_MISSING;
+			}
+		}
+		else {
+			/* Assume tld equal to host */
+			uri->tld = uri->host;
+			uri->tldlen = uri->hostlen;
 		}
 	}
 
 	if (uri->protocol == PROTOCOL_UNKNOWN) {
-		return URI_ERRNO_INVALID_PROTOCOL;
+		if (!(parse_flags & RSPAMD_URL_PARSE_HREF)) {
+			return URI_ERRNO_INVALID_PROTOCOL;
+		}
+		else {
+			/* Hack, hack, hack */
+			uri->protocol = PROTOCOL_HTTP;
+		}
 	}
 
 	return URI_ERRNO_OK;
@@ -2089,7 +2212,8 @@ url_web_end (struct url_callback_data *cb,
 		len = MIN (len, match->newline_pos - pos);
 	}
 
-	if (rspamd_web_parse (NULL, pos, len, &last, FALSE, &flags) != 0) {
+	if (rspamd_web_parse (NULL, pos, len, &last,
+			RSPAMD_URL_PARSE_CHECK, &flags) != 0) {
 		return FALSE;
 	}
 
@@ -2157,7 +2281,8 @@ url_email_end (struct url_callback_data *cb,
 
 	if (!match->prefix || match->prefix[0] == '\0') {
 		/* We have mailto:// at the beginning */
-		if (rspamd_mailto_parse (&u, pos, len, &last, FALSE, &flags) != 0) {
+		if (rspamd_mailto_parse (&u, pos, len, &last,
+				RSPAMD_URL_PARSE_CHECK, &flags) != 0) {
 			return FALSE;
 		}
 
@@ -2236,6 +2361,54 @@ url_email_end (struct url_callback_data *cb,
 	return FALSE;
 }
 
+static gboolean
+url_tel_start (struct url_callback_data *cb,
+			   const gchar *pos,
+			   url_match_t *match)
+{
+	if (!(*pos == '+' || g_ascii_isdigit (*pos))) {
+		/* Urls cannot start with . */
+		return FALSE;
+	}
+
+	match->m_begin = pos;
+
+	return TRUE;
+}
+
+static gboolean
+url_tel_end (struct url_callback_data *cb,
+			 const gchar *pos,
+			 url_match_t *match)
+{
+	UChar32 uc;
+	gint len = cb->end - pos, i = 0;
+
+	if (match->newline_pos && match->st != '<') {
+		/* We should also limit our match end to the newline */
+		len = MIN (len, match->newline_pos - pos);
+	}
+
+	while (i < len) {
+		U8_NEXT (pos, i, len, uc);
+
+		if (uc < 0) {
+			break;
+		}
+
+		if (!(u_isdigit (uc) || u_isspace (uc) ||
+			  IS_OBSCURED_CHAR (uc) || uc == '+' ||
+			  uc == '-' || uc == '.')) {
+			break;
+		}
+	}
+
+	match->m_len = i;
+
+	return TRUE;
+}
+
+
 static gboolean
 rspamd_url_trie_is_match (struct url_matcher *matcher, const gchar *pos,
 		const gchar *end, const gchar *newline_pos)
@@ -2470,7 +2643,9 @@ rspamd_url_trie_generic_callback_common (struct rspamd_multipattern *mp,
 		cb->fin = m.m_begin + m.m_len;
 		url = rspamd_mempool_alloc0 (pool, sizeof (struct rspamd_url));
 		g_strstrip (cb->url_str);
-		rc = rspamd_url_parse (url, cb->url_str, strlen (cb->url_str), pool);
+		rc = rspamd_url_parse (url, cb->url_str,
+				strlen (cb->url_str), pool,
+				RSPAMD_URL_PARSE_TEXT);
 
 		if (rc == URI_ERRNO_OK && url->hostlen > 0) {
 			if (cb->prefix_added) {
@@ -2583,7 +2758,8 @@ rspamd_url_text_part_callback (struct rspamd_url *url, gsize start_offset,
 			rc = rspamd_url_parse (query_url,
 					url_str,
 					strlen (url_str),
-					task->task_pool);
+					task->task_pool,
+					RSPAMD_URL_PARSE_TEXT);
 
 			if (rc == URI_ERRNO_OK &&
 					query_url->hostlen > 0) {
@@ -2737,7 +2913,8 @@ rspamd_url_task_subject_callback (struct rspamd_url *url, gsize start_offset,
 			rc = rspamd_url_parse (query_url,
 					url_str,
 					strlen (url_str),
-					task->task_pool);
+					task->task_pool,
+					RSPAMD_URL_PARSE_TEXT);
 
 			if (rc == URI_ERRNO_OK &&
 					url->hostlen > 0) {
@@ -2794,15 +2971,26 @@ guint
 rspamd_url_hash (gconstpointer u)
 {
 	const struct rspamd_url *url = u;
-	rspamd_cryptobox_fast_hash_state_t st;
-
-	rspamd_cryptobox_fast_hash_init (&st, rspamd_hash_seed ());
 
 	if (url->urllen > 0) {
-		rspamd_cryptobox_fast_hash_update (&st, url->string, url->urllen);
+		return rspamd_cryptobox_fast_hash (url->string, url->urllen,
+				rspamd_hash_seed ());
 	}
 
-	return rspamd_cryptobox_fast_hash_final (&st);
+	return 0;
+}
+
+guint
+rspamd_url_host_hash (gconstpointer u)
+{
+	const struct rspamd_url *url = u;
+
+	if (url->hostlen > 0) {
+		return rspamd_cryptobox_fast_hash (url->host, url->hostlen,
+				rspamd_hash_seed ());
+	}
+
+	return 0;
 }
 
 guint
@@ -2868,6 +3056,22 @@ rspamd_urls_cmp (gconstpointer a, gconstpointer b)
 	return r == 0;
 }
 
+gboolean
+rspamd_urls_host_cmp (gconstpointer a, gconstpointer b)
+{
+	const struct rspamd_url *u1 = a, *u2 = b;
+	int r = 0;
+
+	if (u1->hostlen != u2->hostlen) {
+		return FALSE;
+	}
+	else {
+		r = memcmp (u1->host, u2->host, u1->hostlen);
+	}
+
+	return r == 0;
+}
+
 gsize
 rspamd_url_decode (gchar *dst, const gchar *src, gsize size)
 {
@@ -3078,8 +3282,15 @@ rspamd_url_encode (struct rspamd_url *url, gsize *pdlen,
 	dest = rspamd_mempool_alloc (pool, dlen + 1);
 	d = dest;
 	dend = d + dlen;
-	d += rspamd_snprintf ((gchar *)d, dend - d,
-			"%*s://", url->protocollen, rspamd_url_protocols[url->protocol].name);
+
+	if (url->protocollen > 0 &&
+		(url->protocol >= 0 && url->protocol < G_N_ELEMENTS (rspamd_url_protocols))) {
+		d += rspamd_snprintf ((gchar *) d, dend - d,
+				"%*s://", url->protocollen, rspamd_url_protocols[url->protocol].name);
+	}
+	else {
+		d += rspamd_snprintf ((gchar *) d, dend - d, "http://");
+	}
 
 	if (url->userlen > 0) {
 		ENCODE_URL_COMPONENT ((guchar *)url->user, url->userlen,
@@ -3112,3 +3323,9 @@ rspamd_url_encode (struct rspamd_url *url, gsize *pdlen,
 
 	return (const gchar *)dest;
 }
+
+gboolean
+rspamd_url_is_domain (int c)
+{
+	return is_domain ((guchar)c);
+}

src/libserver/url.h

@@ -27,6 +27,7 @@ enum rspamd_url_flags {
 	RSPAMD_URL_FLAG_HAS_USER = 1 << 14,
 	RSPAMD_URL_FLAG_SCHEMALESS = 1 << 15,
 	RSPAMD_URL_FLAG_UNNORMALISED = 1 << 16,
+	RSPAMD_URL_FLAG_ZW_SPACES = 1 << 17,
 };
 
 struct rspamd_url_tag {
@@ -83,6 +84,7 @@ enum rspamd_url_protocol {
 	PROTOCOL_HTTP,
 	PROTOCOL_HTTPS,
 	PROTOCOL_MAILTO,
+	PROTOCOL_TELEPHONE,
 	PROTOCOL_UNKNOWN
 };
 
@@ -104,6 +106,12 @@ void rspamd_url_text_extract (rspamd_mempool_t *pool,
 	struct rspamd_mime_text_part *part,
 	gboolean is_html);
 
+enum rspamd_url_parse_flags {
+	RSPAMD_URL_PARSE_TEXT = 0,
+	RSPAMD_URL_PARSE_HREF = (1u << 0),
+	RSPAMD_URL_PARSE_CHECK = (1 << 1),
+};
+
 /*
  * Parse a single url into an uri structure
  * @param pool memory pool
@@ -111,9 +119,10 @@ void rspamd_url_text_extract (rspamd_mempool_t *pool,
  * @param uri url object, must be pre allocated
  */
 enum uri_errno rspamd_url_parse (struct rspamd_url *uri,
-	gchar *uristring,
-	gsize len,
-	rspamd_mempool_t *pool);
+								 gchar *uristring,
+								 gsize len,
+								 rspamd_mempool_t *pool,
+								 enum rspamd_url_parse_flags flags);
 
 /*
  * Try to extract url from a text
@@ -194,12 +203,15 @@ void rspamd_url_add_tag (struct rspamd_url *url, const gchar *tag,
 
 guint rspamd_url_hash (gconstpointer u);
 guint rspamd_email_hash (gconstpointer u);
+guint rspamd_url_host_hash (gconstpointer u);
+
 
 /* Compare two emails for building emails hash */
 gboolean rspamd_emails_cmp (gconstpointer a, gconstpointer b);
 
 /* Compare two urls for building emails hash */
 gboolean rspamd_urls_cmp (gconstpointer a, gconstpointer b);
+gboolean rspamd_urls_host_cmp (gconstpointer a, gconstpointer b);
 
 /**
  * Decode URL encoded string in-place and return new length of a string, src and dst are NULL terminated
@@ -220,4 +232,12 @@ gsize rspamd_url_decode (gchar *dst, const gchar *src, gsize size);
 const gchar * rspamd_url_encode (struct rspamd_url *url, gsize *dlen,
 		rspamd_mempool_t *pool);
 
+
+/**
+ * Returns if a character is domain character
+ * @param c
+ * @return
+ */
+gboolean rspamd_url_is_domain (int c);
+
 #endif

src/libstat/backends/redis_backend.c

@@ -151,7 +151,6 @@ rspamd_redis_expand_object (const gchar *pattern,
 	GString *tb;
 	const gchar *rcpt = NULL;
 	gint err_idx;
-	gboolean expansion_errored = FALSE;
 
 	g_assert (ctx != NULL);
 	stcf = ctx->stcf;
@@ -214,9 +213,6 @@ rspamd_redis_expand_object (const gchar *pattern,
 				if (elt) {
 					tlen += strlen (elt);
 				}
-				else {
-					expansion_errored = TRUE;
-				}
 				break;
 			case 'r':
 
@@ -230,9 +226,6 @@ rspamd_redis_expand_object (const gchar *pattern,
 				if (elt) {
 					tlen += strlen (elt);
 				}
-				else {
-					expansion_errored = TRUE;
-				}
 				break;
 			case 'l':
 				if (stcf->label) {
@@ -277,8 +270,8 @@ rspamd_redis_expand_object (const gchar *pattern,
 	}
 
 
-	if (target == NULL || task == NULL || expansion_errored) {
-		return tlen;
+	if (target == NULL || task == NULL) {
+		return -1;
 	}
 
 	*target = rspamd_mempool_alloc (task->task_pool, tlen + 1);
@@ -1305,12 +1298,13 @@ rspamd_redis_parse_classifier_opts (struct redis_stat_ctx *backend,
 	}
 	else {
 		backend->enable_users = FALSE;
+		backend->cbref_user = -1;
 	}
 
 	elt = ucl_object_lookup (obj, "prefix");
 	if (elt == NULL || ucl_object_type (elt) != UCL_STRING) {
 		/* Default non-users statistics */
-		if (backend->enable_users && backend->cbref_user == -1) {
+		if (backend->enable_users || backend->cbref_user != -1) {
 			backend->redis_object = REDIS_DEFAULT_USERS_OBJECT;
 		}
 		else {

src/libstat/stat_api.h

@@ -36,9 +36,9 @@
 #define RSPAMD_STAT_TOKEN_FLAG_NORMALISED (1u << 7)
 #define RSPAMD_STAT_TOKEN_FLAG_STEMMED (1u << 8)
 #define RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE (1u << 9)
-#define RSPAMD_STAT_TOKEN_FLAG_STOP_WORD (1u << 9)
-#define RSPAMD_STAT_TOKEN_FLAG_SKIPPED (1u << 10)
-#define RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES (1u << 11)
+#define RSPAMD_STAT_TOKEN_FLAG_STOP_WORD (1u << 10)
+#define RSPAMD_STAT_TOKEN_FLAG_SKIPPED (1u << 11)
+#define RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES (1u << 12)
 
 typedef struct rspamd_stat_token_s {
 	rspamd_ftok_t original; /* utf8 raw */

src/libutil/addr.c

@@ -104,18 +104,22 @@ rspamd_ip_validate_af (rspamd_inet_addr_t *addr)
 	}
 }
 
+#define RSPAMD_MAYBE_ALLOC_POOL(pool, sz) \
+	(pool != NULL) ? rspamd_mempool_alloc((pool), (sz)) : g_malloc(sz)
+#define RSPAMD_MAYBE_ALLOC0_POOL(pool, sz) \
+	(pool != NULL) ? rspamd_mempool_alloc0((pool), (sz)) : g_malloc0(sz)
 
 static rspamd_inet_addr_t *
-rspamd_inet_addr_create (gint af)
+rspamd_inet_addr_create (gint af, rspamd_mempool_t *pool)
 {
 	rspamd_inet_addr_t *addr;
 
-	addr = g_malloc0 (sizeof (rspamd_inet_addr_t));
+	addr = RSPAMD_MAYBE_ALLOC0_POOL (pool, sizeof(*addr));
 
 	addr->af = af;
 
 	if (af == AF_UNIX) {
-		addr->u.un = g_malloc0 (sizeof (*addr->u.un));
+		addr->u.un = RSPAMD_MAYBE_ALLOC0_POOL(pool, sizeof (*addr->u.un));
 		addr->slen = sizeof (addr->u.un->addr);
 	}
 	else {
@@ -168,6 +172,7 @@ rspamd_ip_check_ipv6 (void)
 			else {
 				ipv6_status = RSPAMD_IPV6_SUPPORTED;
 			}
+
 			close (s);
 		}
 	}
@@ -269,26 +274,26 @@ rspamd_accept_from_socket (gint sock, rspamd_inet_addr_t **target,
 			p = (const guint8 *)&su.s6.sin6_addr;
 
 			if ((p[10] == 0xff && p[11] == 0xff)) {
-				addr = rspamd_inet_addr_create (AF_INET);
+				addr = rspamd_inet_addr_create (AF_INET, NULL);
 				memcpy (&addr->u.in.addr.s4.sin_addr, &p[12],
 						sizeof (struct in_addr));
 			}
 			else {
 				/* Something strange but not mapped v4 address */
-				addr = rspamd_inet_addr_create (AF_INET6);
+				addr = rspamd_inet_addr_create (AF_INET6, NULL);
 				memcpy (&addr->u.in.addr.s6.sin6_addr, &su.s6.sin6_addr,
 						sizeof (struct in6_addr));
 			}
 		}
 		else {
-			addr = rspamd_inet_addr_create (AF_INET6);
+			addr = rspamd_inet_addr_create (AF_INET6, NULL);
 			memcpy (&addr->u.in.addr.s6.sin6_addr, &su.s6.sin6_addr,
 					sizeof (struct in6_addr));
 		}
 
 	}
 	else {
-		addr = rspamd_inet_addr_create (su.sa.sa_family);
+		addr = rspamd_inet_addr_create (su.sa.sa_family, NULL);
 		addr->slen = len;
 
 		if (addr->af == AF_UNIX) {
@@ -339,7 +344,8 @@ out:
 }
 
 static gboolean
-rspamd_parse_unix_path (rspamd_inet_addr_t **target, const char *src)
+rspamd_parse_unix_path (rspamd_inet_addr_t **target, const char *src,
+						rspamd_mempool_t *pool)
 {
 	gchar **tokens, **cur_tok, *p, *pwbuf;
 	glong pwlen;
@@ -349,7 +355,7 @@ rspamd_parse_unix_path (rspamd_inet_addr_t **target, const char *src)
 	bool has_group = false;
 
 	tokens = g_strsplit_set (src, " ,", -1);
-	addr = rspamd_inet_addr_create (AF_UNIX);
+	addr = rspamd_inet_addr_create (AF_UNIX, pool);
 
 	rspamd_strlcpy (addr->u.un->addr.sun_path, tokens[0],
 			sizeof (addr->u.un->addr.sun_path));
@@ -437,7 +443,11 @@ err:
 
 	g_strfreev (tokens);
 	g_free (pwbuf);
-	rspamd_inet_address_free (addr);
+
+	if (!pool) {
+		rspamd_inet_address_free (addr);
+	}
+
 	return FALSE;
 }
 
@@ -518,6 +528,13 @@ rspamd_parse_inet_address_ip6 (const guchar *text, gsize len, gpointer target)
 		len = percent - p; /* Ignore scope */
 	}
 
+	if (len > sizeof ("IPv6:") - 1 &&
+		g_ascii_strncasecmp (p, "IPv6:", sizeof ("IPv6:") - 1) == 0) {
+		/* Special case, SMTP conformant IPv6 address */
+		p += sizeof ("IPv6:") - 1;
+		len -= sizeof ("IPv6:") - 1;
+	}
+
 	for (/* void */; len; len--) {
 		t = *p++;
 
@@ -614,7 +631,8 @@ rspamd_parse_inet_address_ip6 (const guchar *text, gsize len, gpointer target)
 
 /* Checks for ipv6 mapped address */
 static rspamd_inet_addr_t *
-rspamd_inet_address_v6_maybe_map (const struct sockaddr_in6 *sin6)
+rspamd_inet_address_v6_maybe_map (const struct sockaddr_in6 *sin6,
+		rspamd_mempool_t *pool)
 {
 	rspamd_inet_addr_t *addr = NULL;
 	/* 10 zero bytes or 80 bits */
@@ -627,19 +645,19 @@ rspamd_inet_address_v6_maybe_map (const struct sockaddr_in6 *sin6)
 		p = (const guint8 *)&sin6->sin6_addr;
 
 		if ((p[10] == 0xff && p[11] == 0xff)) {
-			addr = rspamd_inet_addr_create (AF_INET);
+			addr = rspamd_inet_addr_create (AF_INET, pool);
 			memcpy (&addr->u.in.addr.s4.sin_addr, &p[12],
 					sizeof (struct in_addr));
 		}
 		else {
 			/* Something strange but not mapped v4 address */
-			addr = rspamd_inet_addr_create (AF_INET6);
+			addr = rspamd_inet_addr_create (AF_INET6, pool);
 			memcpy (&addr->u.in.addr.s6.sin6_addr, &sin6->sin6_addr,
 					sizeof (struct in6_addr));
 		}
 	}
 	else {
-		addr = rspamd_inet_addr_create (AF_INET6);
+		addr = rspamd_inet_addr_create (AF_INET6, pool);
 		memcpy (&addr->u.in.addr.s6.sin6_addr, &sin6->sin6_addr,
 				sizeof (struct in6_addr));
 	}
@@ -682,10 +700,11 @@ rspamd_inet_address_v6_maybe_map_static (const struct sockaddr_in6 *sin6,
 	}
 }
 
-gboolean
-rspamd_parse_inet_address (rspamd_inet_addr_t **target,
-		const char *src,
-		gsize srclen)
+static gboolean
+rspamd_parse_inet_address_common (rspamd_inet_addr_t **target,
+								  const char *src,
+								  gsize srclen,
+								  rspamd_mempool_t *pool)
 {
 	gboolean ret = FALSE;
 	rspamd_inet_addr_t *addr = NULL;
@@ -705,10 +724,11 @@ rspamd_parse_inet_address (rspamd_inet_addr_t **target,
 	rspamd_ip_check_ipv6 ();
 
 	if (src[0] == '/' || src[0] == '.') {
-		return rspamd_parse_unix_path (target, src);
+		return rspamd_parse_unix_path (target, src, pool);
 	}
 
 	if (src[0] == '[') {
+		const gchar *ip_start;
 		/* Ipv6 address in format [::1]:port or just [::1] */
 		end = memchr (src + 1, ']', srclen - 1);
 
@@ -722,11 +742,12 @@ rspamd_parse_inet_address (rspamd_inet_addr_t **target,
 			return FALSE;
 		}
 
-		rspamd_strlcpy (ipbuf, src + 1, iplen + 1);
+		ip_start = src + 1;
+		rspamd_strlcpy (ipbuf, ip_start, iplen + 1);
 
 		if (rspamd_parse_inet_address_ip6 (ipbuf, iplen,
 						&su.s6.sin6_addr)) {
-			addr = rspamd_inet_address_v6_maybe_map (&su.s6);
+			addr = rspamd_inet_address_v6_maybe_map (&su.s6, pool);
 			ret = TRUE;
 		}
 
@@ -742,8 +763,9 @@ rspamd_parse_inet_address (rspamd_inet_addr_t **target,
 			/* This is either port number and ipv4 addr or ipv6 addr */
 			/* Search for another semicolon */
 			if (memchr (end + 1, ':', srclen - (end - src + 1)) &&
-					rspamd_parse_inet_address_ip6 (src, srclen, &su.s6.sin6_addr)) {
-				addr = rspamd_inet_address_v6_maybe_map (&su.s6);
+					rspamd_parse_inet_address_ip6 (src, srclen,
+							&su.s6.sin6_addr)) {
+				addr = rspamd_inet_address_v6_maybe_map (&su.s6, pool);
 				ret = TRUE;
 			}
 			else {
@@ -759,7 +781,7 @@ rspamd_parse_inet_address (rspamd_inet_addr_t **target,
 
 				if (rspamd_parse_inet_address_ip4 (ipbuf, iplen,
 						&su.s4.sin_addr)) {
-					addr = rspamd_inet_addr_create (AF_INET);
+					addr = rspamd_inet_addr_create (AF_INET, pool);
 					memcpy (&addr->u.in.addr.s4.sin_addr, &su.s4.sin_addr,
 							sizeof (struct in_addr));
 					rspamd_strtoul (end + 1, srclen - iplen - 1, &portnum);
@@ -770,13 +792,13 @@ rspamd_parse_inet_address (rspamd_inet_addr_t **target,
 		}
 		else {
 			if (rspamd_parse_inet_address_ip4 (src, srclen, &su.s4.sin_addr)) {
-				addr = rspamd_inet_addr_create (AF_INET);
+				addr = rspamd_inet_addr_create (AF_INET, pool);
 				memcpy (&addr->u.in.addr.s4.sin_addr, &su.s4.sin_addr,
 						sizeof (struct in_addr));
 				ret = TRUE;
 			}
 			else if (rspamd_parse_inet_address_ip6 (src, srclen, &su.s6.sin6_addr)) {
-				addr = rspamd_inet_address_v6_maybe_map (&su.s6);
+				addr = rspamd_inet_address_v6_maybe_map (&su.s6, pool);
 				ret = TRUE;
 			}
 		}
@@ -789,6 +811,28 @@ rspamd_parse_inet_address (rspamd_inet_addr_t **target,
 	return ret;
 }
 
+gboolean
+rspamd_parse_inet_address (rspamd_inet_addr_t **target,
+						   const char *src,
+						   gsize srclen)
+{
+	return rspamd_parse_inet_address_common (target, src, srclen, NULL);
+}
+
+rspamd_inet_addr_t *
+rspamd_parse_inet_address_pool (const char *src,
+								gsize srclen,
+								rspamd_mempool_t *pool)
+{
+	rspamd_inet_addr_t *ret = NULL;
+
+	if (!rspamd_parse_inet_address_common (&ret, src, srclen, pool)) {
+		return NULL;
+	}
+
+	return ret;
+}
+
 gboolean
 rspamd_parse_inet_address_ip (const char *src, gsize srclen,
 		rspamd_inet_addr_t *target)
@@ -1106,7 +1150,7 @@ rspamd_inet_address_recvfrom (gint fd, void *buf, gsize len, gint fl,
 	}
 
 	if (target) {
-		addr = rspamd_inet_addr_create (su.sa.sa_family);
+		addr = rspamd_inet_addr_create (su.sa.sa_family, NULL);
 		addr->slen = slen;
 
 		if (addr->af == AF_UNIX) {
@@ -1457,7 +1501,7 @@ rspamd_inet_address_new (int af, const void *init)
 {
 	rspamd_inet_addr_t *addr;
 
-	addr = rspamd_inet_addr_create (af);
+	addr = rspamd_inet_addr_create (af, NULL);
 
 	if (init != NULL) {
 		if (af == AF_UNIX) {
@@ -1487,7 +1531,7 @@ rspamd_inet_address_from_sa (const struct sockaddr *sa, socklen_t slen)
 	g_assert (sa != NULL);
 	g_assert (slen >= sizeof (struct sockaddr));
 
-	addr = rspamd_inet_addr_create (sa->sa_family);
+	addr = rspamd_inet_addr_create (sa->sa_family, NULL);
 
 	if (sa->sa_family == AF_UNIX) {
 		/* Init is a path */
@@ -1525,12 +1569,12 @@ rspamd_inet_address_from_rnds (const struct rdns_reply_entry *rep)
 	g_assert (rep != NULL);
 
 	if (rep->type == RDNS_REQUEST_A) {
-		addr = rspamd_inet_addr_create (AF_INET);
+		addr = rspamd_inet_addr_create (AF_INET, NULL);
 		memcpy (&addr->u.in.addr.s4.sin_addr, &rep->content.a.addr,
 				sizeof (struct in_addr));
 	}
 	else if (rep->type == RDNS_REQUEST_AAAA) {
-		addr = rspamd_inet_addr_create (AF_INET6);
+		addr = rspamd_inet_addr_create (AF_INET6, NULL);
 		memcpy (&addr->u.in.addr.s6.sin6_addr, &rep->content.aaa.addr,
 						sizeof (struct in6_addr));
 	}
@@ -1639,7 +1683,7 @@ rspamd_inet_address_copy (const rspamd_inet_addr_t *addr)
 		return NULL;
 	}
 
-	n = rspamd_inet_addr_create (addr->af);
+	n = rspamd_inet_addr_create (addr->af, NULL);
 
 	if (n->af == AF_UNIX) {
 		memcpy (n->u.un, addr->u.un, sizeof (*addr->u.un));
@@ -1659,6 +1703,22 @@ rspamd_inet_address_get_af (const rspamd_inet_addr_t *addr)
 	return addr->af;
 }
 
+struct sockaddr*
+rspamd_inet_address_get_sa (const rspamd_inet_addr_t *addr,
+							socklen_t *sz)
+{
+	g_assert (addr != NULL);
+
+	if (addr->af == AF_UNIX) {
+		*sz = addr->slen;
+		return (struct sockaddr *)&addr->u.un->addr;
+	}
+	else {
+		*sz = addr->slen;
+		return (struct sockaddr *)&addr->u.in.addr.sa;
+	}
+}
+
 
 guint
 rspamd_inet_address_hash (gconstpointer a)

src/libutil/addr.h

@@ -112,6 +112,17 @@ gboolean rspamd_parse_inet_address (rspamd_inet_addr_t **target,
 		const char *src,
 		gsize srclen);
 
+/**
+ * Use memory pool allocated inet address
+ * @param src
+ * @param srclen
+ * @param pool
+ * @return
+ */
+rspamd_inet_addr_t* rspamd_parse_inet_address_pool (const char *src,
+													 gsize srclen,
+													 rspamd_mempool_t *pool);
+
 /**
  * Returns string representation of inet address
  * @param addr
@@ -140,6 +151,14 @@ uint16_t rspamd_inet_address_get_port (const rspamd_inet_addr_t *addr);
  */
 gint rspamd_inet_address_get_af (const rspamd_inet_addr_t *addr);
 
+/**
+ * Returns sockaddr and size for this address
+ * @param addr
+ * @param sz
+ * @return
+ */
+struct sockaddr* rspamd_inet_address_get_sa (const rspamd_inet_addr_t *addr,
+		socklen_t *sz);
 
 /**
  * Makes a radix key from inet address

src/libutil/fstring.c

@@ -375,6 +375,19 @@ rspamd_ftok_cmp (const rspamd_ftok_t *s1,
 	return s1->len - s2->len;
 }
 
+gboolean
+rspamd_ftok_starts_with (const rspamd_ftok_t *s1,
+						 const rspamd_ftok_t *s2)
+{
+	g_assert (s1 != NULL && s2 != NULL);
+
+	if (s1->len >= s2->len) {
+		return !!(memcmp (s1->begin, s2->begin, s2->len) == 0);
+	}
+
+	return FALSE;
+}
+
 void
 rspamd_fstring_mapped_ftok_free (gpointer p)
 {

src/libutil/fstring.h

@@ -139,6 +139,15 @@ gint rspamd_ftok_casecmp (const rspamd_ftok_t *s1,
 gint rspamd_ftok_cmp (const rspamd_ftok_t *s1,
 		const rspamd_ftok_t *s2);
 
+/**
+ * Returns true if `s1` starts with `s2`
+ * @param s1
+ * @param s2
+ * @return
+ */
+gboolean rspamd_ftok_starts_with (const rspamd_ftok_t *s1,
+								  const rspamd_ftok_t *s2);
+
 /**
  * Return TRUE if ftok is equal to specified C string
  */

src/libutil/hash.c

@@ -48,6 +48,7 @@ struct rspamd_lru_hash_s {
 enum rspamd_lru_element_flags {
 	RSPAMD_LRU_ELEMENT_NORMAL = 0,
 	RSPAMD_LRU_ELEMENT_VOLATILE = (1 << 0),
+	RSPAMD_LRU_ELEMENT_IMMORTAL = (1 << 1),
 };
 
 struct rspamd_lru_element_s {
@@ -444,7 +445,6 @@ rspamd_lru_hash_evict (rspamd_lru_hash_t *hash, time_t now)
 	 * or, at some probability scan all table and update eviction
 	 * list first
 	 */
-
 	r = rspamd_random_double_fast ();
 
 	if (r < ((double)eviction_candidates) / hash->maxsize) {
@@ -455,6 +455,10 @@ rspamd_lru_hash_evict (rspamd_lru_hash_t *hash, time_t now)
 		kh_foreach_value_ptr (hash, cur, {
 			rspamd_lru_element_t *node = &cur->e;
 
+			if (node->flags & RSPAMD_LRU_ELEMENT_IMMORTAL) {
+				continue;
+			}
+
 			if (node->flags & RSPAMD_LRU_ELEMENT_VOLATILE) {
 				/* If element is expired, just remove it */
 				if (now - cur->creation_time > cur->ttl) {
@@ -596,7 +600,7 @@ rspamd_lru_hash_insert (rspamd_lru_hash_t *hash,
 	node = &vnode->e;
 
 	if (ret == 0) {
-		/* Existing element, be carefull about destructors */
+		/* Existing element, be careful about destructors */
 		if (hash->value_destroy) {
 			/* Remove old data */
 			hash->value_destroy (vnode->e.data);
@@ -629,7 +633,9 @@ rspamd_lru_hash_insert (rspamd_lru_hash_t *hash,
 	if (ret != 0) {
 		/* Also need to check maxsize */
 		if (kh_size (hash) >= hash->maxsize) {
+			node->flags |= RSPAMD_LRU_ELEMENT_IMMORTAL;
 			rspamd_lru_hash_evict (hash, now);
+			node->flags &= ~RSPAMD_LRU_ELEMENT_IMMORTAL;
 		}
 	}
 

src/libutil/map.c

@@ -664,9 +664,10 @@ read_data:
 			if (cbd->data->etag) {
 				/* Remove old etag */
 				rspamd_fstring_free (cbd->data->etag);
-				cbd->data->etag = rspamd_fstring_new_init (etag_hdr->begin,
-						etag_hdr->len);
 			}
+
+			cbd->data->etag = rspamd_fstring_new_init (etag_hdr->begin,
+					etag_hdr->len);
 		}
 		else {
 			if (cbd->data->etag) {

src/libutil/map_helpers.c

@@ -640,12 +640,14 @@ rspamd_map_helper_new_hash (struct rspamd_map *map)
 void
 rspamd_map_helper_destroy_hash (struct rspamd_hash_map_helper *r)
 {
-	if (r == NULL) {
+	if (r == NULL || r->pool == NULL) {
 		return;
 	}
 
+	rspamd_mempool_t *pool = r->pool;
 	kh_destroy (rspamd_map_hash, r->htb);
-	rspamd_mempool_delete (r->pool);
+	memset (r, 0, sizeof (*r));
+	rspamd_mempool_delete (pool);
 }
 
 static void
@@ -696,12 +698,14 @@ rspamd_map_helper_new_radix (struct rspamd_map *map)
 void
 rspamd_map_helper_destroy_radix (struct rspamd_radix_map_helper *r)
 {
-	if (r == NULL) {
+	if (r == NULL || !r->pool) {
 		return;
 	}
 
 	kh_destroy (rspamd_map_hash, r->htb);
-	rspamd_mempool_delete (r->pool);
+	rspamd_mempool_t *pool = r->pool;
+	memset (r, 0, sizeof (*r));
+	rspamd_mempool_delete (pool);
 }
 
 static void
@@ -754,7 +758,7 @@ rspamd_map_helper_destroy_regexp (struct rspamd_regexp_map_helper *re_map)
 	rspamd_regexp_t *re;
 	guint i;
 
-	if (!re_map) {
+	if (!re_map || !re_map->regexps) {
 		return;
 	}
 
@@ -785,7 +789,9 @@ rspamd_map_helper_destroy_regexp (struct rspamd_regexp_map_helper *re_map)
 	}
 #endif
 
-	rspamd_mempool_delete (re_map->pool);
+	rspamd_mempool_t *pool = re_map->pool;
+	memset (re_map, 0, sizeof (*re_map));
+	rspamd_mempool_delete (pool);
 }
 
 gchar *
@@ -1136,7 +1142,7 @@ rspamd_match_regexp_map_single (struct rspamd_regexp_map_helper *map,
 
 	g_assert (in != NULL);
 
-	if (map == NULL || len == 0) {
+	if (map == NULL || len == 0 || map->regexps == NULL) {
 		return NULL;
 	}
 
@@ -1227,7 +1233,7 @@ rspamd_match_regexp_map_all (struct rspamd_regexp_map_helper *map,
 
 	g_assert (in != NULL);
 
-	if (map == NULL || len == 0) {
+	if (map == NULL || map->regexps == NULL || len == 0) {
 		return NULL;
 	}
 
@@ -1288,7 +1294,7 @@ rspamd_match_hash_map (struct rspamd_hash_map_helper *map, const gchar *in)
 	khiter_t k;
 	struct rspamd_map_helper_value *val;
 
-	if (map == NULL) {
+	if (map == NULL || map->htb == NULL) {
 		return NULL;
 	}
 
@@ -1310,7 +1316,7 @@ rspamd_match_radix_map (struct rspamd_radix_map_helper *map,
 {
 	struct rspamd_map_helper_value *val;
 
-	if (map == NULL) {
+	if (map == NULL || map->trie == NULL) {
 		return NULL;
 	}
 
@@ -1332,7 +1338,7 @@ rspamd_match_radix_map_addr (struct rspamd_radix_map_helper *map,
 {
 	struct rspamd_map_helper_value *val;
 
-	if (map == NULL) {
+	if (map == NULL || map->trie == NULL) {
 		return NULL;
 	}
 

src/libutil/mem_pool.c

@@ -853,7 +853,7 @@ __mutex_spin (rspamd_mempool_mutex_t * mutex)
 	/* Spin */
 	while (nanosleep (&ts, &ts) == -1 && errno == EINTR) ;
 #else
-#       error No methods to spin are defined
+#error No methods to spin are defined
 #endif
 	return 1;
 }

src/libutil/str_util.c

@@ -391,6 +391,32 @@ rspamd_strlcpy_fast (gchar *dst, const gchar *src, gsize siz)
 	return (d - dst);
 }
 
+gsize
+rspamd_null_safe_copy (const gchar *src, gsize srclen,
+					   gchar *dest, gsize destlen)
+{
+	gsize copied = 0, si = 0, di = 0;
+
+	if (destlen == 0) {
+		return 0;
+	}
+
+	while (si < srclen && di + 1 < destlen) {
+		if (src[si] != '\0') {
+			dest[di++] = src[si++];
+			copied ++;
+		}
+		else {
+			si ++;
+		}
+	}
+
+	dest[di] = '\0';
+
+	return copied;
+}
+
+
 size_t
 rspamd_strlcpy_safe (gchar *dst, const gchar *src, gsize siz)
 {
@@ -1394,7 +1420,33 @@ rspamd_header_value_fold (const gchar *name,
 	case after_quote:
 		g_string_append_len (res, c, p - c);
 		break;
-
+	case fold_token:
+		/* Here, we have token start at 'c' and token end at 'p' */
+		if (g_ascii_isspace (res->str[res->len - 1])) {
+			g_string_append_len (res, c, p - c);
+		}
+		else {
+			if (*c != '\r' && *c != '\n') {
+				/* We need to add folding as well */
+				switch (how) {
+				case RSPAMD_TASK_NEWLINES_LF:
+					g_string_append_len (res, "\n\t", 2);
+					break;
+				case RSPAMD_TASK_NEWLINES_CR:
+					g_string_append_len (res, "\r\t", 2);
+					break;
+				case RSPAMD_TASK_NEWLINES_CRLF:
+				default:
+					g_string_append_len (res, "\r\n\t", 3);
+					break;
+				}
+				g_string_append_len (res, c, p - c);
+			}
+			else {
+				g_string_append_len (res, c, p - c);
+			}
+		}
+		break;
 	default:
 		g_assert (p == c);
 		break;
@@ -1931,7 +1983,7 @@ rspamd_decode_qp_buf (const gchar *in, gsize inlen,
 	gchar *o, *end, *pos, c;
 	const gchar *p;
 	guchar ret;
-	gsize remain, processed;
+	gssize remain, processed;
 
 	p = in;
 	o = out;
@@ -1967,6 +2019,14 @@ decode:
 
 				continue;
 			}
+			else {
+				/* Hack, hack, hack, treat =<garbadge> as =<garbadge> */
+				if (remain > 0) {
+					*o++ = *(p - 1);
+				}
+
+				continue;
+			}
 
 			if (remain > 0) {
 				c = *p++;
@@ -2412,7 +2472,7 @@ rspamd_get_unicode_normalizer (void)
 }
 
 
-gboolean
+enum rspamd_normalise_result
 rspamd_normalise_unicode_inplace (rspamd_mempool_t *pool, gchar *start,
 		guint *len)
 {
@@ -2422,7 +2482,8 @@ rspamd_normalise_unicode_inplace (rspamd_mempool_t *pool, gchar *start,
 	const UNormalizer2 *norm = rspamd_get_unicode_normalizer ();
 	gint32 nsym, end;
 	UChar *src = NULL, *dest = NULL;
-	gboolean ret = FALSE;
+	enum rspamd_normalise_result ret = 0;
+	gboolean has_invisible = FALSE;
 
 	/* We first need to convert data to UChars :( */
 	src = g_malloc ((*len + 1) * sizeof (*src));
@@ -2432,6 +2493,7 @@ rspamd_normalise_unicode_inplace (rspamd_mempool_t *pool, gchar *start,
 	if (!U_SUCCESS (uc_err)) {
 		msg_warn_pool_check ("cannot normalise URL, cannot convert to unicode: %s",
 				u_errorName (uc_err));
+		ret |= RSPAMD_UNICODE_NORM_ERROR;
 		goto out;
 	}
 
@@ -2441,36 +2503,81 @@ rspamd_normalise_unicode_inplace (rspamd_mempool_t *pool, gchar *start,
 	if (!U_SUCCESS (uc_err)) {
 		msg_warn_pool_check ("cannot normalise URL, cannot check normalisation: %s",
 				u_errorName (uc_err));
+		ret |= RSPAMD_UNICODE_NORM_ERROR;
 		goto out;
 	}
 
-	if (end == nsym) {
-		/* No normalisation needed */
+	for (gint32 i = 0; i < nsym; i ++) {
+		if (IS_ZERO_WIDTH_SPACE (src[i])) {
+			has_invisible = TRUE;
+			break;
+		}
+	}
+
+	uc_err = U_ZERO_ERROR;
+
+	if (end != nsym) {
+		/* No normalisation needed, but we may still have invisible spaces */
+		/* We copy sub(src, 0, end) to dest and normalise the rest */
+		ret |= RSPAMD_UNICODE_NORM_UNNORMAL;
+		dest = g_malloc (nsym * sizeof (*dest));
+		memcpy (dest, src, end * sizeof (*dest));
+		nsym = unorm2_normalizeSecondAndAppend (norm, dest, end, nsym,
+				src + end, nsym - end, &uc_err);
+
+		if (!U_SUCCESS (uc_err)) {
+			if (uc_err != U_BUFFER_OVERFLOW_ERROR) {
+				msg_warn_pool_check ("cannot normalise URL: %s",
+						u_errorName (uc_err));
+				ret |= RSPAMD_UNICODE_NORM_ERROR;
+			}
+
+			goto out;
+		}
+	}
+	else if (!has_invisible) {
 		goto out;
 	}
+	else {
+		dest = src;
+		src = NULL;
+	}
 
-	/* We copy sub(src, 0, end) to dest and normalise the rest */
-	ret = TRUE;
-	dest = g_malloc (nsym * sizeof (*dest));
-	memcpy (dest, src, end * sizeof (*dest));
-	nsym = unorm2_normalizeSecondAndAppend (norm, dest, end, nsym,
-			src + end, nsym - end, &uc_err);
+	if (has_invisible) {
+		/* Also filter zero width spaces */
+		gint32 new_len = 0;
+		UChar *t = dest, *h = dest;
 
-	if (!U_SUCCESS (uc_err)) {
-		if (uc_err != U_BUFFER_OVERFLOW_ERROR) {
-			msg_warn_pool_check ("cannot normalise URL: %s",
-					u_errorName (uc_err));
+		ret |= RSPAMD_UNICODE_NORM_ZERO_SPACES;
+
+		for (gint32 i = 0; i < nsym; i ++) {
+			if (!IS_ZERO_WIDTH_SPACE (*h)) {
+				*t++ = *h++;
+				new_len ++;
+			}
+			else {
+				h ++;
+			}
 		}
 
-		goto out;
+		nsym = new_len;
 	}
 
 	/* We now convert it back to utf */
 	nsym = ucnv_fromUChars (utf8_conv, start, *len, dest, nsym, &uc_err);
 
 	if (!U_SUCCESS (uc_err)) {
-		msg_warn_pool_check ("cannot normalise URL, cannot convert to UTF8: %s",
-				u_errorName (uc_err));
+		msg_warn_pool_check ("cannot normalise URL, cannot convert to UTF8: %s"
+					   " input length: %d chars, unicode length: %d utf16 symbols",
+				u_errorName (uc_err), (gint)*len, (gint)nsym);
+
+		if (uc_err == U_BUFFER_OVERFLOW_ERROR) {
+			ret |= RSPAMD_UNICODE_NORM_OVERFLOW;
+		}
+		else {
+			ret |= RSPAMD_UNICODE_NORM_ERROR;
+		}
+
 		goto out;
 	}
 
@@ -2647,8 +2754,9 @@ rspamd_str_make_utf_valid (const gchar *src, gsize slen, gsize *dstlen)
 	GString *dst;
 	const gchar *last;
 	gchar *dchar;
-	gsize i, valid, prev;
+	gsize valid, prev;
 	UChar32 uc;
+	gint32 i;
 
 	if (src == NULL) {
 		return NULL;
@@ -2696,4 +2804,105 @@ rspamd_str_make_utf_valid (const gchar *src, gsize slen, gsize *dstlen)
 	g_string_free (dst, FALSE);
 
 	return dchar;
+}
+
+gsize
+rspamd_gstring_strip (GString *s, const gchar *strip_chars)
+{
+	const gchar *p, *sc;
+	gsize strip_len = 0, total = 0;
+
+	p = s->str + s->len - 1;
+
+	while (p >= s->str) {
+		gboolean seen = FALSE;
+
+		sc = strip_chars;
+
+		while (*sc != '\0') {
+			if (*p == *sc) {
+				strip_len ++;
+				seen = TRUE;
+				break;
+			}
+
+			sc ++;
+		}
+
+		if (!seen) {
+			break;
+		}
+
+		p --;
+	}
+
+	if (strip_len > 0) {
+		s->len -= strip_len;
+		s->str[s->len] = '\0';
+		total += strip_len;
+	}
+
+	if (s->len > 0) {
+		strip_len = rspamd_memspn (s->str, strip_chars, s->len);
+
+		if (strip_len > 0) {
+			memmove (s->str, s->str + strip_len, s->len - strip_len);
+			s->len -= strip_len;
+			total += strip_len;
+		}
+	}
+
+	return total;
+}
+
+const gchar* rspamd_string_len_strip (const gchar *in,
+									  gsize *len,
+									  const gchar *strip_chars)
+{
+	const gchar *p, *sc;
+	gsize strip_len = 0, old_len = *len;
+
+	p = in + old_len - 1;
+
+	/* Trail */
+	while (p >= in) {
+		gboolean seen = FALSE;
+
+		sc = strip_chars;
+
+		while (*sc != '\0') {
+			if (*p == *sc) {
+				strip_len ++;
+				seen = TRUE;
+				break;
+			}
+
+			sc ++;
+		}
+
+		if (!seen) {
+			break;
+		}
+
+		p --;
+	}
+
+	if (strip_len > 0) {
+		*len -= strip_len;
+	}
+
+	/* Head */
+	old_len = *len;
+
+	if (old_len > 0) {
+		strip_len = rspamd_memspn (in, strip_chars, old_len);
+
+		if (strip_len > 0) {
+			*len -= strip_len;
+
+			return in + strip_len;
+		}
+	}
+
+	return in;
 }

src/libutil/str_util.h

@@ -89,6 +89,18 @@ gsize rspamd_strlcpy_safe (gchar *dst, const gchar *src, gsize siz);
 #  define rspamd_strlcpy rspamd_strlcpy_fast
 #endif
 
+/**
+ * Copies `srclen` characters from `src` to `dst` ignoring \0
+ * @param src
+ * @param srclen
+ * @param dest
+ * @param destlen
+ * @return number of bytes copied
+ */
+gsize
+rspamd_null_safe_copy (const gchar *src, gsize srclen,
+					   gchar *dest, gsize destlen);
+
 /*
  * Try to convert string of length to long
  */
@@ -402,6 +414,14 @@ struct UConverter *rspamd_get_utf8_converter (void);
 struct UNormalizer2;
 const struct UNormalizer2 *rspamd_get_unicode_normalizer (void);
 
+enum rspamd_normalise_result {
+	RSPAMD_UNICODE_NORM_NORMAL = 0,
+	RSPAMD_UNICODE_NORM_UNNORMAL = (1 << 0),
+	RSPAMD_UNICODE_NORM_ZERO_SPACES = (1 << 1),
+	RSPAMD_UNICODE_NORM_ERROR = (1 << 2),
+	RSPAMD_UNICODE_NORM_OVERFLOW = (1 << 3)
+};
+
 /**
  * Gets a string in UTF8 and normalises it to NFKC_Casefold form
  * @param pool optional memory pool used for logging purposes
@@ -409,7 +429,7 @@ const struct UNormalizer2 *rspamd_get_unicode_normalizer (void);
  * @param len
  * @return TRUE if a string has been normalised
  */
-gboolean rspamd_normalise_unicode_inplace (rspamd_mempool_t *pool,
+enum rspamd_normalise_result rspamd_normalise_unicode_inplace (rspamd_mempool_t *pool,
 		gchar *start, guint *len);
 
 enum rspamd_regexp_escape_flags {
@@ -439,4 +459,28 @@ rspamd_str_regexp_escape (const gchar *pattern, gsize slen,
  */
 gchar * rspamd_str_make_utf_valid (const gchar *src, gsize slen, gsize *dstlen);
 
+/**
+ * Strips characters in `strip_chars` from start and end of the GString
+ * @param s
+ * @param strip_chars
+ */
+gsize rspamd_gstring_strip (GString *s, const gchar *strip_chars);
+
+/**
+ * Strips characters in `strip_chars` from start and end of the sized string
+ * @param s
+ * @param strip_chars
+ */
+const gchar* rspamd_string_len_strip (const gchar *in,
+		gsize *len, const gchar *strip_chars);
+
+#define IS_ZERO_WIDTH_SPACE(uc) ((uc) == 0x200B || \
+								(uc) == 0x200C || \
+								(uc) == 0x200D || \
+								(uc) == 0xFEFF)
+#define IS_OBSCURED_CHAR(uc) (((uc) >= 0x200B && (uc) <= 0x200F) || \
+								((uc) >= 0x2028 && (uc) <= 0x202F) || \
+								((uc) >= 0x205F && (uc) <= 0x206F) || \
+								(uc) == 0xFEFF)
+
 #endif /* SRC_LIBUTIL_STR_UTIL_H_ */