Merge pull request #7 from rspamd/master

merge upstream into local master

.circleci/config.yml

@@ -11,36 +11,30 @@ references:
       name: Capturing coverage data
       command: |
         set -e
-        sudo apt-get install -qq lcov
-        gem install coveralls-lcov
-        lcov --no-external -b ../project -d ../project -c --output-file coverage.${CIRCLE_JOB}.info
+        sudo apt-get install -qq python-pip
+        sudo pip install cpp-coveralls
 
-  - &capture_lua_coverage_data
-    run:
-      name: Capturing Lua coverage data
-      command: |
-        set -e
-        if [ ! -z $COVERALLS_REPO_TOKEN ]; then luacov-coveralls -t ${COVERALLS_REPO_TOKEN} || true; fi
-
-  - &restore_coverage_data
-    restore_cache:
-      keys:
-        - coverage-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+        # further, these files will be saved in cache and used in "send-coverage" step
+        # see "merge_and_upload_coverage_data" and "save_cache" records
+        coveralls --dump coverage.${CIRCLE_JOB}.dump
 
   - &merge_and_upload_coverage_data
     run:
       name: Merging and uploading coverage data
       command: |
         set -e
-        if [ -f ~/project/coverage.rspamd-test.info ] && [ -f ~/project/coverage.functional.info ]; then
-            sudo apt-get install -qq lcov
-            lcov -a ~/project/coverage.rspamd-test.info -t rspamd-test -a ~/project/coverage.functional.info -t functional -o coverage.info
-            gem install coveralls-lcov
-            sudo pip install cpp-coveralls
-            sudo luarocks install luacov-coveralls
+        if [ -f ~/project/coverage.rspamd-test.dump ] && [ -f ~/project/coverage.functional.dump ]; then
+            sudo apt-get install -qq python-pip python-dev
+            sudo pip install --upgrade setuptools
+            sudo pip install --upgrade pyOpenSSL
+            sudo pip install cpp-coveralls requests cryptography
+
+            cd ~/project
             if [ ! -z $COVERALLS_REPO_TOKEN ]; then
-              coveralls --lcov-file coverage.info --dump coveralls.dump
-              luacov-coveralls -t ${COVERALLS_REPO_TOKEN} -j coveralls.dump --root=../project
+              # Merge Lua coverage (collected into lua_coverage_report.json) and with C-coverage
+              # (in coverage.rspamd-test.dump, coverage.functional.dump, see &capture_coverage_data)
+              # and finally upload it into coveralls.io
+              test/functional/util/merge_coveralls.py --input coverage.functional.dump coverage.rspamd-test.dump lua_coverage_report.json unit_test_lua.json --output out.josn --token=${COVERALLS_REPO_TOKEN}
             fi
         fi
 
@@ -80,20 +74,21 @@ jobs:
       - run: sudo apt-get install -qq cmake libevent-dev libglib2.0-dev libicu-dev libluajit-5.1-dev libmagic-dev libsqlite3-dev libssl-dev ragel libunwind-dev libunwind8 luarocks
       - run: sudo luarocks install luacheck
       - run: sudo luarocks install luacov
+      - run: sudo luarocks install luacov-coveralls
 
       - run: cd ../build
       - run: make rspamd-test -j`nproc`
       - run: set +e; test/rspamd-test -p /rspamd/lua; echo "export RETURN_CODE=$?" >> $BASH_ENV
+      - run: luacov-coveralls -o unit_test_lua.json --dryrun
 
       - *capture_coverage_data
-      - *capture_lua_coverage_data
 
       # Share coverage data between jobs
       - save_cache:
-          key: coverage-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+          key: coverage-rspamd-test-{{ .Environment.CIRCLE_WORKFLOW_ID }}
           paths:
-            - coverage.rspamd-test.info
-            - luacov.stats.out
+            - coverage.rspamd-test.dump
+            - unit_test_lua.json
 
       - run: (exit $RETURN_CODE)
 
@@ -107,12 +102,13 @@ jobs:
       - run: sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4    # optional, clickhouse key
 
       - run: sudo apt-get update -qq || true
-      - run: sudo apt-get install -qq libluajit-5.1-dev libpcre3-dev luarocks opendkim-tools python-pip redis-server libunwind8 libglib2.0-dev libicu-dev libevent-dev
+      - run: sudo apt-get install -qq libluajit-5.1-dev libpcre3-dev luarocks opendkim-tools python-pip redis-server libunwind8 libglib2.0-dev libicu-dev libevent-dev python-dev
       - run: sudo apt-get install clickhouse-server
 
       - run: sudo pip install demjson psutil robotframework requests http
       - run: sudo luarocks install luacheck
       - run: sudo luarocks install luacov
+      - run: sudo luarocks install luacov-coveralls
 
       - run: cd ../build
       # see coverage notice in "build" stage
@@ -122,9 +118,10 @@ jobs:
 
       # Share coverage data between jobs
       - save_cache:
-          key: coverage-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+          key: coverage-functional-{{ .Environment.CIRCLE_WORKFLOW_ID }}
           paths:
-            - coverage.functional.info
+            - coverage.functional.dump
+            - lua_coverage_report.json
 
       - store_artifacts:
           path: output.xml
@@ -161,10 +158,16 @@ jobs:
     steps:
       - attach_workspace:
           at: *workspace_root
+      - restore_cache:
+          key: coverage-rspamd-test-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+      - restore_cache:
+          key: coverage-functional-{{ .Environment.CIRCLE_WORKFLOW_ID }}
 
-      - *restore_coverage_data
       - *merge_and_upload_coverage_data
 
+      - store_artifacts:
+          path: out.josn
+
 notify:
   webhooks:
     - url: https://coveralls.io/webhook?repo_token={{ .Environment.COVERALLS_REPO_TOKEN }}

.drone.yml

@@ -0,0 +1,76 @@
+---
+workspace:
+  base: /rspamd
+
+pipeline:
+
+  build:
+    # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-build/Dockerfile
+    image: rspamd/ci-ubuntu-build
+    group: build
+    commands:
+      - mkdir /rspamd/build /rspamd/install
+      - cd /rspamd/build
+      - cmake $CI_WORKSPACE -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DCMAKE_INSTALL_PREFIX=/rspamd/install -DCMAKE_RULE_MESSAGES=OFF
+      - ncpu=$(getconf _NPROCESSORS_ONLN)
+      - make -j $ncpu install
+      - make -j $ncpu rspamd-test
+
+  eslint:
+    image: node:10-alpine
+    group: build
+    commands:
+      - npm install
+      - ./node_modules/.bin/eslint -v
+      - ./node_modules/.bin/eslint ./
+
+  rspamd-test:
+    # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test/Dockerfile
+    image: rspamd/ci-ubuntu-test
+    group: tests
+    commands:
+      # rspamd-test and functional test both use luacov.stats.out file and should be started from
+      # different directories (if started in parallel)
+      - cd /rspamd/build/test
+      - set +e
+      - ./rspamd-test -p /rspamd/lua; EXIT_CODE=$?
+      - set -e
+      - luacov-coveralls -o /rspamd/build/unit_test_lua.json --dryrun
+      # coveralls dosn't support layout when object files are located outside git repo root
+      # add hack to disable searching for git
+      - ln -s /bin/true /usr/local/bin/git
+      - coveralls --dump /rspamd/build/coverage.rspamd-test.dump
+      - exit $EXIT_CODE
+
+  functional:
+    # https://github.com/rspamd/rspamd-build-docker/blob/master/ubuntu-test-func/Dockerfile
+    image: rspamd/ci-ubuntu-test-func
+    group: tests
+    commands:
+      - cd /rspamd/build
+      - set +e
+      - RSPAMD_INSTALLROOT=/rspamd/install robot --xunit xunit.xml --exclude isbroken $CI_WORKSPACE/test/functional/cases; EXIT_CODE=$?
+      - set -e
+      # coveralls will not find git repo anyway, see above
+      - ln -s /bin/true /usr/local/bin/git
+      - coveralls --dump coverage.functional.dump
+      - exit $EXIT_CODE
+
+  send-coverage:
+    image: rspamd/ci-ubuntu-test
+    secrets: [ coveralls_repo_token ]
+    commands:
+      - cd /rspamd/build
+      - $CI_WORKSPACE/test/functional/util/merge_coveralls.py --input coverage.functional.dump coverage.rspamd-test.dump unit_test_lua.json lua_coverage_report.json --output out.josn --token=$COVERALLS_REPO_TOKEN
+    when:
+      branch: master
+      # don't send coverage report for pull request
+      event: [push, tag]
+
+  send-test-log:
+    image: drillster/drone-email
+    from: noreply@rspamd.com
+    attachment: /rspamd/build/log.html
+    secrets: [email_host, email_username, email_password]
+    when:
+      status: failure

.eslintrc.json

@@ -23,7 +23,7 @@
             "singleLine": { "afterColon": false }
         }],
         "max-params": ["warn", 6],
-        "max-statements": ["warn", 30],
+        "max-statements": ["warn", 31],
         "max-statements-per-line": ["error", { "max": 2 }],
         "multiline-comment-style": "off",
         "multiline-ternary": ["error", "always-multiline"],
@@ -46,6 +46,7 @@
         "prefer-spread": "off",
         "prefer-template": "off",
         "quote-props" : ["error", "consistent-as-needed"],
+        "quotes": ["error", "double", { "avoidEscape": true }],
         "require-jsdoc": "off",
         "require-unicode-regexp": "off",
         "space-before-function-paren": ["error", {

.luacheckrc

@@ -63,3 +63,7 @@ files['/**/src/rspamadm/*'].globals = {
   'ansicolors',
   'getopt',
 }
+
+files['test/functional/lua/test_coverage.lua'].globals = {
+  '__GLOBAL_COVERAGE_WATCHDOG'
+}

CMakeLists.txt

@@ -1078,10 +1078,16 @@ ENDIF(HAVE_SIGINFO_H)
 
 # Some hack for libevent 2.0
 CHECK_C_SOURCE_COMPILES ("#include <event.h>
-					#if _EVENT_NUMERIC_VERSION < 0x02000000
+					#if !defined(LIBEVENT_VERSION_NUMBER) || LIBEVENT_VERSION_NUMBER < 0x02000000
 					#error Unsupported
 					#endif
 					int main() { return 0;}" HAVE_LIBEVENT2)
+CHECK_C_SOURCE_COMPILES ("#include <event2/event.h>
+					int main() { return EVENT_BASE_FLAG_NO_CACHE_TIME; }"
+		HAVE_EVENT_NO_CACHE_TIME_FLAG)
+LIST(APPEND CMAKE_REQUIRED_LIBRARIES "event")
+CHECK_SYMBOL_EXISTS(event_base_update_cache_time "sys/types.h;event.h"
+		HAVE_EVENT_NO_CACHE_TIME_FUNC)
 
 IF(NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS")
 IF(HAVE_CLOCK_GETTIME)

README.md

@@ -1,6 +1,6 @@
 # <a href="https://rspamd.com"><img src="https://rspamd.com/img/rspamd_logo_black.png" alt="Rspamd" width="220px"/></a>
 
-[![CircleCI](https://circleci.com/gh/rspamd/rspamd/tree/master.svg?style=svg)](https://circleci.com/gh/rspamd/rspamd/tree/master)
+[![DroneCI](https://ci.rspamd.com/api/badges/rspamd/rspamd/status.svg)](https://ci.rspamd.com/rspamd/rspamd)
 
 
 ## Introduction

conf/composites.conf

@@ -88,16 +88,19 @@ composites {
         expression = "RECEIVED_PBL & -RCVD_VIA_SMTP_AUTH";
         description = "Relayed through ZEN PBL IP without sufficient authentication (possible indicating an open relay)";
         score = 2.0;
+        policy = "leave";
     }
     RCVD_DKIM_ARC_DNSWL_MED {
         expression = "(R_DKIM_ALLOW | ARC_ALLOW) & RCVD_IN_DNSWL_MED";
         description = "Sufficiently DKIM/ARC signed and received from IP with medium trust at DNSWL";
-        score = -1.5;
+        score = -0.5;
+        policy = "leave";
     }
     RCVD_DKIM_ARC_DNSWL_HI {
         expression = "(R_DKIM_ALLOW | ARC_ALLOW) & RCVD_IN_DNSWL_HI";
         description = "Sufficiently DKIM/ARC signed and received from IP with high trust at DNSWL";
-        score = -3.5;
+        score = -1.0;
+        policy = "leave";
     }
     AUTOGEN_PHP_SPAMMY {
         expression = "(HAS_X_POS | HAS_PHPMAILER_SIG | HAS_X_PHP_SCRIPT) & (SUBJECT_ENDS_QUESTION | SUBJECT_ENDS_EXCLAIM | MANY_INVISIBLE_PARTS)";
@@ -114,6 +117,11 @@ composites {
         description = "Contains one or more domains trying to disguise owner/destination";
         score = 0.5;
     }
+    BAD_REP_POLICIES {
+      description = "Contains valid policies but are also marked by fuzzy/bayes";
+      expression = "(~g-:policies) & (-g+:fuzzy | -g+:bayes)";
+      score = 0.1;
+    }
 
     .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/composites.conf"
     .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/composites.conf"

conf/logging.inc

@@ -5,7 +5,7 @@ log_format =<<EOD
 id: <$mid>,$if_qid{ qid: <$>,}$if_ip{ ip: $,}$if_user{ user: $,}$if_smtp_from{ from: <$>,}
 (default: $is_spam ($action): [$scores] [$symbols_scores_params]),
 len: $len, time: $time_real real, $time_virtual virtual, dns req: $dns_req,
-digest: <$digest>$if_smtp_rcpts{, rcpts: <$>}$if_mime_rcpts{, mime_rcpts: <$>}$if_filename{, file: $}
+digest: <$digest>$if_smtp_rcpts{, rcpts: <$>}$if_mime_rcpts{, mime_rcpts: <$>}$if_filename{, file: $}$if_forced_action{, forced: $}
 EOD
 
 

conf/modules.d/surbl.conf

@@ -28,6 +28,7 @@ surbl {
     rules {
         "SURBL_MULTI" {
             suffix = "multi.surbl.org";
+            check_dkim = true;
             bits {
                 CRACKED_SURBL = 128; # From February 2016
                 ABUSE_SURBL = 64;
@@ -38,6 +39,7 @@ surbl {
         }
         "URIBL_MULTI" {
             suffix = "multi.uribl.com";
+            check_dkim = true;
             bits {
                 URIBL_BLOCKED = 1;
                 URIBL_BLACK = 2;
@@ -47,6 +49,7 @@ surbl {
         }
         "RSPAMD_URIBL" {
             suffix = "uribl.rspamd.com";
+            check_dkim = true;
             process_script =<<EOD
 function(url, suffix)
   local cr = require "rspamd_cryptobox_hash"
@@ -58,6 +61,7 @@ EOD;
         "DBL" {
             suffix = "dbl.spamhaus.org";
             no_ip = true;
+            check_dkim = true;
 
             ips = {
                 # spam domain

conf/scores.d/statistics_group.conf

@@ -17,7 +17,7 @@
 
 symbols = {
     "BAYES_SPAM" {
-        weight = 4.0;
+        weight = 5.1;
         description = "Message probably spam, probability: ";
     }
     "BAYES_HAM" {

config.h.in

@@ -42,6 +42,8 @@
 #cmakedefine HAVE_IPV6_V6ONLY    1
 #cmakedefine HAVE_LIBAIO_H       1
 #cmakedefine HAVE_LIBEVENT2      1
+#cmakedefine HAVE_EVENT_NO_CACHE_TIME_FLAG 1
+#cmakedefine HAVE_EVENT_NO_CACHE_TIME_FUNC 1
 #cmakedefine HAVE_LIBGEN_H       1
 #cmakedefine HAVE_LIBUTIL_H      1
 #cmakedefine HAVE_LOCALE_H       1

contrib/libucl/ucl.h

@@ -105,6 +105,7 @@ typedef enum ucl_error {
 	UCL_EIO, /**< IO error occurred during parsing */
 	UCL_ESTATE, /**< Invalid state machine state */
 	UCL_ENESTED, /**< Input has too many recursion levels */
+	UCL_EUNPAIRED, /**< Input has too many recursion levels */
 	UCL_EMACRO, /**< Error processing a macro */
 	UCL_EINTERNAL, /**< Internal unclassified error */
 	UCL_ESSL, /**< SSL error */

contrib/libucl/ucl_internal.h

@@ -148,6 +148,7 @@ enum ucl_parser_state {
 	UCL_STATE_OBJECT,
 	UCL_STATE_ARRAY,
 	UCL_STATE_KEY,
+	UCL_STATE_KEY_OBRACE,
 	UCL_STATE_VALUE,
 	UCL_STATE_AFTER_VALUE,
 	UCL_STATE_ARRAY_VALUE,
@@ -185,16 +186,30 @@ struct ucl_macro {
 	UT_hash_handle hh;
 };
 
+enum ucl_stack_flags {
+	UCL_STACK_HAS_OBRACE = (1u << 0),
+	UCL_STACK_MAX = (1u << 1),
+};
+
 struct ucl_stack {
 	ucl_object_t *obj;
 	struct ucl_stack *next;
-	uint64_t level;
+	union {
+		struct {
+			uint16_t level;
+			uint16_t flags;
+			uint32_t line;
+		} params;
+		uint64_t len;
+	} e;
+	struct ucl_chunk *chunk;
 };
 
 struct ucl_chunk {
 	const unsigned char *begin;
 	const unsigned char *end;
 	const unsigned char *pos;
+	char *fname;
 	size_t remain;
 	unsigned int line;
 	unsigned int column;

contrib/libucl/ucl_msgpack.c

@@ -434,7 +434,6 @@ static ssize_t ucl_msgpack_parse_ignore (struct ucl_parser *parser,
 #define MSGPACK_FLAG_EXT (1 << 3)
 #define MSGPACK_FLAG_ASSOC (1 << 4)
 #define MSGPACK_FLAG_KEY (1 << 5)
-#define MSGPACK_CONTAINER_BIT (1ULL << 62)
 
 /*
  * Search tree packed in array
@@ -768,7 +767,6 @@ ucl_msgpack_get_container (struct ucl_parser *parser,
 	assert (obj_parser != NULL);
 
 	if (obj_parser->flags & MSGPACK_FLAG_CONTAINER) {
-		assert ((len & MSGPACK_CONTAINER_BIT) == 0);
 		/*
 		 * Insert new container to the stack
 		 */
@@ -779,6 +777,8 @@ ucl_msgpack_get_container (struct ucl_parser *parser,
 				ucl_create_err (&parser->err, "no memory");
 				return NULL;
 			}
+
+			parser->stack->chunk = parser->chunks;
 		}
 		else {
 			stack = calloc (1, sizeof (struct ucl_stack));
@@ -788,11 +788,12 @@ ucl_msgpack_get_container (struct ucl_parser *parser,
 				return NULL;
 			}
 
+			stack->chunk = parser->chunks;
 			stack->next = parser->stack;
 			parser->stack = stack;
 		}
 
-		parser->stack->level = len | MSGPACK_CONTAINER_BIT;
+		parser->stack->e.len = len;
 
 #ifdef MSGPACK_DEBUG_PARSER
 		stack = parser->stack;
@@ -823,16 +824,11 @@ ucl_msgpack_get_container (struct ucl_parser *parser,
 static bool
 ucl_msgpack_is_container_finished (struct ucl_stack *container)
 {
-	uint64_t level;
-
 	assert (container != NULL);
 
-	if (container->level & MSGPACK_CONTAINER_BIT) {
-		level = container->level & ~MSGPACK_CONTAINER_BIT;
 
-		if (level == 0) {
-			return true;
-		}
+	if (container->e.len == 0) {
+		return true;
 	}
 
 	return false;
@@ -843,12 +839,11 @@ ucl_msgpack_insert_object (struct ucl_parser *parser,
 		const unsigned char *key,
 		size_t keylen, ucl_object_t *obj)
 {
-	uint64_t level;
 	struct ucl_stack *container;
 
 	container = parser->stack;
 	assert (container != NULL);
-	assert (container->level > 0);
+	assert (container->e.len > 0);
 	assert (obj != NULL);
 	assert (container->obj != NULL);
 
@@ -875,10 +870,7 @@ ucl_msgpack_insert_object (struct ucl_parser *parser,
 		return false;
 	}
 
-	if (container->level & MSGPACK_CONTAINER_BIT) {
-		level = container->level & ~MSGPACK_CONTAINER_BIT;
-		container->level = (level - 1) | MSGPACK_CONTAINER_BIT;
-	}
+	container->e.len--;
 
 	return true;
 }
@@ -887,7 +879,7 @@ static struct ucl_stack *
 ucl_msgpack_get_next_container (struct ucl_parser *parser)
 {
 	struct ucl_stack *cur = NULL;
-	uint64_t level;
+	uint64_t len;
 
 	cur = parser->stack;
 
@@ -895,17 +887,16 @@ ucl_msgpack_get_next_container (struct ucl_parser *parser)
 		return NULL;
 	}
 
-	if (cur->level & MSGPACK_CONTAINER_BIT) {
-		level = cur->level & ~MSGPACK_CONTAINER_BIT;
+	len = cur->e.len;
 
-		if (level == 0) {
-			/* We need to switch to the previous container */
-			parser->stack = cur->next;
-			parser->cur_obj = cur->obj;
-			free (cur);
+	if (len == 0) {
+		/* We need to switch to the previous container */
+		parser->stack = cur->next;
+		parser->cur_obj = cur->obj;
+		free (cur);
 
 #ifdef MSGPACK_DEBUG_PARSER
-			cur = parser->stack;
+		cur = parser->stack;
 			while (cur) {
 				fprintf(stderr, "-");
 				cur = cur->next;
@@ -913,8 +904,7 @@ ucl_msgpack_get_next_container (struct ucl_parser *parser)
 			fprintf(stderr, "-%s -> %d\n", parser->cur_obj->type == UCL_OBJECT ? "object" : "array", (int)parser->cur_obj->len);
 #endif
 
-			return ucl_msgpack_get_next_container (parser);
-		}
+		return ucl_msgpack_get_next_container (parser);
 	}
 
 	/*
@@ -1311,8 +1301,7 @@ ucl_msgpack_consume (struct ucl_parser *parser)
 
 	/* Rewind to the top level container */
 	ucl_msgpack_get_next_container (parser);
-	assert (parser->stack == NULL ||
-			(parser->stack->level & MSGPACK_CONTAINER_BIT) == 0);
+	assert (parser->stack == NULL);
 
 	return true;
 }

contrib/libucl/ucl_parser.c

@@ -630,7 +630,7 @@ ucl_copy_or_store_ptr (struct ucl_parser *parser,
  */
 static inline ucl_object_t *
 ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser,
-		bool is_array, int level)
+		bool is_array, uint32_t level, bool has_obrace)
 {
 	struct ucl_stack *st;
 
@@ -666,7 +666,27 @@ ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser,
 	}
 
 	st->obj = obj;
-	st->level = level;
+
+	if (level >= UINT16_MAX) {
+		ucl_set_err (parser, UCL_ENESTED,
+				"objects are nesting too deep (over 65535 limit)",
+				&parser->err);
+		ucl_object_unref (obj);
+		return NULL;
+	}
+
+
+	st->e.params.level = level;
+	st->e.params.line = parser->chunks->line;
+	st->chunk = parser->chunks;
+
+	if (has_obrace) {
+		st->e.params.flags = UCL_STACK_HAS_OBRACE;
+	}
+	else {
+		st->e.params.flags = 0;
+	}
+
 	LL_PREPEND (parser->stack, st);
 	parser->cur_obj = obj;
 
@@ -1014,7 +1034,8 @@ ucl_lex_json_string (struct ucl_parser *parser,
 						ucl_chunk_skipc (chunk, p);
 					}
 					if (p >= chunk->end) {
-						ucl_set_err (parser, UCL_ESYNTAX, "unfinished escape character",
+						ucl_set_err (parser, UCL_ESYNTAX,
+								"unfinished escape character",
 								&parser->err);
 						return false;
 					}
@@ -1040,7 +1061,8 @@ ucl_lex_json_string (struct ucl_parser *parser,
 		ucl_chunk_skipc (chunk, p);
 	}
 
-	ucl_set_err (parser, UCL_ESYNTAX, "no quote at the end of json string",
+	ucl_set_err (parser, UCL_ESYNTAX,
+			"no quote at the end of json string",
 			&parser->err);
 	return false;
 }
@@ -1065,7 +1087,8 @@ ucl_lex_squoted_string (struct ucl_parser *parser,
 			ucl_chunk_skipc (chunk, p);
 
 			if (p >= chunk->end) {
-				ucl_set_err (parser, UCL_ESYNTAX, "unfinished escape character",
+				ucl_set_err (parser, UCL_ESYNTAX,
+						"unfinished escape character",
 						&parser->err);
 				return false;
 			}
@@ -1084,7 +1107,8 @@ ucl_lex_squoted_string (struct ucl_parser *parser,
 		ucl_chunk_skipc (chunk, p);
 	}
 
-	ucl_set_err (parser, UCL_ESYNTAX, "no quote at the end of single quoted string",
+	ucl_set_err (parser, UCL_ESYNTAX,
+			"no quote at the end of single quoted string",
 			&parser->err);
 	return false;
 }
@@ -1706,7 +1730,7 @@ ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk)
 			/* We have a new object */
 			if (parser->stack) {
 				obj = ucl_parser_add_container (obj, parser, false,
-						parser->stack->level);
+						parser->stack->e.params.level, true);
 			}
 			else {
 				return false;
@@ -1727,7 +1751,7 @@ ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk)
 			/* We have a new array */
 			if (parser->stack) {
 				obj = ucl_parser_add_container (obj, parser, true,
-						parser->stack->level);
+						parser->stack->e.params.level, true);
 			}
 			else {
 				return false;
@@ -1906,6 +1930,17 @@ ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk)
 
 					/* Pop all nested objects from a stack */
 					st = parser->stack;
+
+					if (!(st->e.params.flags & UCL_STACK_HAS_OBRACE)) {
+						parser->err_code = UCL_EUNPAIRED;
+						ucl_create_err (&parser->err,
+								"%s:%d object closed with } is not opened with { at line %d",
+								chunk->fname ? chunk->fname : "memory",
+								parser->chunks->line, st->e.params.line);
+
+						return false;
+					}
+
 					parser->stack = st->next;
 					UCL_FREE (sizeof (struct ucl_stack), st);
 
@@ -1916,9 +1951,13 @@ ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk)
 					while (parser->stack != NULL) {
 						st = parser->stack;
 
-						if (st->next == NULL || st->next->level == st->level) {
+						if (st->next == NULL) {
 							break;
 						}
+						else if (st->next->e.params.level == st->e.params.level) {
+							break;
+						}
+
 
 						parser->stack = st->next;
 						parser->cur_obj = st->obj;
@@ -2294,6 +2333,8 @@ ucl_state_machine (struct ucl_parser *parser)
 				return false;
 			}
 			else {
+				bool seen_obrace = false;
+
 				/* Skip any spaces */
 				while (p < chunk->end && ucl_test_character (*p,
 						UCL_CHARACTER_WHITESPACE_UNSAFE)) {
@@ -2305,20 +2346,28 @@ ucl_state_machine (struct ucl_parser *parser)
 				if (*p == '[') {
 					parser->state = UCL_STATE_VALUE;
 					ucl_chunk_skipc (chunk, p);
+					seen_obrace = true;
 				}
 				else {
-					parser->state = UCL_STATE_KEY;
+
 					if (*p == '{') {
 						ucl_chunk_skipc (chunk, p);
+						parser->state = UCL_STATE_KEY_OBRACE;
+						seen_obrace = true;
+					}
+					else {
+						parser->state = UCL_STATE_KEY;
 					}
 				}
 
 				if (parser->top_obj == NULL) {
 					if (parser->state == UCL_STATE_VALUE) {
-						obj = ucl_parser_add_container (NULL, parser, true, 0);
+						obj = ucl_parser_add_container (NULL, parser, true, 0,
+								seen_obrace);
 					}
 					else {
-						obj = ucl_parser_add_container (NULL, parser, false, 0);
+						obj = ucl_parser_add_container (NULL, parser, false, 0,
+								seen_obrace);
 					}
 
 					if (obj == NULL) {
@@ -2332,6 +2381,7 @@ ucl_state_machine (struct ucl_parser *parser)
 			}
 			break;
 		case UCL_STATE_KEY:
+		case UCL_STATE_KEY_OBRACE:
 			/* Skip any spaces */
 			while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
 				ucl_chunk_skipc (chunk, p);
@@ -2362,8 +2412,11 @@ ucl_state_machine (struct ucl_parser *parser)
 			else if (parser->state != UCL_STATE_MACRO_NAME) {
 				if (next_key && parser->stack->obj->type == UCL_OBJECT) {
 					/* Parse more keys and nest objects accordingly */
-					obj = ucl_parser_add_container (parser->cur_obj, parser, false,
-							parser->stack->level + 1);
+					obj = ucl_parser_add_container (parser->cur_obj,
+							parser,
+							false,
+							parser->stack->e.params.level + 1,
+							parser->state == UCL_STATE_KEY_OBRACE);
 					if (obj == NULL) {
 						return false;
 					}
@@ -2415,7 +2468,8 @@ ucl_state_machine (struct ucl_parser *parser)
 				if (!ucl_skip_macro_as_comment (parser, chunk)) {
 					/* We have invalid macro */
 					ucl_create_err (&parser->err,
-							"error on line %d at column %d: invalid macro",
+							"error at %s:%d at column %d: invalid macro",
+							chunk->fname ? chunk->fname : "memory",
 							chunk->line,
 							chunk->column);
 					parser->state = UCL_STATE_ERROR;
@@ -2438,8 +2492,9 @@ ucl_state_machine (struct ucl_parser *parser)
 						HASH_FIND (hh, parser->macroes, c, macro_len, macro);
 						if (macro == NULL) {
 							ucl_create_err (&parser->err,
-									"error on line %d at column %d: "
+									"error at %s:%d at column %d: "
 									"unknown macro: '%.*s', character: '%c'",
+									chunk->fname ? chunk->fname : "memory",
 									chunk->line,
 									chunk->column,
 									(int) (p - c),
@@ -2455,7 +2510,8 @@ ucl_state_machine (struct ucl_parser *parser)
 					else {
 						/* We have invalid macro name */
 						ucl_create_err (&parser->err,
-								"error on line %d at column %d: invalid macro name",
+								"error at %s:%d at column %d: invalid macro name",
+								chunk->fname ? chunk->fname : "memory",
 								chunk->line,
 								chunk->column);
 						parser->state = UCL_STATE_ERROR;
@@ -2554,6 +2610,35 @@ ucl_state_machine (struct ucl_parser *parser)
 		}
 	}
 
+	if (parser->stack != NULL) {
+		struct ucl_stack *st;
+		bool has_error = false;
+
+		LL_FOREACH (parser->stack, st) {
+			if (st->chunk != parser->chunks) {
+				break; /* Not our chunk, give up */
+			}
+			if (st->e.params.flags & UCL_STACK_HAS_OBRACE) {
+				if (parser->err == NULL) {
+					utstring_new (parser->err);
+				}
+
+				utstring_printf (parser->err, "%s:%d unmatched open brace at %d; ",
+						chunk->fname ? chunk->fname : "memory",
+						parser->chunks->line,
+						st->e.params.line);
+
+				has_error = true;
+			}
+		}
+
+		if (has_error) {
+			parser->err_code = UCL_EUNPAIRED;
+
+			return false;
+		}
+	}
+
 	return true;
 }
 
@@ -2788,6 +2873,11 @@ ucl_parser_add_chunk_full (struct ucl_parser *parser, const unsigned char *data,
 		chunk->priority = priority;
 		chunk->strategy = strat;
 		chunk->parse_type = parse_type;
+
+		if (parser->cur_file) {
+			chunk->fname = strdup (parser->cur_file);
+		}
+
 		LL_PREPEND (parser->chunks, chunk);
 		parser->recursion ++;
 
@@ -2868,7 +2958,9 @@ ucl_parser_insert_chunk (struct ucl_parser *parser, const unsigned char *data,
 	parser->state = UCL_STATE_INIT;
 
 	/* Prevent inserted chunks from unintentionally closing the current object */
-	if (parser->stack != NULL && parser->stack->next != NULL) parser->stack->level = parser->stack->next->level;
+	if (parser->stack != NULL && parser->stack->next != NULL) {
+		parser->stack->e.params.level = parser->stack->next->e.params.level;
+	}
 
 	res = ucl_parser_add_chunk_full (parser, data, len, parser->chunks->priority,
 					parser->chunks->strategy, parser->chunks->parse_type);

contrib/libucl/ucl_util.c

@@ -532,6 +532,10 @@ ucl_chunk_free (struct ucl_chunk *chunk)
 			}
 		}
 
+		if (chunk->fname) {
+			free (chunk->fname);
+		}
+
 		UCL_FREE (sizeof (*chunk), chunk);
 	}
 }
@@ -1318,7 +1322,10 @@ ucl_include_file_single (const unsigned char *data, size_t len,
 			return false;
 		}
 		st->obj = nest_obj;
-		st->level = parser->stack->level;
+		st->e.params.level = parser->stack->e.params.level;
+		st->e.params.flags = parser->stack->e.params.flags;
+		st->e.params.line = parser->stack->e.params.line;
+		st->chunk = parser->chunks;
 		LL_PREPEND (parser->stack, st);
 		parser->cur_obj = nest_obj;
 	}

interface/js/app/history.js

@@ -113,6 +113,11 @@ define(["jquery", "footable", "humanize"],
             };
         }
 
+        function getSelector(id) {
+            var e = document.getElementById(id);
+            return e.options[e.selectedIndex].value;
+        }
+
         function get_compare_function() {
             var compare_functions = {
                 magnitude: function (e1, e2) {
@@ -126,11 +131,6 @@ define(["jquery", "footable", "humanize"],
                 }
             };
 
-            function getSelector(id) {
-                var e = document.getElementById(id);
-                return e.options[e.selectedIndex].value;
-            }
-
             return compare_functions[getSelector("selSymOrder")];
         }
 
@@ -346,7 +346,13 @@ define(["jquery", "footable", "humanize"],
                 sortValue: function (val) { return Number(val.options.sortValue); }
             }, {
                 name: "symbols",
-                title: "Symbols",
+                title: "Symbols<br /><br />" +
+                        '<span style="font-weight:normal;">Sort by:</span><br />' +
+                        '<div class="btn-group btn-group-xs btn-sym-order" data-toggle="buttons">' +
+                            '<button type="button" class="btn btn-default btn-sym-magnitude" value="magnitude">Magnitude</button>' +
+                            '<button type="button" class="btn btn-default btn-sym-score" value="score">Value</button>' +
+                            '<button type="button" class="btn btn-default btn-sym-name" value="name">Name</button>' +
+                        "</div>",
                 breakpoints: "all",
                 style: {
                     "font-size": "11px",
@@ -611,7 +617,15 @@ define(["jquery", "footable", "humanize"],
                     "ready.ft.table": drawTooltips,
                     "after.ft.sorting": drawTooltips,
                     "after.ft.paging": drawTooltips,
-                    "after.ft.filtering": drawTooltips
+                    "after.ft.filtering": drawTooltips,
+                    "expand.ft.row": function (e, ft, row) {
+                        setTimeout(function () {
+                            var detail_row = row.$el.next();
+                            var order = getSelector("selSymOrder");
+                            detail_row.find(".btn-sym-" + order)
+                                .addClass("active").siblings().removeClass("active");
+                        }, 5);
+                    }
                 }
             });
         }
@@ -624,14 +638,15 @@ define(["jquery", "footable", "humanize"],
         }
 
         ui.getHistory = function (rspamd, tables) {
-            function waitForRowsDisplayed(callback, iteration) {
+            function waitForRowsDisplayed(rows_total, callback, iteration) {
                 var i = (typeof iteration === "undefined") ? 10 : iteration;
                 var num_rows = $("#historyTable > tbody > tr").length;
-                if (num_rows === rows_per_page) {
+                if (num_rows === rows_per_page ||
+                    num_rows === rows_total) {
                     return callback();
                 } else if (--i) {
                     setTimeout(function () {
-                        waitForRowsDisplayed(callback, i);
+                        waitForRowsDisplayed(rows_total, callback, i);
                     }, 500);
                 }
                 return null;
@@ -676,7 +691,7 @@ define(["jquery", "footable", "humanize"],
                             tables.history.rows.load(items);
                             if (version) { // Non-legacy
                                 // Is there a way to get an event when all rows are loaded?
-                                waitForRowsDisplayed(function () {
+                                waitForRowsDisplayed(items.length, function () {
                                     drawTooltips();
                                 });
                             }
@@ -697,18 +712,29 @@ define(["jquery", "footable", "humanize"],
         };
 
         ui.setup = function (rspamd, tables) {
-            $("#updateHistory").off("click");
-            $("#updateHistory").on("click", function (e) {
-                e.preventDefault();
-                ui.getHistory(rspamd, tables);
-            });
-            $("#selSymOrder").unbind().change(function () {
+            function change_symbols_order(order) {
+                $(".btn-sym-" + order).addClass("active").siblings().removeClass("active");
                 var compare_function = get_compare_function();
                 $.each(tables.history.rows.all, function (i, row) {
                     var cell_val = sort_symbols(symbols[i], compare_function);
                     row.cells[8].val(cell_val, false, true);
                 });
                 drawTooltips();
+            }
+
+            $("#updateHistory").off("click");
+            $("#updateHistory").on("click", function (e) {
+                e.preventDefault();
+                ui.getHistory(rspamd, tables);
+            });
+            $("#selSymOrder").unbind().change(function () {
+                var order = this.value;
+                change_symbols_order(order);
+            });
+            $(document).on("click", ".btn-sym-order button", function () {
+                var order = this.value;
+                $("#selSymOrder").val(order);
+                change_symbols_order(order);
             });
 
             // @reset history log

lualib/lua_nn.lua

@@ -20,9 +20,13 @@ local exports = {}
 
 local lua_nn_models = {}
 
-if rspamd_config:has_torch() then
-  torch = require "torch"
-  torch.setnumthreads(1)
+local conf_section = rspamd_config:get_all_opt("nn_models")
+
+if conf_section then
+  if rspamd_config:has_torch() then
+    torch = require "torch"
+    torch.setnumthreads(1)
+  end
 end
 
 if torch then
@@ -43,10 +47,9 @@ if torch then
         end
       end
     end
-    local section = rspamd_config:get_all_opt("nn_models")
 
-    if section and type(section) == 'table' then
-      for k,v in pairs(section) do
+    if conf_section and type(conf_section) == 'table' then
+      for k,v in pairs(conf_section) do
         if not rspamd_config:add_map(v, "nn map " .. k, gen_process_callback(k)) then
           rspamd_logger.warnx(rspamd_config, 'cannot load NN map %1', k)
         end

lualib/lua_util.lua

@@ -656,9 +656,14 @@ exports.extract_specific_urls = function(params_or_task, lim, need_emails, filte
   for i=1,ntlds / 2 do
     local tld1 = tlds[tlds_keys[i]]
     local tld2 = tlds[tlds_keys[ntlds - i]]
-    table.insert(res, table.remove(tld1))
-    table.insert(res, table.remove(tld2))
-    limit = limit - 2
+    if #tld1 > 0 then
+      table.insert(res, table.remove(tld1))
+      limit = limit - 1
+    end
+    if #tld2 > 0 then
+      table.insert(res, table.remove(tld2))
+      limit = limit - 1
+    end
 
     if limit <= 0 then
       break

lualib/rspamadm/confighelp.lua

@@ -20,7 +20,7 @@ parser:flag "--no-color"
 parser:flag "--short"
     :description "Show only option names"
 parser:flag "--no-examples"
-    :description "Do not show examples (impied by --short)"
+    :description "Do not show examples (implied by --short)"
 
 local function maybe_print_color(key)
   if not opts['no-color'] then

rules/rspamd.lua

@@ -21,7 +21,7 @@ require "global_functions" ()
 config['regexp'] = {}
 rspamd_maps = {} -- Global maps
 
-local local_conf = rspamd_paths['CONFDIR']
+local local_conf = rspamd_paths['LOCAL_CONFDIR']
 local local_rules = rspamd_paths['RULESDIR']
 local rspamd_util = require "rspamd_util"
 
@@ -65,4 +65,4 @@ if rmaps and type(rmaps) == 'table' then
 end
 
 local rspamd_nn = require "lua_nn"
-rspamd_nn.load_rspamd_nn() -- Load defined models
+rspamd_nn.load_rspamd_nn() -- Load defined models if any

src/controller.c

@@ -1047,12 +1047,18 @@ rspamd_controller_handle_get_map (struct rspamd_http_connection_entry *conn_ent,
 		reply->date = time (NULL);
 		reply->code = 200;
 
-		if (!rspamd_http_message_set_body_from_fd (reply, fd)) {
-			close (fd);
-			rspamd_http_message_unref (reply);
-			msg_err_session ("cannot read map %s: %s", bk->uri, strerror (errno));
-			rspamd_controller_send_error (conn_ent, 500, "Map read error");
-			return 0;
+		if (st.st_size > 0) {
+			if (!rspamd_http_message_set_body_from_fd (reply, fd)) {
+				close (fd);
+				rspamd_http_message_unref (reply);
+				msg_err_session ("cannot read map %s: %s", bk->uri, strerror (errno));
+				rspamd_controller_send_error (conn_ent, 500, "Map read error");
+				return 0;
+			}
+		}
+		else {
+			rspamd_fstring_t *empty_body = rspamd_fstring_new_init ("", 0);
+			rspamd_http_message_set_body_from_fstring_steal (reply, empty_body);
 		}
 
 		close (fd);

src/libcryptobox/cryptobox.c

@@ -374,7 +374,8 @@ rspamd_cryptobox_init (void)
 	ctx->blake2_impl = blake2b_load ();
 	ctx->ed25519_impl = ed25519_load ();
 	ctx->base64_impl = base64_load ();
-#ifdef HAVE_USABLE_OPENSSL
+#if defined(HAVE_USABLE_OPENSSL) && (OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER))
+	/* Needed for old openssl api, not sure about LibreSSL */
 	ERR_load_EC_strings ();
 	ERR_load_RAND_strings ();
 	ERR_load_EVP_strings ();

src/libmime/email_addr.c

@@ -193,6 +193,36 @@ rspamd_email_address_parse_heuristic (const char *data, size_t len,
 	return ret;
 }
 
+static inline gboolean
+rspamd_email_address_check_and_add (const gchar *start, gsize len,
+									GPtrArray *res,
+									rspamd_mempool_t *pool,
+									GString *ns)
+{
+	struct rspamd_email_address addr;
+
+	/* The whole email is likely address */
+	rspamd_smtp_addr_parse (start, len, &addr);
+
+	if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) {
+		rspamd_email_address_add (pool, res, &addr, ns);
+	}
+	else {
+		/* Try heuristic */
+		if (rspamd_email_address_parse_heuristic (start,
+				len, &addr)) {
+			rspamd_email_address_add (pool, res, &addr, ns);
+
+			return TRUE;
+		}
+		else {
+			return FALSE;
+		}
+	}
+
+	return TRUE;
+}
+
 GPtrArray *
 rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 		const gchar *hdr, guint len,
@@ -200,7 +230,7 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 {
 	GPtrArray *res = src;
 	gboolean seen_at = FALSE;
-	struct rspamd_email_address addr;
+
 	const gchar *p = hdr, *end = hdr + len, *c = hdr, *t;
 	GString *ns;
 	gint obraces, ebraces;
@@ -264,21 +294,9 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 						t --;
 					}
 
-					rspamd_smtp_addr_parse (c, t - c + 1, &addr);
-
-					if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) {
-						rspamd_email_address_add (pool, res, &addr, ns);
-					}
-					else {
-						/* Try heuristic */
-						if (seen_at &&
-								rspamd_email_address_parse_heuristic (c,
-										t - c + 1, &addr)) {
-							rspamd_email_address_add (pool, res, &addr, ns);
-						}
-						else {
-							rspamd_email_address_add (pool, res, NULL, ns);
-						}
+					if (!rspamd_email_address_check_and_add (c, t - c + 1,
+							res, pool, ns)) {
+						rspamd_email_address_add (pool, res, NULL, ns);
 					}
 
 					/* Cleanup for the next use */
@@ -301,6 +319,16 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 					}
 
 					g_string_append_len (ns, c, t - c + 1);
+
+					if (seen_at) {
+						if (!rspamd_email_address_check_and_add (c, t - c + 1,
+								res, pool, ns)) {
+							rspamd_email_address_add (pool, res, NULL, ns);
+						}
+
+						g_string_set_size (ns, 0);
+						seen_at = FALSE;
+					}
 				}
 
 				c = p;
@@ -324,21 +352,9 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 			break;
 		case parse_addr:
 			if (*p == '>') {
-				rspamd_smtp_addr_parse (c, p - c + 1, &addr);
-
-				if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) {
-					rspamd_email_address_add (pool, res, &addr, ns);
-				}
-				else {
-					/* Try heuristic */
-					if (seen_at &&
-							rspamd_email_address_parse_heuristic (c,
-									p - c + 1, &addr)) {
-						rspamd_email_address_add (pool, res, &addr, ns);
-					}
-					else {
-						rspamd_email_address_add (pool, res, NULL, ns);
-					}
+				if (!rspamd_email_address_check_and_add (c, p - c + 1,
+						res, pool, ns)) {
+					rspamd_email_address_add (pool, res, NULL, ns);
 				}
 
 				/* Cleanup for the next use */
@@ -377,19 +393,22 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 
 			if (obraces == ebraces) {
 				if (next_state == parse_name) {
-					/* Include comment in name */
-					if (p > c) {
-						t = p - 1;
+					if (ns->len > 0) {
+						/* Include comment in name if it has been seen */
+						if (p > c) {
+							t = p - 1;
 
-						while (t > c && g_ascii_isspace (*t)) {
-							t --;
-						}
+							while (t > c && g_ascii_isspace (*t)) {
+								t --;
+							}
 
-						g_string_append_len (ns, c, t - c + 1);
+							g_string_append_len (ns, c, t - c + 1);
+						}
 					}
 
-					c = p;
+					c = p + 1;
 				}
+
 				state = next_state;
 			}
 			p ++;
@@ -406,43 +425,35 @@ rspamd_email_address_from_mime (rspamd_mempool_t *pool,
 				p --;
 			}
 
-			if (seen_at) {
-				/* The whole email is likely address */
-				rspamd_smtp_addr_parse (c, p - c, &addr);
-
-				if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) {
-					rspamd_email_address_add (pool, res, &addr, ns);
-				}
-				else {
-					/* Try heuristic */
-					if (rspamd_email_address_parse_heuristic (c,
-									p - c, &addr)) {
-						rspamd_email_address_add (pool, res, &addr, ns);
+			if (p > c) {
+				if (seen_at) {
+					/* The whole email is likely address */
+					if (!rspamd_email_address_check_and_add (c, p - c,
+							res, pool, ns)) {
+						if (res->len == 0) {
+							rspamd_email_address_add (pool, res, NULL, ns);
+						}
 					}
-					else {
+				} else {
+					/* No @ seen */
+					g_string_append_len (ns, c, p - c);
+
+					if (res->len == 0) {
 						rspamd_email_address_add (pool, res, NULL, ns);
 					}
 				}
 			}
-			else {
-				/* No @ seen */
-				g_string_append_len (ns, c, p - c);
+			else if (res->len == 0) {
 				rspamd_email_address_add (pool, res, NULL, ns);
 			}
 		}
 		break;
 	case parse_addr:
 		if (p > c) {
-			rspamd_smtp_addr_parse (c, p - c, &addr);
-
-			if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) {
-				rspamd_email_address_add (pool, res, &addr, ns);
-			}
-			else {
-				/* Try heuristic */
-				if (rspamd_email_address_parse_heuristic (c, p - c,
-						&addr)) {
-					rspamd_email_address_add (pool, res, &addr, ns);
+			if (!rspamd_email_address_check_and_add (c, p - c,
+					res, pool, ns)) {
+				if (res->len == 0) {
+					rspamd_email_address_add (pool, res, NULL, ns);
 				}
 			}
 		}

src/libmime/filter.c

@@ -60,6 +60,7 @@ rspamd_create_metric_result (struct rspamd_task *task)
 	metric_res->sym_groups = kh_init (rspamd_symbols_group_hash);
 	metric_res->grow_factor = 0;
 	metric_res->score = 0;
+	metric_res->passthrough_result = NULL;
 
 	/* Optimize allocation */
 	kh_resize (rspamd_symbols_group_hash, metric_res->sym_groups, 4);
@@ -71,8 +72,15 @@ rspamd_create_metric_result (struct rspamd_task *task)
 		kh_resize (rspamd_symbols_hash, metric_res->symbols, 4);
 	}
 
-	for (i = 0; i < METRIC_ACTION_MAX; i++) {
-		metric_res->actions_limits[i] = task->cfg->actions[i].score;
+	if (task->cfg) {
+		for (i = 0; i < METRIC_ACTION_MAX; i++) {
+			metric_res->actions_limits[i] = task->cfg->actions[i].score;
+		}
+	}
+	else {
+		for (i = 0; i < METRIC_ACTION_MAX; i++) {
+			metric_res->actions_limits[i] = NAN;
+		}
 	}
 
 	rspamd_mempool_add_destructor (task->task_pool,
@@ -82,6 +90,49 @@ rspamd_create_metric_result (struct rspamd_task *task)
 	return metric_res;
 }
 
+static inline int
+rspamd_pr_sort (const struct rspamd_passthrough_result *pra,
+				const struct rspamd_passthrough_result *prb)
+{
+	return prb->priority - pra->priority;
+}
+
+void
+rspamd_add_passthrough_result (struct rspamd_task *task,
+									enum rspamd_action_type action,
+									guint priority,
+									double target_score,
+									const gchar *message,
+									const gchar *module)
+{
+	struct rspamd_metric_result *metric_res;
+	struct rspamd_passthrough_result *pr;
+
+	metric_res = task->result;
+
+	pr = rspamd_mempool_alloc (task->task_pool, sizeof (*pr));
+	pr->action = action;
+	pr->priority = priority;
+	pr->message = message;
+	pr->module = module;
+	pr->target_score = target_score;
+
+	DL_APPEND (metric_res->passthrough_result, pr);
+	DL_SORT (metric_res->passthrough_result, rspamd_pr_sort);
+
+	if (!isnan (target_score)) {
+		msg_info_task ("<%s>: set pre-result to %s (%.2f): '%s' from %s(%d)",
+				task->message_id, rspamd_action_to_str (action), target_score,
+				message, module, priority);
+	}
+
+	else {
+		msg_info_task ("<%s>: set pre-result to %s (no score): '%s' from %s(%d)",
+				task->message_id, rspamd_action_to_str (action),
+				message, module, priority);
+	}
+}
+
 static inline gdouble
 rspamd_check_group_score (struct rspamd_task *task,
 		const gchar *symbol,
@@ -123,7 +174,7 @@ insert_metric_result (struct rspamd_task *task,
 	gboolean single = !!(flags & RSPAMD_SYMBOL_INSERT_SINGLE);
 	gchar *sym_cpy;
 
-	metric_res = rspamd_create_metric_result (task);
+	metric_res = task->result;
 
 	if (!isfinite (weight)) {
 		msg_warn_task ("detected %s score for symbol %s, replace it with zero",
@@ -191,11 +242,10 @@ insert_metric_result (struct rspamd_task *task,
 			k = kh_get (rspamd_options_hash, s->options, opt);
 
 			if (k == kh_end (s->options)) {
-				single = TRUE;
+				rspamd_task_add_result_option (task, s, opt);
 			}
 			else {
 				s->nshots ++;
-				rspamd_task_add_result_option (task, s, opt);
 			}
 		}
 		else {
@@ -417,12 +467,13 @@ enum rspamd_action_type
 rspamd_check_action_metric (struct rspamd_task *task, struct rspamd_metric_result *mres)
 {
 	struct rspamd_action *action, *selected_action = NULL;
+	struct rspamd_passthrough_result *pr;
 	double max_score = -(G_MAXDOUBLE), sc;
 	int i;
 	gboolean set_action = FALSE;
 
 	/* We are not certain about the results during processing */
-	if (task->pre_result.action == METRIC_ACTION_MAX) {
+	if (task->result->passthrough_result == NULL) {
 		for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) {
 			action = &task->cfg->actions[i];
 			sc = mres->actions_limits[i];
@@ -442,26 +493,13 @@ rspamd_check_action_metric (struct rspamd_task *task, struct rspamd_metric_resul
 		}
 	}
 	else {
-		sc = NAN;
-
-		for (i = task->pre_result.action; i < METRIC_ACTION_MAX; i ++) {
-			selected_action = &task->cfg->actions[i];
-			sc = mres->actions_limits[i];
-
-			if (isnan (sc)) {
-				if (i == task->pre_result.action) {
-					/* No scores defined, just avoid NaN */
-					sc = 0;
-					break;
-				}
-			}
-			else {
-				break;
-			}
-		}
+		/* Peek the highest priority result */
+		pr = task->result->passthrough_result;
+		sc = pr->target_score;
+		selected_action = &task->cfg->actions[pr->action];
 
 		if (!isnan (sc)) {
-			if (task->pre_result.action == METRIC_ACTION_NOACTION) {
+			if (pr->action == METRIC_ACTION_NOACTION) {
 				mres->score = MIN (sc, mres->score);
 			}
 			else {

src/libmime/filter.h

@@ -58,9 +58,25 @@ KHASH_INIT (rspamd_symbols_group_hash,
 		1,
 		rspamd_ptr_hash_func,
 		rspamd_ptr_equal_func);
+
+#define RSPAMD_PASSTHROUGH_NORMAL 1
+#define RSPAMD_PASSTHROUGH_LOW 0
+#define RSPAMD_PASSTHROUGH_HIGH 2
+#define RSPAMD_PASSTHROUGH_CRITICAL 3
+
+struct rspamd_passthrough_result {
+	enum rspamd_action_type action;
+	guint priority;
+	double target_score;
+	const gchar *message;
+	const gchar *module;
+	struct rspamd_passthrough_result *prev, *next;
+};
+
 struct rspamd_metric_result {
-	double score;                                   /**< total score							*/
+	double score;									/**< total score							*/
 	double grow_factor;								/**< current grow factor					*/
+	struct rspamd_passthrough_result *passthrough_result;
 	khash_t(rspamd_symbols_hash) *symbols;			/**< symbols of metric						*/
 	khash_t(rspamd_symbols_group_hash) *sym_groups; /**< groups of symbols						*/
 	gdouble actions_limits[METRIC_ACTION_MAX];		/**< set of actions for this metric			*/
@@ -73,6 +89,22 @@ struct rspamd_metric_result {
  */
 struct rspamd_metric_result * rspamd_create_metric_result (struct rspamd_task *task);
 
+/**
+ * Adds a new passthrough result to a task
+ * @param task
+ * @param action
+ * @param priority
+ * @param target_score
+ * @param message
+ * @param module
+ */
+void rspamd_add_passthrough_result (struct rspamd_task *task,
+									enum rspamd_action_type action,
+									guint priority,
+									double target_score,
+									const gchar *message,
+									const gchar *module);
+
 enum rspamd_symbol_insert_flags {
 	RSPAMD_SYMBOL_INSERT_DEFAULT = 0,
 	RSPAMD_SYMBOL_INSERT_SINGLE = (1 << 0),

src/libmime/lang_detection.c

@@ -87,6 +87,7 @@ struct rspamd_language_elt {
 	enum rspamd_language_elt_flags flags;
 	enum rspamd_language_category category;
 	guint trigramms_words;
+	guint stop_words;
 	gdouble mean;
 	gdouble std;
 	guint occurencies; /* total number of parts with this language */
@@ -447,6 +448,7 @@ rspamd_language_detector_read_file (struct rspamd_config *cfg,
 			while ((w = ucl_object_iterate (specific_stop_words, &it, true)) != NULL) {
 				rspamd_multipattern_add_pattern (d->stop_words[cat].mp,
 						ucl_object_tostring (w), 0);
+				nelt->stop_words ++;
 				nstop ++;
 			}
 
@@ -592,7 +594,7 @@ rspamd_language_detector_read_file (struct rspamd_config *cfg,
 			(gint)nelt->trigramms_words,
 			total,
 			std, mean,
-			skipped, loaded, nstop,
+			skipped, loaded, nelt->stop_words,
 			rspamd_language_detector_print_flags (nelt));
 
 	g_ptr_array_add (d->languages, nelt);
@@ -1446,8 +1448,24 @@ rspamd_language_detector_try_uniscript (struct rspamd_task *task,
 	return FALSE;
 }
 
+static guint
+rspamd_langelt_hash_func (gconstpointer key)
+{
+	const struct rspamd_language_elt *elt = (const struct rspamd_language_elt *)key;
+	return rspamd_cryptobox_fast_hash (elt->name, strlen (elt->name),
+			rspamd_hash_seed ());
+}
 
-KHASH_MAP_INIT_STR (rspamd_sw_hash, int);
+static gboolean
+rspamd_langelt_equal_func (gconstpointer v, gconstpointer v2)
+{
+	const struct rspamd_language_elt *elt1 = (const struct rspamd_language_elt *)v,
+			*elt2 = (const struct rspamd_language_elt *)v2;
+	return strcmp (elt1->name, elt2->name) == 0;
+}
+
+KHASH_INIT (rspamd_sw_hash, struct rspamd_language_elt *, int, 1,
+		rspamd_langelt_hash_func, rspamd_langelt_equal_func);
 
 struct rspamd_sw_cbdata {
 	khash_t (rspamd_sw_hash) *res;
@@ -1480,7 +1498,7 @@ rspamd_language_detector_sw_cb (struct rspamd_multipattern *mp,
 								  void *context)
 {
 	/* Check if boundary */
-	const gchar *prev, *next;
+	const gchar *prev = text, *next = text + len;
 	struct rspamd_stop_word_range *r;
 	struct rspamd_sw_cbdata *cbdata = (struct rspamd_sw_cbdata *)context;
 	khiter_t k;
@@ -1492,8 +1510,9 @@ rspamd_language_detector_sw_cb (struct rspamd_multipattern *mp,
 			return 0;
 		}
 	}
-	else if (match_pos < len) {
-		next = text + match_pos + 1;
+
+	if (match_pos < len) {
+		next = text + match_pos;
 
 		if (!(g_ascii_isspace (*next) || g_ascii_ispunct (*next))) {
 			return 0;
@@ -1503,10 +1522,9 @@ rspamd_language_detector_sw_cb (struct rspamd_multipattern *mp,
 	/* We have a word on the boundary, check range */
 	r = bsearch (GINT_TO_POINTER (strnum), cbdata->ranges->data,
 			cbdata->ranges->len, sizeof (*r), rspamd_ranges_cmp);
-
 	g_assert (r != NULL);
 
-	k = kh_get (rspamd_sw_hash, cbdata->res, r->elt->name);
+	k = kh_get (rspamd_sw_hash, cbdata->res, r->elt);
 
 	if (k != kh_end (cbdata->res)) {
 		kh_value (cbdata->res, k) ++;
@@ -1514,7 +1532,7 @@ rspamd_language_detector_sw_cb (struct rspamd_multipattern *mp,
 	else {
 		gint tt;
 
-		k = kh_put (rspamd_sw_hash, cbdata->res, r->elt->name, &tt);
+		k = kh_put (rspamd_sw_hash, cbdata->res, r->elt, &tt);
 		kh_value (cbdata->res, k) = 1;
 	}
 
@@ -1540,19 +1558,24 @@ rspamd_language_detector_try_stop_words (struct rspamd_task *task,
 			&cbdata, NULL);
 
 	if (kh_size (cbdata.res) > 0) {
-		gint max = G_MININT, cur_matches;
-		const gchar *sel = NULL, *cur_lang;
+		gint cur_matches;
+		double max_rate = G_MINDOUBLE;
+		const gchar *sel = NULL;
+		struct rspamd_language_elt *cur_lang;
 
 		kh_foreach (cbdata.res, cur_lang, cur_matches, {
-			if (cur_matches > max) {
-				max = cur_matches;
-				sel = cur_lang;
+			double rate = (double)cur_matches / (double)cur_lang->stop_words;
+			if (rate > max_rate) {
+				max_rate = rate;
+				sel = cur_lang->name;
 			}
+			msg_debug_lang_det ("found %d stop words from %s: %3f rate",
+					cur_matches, cur_lang->name, rate);
 		});
 
-		if (max > 0 && sel) {
-			msg_debug_lang_det ("set language based on stop words script %s, %d found",
-					sel, max);
+		if (max_rate > 0 && sel) {
+			msg_debug_lang_det ("set language based on stop words script %s, %.3f found",
+					sel, max_rate);
 			rspamd_language_detector_set_language (task, part,
 					sel);
 

src/libmime/message.c

@@ -32,6 +32,8 @@
 #include "libstemmer.h"
 #endif
 
+#include <math.h>
+
 #define GTUBE_SYMBOL "GTUBE"
 
 #define SET_PART_RAW(part) ((part)->flags &= ~RSPAMD_MIME_TEXT_PART_FLAG_UTF)
@@ -109,7 +111,7 @@ rspamd_mime_part_extract_words (struct rspamd_task *task,
 #endif
 
 			if (w->len > 0 && (w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT)) {
-				avg_len = avg_len + (w->len - avg_len) / (double) i;
+				avg_len = avg_len + (w->len - avg_len) / (double) (i + 1);
 
 				if (r != NULL) {
 					nlen = strlen (r);
@@ -119,19 +121,22 @@ rspamd_mime_part_extract_words (struct rspamd_task *task,
 
 					if (IS_PART_UTF (part)) {
 						rspamd_str_lc_utf8 (temp_word, nlen);
-					} else {
+					}
+					else {
 						rspamd_str_lc (temp_word, nlen);
 					}
 
 					w->begin = temp_word;
 					w->len = nlen;
-				} else {
+				}
+				else {
 					temp_word = rspamd_mempool_alloc (task->task_pool, w->len);
 					memcpy (temp_word, w->begin, w->len);
 
 					if (IS_PART_UTF (part)) {
 						rspamd_str_lc_utf8 (temp_word, w->len);
-					} else {
+					}
+					else {
 						rspamd_str_lc (temp_word, w->len);
 					}
 
@@ -168,7 +173,8 @@ rspamd_mime_part_extract_words (struct rspamd_task *task,
 				*avg_len_p = total_len;
 				rspamd_mempool_set_variable (task->task_pool,
 						RSPAMD_MEMPOOL_AVG_WORDS_LEN, avg_len_p, NULL);
-			} else {
+			}
+			else {
 				*avg_len_p += total_len;
 			}
 
@@ -181,7 +187,8 @@ rspamd_mime_part_extract_words (struct rspamd_task *task,
 				*short_len_p = short_len;
 				rspamd_mempool_set_variable (task->task_pool,
 						RSPAMD_MEMPOOL_SHORT_WORDS_CNT, avg_len_p, NULL);
-			} else {
+			}
+			else {
 				*short_len_p += short_len;
 			}
 		}
@@ -849,22 +856,18 @@ rspamd_message_process_text_part_maybe (struct rspamd_task *task,
 
 	act = rspamd_check_gtube (task, text_part);
 	if (act != METRIC_ACTION_NOACTION) {
-		struct rspamd_metric_result *mres;
+		struct rspamd_metric_result *mres = task->result;
+		gdouble score = NAN;
 
-		mres = rspamd_create_metric_result (task);
-
-		if (mres != NULL) {
-			if (act == METRIC_ACTION_REJECT) {
-				mres->score = rspamd_task_get_required_score (task, mres);
-			}
-			else {
-				mres->score = mres->actions_limits[act];
-			}
+		if (act == METRIC_ACTION_REJECT) {
+			score = rspamd_task_get_required_score (task, mres);
+		}
+		else {
+			score = mres->actions_limits[act];
 		}
 
-		task->result = mres;
-		task->pre_result.action = act;
-		task->pre_result.str = "Gtube pattern";
+		rspamd_add_passthrough_result (task, act, RSPAMD_PASSTHROUGH_CRITICAL,
+				score, "Gtube pattern", "GTUBE");
 
 		if (ucl_object_lookup (task->messages, "smtp_message") == NULL) {
 			ucl_object_replace_key (task->messages,

src/libserver/cfg_file.h

@@ -236,6 +236,7 @@ enum rspamd_log_format_type {
 	RSPAMD_LOG_LUA,
 	RSPAMD_LOG_DIGEST,
 	RSPAMD_LOG_FILENAME,
+	RSPAMD_LOG_FORCED_ACTION,
 };
 
 enum rspamd_log_format_flags {

src/libserver/cfg_utils.c

@@ -258,10 +258,12 @@ rspamd_config_free (struct rspamd_config *cfg)
 	}
 #endif
 
-	rspamd_mempool_delete (cfg->cfg_pool);
 	if (cfg->monitored_ctx) {
 		rspamd_monitored_ctx_destroy (cfg->monitored_ctx);
 	}
+
+	rspamd_mempool_delete (cfg->cfg_pool);
+
 	if (cfg->checksum) {
 		g_free (cfg->checksum);
 	}
@@ -444,6 +446,9 @@ rspamd_config_process_var (struct rspamd_config *cfg, const rspamd_ftok_t *var,
 	else if (rspamd_ftok_cstr_equal (&tok, "filename", TRUE)) {
 		type = RSPAMD_LOG_FILENAME;
 	}
+	else if (rspamd_ftok_cstr_equal (&tok, "forced_action", TRUE)) {
+		type = RSPAMD_LOG_FORCED_ACTION;
+	}
 	else {
 		msg_err_config ("unknown log variable: %T", &tok);
 		return FALSE;

src/libserver/composites.c

@@ -21,6 +21,8 @@
 #include "filter.h"
 #include "composites.h"
 
+#include <math.h>
+
 #define msg_err_composites(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
         "composites", task->task_pool->tag.uid, \
         G_STRFUNC, \
@@ -94,7 +96,7 @@ rspamd_composite_expr_parse (const gchar *line, gsize len,
 	/*
 	 * Composites are just sequences of symbols
 	 */
-	clen = strcspn (line, ", \t()><+!|&\n");
+	clen = strcspn (line, ", \t()><!|&\n");
 	if (clen == 0) {
 		/* Invalid composite atom */
 		g_set_error (err, rspamd_composites_quark (), 100, "Invalid composite: %s",
@@ -172,20 +174,98 @@ rspamd_composite_process_single_symbol (struct composites_data *cd,
 	return rc;
 }
 
+static void
+rspamd_composite_process_symbol_removal (rspamd_expression_atom_t *atom,
+										 struct composites_data *cd,
+										 struct rspamd_symbol_result *ms,
+										 const gchar *beg)
+{
+	gchar t;
+	struct symbol_remove_data *rd, *nrd;
+	struct rspamd_task *task = cd->task;
+
+	if (ms == NULL) {
+		return;
+	}
+
+	/*
+	 * At this point we know that we need to do something about this symbol,
+	 * however, we don't know whether we need to delete it unfortunately,
+	 * that depends on the later decisions when the complete expression is
+	 * evaluated.
+	 */
+	rd = g_hash_table_lookup (cd->symbols_to_remove, ms->name);
+
+	nrd = rspamd_mempool_alloc (cd->task->task_pool, sizeof (*nrd));
+	nrd->sym = ms->name;
+
+	/* By default remove symbols */
+	switch (cd->composite->policy) {
+	case RSPAMD_COMPOSITE_POLICY_REMOVE_ALL:
+	default:
+		nrd->action = (RSPAMD_COMPOSITE_REMOVE_SYMBOL|RSPAMD_COMPOSITE_REMOVE_WEIGHT);
+		break;
+	case RSPAMD_COMPOSITE_POLICY_REMOVE_SYMBOL:
+		nrd->action = RSPAMD_COMPOSITE_REMOVE_SYMBOL;
+		break;
+	case RSPAMD_COMPOSITE_POLICY_REMOVE_WEIGHT:
+		nrd->action = RSPAMD_COMPOSITE_REMOVE_WEIGHT;
+		break;
+	case RSPAMD_COMPOSITE_POLICY_LEAVE:
+		nrd->action = 0;
+		break;
+	}
+
+	for (;;) {
+		t = *beg;
+
+		if (t == '~') {
+			nrd->action &= ~RSPAMD_COMPOSITE_REMOVE_SYMBOL;
+		}
+		else if (t == '-') {
+			nrd->action &= ~(RSPAMD_COMPOSITE_REMOVE_WEIGHT|
+							 RSPAMD_COMPOSITE_REMOVE_SYMBOL);
+		}
+		else if (t == '^') {
+			nrd->action |= RSPAMD_COMPOSITE_REMOVE_FORCED;
+		}
+		else {
+			break;
+		}
+
+		beg ++;
+	}
+
+	nrd->comp = cd->composite;
+	nrd->parent = atom->parent;
+
+	if (rd == NULL) {
+		DL_APPEND (rd, nrd);
+		g_hash_table_insert (cd->symbols_to_remove, (gpointer)ms->name, rd);
+		msg_debug_composites ("added symbol %s to removal: %d policy, from composite %s",
+				ms->name, nrd->action, cd->composite->sym);
+	}
+	else {
+		DL_APPEND (rd, nrd);
+		msg_debug_composites ("append symbol %s to removal: %d policy, from composite %s",
+				ms->name, nrd->action, cd->composite->sym);
+	}
+}
+
 static gdouble
-rspamd_composite_expr_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom)
+rspamd_composite_expr_process (struct rspamd_expr_process_data *process_data,
+		rspamd_expression_atom_t *atom)
 {
 	struct composites_data *cd = process_data->cd;
 	const gchar *beg = atom->data, *sym = NULL;
-	gchar t;
-	struct symbol_remove_data *rd, *nrd;
+
 	struct rspamd_symbol_result *ms = NULL;
 	struct rspamd_symbols_group *gr;
 	struct rspamd_symbol *sdef;
 	struct rspamd_task *task = cd->task;
 	GHashTableIter it;
 	gpointer k, v;
-	gdouble rc = 0;
+	gdouble rc = 0, max = 0;
 
 	if (isset (cd->checked, cd->composite->id * 2)) {
 		/* We have already checked this composite, so just return its value */
@@ -225,76 +305,94 @@ rspamd_composite_expr_process (struct rspamd_expr_process_data *process_data, rs
 				rc = rspamd_composite_process_single_symbol (cd, sdef->name, &ms);
 
 				if (rc) {
-					break;
+					rspamd_composite_process_symbol_removal (atom,
+							cd,
+							ms,
+							beg);
+
+					if (fabs (rc) > max) {
+						max = fabs (rc);
+					}
 				}
 			}
 		}
+
+		rc = max;
 	}
-	else {
-		rc = rspamd_composite_process_single_symbol (cd, sym, &ms);
-	}
+	else if (strncmp (sym, "g+:", 3) == 0) {
+		/* Group, positive symbols only */
+		gr = g_hash_table_lookup (cd->task->cfg->groups, sym + 3);
 
-	if (rc != 0 && ms) {
-		/*
-		 * At this point we know that we need to do something about this symbol,
-		 * however, we don't know whether we need to delete it unfortunately,
-		 * that depends on the later decisions when the complete expression is
-		 * evaluated.
-		 */
-		rd = g_hash_table_lookup (cd->symbols_to_remove, ms->name);
+		if (gr != NULL) {
+			g_hash_table_iter_init (&it, gr->symbols);
 
-		nrd = rspamd_mempool_alloc (cd->task->task_pool, sizeof (*nrd));
-		nrd->sym = ms->name;
+			while (g_hash_table_iter_next (&it, &k, &v)) {
+				sdef = v;
 
-		/* By default remove symbols */
-		switch (cd->composite->policy) {
-		case RSPAMD_COMPOSITE_POLICY_REMOVE_ALL:
-		default:
-			nrd->action = (RSPAMD_COMPOSITE_REMOVE_SYMBOL|RSPAMD_COMPOSITE_REMOVE_WEIGHT);
-			break;
-		case RSPAMD_COMPOSITE_POLICY_REMOVE_SYMBOL:
-			nrd->action = RSPAMD_COMPOSITE_REMOVE_SYMBOL;
-			break;
-		case RSPAMD_COMPOSITE_POLICY_REMOVE_WEIGHT:
-			nrd->action = RSPAMD_COMPOSITE_REMOVE_WEIGHT;
-			break;
-		case RSPAMD_COMPOSITE_POLICY_LEAVE:
-			nrd->action = 0;
-			break;
+				if (sdef->score > 0) {
+					rc = rspamd_composite_process_single_symbol (cd,
+							sdef->name,
+							&ms);
+
+					if (rc) {
+						rspamd_composite_process_symbol_removal (atom,
+								cd,
+								ms,
+								beg);
+
+						if (fabs (rc) > max) {
+							max = fabs (rc);
+						}
+					}
+				}
+			}
+
+			rc = max;
 		}
+	}
+	else if (strncmp (sym, "g-:", 3) == 0) {
+		/* Group, negative symbols only */
+		gr = g_hash_table_lookup (cd->task->cfg->groups, sym + 3);
 
-		for (;;) {
-			t = *beg;
+		if (gr != NULL) {
+			g_hash_table_iter_init (&it, gr->symbols);
 
-			if (t == '~') {
-				nrd->action &= ~RSPAMD_COMPOSITE_REMOVE_SYMBOL;
-			}
-			else if (t == '-') {
-				nrd->action &= ~(RSPAMD_COMPOSITE_REMOVE_WEIGHT|
-						RSPAMD_COMPOSITE_REMOVE_SYMBOL);
-			}
-			else if (t == '^') {
-				nrd->action |= RSPAMD_COMPOSITE_REMOVE_FORCED;
-			}
-			else {
-				break;
-			}
+			while (g_hash_table_iter_next (&it, &k, &v)) {
+				sdef = v;
 
-			beg ++;
-		}
+				if (sdef->score < 0) {
+					rc = rspamd_composite_process_single_symbol (cd, sdef->name, &ms);
 
-		nrd->comp = cd->composite;
-		nrd->parent = atom->parent;
+					if (rc) {
+						rspamd_composite_process_symbol_removal (atom,
+								cd,
+								ms,
+								beg);
 
-		if (rd == NULL) {
-			DL_APPEND (rd, nrd);
-			g_hash_table_insert (cd->symbols_to_remove, (gpointer)ms->name, rd);
+						if (fabs (rc) > max) {
+							max = fabs (rc);
+						}
+					}
+				}
+			}
+
+			rc = max;
 		}
-		else {
-			DL_APPEND (rd, nrd);
+	}
+	else {
+		rc = rspamd_composite_process_single_symbol (cd, sym, &ms);
+
+		if (rc) {
+			rspamd_composite_process_symbol_removal (atom,
+					cd,
+					ms,
+					beg);
 		}
 	}
 
+	msg_debug_composites ("final result for composite %s is %.2f",
+			cd->composite->sym, rc);
+
 	return rc;
 }
 

src/libserver/fuzzy_backend_redis.c

@@ -126,7 +126,7 @@ rspamd_fuzzy_redis_session_dtor (struct rspamd_fuzzy_redis_session *session,
 				ac, is_fatal);
 	}
 
-	if (event_get_base (&session->timeout)) {
+	if (rspamd_event_pending (&session->timeout, EV_TIMEOUT)) {
 		event_del (&session->timeout);
 	}
 

src/libserver/html.c

@@ -1512,6 +1512,10 @@ rspamd_html_process_img_tag (rspamd_mempool_t *pool, struct html_tag *tag,
 			}
 			else {
 				img->flags |= RSPAMD_HTML_FLAG_IMAGE_EXTERNAL;
+				if (img->src) {
+					img->url = rspamd_html_process_url (pool,
+							img->src, fstr.len, NULL);
+				}
 			}
 		}
 		else if (comp->type == RSPAMD_HTML_COMPONENT_HEIGHT) {

src/libserver/html.h

@@ -48,6 +48,7 @@ struct html_image {
 	guint width;
 	guint flags;
 	gchar *src;
+	struct rspamd_url *url;
 	struct html_tag *tag;
 };
 

src/libserver/milter.c

@@ -185,7 +185,7 @@ rspamd_milter_session_dtor (struct rspamd_milter_session *session)
 		priv = session->priv;
 		msg_debug_milter ("destroying milter session");
 
-		if (event_get_base (&priv->ev)) {
+		if (rspamd_event_pending (&priv->ev, EV_TIMEOUT|EV_WRITE|EV_READ)) {
 			event_del (&priv->ev);
 		}
 
@@ -265,7 +265,7 @@ static inline void
 rspamd_milter_plan_io (struct rspamd_milter_session *session,
 		struct rspamd_milter_private *priv, gshort what)
 {
-	if (event_get_base (&priv->ev)) {
+	if (rspamd_event_pending (&priv->ev, EV_TIMEOUT|EV_WRITE|EV_READ)) {
 		event_del (&priv->ev);
 	}
 

src/libserver/monitored.c

@@ -589,7 +589,7 @@ rspamd_monitored_stop (struct rspamd_monitored *m)
 {
 	g_assert (m != NULL);
 
-	if (event_get_base (&m->periodic)) {
+	if (rspamd_event_pending (&m->periodic, EV_TIMEOUT)) {
 		event_del (&m->periodic);
 	}
 }
@@ -606,7 +606,7 @@ rspamd_monitored_start (struct rspamd_monitored *m)
 			0.0);
 	double_to_tv (jittered, &tv);
 
-	if (event_get_base (&m->periodic)) {
+	if (rspamd_event_pending (&m->periodic, EV_TIMEOUT)) {
 		event_del (&m->periodic);
 	}
 
@@ -626,8 +626,8 @@ rspamd_monitored_ctx_destroy (struct rspamd_monitored_ctx *ctx)
 	for (i = 0; i < ctx->elts->len; i ++) {
 		m = g_ptr_array_index (ctx->elts, i);
 		rspamd_monitored_stop (m);
-		g_free (m->url);
 		m->proc.monitored_dtor (m, m->ctx, m->proc.ud);
+		g_free (m->url);
 		g_free (m);
 	}
 

src/libserver/redis_pool.c

@@ -107,7 +107,7 @@ rspamd_redis_pool_conn_dtor (struct rspamd_redis_pool_connection *conn)
 				redisAsyncContext *ac = conn->ctx;
 
 				conn->ctx = NULL;
-				g_hash_table_remove (conn->elt->pool->elts_by_ctx, conn->ctx);
+				g_hash_table_remove (conn->elt->pool->elts_by_ctx, ac);
 				ac->onDisconnect = NULL;
 				redisAsyncFree (ac);
 			}
@@ -120,7 +120,7 @@ rspamd_redis_pool_conn_dtor (struct rspamd_redis_pool_connection *conn)
 	else {
 		msg_debug_rpool ("inactive connection removed");
 
-		if (event_get_base (&conn->timeout)) {
+		if (rspamd_event_pending (&conn->timeout, EV_TIMEOUT)) {
 			event_del (&conn->timeout);
 		}
 
@@ -129,7 +129,7 @@ rspamd_redis_pool_conn_dtor (struct rspamd_redis_pool_connection *conn)
 
 			/* To prevent on_disconnect here */
 			conn->active = TRUE;
-			g_hash_table_remove (conn->elt->pool->elts_by_ctx, conn->ctx);
+			g_hash_table_remove (conn->elt->pool->elts_by_ctx, ac);
 			conn->ctx = NULL;
 			ac->onDisconnect = NULL;
 			redisAsyncFree (ac);
@@ -178,8 +178,8 @@ rspamd_redis_conn_timeout (gint fd, short what, gpointer p)
 	struct rspamd_redis_pool_connection *conn = p;
 
 	g_assert (!conn->active);
-	msg_debug_rpool ("scheduled removal of connection, refcount: %d",
-			conn->ref.refcount);
+	msg_debug_rpool ("scheduled removal of connection %p, refcount: %d",
+			conn->ctx, conn->ref.refcount);
 	REF_RELEASE (conn);
 }
 
@@ -201,8 +201,8 @@ rspamd_redis_pool_schedule_timeout (struct rspamd_redis_pool_connection *conn)
 		real_timeout = rspamd_time_jitter (real_timeout, real_timeout / 2.0);
 	}
 
-	msg_debug_rpool ("scheduled connection cleanup in %.1f seconds",
-			real_timeout);
+	msg_debug_rpool ("scheduled connection %p cleanup in %.1f seconds",
+			conn->ctx, real_timeout);
 	double_to_tv (real_timeout, &tv);
 	event_set (&conn->timeout, -1, EV_TIMEOUT, rspamd_redis_conn_timeout, conn);
 	event_base_set (conn->elt->pool->ev_base, &conn->timeout);
@@ -268,7 +268,7 @@ rspamd_redis_pool_new_connection (struct rspamd_redis_pool *pool,
 			conn->ctx = ctx;
 			rspamd_random_hex (conn->tag, sizeof (conn->tag));
 			REF_INIT_RETAIN (conn, rspamd_redis_pool_conn_dtor);
-			msg_debug_rpool ("created new connection to %s:%d", ip, port);
+			msg_debug_rpool ("created new connection to %s:%d: %p", ip, port, ctx);
 
 			redisLibeventAttach (ctx, pool->ev_base);
 			redisAsyncSetDisconnectCallback (ctx, rspamd_redis_pool_on_disconnect,
@@ -355,7 +355,8 @@ rspamd_redis_pool_connect (struct rspamd_redis_pool *pool,
 				event_del (&conn->timeout);
 				conn->active = TRUE;
 				g_queue_push_tail_link (elt->active, conn_entry);
-				msg_debug_rpool ("reused existing connection to %s:%d", ip, port);
+				msg_debug_rpool ("reused existing connection to %s:%d: %p",
+						ip, port, conn->ctx);
 			}
 			else {
 				g_list_free (conn->entry);
@@ -407,7 +408,7 @@ rspamd_redis_pool_release_connection (struct rspamd_redis_pool *pool,
 
 		if (is_fatal || ctx->err != REDIS_OK) {
 			/* We need to terminate connection forcefully */
-			msg_debug_rpool ("closed connection forcefully");
+			msg_debug_rpool ("closed connection %p forcefully", conn->ctx);
 			REF_RELEASE (conn);
 		}
 		else {
@@ -418,10 +419,11 @@ rspamd_redis_pool_release_connection (struct rspamd_redis_pool *pool,
 				g_queue_push_head_link (conn->elt->inactive, conn->entry);
 				conn->active = FALSE;
 				rspamd_redis_pool_schedule_timeout (conn);
-				msg_debug_rpool ("mark connection inactive");
+				msg_debug_rpool ("mark connection %p inactive", conn->ctx);
 			}
 			else {
-				msg_debug_rpool ("closed connection due to callbacks leftover");
+				msg_debug_rpool ("closed connection %p due to callbacks left",
+					conn->ctx);
 				REF_RELEASE (conn);
 			}
 		}

src/libserver/symbols_cache.c

@@ -714,14 +714,25 @@ rspamd_symbols_cache_save_items (struct symbols_cache *cache, const gchar *name)
 	gpointer k, v;
 	gint fd;
 	bool ret;
+	gchar path[PATH_MAX];
 
-	(void)unlink (name);
-	fd = open (name, O_CREAT | O_TRUNC | O_WRONLY | O_EXCL, 00644);
+	rspamd_snprintf (path, sizeof (path), "%s.new", name);
 
-	if (fd == -1) {
-		msg_info_cache ("cannot open file %s, error %d, %s", name,
-				errno, strerror (errno));
-		return FALSE;
+	for (;;) {
+		fd = open (path, O_CREAT | O_WRONLY | O_EXCL, 00644);
+
+		if (fd == -1) {
+			if (errno == EEXIST) {
+				/* Some other process is already writing data, give up silently */
+				return TRUE;
+			}
+
+			msg_info_cache ("cannot open file %s, error %d, %s", path,
+					errno, strerror (errno));
+			return FALSE;
+		}
+
+		break;
 	}
 
 	rspamd_file_lock (fd, FALSE);
@@ -731,7 +742,7 @@ rspamd_symbols_cache_save_items (struct symbols_cache *cache, const gchar *name)
 			sizeof (rspamd_symbols_cache_magic));
 
 	if (write (fd, &hdr, sizeof (hdr)) == -1) {
-		msg_info_cache ("cannot write to file %s, error %d, %s", name,
+		msg_info_cache ("cannot write to file %s, error %d, %s", path,
 				errno, strerror (errno));
 		rspamd_file_unlock (fd, FALSE);
 		close (fd);
@@ -773,6 +784,13 @@ rspamd_symbols_cache_save_items (struct symbols_cache *cache, const gchar *name)
 	rspamd_file_unlock (fd, FALSE);
 	close (fd);
 
+	if (rename (path, name) == -1) {
+		msg_info_cache ("cannot rename %s -> %s, error %d, %s", path, name,
+				errno, strerror (errno));
+		(void)unlink (path);
+		ret = FALSE;
+	}
+
 	return ret;
 }
 
@@ -1285,6 +1303,9 @@ rspamd_symbols_cache_watcher_cb (gpointer sessiond, gpointer ud)
 	setbit (checkpoint->processed_bits, item->id * 2 + 1);
 
 	if (checkpoint->pass > 0) {
+#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC
+		event_base_update_cache_time (task->ev_base);
+#endif
 		for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
 			it = g_ptr_array_index (checkpoint->waitq, i);
 
@@ -1364,9 +1385,26 @@ rspamd_symbols_cache_check_symbol (struct rspamd_task *task,
 					rspamd_symbols_cache_watcher_cb,
 					item);
 			msg_debug_task ("execute %s, %d", item->symbol, item->id);
+#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC
+			struct timeval tv;
+
+			event_base_update_cache_time (task->ev_base);
+			event_base_gettimeofday_cached (task->ev_base, &tv);
+			t1 = tv_to_double (&tv);
+#else
 			t1 = rspamd_get_ticks (FALSE);
+#endif
+
 			item->func (task, item->user_data);
+
+#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC
+			event_base_update_cache_time (task->ev_base);
+			event_base_gettimeofday_cached (task->ev_base, &tv);
+			t2 = tv_to_double (&tv);
+#else
 			t2 = rspamd_get_ticks (FALSE);
+#endif
+
 			diff = (t2 - t1);
 
 			if (G_UNLIKELY (RSPAMD_TASK_IS_PROFILING (task))) {
@@ -1569,7 +1607,7 @@ rspamd_symbols_cache_make_checkpoint (struct rspamd_task *task,
 	checkpoint->pass = RSPAMD_CACHE_PASS_INIT;
 	task->checkpoint = checkpoint;
 
-	task->result = rspamd_create_metric_result (task);
+	task->result = task->result;
 
 	return checkpoint;
 }

src/libserver/task.c

@@ -145,7 +145,7 @@ rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg,
 
 	new_task->sock = -1;
 	new_task->flags |= (RSPAMD_TASK_FLAG_MIME|RSPAMD_TASK_FLAG_JSON);
-	new_task->pre_result.action = METRIC_ACTION_MAX;
+	new_task->result = rspamd_create_metric_result (new_task);
 
 	new_task->message_id = new_task->queue_id = "undef";
 	new_task->messages = ucl_object_typed_new (UCL_OBJECT);
@@ -296,7 +296,7 @@ rspamd_task_free (struct rspamd_task *task)
 			g_error_free (task->err);
 		}
 
-		if (event_get_base (&task->timeout_ev) != NULL) {
+		if (rspamd_event_pending (&task->timeout_ev, EV_TIMEOUT)) {
 			event_del (&task->timeout_ev);
 		}
 
@@ -1010,6 +1010,11 @@ rspamd_task_log_check_condition (struct rspamd_task *task,
 			ret = TRUE;
 		}
 		break;
+	case RSPAMD_LOG_FORCED_ACTION:
+		if (task->result->passthrough_result) {
+			ret = TRUE;
+		}
+		break;
 	default:
 		ret = TRUE;
 		break;
@@ -1292,7 +1297,7 @@ rspamd_task_log_variable (struct rspamd_task *task,
 {
 	rspamd_fstring_t *res = logbuf;
 	rspamd_ftok_t var = {.begin = NULL, .len = 0};
-	static gchar numbuf[64];
+	static gchar numbuf[128];
 	static const gchar undef[] = "undef";
 
 	switch (lf->type) {
@@ -1412,6 +1417,29 @@ rspamd_task_log_variable (struct rspamd_task *task,
 			var.len = sizeof (undef) - 1;
 		}
 		break;
+	case RSPAMD_LOG_FORCED_ACTION:
+		if (task->result->passthrough_result) {
+			struct rspamd_passthrough_result *pr = task->result->passthrough_result;
+
+			if (!isnan (pr->target_score)) {
+				var.len = rspamd_snprintf (numbuf, sizeof (numbuf),
+						"%s \"%s\"; score=%.2f (set by %s)",
+						rspamd_action_to_str (pr->action),
+						pr->message, pr->target_score, pr->module);
+			}
+			else {
+				var.len = rspamd_snprintf (numbuf, sizeof (numbuf),
+						"%s \"%s\"; score=nan (set by %s)",
+						rspamd_action_to_str (pr->action),
+						pr->message, pr->module);
+			}
+			var.begin = numbuf;
+		}
+		else {
+			var.begin = undef;
+			var.len = sizeof (undef) - 1;
+		}
+		break;
 	default:
 		var = rspamd_task_log_metric_res (task, lf);
 		break;

src/libserver/task.h

@@ -201,12 +201,6 @@ struct rspamd_task {
 	struct event *guard_ev;							/**< Event for input sanity guard 					*/
 
 	gpointer checkpoint;							/**< Opaque checkpoint data							*/
-
-	struct {
-		gint action;								/**< Action of pre filters							*/
-		gchar *str;									/**< String describing action						*/
-	} pre_result;									/**< Result of pre-filters							*/
-
 	ucl_object_t *settings;							/**< Settings applied to task						*/
 
 	const gchar *classifier;						/**< Classifier to learn (if needed)				*/

src/libserver/worker_util.c

@@ -359,11 +359,11 @@ rspamd_worker_stop_accept (struct rspamd_worker *worker)
 	while (cur) {
 		events = cur->data;
 
-		if (event_get_base (&events[0])) {
+		if (rspamd_event_pending (&events[0], EV_TIMEOUT|EV_READ|EV_WRITE)) {
 			event_del (&events[0]);
 		}
 
-		if (event_get_base (&events[1])) {
+		if (rspamd_event_pending (&events[1], EV_TIMEOUT|EV_READ|EV_WRITE)) {
 			event_del (&events[1]);
 		}
 

src/libstat/backends/redis_backend.c

@@ -994,7 +994,7 @@ rspamd_redis_fin (gpointer data)
 
 	rt->has_event = FALSE;
 	/* Stop timeout */
-	if (event_get_base (&rt->timeout_event)) {
+	if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
 		event_del (&rt->timeout_event);
 	}
 
@@ -1014,7 +1014,7 @@ rspamd_redis_fin_learn (gpointer data)
 
 	rt->has_event = FALSE;
 	/* Stop timeout */
-	if (event_get_base (&rt->timeout_event)) {
+	if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
 		event_del (&rt->timeout_event);
 	}
 
@@ -1597,7 +1597,7 @@ rspamd_redis_process_tokens (struct rspamd_task *task,
 		rspamd_session_add_event (task->s, NULL, rspamd_redis_fin, rt, rspamd_redis_stat_quark ());
 		rt->has_event = TRUE;
 
-		if (event_get_base (&rt->timeout_event)) {
+		if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
 			event_del (&rt->timeout_event);
 		}
 		event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
@@ -1634,7 +1634,7 @@ rspamd_redis_finalize_process (struct rspamd_task *task, gpointer runtime,
 	struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime);
 	redisAsyncContext *redis;
 
-	if (event_get_base (&rt->timeout_event)) {
+	if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
 		event_del (&rt->timeout_event);
 	}
 
@@ -1802,7 +1802,7 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens,
 		rt->has_event = TRUE;
 
 		/* Set timeout */
-		if (event_get_base (&rt->timeout_event)) {
+		if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
 			event_del (&rt->timeout_event);
 		}
 		event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
@@ -1827,7 +1827,7 @@ rspamd_redis_finalize_learn (struct rspamd_task *task, gpointer runtime,
 	struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime);
 	redisAsyncContext *redis;
 
-	if (event_get_base (&rt->timeout_event)) {
+	if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
 		event_del (&rt->timeout_event);
 	}
 

src/libstat/learn_cache/redis_cache.c

@@ -73,7 +73,7 @@ rspamd_redis_cache_fin (gpointer data)
 	redisAsyncContext *redis;
 
 	rt->has_event = FALSE;
-	if (event_get_base (&rt->timeout_event)) {
+	if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
 		event_del (&rt->timeout_event);
 	}
 

src/libutil/expression.c

@@ -613,78 +613,89 @@ rspamd_parse_expression (const gchar *line, gsize len,
 		case PARSE_ATOM:
 			if (g_ascii_isspace (*p)) {
 				state = SKIP_SPACES;
+				continue;
 			}
 			else if (rspamd_expr_is_operation_symbol (*p)) {
+				if (p + 1 < end) {
+					gchar t = *(p + 1);
+
+					if (t != ':') {
+						state = PARSE_OP;
+						continue;
+					}
+				}
+				else {
+					state = PARSE_OP;
+					continue;
+				}
+			}
+
+			/*
+			 * First of all, we check some pre-conditions:
+			 * 1) if we have 'and ' or 'or ' or 'not ' strings, they are op
+			 * 2) if we have full numeric string, then we check for
+			 * the following expression:
+			 *  ^\d+\s*[><]$
+			 */
+			if ((gulong)(end - p) > sizeof ("and ") &&
+				(g_ascii_strncasecmp (p, "and ", sizeof ("and ") - 1) == 0 ||
+				 g_ascii_strncasecmp (p, "not ", sizeof ("not ") - 1) == 0 )) {
+				state = PARSE_OP;
+			}
+			else if ((gulong)(end - p) > sizeof ("or ") &&
+					 g_ascii_strncasecmp (p, "or ", sizeof ("or ") - 1) == 0) {
 				state = PARSE_OP;
 			}
 			else {
 				/*
-				 * First of all, we check some pre-conditions:
-				 * 1) if we have 'and ' or 'or ' or 'not ' strings, they are op
-				 * 2) if we have full numeric string, then we check for
-				 * the following expression:
-				 *  ^\d+\s*[><]$
+				 * If we have any comparison operator in the stack, then try
+				 * to parse limit
 				 */
-				if ((gulong)(end - p) > sizeof ("and ") &&
-					(g_ascii_strncasecmp (p, "and ", sizeof ("and ") - 1) == 0 ||
-					g_ascii_strncasecmp (p, "not ", sizeof ("not ") - 1) == 0 )) {
-					state = PARSE_OP;
-				}
-				else if ((gulong)(end - p) > sizeof ("or ") &&
-					g_ascii_strncasecmp (p, "or ", sizeof ("or ") - 1) == 0) {
-					state = PARSE_OP;
-				}
-				else {
-					/*
-					 * If we have any comparison operator in the stack, then try
-					 * to parse limit
-					 */
-					op = GPOINTER_TO_INT (rspamd_expr_stack_peek (e));
-
-					if (op >= OP_LT && op <= OP_GE) {
-						if (rspamd_regexp_search (num_re,
-								p,
-								end - p,
-								NULL,
-								NULL,
-								FALSE,
-								NULL)) {
-							c = p;
-							state = PARSE_LIM;
-							continue;
-						}
+				op = GPOINTER_TO_INT (rspamd_expr_stack_peek (e));
+
+				if (op >= OP_LT && op <= OP_GE) {
+					if (rspamd_regexp_search (num_re,
+							p,
+							end - p,
+							NULL,
+							NULL,
+							FALSE,
+							NULL)) {
+						c = p;
+						state = PARSE_LIM;
+						continue;
 					}
+				}
 
-					/* Try to parse atom */
-					atom = subr->parse (p, end - p, pool, subr_data, err);
-					if (atom == NULL || atom->len == 0) {
-						/* We couldn't parse the atom, so go out */
-						if (err != NULL && *err == NULL) {
-							g_set_error (err,
-									rspamd_expr_quark (),
-									500,
-									"Cannot parse atom: callback function failed"
-											" to parse '%.*s'",
-									(int) (end - p),
-									p);
-						}
-						goto err;
+				/* Try to parse atom */
+				atom = subr->parse (p, end - p, pool, subr_data, err);
+				if (atom == NULL || atom->len == 0) {
+					/* We couldn't parse the atom, so go out */
+					if (err != NULL && *err == NULL) {
+						g_set_error (err,
+								rspamd_expr_quark (),
+								500,
+								"Cannot parse atom: callback function failed"
+								" to parse '%.*s'",
+								(int) (end - p),
+								p);
 					}
+					goto err;
+				}
 
-					if (atom->str == NULL) {
-						atom->str = p;
-					}
+				if (atom->str == NULL) {
+					atom->str = p;
+				}
 
-					p = p + atom->len;
+				p = p + atom->len;
 
-					/* Push to output */
-					elt.type = ELT_ATOM;
-					elt.p.atom = atom;
-					g_array_append_val (e->expressions, elt);
-					rspamd_expr_stack_elt_push (operand_stack,
-							g_node_new (rspamd_expr_dup_elt (pool, &elt)));
+				/* Push to output */
+				elt.type = ELT_ATOM;
+				elt.p.atom = atom;
+				g_array_append_val (e->expressions, elt);
+				rspamd_expr_stack_elt_push (operand_stack,
+						g_node_new (rspamd_expr_dup_elt (pool, &elt)));
 
-				}
 			}
 			break;
 		case PARSE_LIM:

src/libutil/http.c

@@ -629,7 +629,7 @@ rspamd_http_on_headers_complete (http_parser * parser)
 
 	if (msg->method == HTTP_HEAD) {
 		/* We don't care about the rest */
-		if (event_pending (&priv->ev, EV_READ, NULL)) {
+		if (rspamd_event_pending (&priv->ev, EV_READ)) {
 			event_del (&priv->ev);
 		}
 
@@ -804,7 +804,7 @@ rspamd_http_on_headers_complete_decrypted (http_parser *parser)
 
 	if (msg->method == HTTP_HEAD) {
 		/* We don't care about the rest */
-		if (event_pending (&priv->ev, EV_READ, NULL)) {
+		if (rspamd_event_pending (&priv->ev, EV_READ)) {
 			event_del (&priv->ev);
 		}
 
@@ -949,7 +949,7 @@ rspamd_http_on_message_complete (http_parser * parser)
 	}
 
 	if (ret == 0) {
-		if (event_pending (&priv->ev, EV_READ, NULL)) {
+		if (rspamd_event_pending (&priv->ev, EV_READ)) {
 			event_del (&priv->ev);
 		}
 
@@ -1370,7 +1370,7 @@ rspamd_http_connection_reset (struct rspamd_http_connection *conn)
 
 	if (!(priv->flags & RSPAMD_HTTP_CONN_FLAG_RESETED)) {
 
-		if (event_get_base (&priv->ev)) {
+		if (rspamd_event_pending (&priv->ev, EV_READ|EV_WRITE|EV_TIMEOUT)) {
 			event_del (&priv->ev);
 		}
 
@@ -2296,7 +2296,7 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn
 
 	priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_RESETED;
 
-	if (base != NULL && event_get_base (&priv->ev) == base) {
+	if (rspamd_event_pending (&priv->ev, EV_TIMEOUT|EV_WRITE|EV_READ)) {
 		event_del (&priv->ev);
 	}
 

src/libutil/logger.c

@@ -887,6 +887,28 @@ syslog_log_function (const gchar *module, const gchar *id,
 #endif
 }
 
+
+static inline void
+log_time (gdouble now, rspamd_logger_t *rspamd_log, gchar *timebuf,
+		size_t len)
+{
+	time_t sec = (time_t)now;
+	gsize r;
+	struct tm tms;
+
+	rspamd_localtime (sec, &tms);
+	r = strftime (timebuf, len, "%F %H:%M:%S", &tms);
+
+	if (rspamd_log->flags & RSPAMD_LOG_FLAG_USEC) {
+		gchar usec_buf[16];
+
+		rspamd_snprintf (usec_buf, sizeof (usec_buf), "%.5f",
+				now - (gdouble)sec);
+		rspamd_snprintf (timebuf + r, len - r,
+				"%s", usec_buf + 1);
+	}
+}
+
 /**
  * Main file interface for logging
  */
@@ -901,7 +923,7 @@ file_log_function (const gchar *module, const gchar *id,
 	gchar tmpbuf[256];
 	gchar *m;
 	gdouble now;
-	struct tm tms;
+
 	struct iovec iov[5];
 	gulong r = 0, mr = 0;
 	guint64 cksum;
@@ -1044,20 +1066,7 @@ file_log_function (const gchar *module, const gchar *id,
 
 		/* Format time */
 		if (!(rspamd_log->flags & RSPAMD_LOG_FLAG_SYSTEMD)) {
-			time_t sec = now;
-			gsize r;
-
-			rspamd_localtime (sec, &tms);
-			r = strftime (timebuf, sizeof (timebuf), "%F %H:%M:%S", &tms);
-
-			if (rspamd_log->flags & RSPAMD_LOG_FLAG_USEC) {
-				gchar usec_buf[16];
-
-				rspamd_snprintf (usec_buf, sizeof (usec_buf), "%.5f",
-						now - (gdouble)sec);
-				rspamd_snprintf (timebuf + r, sizeof (timebuf) - r,
-						"%s", usec_buf + 1);
-			}
+			log_time (now, rspamd_log, timebuf, sizeof (timebuf));
 		}
 
 		cptype = g_quark_to_string (rspamd_log->process_type);
@@ -1164,17 +1173,24 @@ file_log_function (const gchar *module, const gchar *id,
 
 			iov[0].iov_base = (void *) tmpbuf;
 			iov[0].iov_len = r;
-			iov[1].iov_base = (void *) message;
-			iov[1].iov_len = mlen;
-			r = 2;
+			r = 1;
 		}
 		else {
-			iov[0].iov_base = (void *) message;
-			iov[0].iov_len = mlen;
-			r = 1;
+			r = 0;
 		}
 
 
+		if (rspamd_log->log_level == G_LOG_LEVEL_DEBUG) {
+			log_time (rspamd_get_calendar_ticks (),
+					rspamd_log, timebuf, sizeof (timebuf));
+			iov[r].iov_base = (void *) timebuf;
+			iov[r++].iov_len = strlen (timebuf);
+			iov[r].iov_base = (void *) " ";
+			iov[r++].iov_len = 1;
+		}
+
+		iov[r].iov_base = (void *) message;
+		iov[r++].iov_len = mlen;
 		iov[r].iov_base = (void *) &lf_chr;
 		iov[r++].iov_len = 1;
 

src/libutil/ssl_util.c

@@ -534,7 +534,7 @@ rspamd_ssl_connect_fd (struct rspamd_ssl_connection *conn, gint fd,
 	if (ret == 1) {
 		conn->state = ssl_conn_connected;
 
-		if (event_get_base (ev)) {
+		if (rspamd_event_pending (ev, EV_TIMEOUT|EV_WRITE|EV_READ)) {
 			event_del (ev);
 		}
 
@@ -561,7 +561,7 @@ rspamd_ssl_connect_fd (struct rspamd_ssl_connection *conn, gint fd,
 			return FALSE;
 		}
 
-		if (event_get_base (ev)) {
+		if (rspamd_event_pending (ev, EV_TIMEOUT|EV_WRITE|EV_READ)) {
 			event_del (ev);
 		}
 

src/libutil/upstream.c

@@ -837,7 +837,7 @@ rspamd_upstream_restore_cb (gpointer elt, gpointer ls)
 	/* Here the upstreams list is already locked */
 	RSPAMD_UPSTREAM_LOCK (up->lock);
 
-	if (event_get_base (&up->ev)) {
+	if (rspamd_event_pending (&up->ev, EV_TIMEOUT)) {
 		event_del (&up->ev);
 	}
 	g_ptr_array_add (ups->alive, up);

src/libutil/util.c

@@ -2468,6 +2468,16 @@ event_get_base (struct event *ev)
 }
 #endif
 
+int
+rspamd_event_pending (struct event *ev, short what)
+{
+	if (ev->ev_base == NULL) {
+		return 0;
+	}
+
+	return event_pending (ev, what, NULL);
+}
+
 int
 rspamd_file_xopen (const char *fname, int oflags, guint mode,
 		gboolean allow_symlink)

src/libutil/util.h

@@ -11,6 +11,7 @@
 #ifdef HAVE_NETDB_H
 #include <netdb.h>
 #endif
+
 #include <event.h>
 #include <time.h>
 
@@ -429,6 +430,9 @@ struct event_base * event_get_base (struct event *ev);
     event_set((ev), (x), EV_SIGNAL|EV_PERSIST, (cb), (arg))
 #endif
 
+/* Avoid stupidity in libevent > 1.4 */
+int rspamd_event_pending (struct event *ev, short what);
+
 /**
  * Open file without following symlinks or special stuff
  * @param fname filename

src/lua/lua_common.c

@@ -580,10 +580,15 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
 	/* Set known paths as rspamd_paths global */
 	lua_getglobal (L, "rspamd_paths");
 	if (lua_isnil (L, -1)) {
-		const gchar *confdir = RSPAMD_CONFDIR, *rundir = RSPAMD_RUNDIR,
-				*dbdir = RSPAMD_DBDIR, *logdir = RSPAMD_LOGDIR,
-				*wwwdir = RSPAMD_WWWDIR, *pluginsdir = RSPAMD_PLUGINSDIR,
-				*rulesdir = RSPAMD_RULESDIR, *lualibdir = RSPAMD_LUALIBDIR,
+		const gchar *confdir = RSPAMD_CONFDIR,
+				*local_confdir = RSPAMD_LOCAL_CONFDIR,
+				*rundir = RSPAMD_RUNDIR,
+				*dbdir = RSPAMD_DBDIR,
+				*logdir = RSPAMD_LOGDIR,
+				*wwwdir = RSPAMD_WWWDIR,
+				*pluginsdir = RSPAMD_PLUGINSDIR,
+				*rulesdir = RSPAMD_RULESDIR,
+				*lualibdir = RSPAMD_LUALIBDIR,
 				*prefix = RSPAMD_PREFIX;
 		const gchar *t;
 
@@ -628,6 +633,11 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
 			confdir = t;
 		}
 
+		t = getenv ("LOCAL_CONFDIR");
+		if (t) {
+			local_confdir = t;
+		}
+
 
 		if (vars) {
 			t = g_hash_table_lookup (vars, "PLUGINSDIR");
@@ -660,6 +670,11 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
 				confdir = t;
 			}
 
+			t = g_hash_table_lookup (vars, "LOCAL_CONFDIR");
+			if (t) {
+				local_confdir = t;
+			}
+
 			t = g_hash_table_lookup (vars, "DBDIR");
 			if (t) {
 				dbdir = t;
@@ -674,6 +689,7 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
 		lua_createtable (L, 0, 9);
 
 		rspamd_lua_table_set (L, RSPAMD_CONFDIR_INDEX, confdir);
+		rspamd_lua_table_set (L, RSPAMD_LOCAL_CONFDIR_INDEX, local_confdir);
 		rspamd_lua_table_set (L, RSPAMD_RUNDIR_INDEX, rundir);
 		rspamd_lua_table_set (L, RSPAMD_DBDIR_INDEX, dbdir);
 		rspamd_lua_table_set (L, RSPAMD_LOGDIR_INDEX, logdir);

src/lua/lua_common.h

@@ -425,6 +425,7 @@ gboolean rspamd_lua_require_function (lua_State *L, const gchar *modname,
 
 /* Paths defs */
 #define RSPAMD_CONFDIR_INDEX "CONFDIR"
+#define RSPAMD_LOCAL_CONFDIR_INDEX "LOCAL_CONFDIR"
 #define RSPAMD_RUNDIR_INDEX "RUNDIR"
 #define RSPAMD_DBDIR_INDEX "DBDIR"
 #define RSPAMD_LOGDIR_INDEX "LOGDIR"

src/lua/lua_config.c

@@ -2873,7 +2873,6 @@ lua_periodic_callback (gint unused_fd, short what, gpointer ud)
 	*pev_base = periodic->ev_base;
 
 	event_del (&periodic->ev);
-
 	lua_thread_call (thread, 2);
 }
 
@@ -2888,6 +2887,10 @@ lua_periodic_callback_finish (struct thread_entry *thread, int ret)
 
 	L = thread->lua_state;
 
+#ifdef HAVE_EVENT_NO_CACHE_TIME_FUNC
+	event_base_update_cache_time (periodic->ev_base);
+#endif
+
 	if (ret == 0) {
 		if (lua_type (L, -1) == LUA_TBOOLEAN) {
 			plan_more = lua_toboolean (L, -1);
@@ -3453,6 +3456,7 @@ lua_config_load_ucl (lua_State *L)
 
 		if (lua_istable (L, -1)) {
 			LUA_TABLE_TO_HASH(paths, RSPAMD_CONFDIR_INDEX);
+			LUA_TABLE_TO_HASH(paths, RSPAMD_LOCAL_CONFDIR_INDEX);
 			LUA_TABLE_TO_HASH(paths, RSPAMD_RUNDIR_INDEX);
 			LUA_TABLE_TO_HASH(paths, RSPAMD_DBDIR_INDEX);
 			LUA_TABLE_TO_HASH(paths, RSPAMD_LOGDIR_INDEX);

src/lua/lua_html.c

@@ -260,6 +260,7 @@ lua_html_push_image (lua_State *L, struct html_image *img)
 {
 	LUA_TRACE_POINT;
 	struct html_tag **ptag;
+	struct rspamd_url **purl;
 
 	lua_newtable (L);
 
@@ -269,6 +270,14 @@ lua_html_push_image (lua_State *L, struct html_image *img)
 		lua_settable (L, -3);
 	}
 
+	if (img->url) {
+		lua_pushstring (L, "url");
+		purl = lua_newuserdata (L, sizeof (gpointer));
+		*purl = img->url;
+		rspamd_lua_setclass (L, "rspamd{url}", -1);
+		lua_settable (L, -3);
+	}
+
 	if (img->tag) {
 		lua_pushstring (L, "tag");
 		ptag = lua_newuserdata (L, sizeof (gpointer));

src/lua/lua_redis.c

@@ -182,7 +182,9 @@ lua_redis_dtor (struct lua_redis_ctx *ctx)
 	if (ud->ctx) {
 
 		LL_FOREACH_SAFE (ud->specific, cur, tmp) {
-			event_del (&cur->timeout);
+			if (rspamd_event_pending (&cur->timeout, EV_TIMEOUT)) {
+				event_del (&cur->timeout);
+			}
 
 			if (!(cur->flags & LUA_REDIS_SPECIFIC_REPLIED)) {
 				is_successful = FALSE;
@@ -196,7 +198,7 @@ lua_redis_dtor (struct lua_redis_ctx *ctx)
 		ud->terminated = 1;
 		ac = ud->ctx;
 		ud->ctx = NULL;
-		rspamd_redis_pool_release_connection (ud->pool, ac, is_successful);
+		rspamd_redis_pool_release_connection (ud->pool, ac, !is_successful);
 	}
 
 	LL_FOREACH_SAFE (ud->specific, cur, tmp) {
@@ -240,7 +242,11 @@ lua_redis_fin (void *arg)
 	struct lua_redis_ctx *ctx;
 
 	ctx = sp_ud->ctx;
-	event_del (&sp_ud->timeout);
+
+	if (rspamd_event_pending (&sp_ud->timeout, EV_TIMEOUT)) {
+		event_del (&sp_ud->timeout);
+	}
+
 	msg_debug ("finished redis query %p from session %p", sp_ud, ctx);
 	sp_ud->flags |= LUA_REDIS_SPECIFIC_FINISHED;
 
@@ -400,7 +406,7 @@ lua_redis_callback (redisAsyncContext *c, gpointer r, gpointer priv)
 		return;
 	}
 
-	msg_debug ("got reply from redis %p for query %p", ctx, sp_ud);
+	msg_debug ("got reply from redis %p for query %p", sp_ud->c->ctx, sp_ud);
 
 	REDIS_RETAIN (ctx);
 
@@ -480,6 +486,8 @@ lua_redis_push_results (struct lua_redis_ctx *ctx, lua_State *L)
 static void
 lua_redis_cleanup_events (struct lua_redis_ctx *ctx)
 {
+	REDIS_RETAIN (ctx); /* To avoid preliminary destruction */
+
 	while (!g_queue_is_empty (ctx->events_cleanup)) {
 		struct lua_redis_result *result = g_queue_pop_head (ctx->events_cleanup);
 
@@ -488,6 +496,8 @@ lua_redis_cleanup_events (struct lua_redis_ctx *ctx)
 
 		g_free (result);
 	}
+
+	REDIS_RELEASE (ctx);
 }
 
 /**
@@ -522,9 +532,11 @@ lua_redis_callback_sync (redisAsyncContext *ac, gpointer r, gpointer priv)
 		return;
 	}
 
-	event_del (&sp_ud->timeout);
+	if (rspamd_event_pending (&sp_ud->timeout, EV_TIMEOUT)) {
+		event_del (&sp_ud->timeout);
+	}
 
-	msg_debug ("got reply from redis %p for query %p", ctx, sp_ud);
+	msg_debug ("got reply from redis: %p for query %p", ac, sp_ud);
 
 	struct lua_redis_result *result = g_malloc0 (sizeof *result);
 
@@ -557,7 +569,8 @@ lua_redis_callback_sync (redisAsyncContext *ac, gpointer r, gpointer priv)
 	/* if error happened, we should terminate the connection,
 	   and release it */
 
-	if (result->is_error) {
+	if (result->is_error && sp_ud->c->ctx) {
+		ac = sp_ud->c->ctx;
 		/* Set to NULL to avoid double free in dtor */
 		sp_ud->c->ctx = NULL;
 		ctx->flags |= LUA_REDIS_TERMINATED;
@@ -585,9 +598,7 @@ lua_redis_callback_sync (redisAsyncContext *ac, gpointer r, gpointer priv)
 			ctx->thread = NULL;
 
 			results = lua_redis_push_results (ctx, thread->lua_state);
-
 			lua_thread_resume (thread, results);
-
 			lua_redis_cleanup_events (ctx);
 		}
 	}
@@ -600,19 +611,24 @@ lua_redis_timeout_sync (int fd, short what, gpointer priv)
 	struct lua_redis_ctx *ctx = sp_ud->ctx;
 	redisAsyncContext *ac;
 
-	ac = sp_ud->c->ctx;
+	msg_debug ("timeout while querying redis server: %p, redis: %p", sp_ud,
+			sp_ud->c->ctx);
 
-	/* Set to NULL to avoid double free in dtor */
-	sp_ud->c->ctx = NULL;
-	ac->err = REDIS_ERR_IO;
-	errno = ETIMEDOUT;
-	ctx->flags |= LUA_REDIS_TERMINATED;
+	if (sp_ud->c->ctx) {
+		ac = sp_ud->c->ctx;
+
+		/* Set to NULL to avoid double free in dtor */
+		sp_ud->c->ctx = NULL;
+		ac->err = REDIS_ERR_IO;
+		errno = ETIMEDOUT;
+		ctx->flags |= LUA_REDIS_TERMINATED;
 
-	/*
-	 * This will call all callbacks pending so the entire context
-	 * will be destructed
-	 */
-	rspamd_redis_pool_release_connection (sp_ud->c->pool, ac, TRUE);
+		/*
+		 * This will call all callbacks pending so the entire context
+		 * will be destructed
+		 */
+		rspamd_redis_pool_release_connection (sp_ud->c->pool, ac, TRUE);
+	}
 }
 
 static void
@@ -629,7 +645,8 @@ lua_redis_timeout (int fd, short what, gpointer u)
 	ctx = sp_ud->ctx;
 
 	REDIS_RETAIN (ctx);
-	msg_debug ("timeout while querying redis server");
+	msg_debug ("timeout while querying redis server: %p, redis: %p", sp_ud,
+			sp_ud->c->ctx);
 	lua_redis_push_error ("timeout while connecting the server", ctx, sp_ud, TRUE);
 
 	if (sp_ud->c->ctx) {
@@ -989,7 +1006,7 @@ lua_redis_make_request (lua_State *L)
 		lua_pop (L, 1);
 
 		lua_pushstring (L, "timeout");
-		lua_gettable (L, -2);
+		lua_gettable (L, 1);
 		if (lua_type (L, -1) == LUA_TNUMBER) {
 			timeout = lua_tonumber (L, -1);
 		}
@@ -998,7 +1015,7 @@ lua_redis_make_request (lua_State *L)
 
 
 		lua_pushstring (L, "args");
-		lua_gettable (L, -2);
+		lua_gettable (L, 1);
 		lua_redis_parse_args (L, -1, cmd, &sp_ud->args, &sp_ud->arglens,
 				&sp_ud->nargs);
 		lua_pop (L, 1);
@@ -1262,15 +1279,15 @@ lua_redis_connect_sync (lua_State *L)
 
 	ctx = rspamd_lua_redis_prepare_connection (L, NULL, FALSE);
 
-
 	if (ctx) {
-
-		lua_pushstring (L, "timeout");
-		lua_gettable (L, -2);
-		if (lua_type (L, -1) == LUA_TNUMBER) {
-			timeout = lua_tonumber (L, -1);
+		if (lua_istable (L, 1)) {
+			lua_pushstring (L, "timeout");
+			lua_gettable (L, 1);
+			if (lua_type (L, -1) == LUA_TNUMBER) {
+				timeout = lua_tonumber (L, -1);
+			}
+			lua_pop (L, 1);
 		}
-		lua_pop (L, 1);
 
 		ctx->async.timeout = timeout;
 

src/lua/lua_task.c

@@ -1567,8 +1567,10 @@ lua_task_set_pre_result (lua_State * L)
 {
 	LUA_TRACE_POINT;
 	struct rspamd_task *task = lua_check_task (L, 1);
-	gchar *action_str;
+	const gchar *message = NULL, *module = NULL;
+	gdouble score = NAN;
 	gint action = METRIC_ACTION_MAX;
+	guint priority = RSPAMD_PASSTHROUGH_NORMAL;
 
 	if (task != NULL) {
 
@@ -1584,29 +1586,38 @@ lua_task_set_pre_result (lua_State * L)
 			rspamd_action_from_str (lua_tostring (L, 2), &action);
 		}
 
-		if (action < METRIC_ACTION_MAX && action >= METRIC_ACTION_REJECT) {
-			/* We also need to set the default metric to that result */
-			if (!task->result) {
-				task->result = rspamd_create_metric_result (task);
-			}
+		if (lua_type (L, 3) == LUA_TSTRING) {
+			message = lua_tostring (L, 3);
 
-			task->pre_result.action = action;
+			/* Keep compatibility here :( */
+			ucl_object_replace_key (task->messages,
+					ucl_object_fromstring (message), "smtp_message", 0,
+					false);
+		}
+		else {
+			message = "unknown reason";
+		}
 
-			if (lua_gettop (L) >= 3) {
-				action_str = rspamd_mempool_strdup (task->task_pool,
-						luaL_checkstring (L, 3));
-				task->pre_result.str = action_str;
-				ucl_object_replace_key (task->messages,
-						ucl_object_fromstring (action_str), "smtp_message", 0,
-						false);
-			}
-			else {
-				task->pre_result.str = "unknown";
-			}
+		if (lua_type (L, 4) == LUA_TSTRING) {
+			module = lua_tostring (L, 4);
+		}
+		else {
+			module = "Unknown lua";
+		}
 
-			msg_info_task ("<%s>: set pre-result to %s: '%s'",
-						task->message_id, rspamd_action_to_str (action),
-						task->pre_result.str);
+		if (lua_type (L, 5) == LUA_TNUMBER) {
+			score = lua_tonumber (L, 5);
+		}
+
+		if (lua_type (L, 6) == LUA_TNUMBER) {
+			priority = lua_tonumber (L, 6);
+		}
+
+		if (action < METRIC_ACTION_MAX && action >= METRIC_ACTION_REJECT) {
+
+			rspamd_add_passthrough_result (task, action, priority,
+					score, rspamd_mempool_strdup (task->task_pool, message),
+					rspamd_mempool_strdup (task->task_pool, module));
 
 			/* Don't classify or filter message if pre-filter sets results */
 			task->processed_stages |= (RSPAMD_TASK_STAGE_FILTERS |
@@ -1632,7 +1643,7 @@ lua_task_has_pre_result (lua_State * L)
 	struct rspamd_task *task = lua_check_task (L, 1);
 
 	if (task) {
-		lua_pushboolean (L, task->pre_result.action != METRIC_ACTION_MAX);
+		lua_pushboolean (L, task->result->passthrough_result != NULL);
 	}
 	else {
 		return luaL_error (L, "invalid arguments");
@@ -4127,18 +4138,23 @@ lua_task_set_settings (lua_State *L)
 			/* Adjust desired actions */
 			mres = task->result;
 
-			if (mres == NULL) {
-				mres = rspamd_create_metric_result (task);
-			}
-
 			for (i = 0; i < METRIC_ACTION_MAX; i++) {
 				elt = ucl_object_lookup_any (act, rspamd_action_to_str (i),
 						rspamd_action_to_str_alt (i), NULL);
 
 				if (elt) {
-					mres->actions_limits[i] = ucl_object_todouble (elt);
-					msg_debug_task ("adjusted action %s to %.2f",
-							ucl_object_key (elt), mres->actions_limits[i]);
+
+					if (ucl_object_type (elt) == UCL_FLOAT ||
+								ucl_object_type (elt) == UCL_INT) {
+						mres->actions_limits[i] = ucl_object_todouble (elt);
+						msg_debug_task ("adjusted action %s to %.2f",
+								ucl_object_key (elt), mres->actions_limits[i]);
+					}
+					else if (ucl_object_type (elt) == UCL_NULL) {
+						mres->actions_limits[i] = NAN;
+						msg_info_task ("disabled action %s due to settings",
+								ucl_object_key (elt));
+					}
 				}
 			}
 		}
@@ -4534,9 +4550,7 @@ lua_task_get_metric_action (lua_State *L)
 	enum rspamd_action_type action;
 
 	if (task) {
-		if ((metric_res = task->result) == NULL) {
-			metric_res = rspamd_create_metric_result (task);
-		}
+		metric_res = task->result;
 
 		action = rspamd_check_action_metric (task, metric_res);
 		lua_pushstring (L, rspamd_action_to_str (action));

src/plugins/lua/antivirus.lua

@@ -123,7 +123,7 @@ local function yield_result(task, rule, vname)
         lua_util.template(rule.message or 'Rejected', {
           SCANNER = rule['type'],
           VIRUS = vname,
-        }))
+        }), N)
   end
 end
 

src/plugins/lua/clickhouse.lua

@@ -493,18 +493,22 @@ local function clickhouse_collect(task)
   local urls_urls = {}
   if task:has_urls(false) then
     for _,u in ipairs(task:get_urls(false)) do
-      table.insert(urls_tlds, u:get_tld())
+      urls_tlds[u:get_tld()] = true
       if settings['full_urls'] then
-        table.insert(urls_urls, u:get_text())
+        urls_urls[u:get_text()] = true
       else
-        table.insert(urls_urls, u:get_host())
+        urls_urls[u:get_host()] = true
       end
     end
   end
 
+  local flatten_urls = function(...)
+    return fun.totable(fun.map(function(k,_) return k end, ...))
+  end
+
   if #urls_tlds > 0 then
-    table.insert(row, urls_tlds)
-    table.insert(row, urls_urls)
+    table.insert(row, flatten_urls(urls_tlds))
+    table.insert(row, flatten_urls(urls_urls))
   else
     table.insert(row, {})
     table.insert(row, {})
@@ -512,8 +516,8 @@ local function clickhouse_collect(task)
 
   -- Emails step
   if task:has_urls(true) then
-    table.insert(row, fun.totable(fun.map(function(u)
-      return string.format('%s@%s', u:get_user(), u:get_host())
+    table.insert(row, flatten_urls(fun.map(function(u)
+      return string.format('%s@%s', u:get_user(), u:get_host()),true
     end, task:get_emails())))
   else
     table.insert(row, {})

src/plugins/lua/dmarc.lua

@@ -195,372 +195,500 @@ local function dmarc_report(task, spf_ok, dkim_ok, disposition,
   return res
 end
 
-local function dmarc_callback(task)
-  local function maybe_force_action(disposition)
+local function maybe_force_action(task, disposition)
+  if disposition then
     local force_action = dmarc_actions[disposition]
     if force_action then
       -- Don't do anything if pre-result has been already set
       if task:has_pre_result() then return end
-      task:set_pre_result(force_action, 'Action set by DMARC')
+      task:set_pre_result(force_action, 'Action set by DMARC', N)
     end
   end
-  local from = task:get_from(2)
-  local hfromdom = ((from or E)[1] or E).domain
-  local dmarc_domain, spf_domain
-  local ip_addr = task:get_ip()
-  local dkim_results = {}
-  local dmarc_checks = task:get_mempool():get_variable('dmarc_checks', 'int') or 0
+end
 
-  if dmarc_checks ~= 2 then
-    rspamd_logger.infox(task, "skip DMARC checks as either SPF or DKIM were not checked");
-    return
-  end
+--[[
+-- Used to check dmarc record, check elements and produce dmarc policy processed
+-- result.
+-- Returns:
+--     false,false - record is garbadge
+--     false,error_message - record is invalid
+--     true,policy_table - record is valid and parsed
+]]
+local function dmarc_check_record(task, record, is_tld)
+  local failed_policy
+  local result = {
+    dmarc_policy = 'none'
+  }
+
+  local elts = dmarc_grammar:match(record)
+  lua_util.debugm(N, task, "got DMARC record: %s, tld_flag=%s, processed=%s",
+      record, is_tld, elts)
+
+  if elts then
+    local dkim_pol = elts['adkim']
+    if dkim_pol then
+      if dkim_pol == 's' then
+        result.strict_dkim = true
+      elseif dkim_pol ~= 'r' then
+        failed_policy = 'adkim tag has invalid value: ' .. dkim_pol
+        return false,failed_policy
+      end
+    end
 
-  if ((not check_authed and task:get_user()) or
-      (not check_local and ip_addr and ip_addr:is_local())) then
-    rspamd_logger.infox(task, "skip DMARC checks for local networks and authorized users");
-    return
-  end
-  if hfromdom and hfromdom ~= '' and not (from or E)[2] then
-    dmarc_domain = rspamd_util.get_tld(hfromdom)
-  elseif (from or E)[2] then
-    task:insert_result(dmarc_symbols['na'], 1.0, 'Duplicate From header')
-    return maybe_force_action('na')
-  elseif (from or E)[1] then
-    task:insert_result(dmarc_symbols['na'], 1.0, 'No domain in From header')
-    return maybe_force_action('na')
+    local spf_pol = elts['aspf']
+    if spf_pol then
+      if spf_pol == 's' then
+        result.strict_spf = true
+      elseif spf_pol ~= 'r' then
+        failed_policy = 'aspf tag has invalid value: ' .. spf_pol
+        return false,failed_policy
+      end
+    end
+
+    local policy = elts['p']
+    if policy then
+      if (policy == 'reject') then
+        result.dmarc_policy = 'reject'
+      elseif (policy == 'quarantine') then
+        result.dmarc_policy = 'quarantine'
+      elseif (policy ~= 'none') then
+        failed_policy = 'p tag has invalid value: ' .. policy
+        return false,failed_policy
+      end
+    end
+
+    -- Adjust policy if we are in tld mode
+    local subdomain_policy = elts['sp']
+    if elts['sp'] and is_tld then
+      result.subdomain_policy = elts['sp']
+
+      if (subdomain_policy == 'reject') then
+        result.dmarc_policy = 'reject'
+      elseif (subdomain_policy == 'quarantine') then
+        result.dmarc_policy = 'quarantine'
+      elseif (subdomain_policy == 'none') then
+        result.dmarc_policy = 'none'
+      elseif (subdomain_policy ~= 'none') then
+        failed_policy = 'sp tag has invalid value: ' .. subdomain_policy
+        return false,failed_policy
+      end
+    end
+    result.pct = elts['pct']
+    if result.pct then
+      result.pct = tonumber(result.pct)
+    end
+
+    if elts.rua then
+      result.rua = elts['rua']
+    end
   else
-    task:insert_result(dmarc_symbols['na'], 1.0, 'No From header')
-    return maybe_force_action('na')
+    return false,false -- Ignore garbadge
   end
 
-  local function dmarc_report_cb(err)
-    if not err then
-      rspamd_logger.infox(task, '<%1> dmarc report saved for %2',
-        task:get_message_id(), hfromdom)
+  return true, result
+end
+
+local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld)
+  local reason = {}
+
+  -- Check dkim and spf symbols
+  local spf_ok = false
+  local dkim_ok = false
+  local spf_tmpfail = false
+  local dkim_tmpfail = false
+
+  local spf_domain = ((task:get_from(1) or E)[1] or E).domain
+
+  if not spf_domain or spf_domain == '' then
+    spf_domain = task:get_helo() or ''
+  end
+
+  if task:has_symbol(symbols['spf_allow_symbol']) then
+    if policy.strict_spf then
+      if rspamd_util.strequal_caseless(spf_domain, hdrfromdom) then
+        spf_ok = true
+      else
+        table.insert(reason, "SPF not aligned (strict)")
+      end
     else
-      rspamd_logger.errx(task, '<%1> dmarc report is not saved for %2: %3',
-        task:get_message_id(), hfromdom, err)
+      local spf_tld = rspamd_util.get_tld(spf_domain)
+      if rspamd_util.strequal_caseless(spf_tld, dmarc_esld) then
+        spf_ok = true
+      else
+        table.insert(reason, "SPF not aligned (relaxed)")
+      end
     end
+  else
+    if task:has_symbol(symbols['spf_tempfail_symbol']) then
+      if policy.strict_spf then
+        if rspamd_util.strequal_caseless(spf_domain, hdrfromdom) then
+          spf_tmpfail = true
+        end
+      else
+        local spf_tld = rspamd_util.get_tld(spf_domain)
+        if rspamd_util.strequal_caseless(spf_tld, dmarc_esld) then
+          spf_tmpfail = true
+        end
+      end
+    end
+
+    table.insert(reason, "No valid SPF")
   end
 
-  local function dmarc_dns_cb(_, to_resolve, results, err)
 
-    local lookup_domain = string.sub(to_resolve, 8)
-    if err and (err ~= 'requested record is not found' and err ~= 'no records with this name') then
-      task:insert_result(dmarc_symbols['dnsfail'], 1.0, lookup_domain .. ' : ' .. err)
-      return maybe_force_action('dnsfail')
-    elseif err and (err == 'requested record is not found' or err == 'no records with this name') and
-      lookup_domain == dmarc_domain then
-      task:insert_result(dmarc_symbols['na'], 1.0, lookup_domain)
-      return maybe_force_action('na')
-    end
+  local opts = ((task:get_symbol('DKIM_TRACE') or E)[1] or E).options
+  local dkim_results = {
+    pass = {},
+    temperror = {},
+    permerror = {},
+    fail = {},
+  }
 
-    if not results then
-      if lookup_domain ~= dmarc_domain then
-        local resolve_name = '_dmarc.' .. dmarc_domain
-        task:get_resolver():resolve_txt({
-          task=task,
-          name = resolve_name,
-          callback = dmarc_dns_cb,
-          forced = true})
-        return
-      end
 
-      task:insert_result(dmarc_symbols['na'], 1.0, lookup_domain)
-      return maybe_force_action('na')
-    end
+  if opts then
+    dkim_results.pass = {}
+    local dkim_violated
 
-    local pct
-    local reason = {}
-    local strict_spf = false
-    local strict_dkim = false
-    local dmarc_policy = 'none'
-    local found_policy = false
-    local failed_policy
-    local rua
-
-    for _,r in ipairs(results) do
-      if failed_policy then break end
-      local function try()
-        local elts = dmarc_grammar:match(r)
-        if not elts then
-          return
+    for _,opt in ipairs(opts) do
+      local check_res = string.sub(opt, -1)
+      local domain = string.sub(opt, 1, -3)
+
+      if check_res == '+' then
+        table.insert(dkim_results.pass, domain)
+
+        if policy.strict_dkim then
+          if rspamd_util.strequal_caseless(hdrfromdom, domain) then
+            dkim_ok = true
+          else
+            dkim_violated = "DKIM not aligned (strict)"
+          end
         else
-          if found_policy then
-            failed_policy = 'Multiple policies defined in DNS'
-            return
+          local dkim_tld = rspamd_util.get_tld(domain)
+
+          if rspamd_util.strequal_caseless(dkim_tld, dmarc_esld) then
+            dkim_ok = true
           else
-            found_policy = true
+            dkim_violated = "DKIM not aligned (relaxed)"
           end
         end
-
-        if elts then
-          local dkim_pol = elts['adkim']
-          if dkim_pol then
-            if dkim_pol == 's' then
-              strict_dkim = true
-            elseif dkim_pol ~= 'r' then
-              failed_policy = 'adkim tag has invalid value: ' .. dkim_pol
-              return
+      elseif check_res == '?' then
+        -- Check for dkim tempfail
+        if not dkim_ok then
+          if policy.strict_dkim then
+            if rspamd_util.strequal_caseless(hdrfromdom, domain) then
+              dkim_tmpfail = true
             end
-          end
+          else
+            local dkim_tld = rspamd_util.get_tld(domain)
 
-          local spf_pol = elts['aspf']
-          if spf_pol then
-            if spf_pol == 's' then
-              strict_spf = true
-            elseif spf_pol ~= 'r' then
-              failed_policy = 'aspf tag has invalid value: ' .. spf_pol
-              return
+            if rspamd_util.strequal_caseless(dkim_tld, dmarc_esld) then
+              dkim_tmpfail = true
             end
           end
+        end
+        table.insert(dkim_results.temperror, domain)
+      elseif check_res == '-' then
+        table.insert(dkim_results.fail, domain)
+      else
+        table.insert(dkim_results.permerror, domain)
+      end
+    end
 
-          local policy = elts['p']
-          if policy then
-            if (policy == 'reject') then
-              dmarc_policy = 'reject'
-            elseif (policy == 'quarantine') then
-              dmarc_policy = 'quarantine'
-            elseif (policy ~= 'none') then
-              failed_policy = 'p tag has invalid value: ' .. policy
-              return
-            end
-          end
+    if not dkim_ok and dkim_violated then
+      table.insert(reason, dkim_violated)
+    end
+  else
+    table.insert(reason, "No valid DKIM")
+  end
 
-          local subdomain_policy = elts['sp']
-          if subdomain_policy and lookup_domain == dmarc_domain then
-            if (subdomain_policy == 'reject') then
-              if dmarc_domain ~= hfromdom then
-                dmarc_policy = 'reject'
-              end
-            elseif (subdomain_policy == 'quarantine') then
-              if dmarc_domain ~= hfromdom then
-                dmarc_policy = 'quarantine'
-              end
-            elseif (subdomain_policy == 'none') then
-              if dmarc_domain ~= hfromdom then
-                dmarc_policy = 'none'
-              end
-            elseif (subdomain_policy ~= 'none') then
-              failed_policy = 'sp tag has invalid value: ' .. subdomain_policy
-              return
-            end
-          end
+  lua_util.debugm(N, task, "validated dmarc policy for %s: %s; dkim_ok=%s, dkim_tempfail=%s, spf_ok=%s, spf_tempfail=%s",
+      policy.domain, policy.dmarc_policy,
+      dkim_ok, dkim_tmpfail,
+      spf_ok, spf_tmpfail)
 
-          pct = elts['pct']
-          if pct then
-            pct = tonumber(pct)
-          end
+  local disposition = 'none'
+  local sampled_out = false
 
-          if not rua then
-            rua = elts['rua']
-          end
+  local function handle_dmarc_failure(what, reason_str)
+    if not policy.pct or policy.pct == 100 then
+      task:insert_result(dmarc_symbols[what], 1.0,
+          policy.domain .. ' : ' .. reason_str, policy.dmarc_policy)
+      disposition = what
+    else
+      if (math.random(100) > policy.pct) then
+        if (not no_sampling_domains or
+            not no_sampling_domains:get_key(policy.domain)) then
+          task:insert_result(dmarc_symbols['softfail'], 1.0,
+              policy.domain .. ' : ' .. reason_str, policy.dmarc_policy, "sampled_out")
+          sampled_out = true
+        else
+          task:insert_result(dmarc_symbols[what], 1.0,
+              policy.domain .. ' : ' .. reason_str, policy.dmarc_policy, "local_policy")
+          disposition = what
         end
+      else
+        task:insert_result(dmarc_symbols[what], 1.0,
+            policy.domain .. ' : ' .. reason_str, policy.dmarc_policy)
+        disposition = what
       end
-      try()
     end
 
-    if not found_policy then
-      if lookup_domain ~= dmarc_domain then
-        local resolve_name = '_dmarc.' .. dmarc_domain
-        task:get_resolver():resolve_txt({
-          task=task,
-          name = resolve_name,
-          callback = dmarc_dns_cb,
-          forced = true})
+    maybe_force_action(task, disposition)
+  end
 
-        return
+  if spf_ok or dkim_ok then
+    --[[
+    https://tools.ietf.org/html/rfc7489#section-6.6.2
+    DMARC evaluation can only yield a "pass" result after one of the
+    underlying authentication mechanisms passes for an aligned
+    identifier.
+    ]]--
+    task:insert_result(dmarc_symbols['allow'], 1.0, policy.domain,
+        policy.dmarc_policy)
+  else
+    --[[
+    https://tools.ietf.org/html/rfc7489#section-6.6.2
+
+    If neither passes and one or both of them fail due to a
+    temporary error, the Receiver evaluating the message is unable to
+    conclude that the DMARC mechanism had a permanent failure; they
+    therefore cannot apply the advertised DMARC policy.
+    ]]--
+    if spf_tmpfail or dkim_tmpfail then
+      task:insert_result(dmarc_symbols['dnsfail'], 1.0, policy.domain..
+          ' : ' .. 'SPF/DKIM temp error', policy.dmarc_policy)
+    else
+      -- We can now check the failed policy and maybe send report data elt
+      local reason_str = table.concat(reason, ', ')
+
+      if policy.dmarc_policy == 'quarantine' then
+        handle_dmarc_failure('quarantine', reason_str)
+      elseif policy.dmarc_policy == 'reject' then
+        handle_dmarc_failure('reject', reason_str)
       else
-        task:insert_result(dmarc_symbols['na'], 1.0, lookup_domain)
-        return maybe_force_action('na')
+        task:insert_result(dmarc_symbols['softfail'], 1.0,
+            policy.domain .. ' : ' .. reason_str,
+            policy.dmarc_policy)
       end
     end
+  end
 
-    local res = 0.5
-    if failed_policy then
-      task:insert_result(dmarc_symbols['badpolicy'], res, lookup_domain .. ' : ' .. failed_policy)
-      return maybe_force_action('badpolicy')
+  if policy.rua and redis_params and dmarc_reporting then
+    if no_reporting_domains then
+      if no_reporting_domains:get_key(policy.domain) or
+          no_reporting_domains:get_key(rspamd_util.get_tld(policy.domain)) then
+        rspamd_logger.infox(task, 'DMARC reporting suppressed for %1', policy.domain)
+        return
+      end
     end
 
-    -- Check dkim and spf symbols
-    local spf_ok = false
-    local dkim_ok = false
-    spf_domain = ((task:get_from(1) or E)[1] or E).domain
-    if not spf_domain or spf_domain == '' then
-      spf_domain = task:get_helo() or ''
+    local function dmarc_report_cb(err)
+      if not err then
+        rspamd_logger.infox(task, '<%1> dmarc report saved for %2',
+            task:get_message_id(), hdrfromdom)
+      else
+        rspamd_logger.errx(task, '<%1> dmarc report is not saved for %2: %3',
+            task:get_message_id(), hdrfromdom, err)
+      end
     end
 
-    if task:has_symbol(symbols['spf_allow_symbol']) then
-      if strict_spf and rspamd_util.strequal_caseless(spf_domain, hfromdom) then
-        spf_ok = true
-      elseif strict_spf then
-        table.insert(reason, "SPF not aligned (strict)")
-      end
-      if not strict_spf then
-        local spf_tld = rspamd_util.get_tld(spf_domain)
-        if rspamd_util.strequal_caseless(spf_tld, dmarc_domain) then
-          spf_ok = true
-        else
-          table.insert(reason, "SPF not aligned (relaxed)")
-        end
-      end
+    local spf_result
+    if spf_ok then
+      spf_result = 'pass'
+    elseif spf_tmpfail then
+      spf_result = 'temperror'
     else
-      table.insert(reason, "No valid SPF")
-    end
-    local das = task:get_symbol(symbols['dkim_allow_symbol'])
-    if ((das or E)[1] or E).options then
-      dkim_results.pass = {}
-      for _,domain in ipairs(das[1]['options']) do
-        table.insert(dkim_results.pass, domain)
-        if strict_dkim and rspamd_util.strequal_caseless(hfromdom, domain) then
-          dkim_ok = true
-        elseif strict_dkim then
-          table.insert(reason, "DKIM not aligned (strict)")
-        end
-        if not strict_dkim then
-          local dkim_tld = rspamd_util.get_tld(domain)
-          if rspamd_util.strequal_caseless(dkim_tld, dmarc_domain) then
-            dkim_ok = true
-          else
-            table.insert(reason, "DKIM not aligned (relaxed)")
-          end
-        end
+      if task:get_symbol(symbols.spf_deny_symbol) then
+        spf_result = 'fail'
+      elseif task:get_symbol(symbols.spf_softfail_symbol) then
+        spf_result = 'softfail'
+      elseif task:get_symbol(symbols.spf_neutral_symbol) then
+        spf_result = 'neutral'
+      elseif task:get_symbol(symbols.spf_permfail_symbol) then
+        spf_result = 'permerror'
+      else
+        spf_result = 'none'
       end
-    else
-      table.insert(reason, "No valid DKIM")
     end
 
-    local disposition = 'none'
-    local sampled_out = false
-    local spf_tmpfail, dkim_tmpfail
-
-    if not (spf_ok or dkim_ok) then
-      local reason_str = table.concat(reason, ", ")
-      res = 1.0
-      spf_tmpfail = task:get_symbol(symbols['spf_tempfail_symbol'])
-      dkim_tmpfail = task:get_symbol(symbols['dkim_tempfail_symbol'])
-      if (spf_tmpfail or dkim_tmpfail) then
-        if ((dkim_tmpfail or E)[1] or E).options then
-          dkim_results.tempfail = {}
-          for _,domain in ipairs(dkim_tmpfail[1]['options']) do
-            table.insert(dkim_results.tempfail, domain)
-          end
-        end
-        task:insert_result(dmarc_symbols['dnsfail'], 1.0, lookup_domain .. ' : ' .. 'SPF/DKIM temp error', dmarc_policy)
-        return maybe_force_action('dnsfail')
+    -- Prepare and send redis report element
+    local period = os.date('%Y%m%d',
+        task:get_date({format = 'connect', gmt = true}))
+    local dmarc_domain_key = table.concat(
+        {redis_keys.report_prefix, hdrfromdom, period}, redis_keys.join_char)
+    local report_data = dmarc_report(task,
+        spf_ok and 'pass' or 'fail',
+        dkim_ok and 'pass' or 'fail',
+        disposition,
+        sampled_out,
+        hdrfromdom,
+        spf_domain,
+        dkim_results,
+        spf_result)
+
+    local idx_key = table.concat({redis_keys.index_prefix, period},
+        redis_keys.join_char)
+
+    if report_data then
+      rspamd_redis.exec_redis_script(take_report_id,
+          {task = task, is_write = true},
+          dmarc_report_cb,
+          {idx_key, dmarc_domain_key},
+          {hdrfromdom, report_data})
+    end
+  end
+end
+
+local function dmarc_callback(task)
+  local from = task:get_from(2)
+  local hfromdom = ((from or E)[1] or E).domain
+  local dmarc_domain
+  local ip_addr = task:get_ip()
+  local dmarc_checks = task:get_mempool():get_variable('dmarc_checks', 'int') or 0
+  local seen_invalid = false
+
+  if dmarc_checks ~= 2 then
+    rspamd_logger.infox(task, "skip DMARC checks as either SPF or DKIM were not checked");
+    return
+  end
+
+  if ((not check_authed and task:get_user()) or
+      (not check_local and ip_addr and ip_addr:is_local())) then
+    rspamd_logger.infox(task, "skip DMARC checks for local networks and authorized users");
+    return
+  end
+
+  -- Do some initial sanity checks, detect tld domain if different
+  if hfromdom and hfromdom ~= '' and not (from or E)[2] then
+    dmarc_domain = rspamd_util.get_tld(hfromdom)
+  elseif (from or E)[2] then
+    task:insert_result(dmarc_symbols['na'], 1.0, 'Duplicate From header')
+    return maybe_force_action(task, 'na')
+  elseif (from or E)[1] then
+    task:insert_result(dmarc_symbols['na'], 1.0, 'No domain in From header')
+    return maybe_force_action(task,'na')
+  else
+    task:insert_result(dmarc_symbols['na'], 1.0, 'No From header')
+    return maybe_force_action(task,'na')
+  end
+
+
+  local dns_checks_inflight = 0
+  local dmarc_domain_policy = {}
+  local dmarc_tld_policy = {}
+
+  local function process_dmarc_policy(policy, final)
+    lua_util.debugm(N, task, "validate DMARC policy (final=%s): %s",
+        true, policy)
+    if policy.err and policy.symbol then
+      -- In case of fatal errors or final check for tld, we give up and
+      -- insert result
+      if final or policy.fatal then
+        task:insert_result(policy.symbol, 1.0, policy.err)
+        maybe_force_action(task, policy.disposition)
+
+        return true
       end
-      if dmarc_policy == 'quarantine' then
-        if not pct or pct == 100 then
-          task:insert_result(dmarc_symbols['quarantine'], res, lookup_domain .. ' : ' .. reason_str, dmarc_policy)
-          disposition = "quarantine"
-        else
-          if (math.random(100) > pct) then
-            if (not no_sampling_domains or not no_sampling_domains:get_key(dmarc_domain)) then
-              task:insert_result(dmarc_symbols['softfail'], res, lookup_domain .. ' : ' .. reason_str, dmarc_policy, "sampled_out")
-              sampled_out = true
-            else
-              task:insert_result(dmarc_symbols['quarantine'], res, lookup_domain .. ' : ' .. reason_str, dmarc_policy, "local_policy")
-              disposition = "quarantine"
-            end
+    elseif policy.dmarc_policy then
+      dmarc_validate_policy(task, policy, hfromdom, dmarc_domain)
+
+      return true -- We have a more specific version, use it
+    end
+
+    return false -- Missing record
+  end
+
+  local function gen_dmarc_cb(lookup_domain, is_tld)
+    local policy_target = dmarc_domain_policy
+    if is_tld then
+      policy_target = dmarc_tld_policy
+    end
+
+    return function (_, _, results, err)
+      dns_checks_inflight = dns_checks_inflight - 1
+
+      if not seen_invalid then
+        policy_target.domain = lookup_domain
+
+        if err then
+          if (err ~= 'requested record is not found' and
+              err ~= 'no records with this name') then
+            policy_target.err = lookup_domain .. ' : ' .. err
+            policy_target.symbol = dmarc_symbols['dnsfail']
           else
-            task:insert_result(dmarc_symbols['quarantine'], res, lookup_domain .. ' : ' .. reason_str, dmarc_policy)
-            disposition = "quarantine"
+            policy_target.err = lookup_domain
+            policy_target.symbol = dmarc_symbols['na']
           end
-        end
-      elseif dmarc_policy == 'reject' then
-        if not pct or pct == 100 then
-          task:insert_result(dmarc_symbols['reject'], res, lookup_domain .. ' : ' .. reason_str, dmarc_policy)
-          disposition = "reject"
         else
-          if (math.random(100) > pct) then
-            if (not no_sampling_domains or not no_sampling_domains:get_key(dmarc_domain)) then
-              task:insert_result(dmarc_symbols['quarantine'], res, lookup_domain .. ' : ' .. reason_str, dmarc_policy, "sampled_out")
-              disposition = "quarantine"
-              sampled_out = true
+          local has_valid_policy = false
+
+          for _,rec in ipairs(results) do
+            local ret,results_or_err = dmarc_check_record(task, rec, is_tld)
+
+            if not ret then
+              if results_or_err then
+                -- We have a fatal parsing error, give up
+                policy_target.err = lookup_domain .. ' : ' .. results_or_err
+                policy_target.symbol = dmarc_symbols['badpolicy']
+                policy_target.fatal = true
+                seen_invalid = true
+              end
             else
-              task:insert_result(dmarc_symbols['reject'], res, lookup_domain .. ' : ' .. reason_str, dmarc_policy, "local_policy")
-              disposition = "reject"
+              if has_valid_policy then
+                policy_target.err = lookup_domain .. ' : ' ..
+                    'Multiple policies defined in DNS'
+                policy_target.symbol = dmarc_symbols['badpolicy']
+                policy_target.fatal = true
+                seen_invalid = true
+              end
+              has_valid_policy = true
+
+              for k,v in pairs(results_or_err) do
+                policy_target[k] = v
+              end
             end
-          else
-            task:insert_result(dmarc_symbols['reject'], res, lookup_domain .. ' : ' .. reason_str, dmarc_policy)
-            disposition = "reject"
           end
         end
-      else
-        task:insert_result(dmarc_symbols['softfail'], res, lookup_domain .. ' : ' .. reason_str, dmarc_policy)
-      end
-    else
-      task:insert_result(dmarc_symbols['allow'], res, lookup_domain, dmarc_policy)
-    end
-
-    if rua and redis_params and dmarc_reporting then
-
-      if no_reporting_domains then
-        if no_reporting_domains:get_key(dmarc_domain) or no_reporting_domains:get_key(rspamd_util.get_tld(dmarc_domain)) then
-          rspamd_logger.infox(task, 'DMARC reporting suppressed for %1', dmarc_domain)
-          return maybe_force_action(disposition)
-        end
       end
 
-      local spf_result
-      if spf_ok then
-        spf_result = 'pass'
-      elseif spf_tmpfail then
-        spf_result = 'temperror'
-      else
-        if task:get_symbol(symbols.spf_deny_symbol) then
-          spf_result = 'fail'
-        elseif task:get_symbol(symbols.spf_softfail_symbol) then
-          spf_result = 'softfail'
-        elseif task:get_symbol(symbols.spf_neutral_symbol) then
-          spf_result = 'neutral'
-        elseif task:get_symbol(symbols.spf_permfail_symbol) then
-          spf_result = 'permerror'
-        else
-          spf_result = 'none'
-        end
-      end
-      local dkim_deny = ((task:get_symbol(symbols.dkim_deny_symbol) or E)[1] or E).options
-      if dkim_deny then
-        dkim_results.fail = {}
-        for _, domain in ipairs(dkim_deny) do
-          table.insert(dkim_results.fail, domain)
-        end
-      end
-      local dkim_permerror = ((task:get_symbol(symbols.dkim_permfail_symbol) or E)[1] or E).options
-      if dkim_permerror then
-        dkim_results.permerror = {}
-        for _, domain in ipairs(dkim_permerror) do
-          table.insert(dkim_results.permerror, domain)
+      if dns_checks_inflight == 0 then
+        lua_util.debugm(N, task, "finished DNS queries, validate policies")
+        -- We have checked both tld and real domain (if different)
+        if not process_dmarc_policy(dmarc_domain_policy, false) then
+          -- Try tld policy as well
+          if not process_dmarc_policy(dmarc_tld_policy, true) then
+            process_dmarc_policy(dmarc_domain_policy, true)
+          end
         end
       end
-      -- Prepare and send redis report element
-      local period = os.date('%Y%m%d', task:get_date({format = 'connect', gmt = true}))
-      local dmarc_domain_key = table.concat({redis_keys.report_prefix, hfromdom, period}, redis_keys.join_char)
-      local report_data = dmarc_report(task, spf_ok and 'pass' or 'fail', dkim_ok and 'pass' or 'fail', disposition, sampled_out,
-        hfromdom, spf_domain, dkim_results, spf_result)
-      local idx_key = table.concat({redis_keys.index_prefix, period}, redis_keys.join_char)
-
-      if report_data then
-        rspamd_redis.exec_redis_script(take_report_id, {task = task, is_write = true}, dmarc_report_cb,
-          {idx_key, dmarc_domain_key}, {hfromdom, report_data})
-      end
     end
-
-    return maybe_force_action(disposition)
-
   end
 
-  -- Do initial request
   local resolve_name = '_dmarc.' .. hfromdom
+
   task:get_resolver():resolve_txt({
     task=task,
     name = resolve_name,
-    callback = dmarc_dns_cb,
-    forced = true})
+    callback = gen_dmarc_cb(hfromdom, false),
+    forced = true
+  })
+  dns_checks_inflight = dns_checks_inflight + 1
+
+  if dmarc_domain ~= hfromdom then
+    resolve_name = '_dmarc.' .. dmarc_domain
+
+    task:get_resolver():resolve_txt({
+      task=task,
+      name = resolve_name,
+      callback = gen_dmarc_cb(dmarc_domain, true),
+      forced = true
+    })
+
+    dns_checks_inflight = dns_checks_inflight + 1
+  end
 end
 
+
 local function try_opts(where)
   local ret = false
   local opts = rspamd_config:get_all_opt(where)
@@ -595,6 +723,7 @@ if opts['symbols'] then
   end
 end
 
+-- XXX: rework this shitty code some day please
 if opts['reporting'] == true then
   redis_params = rspamd_parse_redis_server('dmarc')
   if not redis_params then

src/plugins/lua/emails.lua

@@ -52,7 +52,8 @@ local function check_email_rule(task, rule, addr)
     local function emails_dns_cb(_, _, results, err)
       if err and (err ~= 'requested record is not found'
           and err ~= 'no records with this name') then
-        logger.errx(task, 'Error querying DNS(%s): %s', to_resolve, err)
+        logger.errx(task, 'Error querying DNS(%s.%s): %s', to_resolve,
+            rule['dnsbl'], err)
       elseif results then
         local expected_found = false
         local symbol = rule['symbol']

src/plugins/lua/force_actions.lua

@@ -79,9 +79,9 @@ local function gen_cb(expr, act, pool, message, subject, raction, honor, limit)
         task:set_metric_subject(subject)
       end
       if type(message) == 'string' then
-        task:set_pre_result(act, message)
+        task:set_pre_result(act, message, N)
       else
-        task:set_pre_result(act)
+        task:set_pre_result(act, nil, N)
       end
       return true, act
     end

src/plugins/lua/greylist.lua

@@ -157,13 +157,13 @@ local function greylist_message(task, end_time, why)
   if rspamd_lua_utils.is_rspamc_or_controller(task) then return end
   if settings.message_func then
     task:set_pre_result(settings['action'],
-      settings.message_func(task, end_time))
+      settings.message_func(task, end_time), N)
   else
     local message = settings['message']
     if settings.report_time then
       message = string.format("%s: %s", message, end_time)
     end
-    task:set_pre_result(settings['action'], message)
+    task:set_pre_result(settings['action'], message, N)
   end
 
   task:set_flag('greylisted')

src/plugins/lua/metadata_exporter.lua

@@ -229,7 +229,7 @@ local selectors = {
 local function maybe_defer(task, rule)
   if rule.defer then
     rspamd_logger.warnx(task, 'deferring message')
-    task:set_pre_result('soft reject', 'deferred')
+    task:set_pre_result('soft reject', 'deferred', N)
   end
 end
 

src/plugins/lua/multimap.lua

@@ -555,9 +555,9 @@ local function multimap_callback(task, rule)
             r['message'] = r.message_func(task, r['symbol'], opt)
           end
           if r['message'] then
-            task:set_pre_result(r['action'], r['message'])
+            task:set_pre_result(r['action'], r['message'], N)
           else
-            task:set_pre_result(r['action'], 'Matched map: ' .. r['symbol'])
+            task:set_pre_result(r['action'], 'Matched map: ' .. r['symbol'], N)
           end
         end
       end
@@ -731,7 +731,7 @@ local function multimap_callback(task, rule)
           elseif is_ok then
             task:insert_result(rule['symbol'], 1, rule['map'])
             if pre_filter then
-              task:set_pre_result(rule['action'], 'Matched map: ' .. rule['symbol'])
+              task:set_pre_result(rule['action'], 'Matched map: ' .. rule['symbol'], N)
             end
           end
 

src/plugins/lua/neural.lua

@@ -659,7 +659,7 @@ local function train_ann(rule, _, ev_base, elt, worker)
           dataset.size = function() return #dataset end
 
           local function train_torch()
-            if rule.train.learn_threads > 1 then
+            if rule.train.learn_threads then
               torch.setnumthreads(rule.train.learn_threads)
             end
             local criterion = nn.MSECriterion()
@@ -950,6 +950,8 @@ else
   else
     torch = require "torch"
     nn = require "nn"
+
+    torch.setnumthreads(1)
   end
 
   local id = rspamd_config:register_symbol({

src/plugins/lua/ratelimit.lua

@@ -556,7 +556,7 @@ local function ratelimit_cb(task)
               bucket.burst, bucket.rate,
               data[2], data[3], data[4])
           task:set_pre_result('soft reject',
-              message_func(task, lim_name, prefix, bucket))
+              message_func(task, lim_name, prefix, bucket), N)
         end
       end
     end

src/plugins/lua/replies.lua

@@ -65,7 +65,7 @@ local function replies_check(task)
             (settings.use_local and ip_addr and ip_addr:is_local()) then
           rspamd_logger.infox(task, "not forcing action for local network or authorized user");
         else
-          task:set_pre_result(settings['action'], settings['message'])
+          task:set_pre_result(settings['action'], settings['message'], N)
         end
       end
     end

src/plugins/lua/spamtrap.lua

@@ -66,7 +66,7 @@ local function spamtrap_cb(task)
       rspamd_logger.infox(task, 'spamtrap found: <%s>', rcpt)
       if settings.smtp_message then
         task:set_pre_result(settings['action'],
-          lua_util.template(settings.smtp_message, { rcpt = rcpt}))
+          lua_util.template(settings.smtp_message, { rcpt = rcpt}), 'spamtrap')
       else
         local smtp_message = 'unknown error'
         if settings.action == 'no action' then
@@ -74,7 +74,7 @@ local function spamtrap_cb(task)
         elseif settings.action == 'reject' then
           smtp_message = 'message rejected'
         end
-        task:set_pre_result(settings['action'], smtp_message)
+        task:set_pre_result(settings['action'], smtp_message, 'spamtrap')
       end
     end
   end

src/plugins/lua/whitelist.lua

@@ -207,7 +207,6 @@ local function whitelist_cb(symbol, rule, task)
 
         if not found_bl then
           final_mult = val[2]
-          lua_util.debugm(N, task, "hui4 final mult: %s", final_mult)
         end
       end
     end
@@ -219,7 +218,6 @@ local function whitelist_cb(symbol, rule, task)
         table.insert(opts, dom .. ':d:-')
         found_bl = true
         final_mult = val[2]
-        lua_util.debugm(N, task, "hui2 final mult: %s", final_mult)
       end
     end
   end
@@ -229,7 +227,6 @@ local function whitelist_cb(symbol, rule, task)
       if val[1] == 'both' or val[1] == 'bl' then
         found_bl = true
         final_mult = val[2]
-        lua_util.debugm(N, task, "hui3 final mult: %s", final_mult)
         table.insert(opts, string.format("%s:%s:-", dom, what))
       end
     else
@@ -238,7 +235,6 @@ local function whitelist_cb(symbol, rule, task)
         table.insert(opts, string.format("%s:%s:+", dom, what))
         if not found_bl then
           final_mult = val[2]
-          lua_util.debugm(N, task, "hui1 final mult: %s", final_mult)
         end
       end
     end

src/plugins/surbl.c

@@ -70,6 +70,7 @@ INIT_LOG_MODULE(surbl)
 #define SURBL_OPTION_NOIP (1 << 0)
 #define SURBL_OPTION_RESOLVEIP (1 << 1)
 #define SURBL_OPTION_CHECKIMAGES (1 << 2)
+#define SURBL_OPTION_CHECKDKIM (1 << 3)
 #define MAX_LEVELS 10
 
 struct surbl_ctx {
@@ -574,6 +575,15 @@ surbl_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
 			0,
 			NULL,
 			0);
+	rspamd_rcl_add_doc_by_path (cfg,
+			"surbl.rule",
+			"Check domains in valid DKIM signatures",
+			"check_dkim",
+			UCL_BOOLEAN,
+			NULL,
+			0,
+			NULL,
+			0);
 
 	return 0;
 }
@@ -730,6 +740,13 @@ surbl_module_parse_rule (const ucl_object_t* value, struct rspamd_config* cfg)
 			}
 		}
 
+		cur = ucl_object_lookup (cur_rule, "check_dkim");
+		if (cur != NULL && cur->type == UCL_BOOLEAN) {
+			if (ucl_object_toboolean (cur)) {
+				new_suffix->options |= SURBL_OPTION_CHECKDKIM;
+			}
+		}
+
 		if ((new_suffix->options & (SURBL_OPTION_RESOLVEIP | SURBL_OPTION_NOIP))
 				== (SURBL_OPTION_NOIP | SURBL_OPTION_RESOLVEIP)) {
 			/* Mutually exclusive options */
@@ -1057,12 +1074,20 @@ surbl_module_config (struct rspamd_config *cfg)
 			surbl_module_ctx->suffixes);
 	}
 
+
 	cur_opt = surbl_module_ctx->suffixes;
 	while (cur_opt) {
 		cur_suffix = cur_opt->data;
+
 		if (cur_suffix->bits != NULL || cur_suffix->ips != NULL) {
 			register_bit_symbols (cfg, cur_suffix, cur_suffix->callback_id);
 		}
+
+		if (cur_suffix->options & SURBL_OPTION_CHECKDKIM) {
+			rspamd_symbols_cache_add_dependency (cfg->cache,
+					cur_suffix->callback_id, "DKIM_TRACE");
+		}
+
 		cur_opt = g_list_next (cur_opt);
 	}
 
@@ -1881,15 +1906,35 @@ surbl_test_url (struct rspamd_task *task, void *user_data)
 					img = g_ptr_array_index (part->html->images, j);
 
 					if ((img->flags & RSPAMD_HTML_FLAG_IMAGE_EXTERNAL)
-							&& img->src) {
-						url = rspamd_html_process_url (task->task_pool,
-								img->src, strlen (img->src), NULL);
-
-						if (url) {
-							surbl_tree_url_callback (url, url, param);
-							msg_debug_surbl ("checked image url %s over %s",
-									img->src, suffix->suffix);
-						}
+							&& img->url) {
+						surbl_tree_url_callback (img->url, img->url, param);
+						msg_debug_surbl ("checked image url %s over %s",
+								img->src, suffix->suffix);
+					}
+				}
+			}
+		}
+	}
+
+	if (suffix->options & SURBL_OPTION_CHECKDKIM) {
+		struct rspamd_symbol_result *s;
+		struct rspamd_symbol_option *opt;
+
+		s = rspamd_task_find_symbol_result (task, "DKIM_TRACE");
+
+		if (s && s->opts_head) {
+			DL_FOREACH (s->opts_head, opt) {
+				gsize len = strlen (opt->option);
+				gchar *p = opt->option + len - 1;
+
+				if (*p == '+') {
+					url = rspamd_html_process_url (task->task_pool,
+							opt->option, len - 2, NULL);
+
+					if (url) {
+						surbl_tree_url_callback (url, url, param);
+						msg_debug_surbl ("checked dkim url %s over %s",
+								url->string, suffix->suffix);
 					}
 				}
 			}

src/rspamadm/lua_repl.c

@@ -431,7 +431,7 @@ rspamadm_lua_message_handler (lua_State *L, gint argc, gchar **argv)
 			rspamd_printf ("cannot open %s: %s\n", argv[i], strerror (errno));
 		}
 		else {
-			task = rspamd_task_new (NULL, NULL, NULL, NULL);
+			task = rspamd_task_new (NULL, rspamd_main->cfg, NULL, NULL);
 
 			if (!rspamd_task_load_message (task, NULL, map, len)) {
 				rspamd_printf ("cannot load %s\n", argv[i]);
@@ -803,7 +803,7 @@ rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd)
 			exit (EXIT_FAILURE);
 		}
 
-		ev_base = event_init ();
+		ev_base = rspamd_main->ev_base;
 		ctx = g_malloc0  (sizeof (*ctx));
 		http = rspamd_http_router_new (rspamadm_lua_error_handler,
 						rspamadm_lua_finish_handler,

src/rspamadm/rspamadm.c

@@ -368,7 +368,15 @@ main (gint argc, gchar **argv, gchar **env)
 	rspamd_main->type = process_quark;
 	rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
 			"rspamadm");
+
+#ifdef HAVE_EVENT_NO_CACHE_TIME_FLAG
+	struct event_config *ev_cfg;
+	ev_cfg = event_config_new ();
+	event_config_set_flag (ev_cfg, EVENT_BASE_FLAG_NO_CACHE_TIME);
+	rspamd_main->ev_base = event_base_new_with_config (ev_cfg);
+#else
 	rspamd_main->ev_base = event_init ();
+#endif
 
 	rspamadm_fill_internal_commands (all_commands);
 	help_command.command_data = all_commands;
@@ -412,6 +420,7 @@ main (gint argc, gchar **argv, gchar **env)
 	/* Setup logger */
 	if (verbose) {
 		cfg->log_level = G_LOG_LEVEL_DEBUG;
+		cfg->log_flags |= RSPAMD_LOG_FLAG_USEC;
 	}
 	else {
 		cfg->log_level = G_LOG_LEVEL_MESSAGE;
@@ -419,7 +428,7 @@ main (gint argc, gchar **argv, gchar **env)
 
 	cfg->log_type = RSPAMD_LOG_CONSOLE;
 	/* Avoid timestamps printing */
-	cfg->log_flags = RSPAMD_LOG_FLAG_RSPAMADM;
+	cfg->log_flags |= RSPAMD_LOG_FLAG_RSPAMADM;
 	rspamd_set_logger (cfg, process_quark, &rspamd_main->logger,
 			rspamd_main->server_pool);
 	(void) rspamd_log_open (rspamd_main->logger);
@@ -536,6 +545,9 @@ main (gint argc, gchar **argv, gchar **env)
 	}
 
 	event_base_loopexit (rspamd_main->ev_base, NULL);
+#ifdef HAVE_EVENT_NO_CACHE_TIME_FLAG
+	event_config_free (ev_cfg);
+#endif
 
 	REF_RELEASE (rspamd_main->cfg);
 	rspamd_log_close (rspamd_main->logger, TRUE);

test/coverage.md

@@ -0,0 +1,58 @@
+Coverage collection explained
+=============================
+
+Hi mate. In short, you don't wanna know this. Believe me, you don't. Please, close this file and forget about it.
+
+Surely? You still here?
+
+Please, stop it until it's too late.
+
+You were warned.
+
+
+Preamble
+--------
+RSPAMD is written mainly in two languages: C and Lua. Coverage for each of them is being collected using different 
+tools and approaches and is sent into [coveralls.io](https://coveralls.io).
+Each approach is not quite compatible to other tools. This document describes how we crutch them to work together.
+
+
+C coverage
+----------
+In general, pretty boring. When you run `cmake` with "-DENABLE_COVERAGE=ON" flag, it adds "--coverage" flag to both
+CFLAGS and LDFLAGS. So that each run of generated binary will create `*.gcda` file containing coverage data.
+
+However, there are some moment to highlight:
+
+- RSPAMD is run under "nobody" user. Hence, directories and files should be writable for this user. 
+- To make it possible, we explicitly run `umask 0000` in "build" and "functional" stages in .circleci/config.yml
+- After run, we persist coverage data in "coverage.${CIRCLE\_JOB}.dump" during this build flow, see `capture_coverage_data`,
+  to use it on the final stage.
+- we user `cpp-coverals` because it is able to save data for coveralls without actually sending it. We send on our own
+  along with Lua-coverage.
+
+Lua coverage
+------------
+Lua coverage is collected for unit-tests and functional test suite. 
+First part contains nothing interesting, just see `test/lua/tests.lua`.
+
+"Functional" part is completely unobvious.
+
+1. Coverage collecting is initiated and dumped in `test/functional/lua/test_coverage.lua` (there are a lot of comments inside).
+   This file should be included on the very early stage of test run. Usually it's included via config.
+2. Coverage is dumped into ${TMPDIR}/%{woker_name}.luacov.stats.out
+3. All worker coverage reports are merged into `lua_coverage_report.json` (see `collect_lua_coverage()`)
+4. finally, `lua_coverage_report.json` is persisted in build flow (see `functional` stage)
+
+Altogether
+----------
+
+Finally, we get all the reports:
+
+- `coverage.functional.dump` 
+- `coverage.rspamd-test.dump`
+- `lua_coverage_report.json`
+- `unit_test_lua.json`
+
+and merge them and send the resulting report using `test/functional/util/merge_coveralls.py`. Also, this scripts maps installed
+paths into corresponding repository paths and removes unneeded files (i.e. test sources).

test/functional/cases/115_dmarc.robot

@@ -87,11 +87,11 @@ DKIM PERMFAIL BAD RECORD
   ...  -i  37.48.67.26
   Check Rspamc  ${result}  R_DKIM_PERMFAIL
 
-DKIM TEMPFAIL SERVFAIL
+DKIM TEMPFAIL SERVFAIL UNALIGNED
   ${result} =  Scan Message With Rspamc  ${TESTDIR}/messages/dmarc/bad_dkim3.eml
   ...  -i  37.48.67.26
   Check Rspamc  ${result}  R_DKIM_TEMPFAIL
-  Should Contain  ${result.stdout}  DMARC_DNSFAIL
+  Should Contain  ${result.stdout}  DMARC_POLICY_SOFTFAIL
 
 DKIM NA NOSIG
   ${result} =  Scan Message With Rspamc  ${TESTDIR}/messages/utf.eml
@@ -103,11 +103,11 @@ SPF PERMFAIL UNRESOLVEABLE INCLUDE
   ...  -i  37.48.67.26  -F  x@fail3.org.org.za
   Check Rspamc  ${result}  R_SPF_PERMFAIL
 
-SPF DNSFAIL FAILED INCLUDE
+SPF DNSFAIL FAILED INCLUDE UNALIGNED
   ${result} =  Scan Message With Rspamc  ${TESTDIR}/messages/dmarc/bad_dkim1.eml
   ...  -i  8.8.8.8  -F  x@fail2.org.org.za
   Check Rspamc  ${result}  R_SPF_DNSFAIL
-  Should Contain  ${result.stdout}  DMARC_DNSFAIL
+  Should Contain  ${result.stdout}  DMARC_POLICY_SOFTFAIL
 
 SPF ALLOW UNRESOLVEABLE INCLUDE
   ${result} =  Scan Message With Rspamc  ${TESTDIR}/messages/dmarc/bad_dkim1.eml

test/functional/configs/clickhouse.conf

@@ -56,6 +56,7 @@ worker {
         secure_ip = ["127.0.0.1", "::1"];
         stats_path = "${TMPDIR}/stats.ucl"
 }
+lua = "${TESTDIR}/lua/test_coverage.lua";
 modules {
     path = "${TESTDIR}/../../src/plugins/lua/"
 }

test/functional/configs/dkim.conf

@@ -78,3 +78,4 @@ EOD;
   trusted_only = false;
   skip_multi = false;
 }
+lua = "${TESTDIR}/lua/test_coverage.lua";

test/functional/configs/fuzzy.conf

@@ -1,6 +1,7 @@
 redis {
   servers = "${REDIS_ADDR}:${REDIS_PORT}";
 }
+lua = "${TESTDIR}/lua/test_coverage.lua";
 options = {
 	filters = "fuzzy_check";
 	pidfile = "${TMPDIR}/rspamd.pid";

test/functional/configs/lua_test.conf

@@ -47,5 +47,5 @@ worker {
 	secure_ip = ["127.0.0.1", "::1"];
 	stats_path = "${TMPDIR}/stats.ucl"
 }
-
+lua = "${TESTDIR}/lua/test_coverage.lua";
 lua = ${LUA_SCRIPT};

test/functional/configs/milter.conf

@@ -50,6 +50,7 @@ worker {
 modules {
     path = "${TESTDIR}/../../src/plugins/lua/"
 }
+lua = "${TESTDIR}/lua/test_coverage.lua";
 lua = "${INSTALLROOT}/share/rspamd/rules/rspamd.lua"
 lua = "${TESTDIR}/lua/params.lua"
 milter_headers {

test/functional/configs/password.conf

@@ -42,3 +42,4 @@ worker {
 	enable_password = ${ENABLE_PASSWORD};
 	stats_path = "${TMPDIR}/stats.ucl";
 }
+lua = "${TESTDIR}/lua/test_coverage.lua";

test/functional/configs/plugins.conf

@@ -507,5 +507,6 @@ worker {
 modules {
     path = "${TESTDIR}/../../src/plugins/lua/"
 }
+lua = "${TESTDIR}/lua/test_coverage.lua";
 lua = "${INSTALLROOT}/share/rspamd/rules/rspamd.lua"
 ${PLUGIN_CONFIG}

test/functional/configs/proxy.conf

@@ -20,3 +20,4 @@ worker "rspamd_proxy" {
     }
     count = 1;
 }
+lua = "${TESTDIR}/lua/test_coverage.lua";

test/functional/configs/spamassassin.conf

@@ -1,4 +1,5 @@
 spamassassin {
 	rules = "${TESTDIR}/configs/spamassassin.rules"
 }
+lua = "${TESTDIR}/lua/test_coverage.lua";
 lua = "${TESTDIR}/lua/simple.lua"

test/functional/configs/stats.conf

@@ -72,5 +72,6 @@ classifier {
 		${REDIS_SERVER}
 	}
 }
+lua = "${TESTDIR}/lua/test_coverage.lua";
 
 settings {}

test/functional/lib/rspamd.py

@@ -3,6 +3,7 @@ import grp
 import os
 import os.path
 import psutil
+import glob
 import pwd
 import re
 import shutil
@@ -12,6 +13,7 @@ import errno
 import sys
 import tempfile
 import time
+import subprocess
 from robot.libraries.BuiltIn import BuiltIn
 from robot.api import logger
 
@@ -227,3 +229,107 @@ def get_file_if_exists(file_path):
             return myfile.read()
     return None
 
+# copy-paste from 
+# https://hg.python.org/cpython/file/6860263c05b3/Lib/shutil.py#l1068
+# As soon as we move to Python 3, this should be removed in favor of shutil.which()
+def python3_which(cmd, mode=os.F_OK | os.X_OK, path=None):
+    """Given a command, mode, and a PATH string, return the path which
+    conforms to the given mode on the PATH, or None if there is no such
+    file.
+
+    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+    of os.environ.get("PATH"), or can be overridden with a custom search
+    path.
+    """
+
+    # Check that a given file can be accessed with the correct mode.
+    # Additionally check that `file` is not a directory, as on Windows
+    # directories pass the os.access check.
+    def _access_check(fn, mode):
+        return (os.path.exists(fn) and os.access(fn, mode)
+                and not os.path.isdir(fn))
+
+    # If we're given a path with a directory part, look it up directly rather
+    # than referring to PATH directories. This includes checking relative to the
+    # current directory, e.g. ./script
+    if os.path.dirname(cmd):
+        if _access_check(cmd, mode):
+            return cmd
+        return None
+
+    if path is None:
+        path = os.environ.get("PATH", os.defpath)
+    if not path:
+        return None
+    path = path.split(os.pathsep)
+
+    if sys.platform == "win32":
+        # The current directory takes precedence on Windows.
+        if not os.curdir in path:
+            path.insert(0, os.curdir)
+
+        # PATHEXT is necessary to check on Windows.
+        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+        # See if the given file matches any of the expected path extensions.
+        # This will allow us to short circuit when given "python.exe".
+        # If it does match, only test that one, otherwise we have to try
+        # others.
+        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+            files = [cmd]
+        else:
+            files = [cmd + ext for ext in pathext]
+    else:
+        # On other platforms you don't have things like PATHEXT to tell you
+        # what file suffixes are executable, so just pass on cmd as-is.
+        files = [cmd]
+
+    seen = set()
+    for dir in path:
+        normdir = os.path.normcase(dir)
+        if not normdir in seen:
+            seen.add(normdir)
+            for thefile in files:
+                name = os.path.join(dir, thefile)
+                if _access_check(name, mode):
+                    return name
+    return None
+
+
+def collect_lua_coverage():
+    if python3_which("luacov-coveralls") is None:
+        logger.info("luacov-coveralls not found, will not collect Lua coverage")
+        return
+
+    # decided not to do optional coverage so far
+    #if not 'ENABLE_LUA_COVERAGE' in os.environ['HOME']:
+    #    logger.info("ENABLE_LUA_COVERAGE is not present in env, will not collect Lua coverage")
+    #    return
+
+    current_directory = os.getcwd()
+    report_file = current_directory + "/lua_coverage_report.json"
+    old_report = current_directory + "/lua_coverage_report.json.old"
+
+    tmp_dir = BuiltIn().get_variable_value("${TMPDIR}")
+    coverage_files = glob.glob('%s/*.luacov.stats.out' % (tmp_dir))
+
+    for stat_file in coverage_files:
+        shutil.move(stat_file, "luacov.stats.out")
+        # logger.console("statfile: " + stat_file)
+
+        if (os.path.isfile(report_file)):
+            shutil.move(report_file, old_report)
+            p = subprocess.Popen(["luacov-coveralls", "-o", report_file, "-j", old_report, "--merge", "--dryrun"], 
+                                 stdout = subprocess.PIPE, stderr= subprocess.PIPE)
+            output,error = p.communicate()
+
+            logger.info("luacov-coveralls stdout: " + output)
+            logger.info("luacov-coveralls stderr: " + error)
+            os.remove(old_report)
+        else:
+            p = subprocess.Popen(["luacov-coveralls", "-o", report_file, "--dryrun"], stdout = subprocess.PIPE, stderr= subprocess.PIPE)
+            output,error = p.communicate()
+
+            logger.info("luacov-coveralls stdout: " + output)
+            logger.info("luacov-coveralls stderr: " + error)
+        os.remove("luacov.stats.out")
+

test/functional/lib/rspamd.robot

@@ -72,6 +72,7 @@ Generic Teardown
   Shutdown Process With Children  ${RSPAMD_PID}
   Log does not contain segfault record
   Save Run Results  ${TMPDIR}  rspamd.log redis.log rspamd.conf clickhouse-server.log clickhouse-server.err.log clickhouse-config.xml
+  Collect Lua Coverage
   Cleanup Temporary Directory  ${TMPDIR}
 
 Log does not contain segfault record

test/functional/lua/test_coverage.lua

@@ -0,0 +1,46 @@
+--[[
+-- This should be the very first file executed during a test
+-- otherwise coverage will be partly missed
+--]]
+local logger = require "rspamd_logger"
+local mempool = require "rspamd_mempool"
+local loaded, luacov = pcall(require, 'luacov.runner')
+if not loaded then
+  logger.errx('luacov is not loaded, will not collect coverage')
+  return
+end
+
+luacov.init()
+
+local pool = mempool.create()
+-- we don't need the pool, we need userdata to put __gc() on it
+-- __gc() is not called for tables, that't why there is such trick
+-- so, we are free to clean memory, let's do this :)
+pool:destroy()
+
+local woker_name
+
+rspamd_config:add_on_load(function(cfg, ev_base, worker)
+  woker_name = worker:get_name()
+  local stats_path = rspamd_paths["DBDIR"] .. '/' .. woker_name .. '.luacov.stats.out'
+  local config = luacov.load_config()
+  config.statsfile = stats_path
+end)
+
+-- use global variable to prevent the object from being GC'ed too early
+__GLOBAL_COVERAGE_WATCHDOG = {pool = pool}
+
+local mt = {
+  __gc = function()
+  --[[
+  -- We could've used finish_script but in that case some coverage would be missed:
+  -- pool destructors are executed after finish_scripts (when Lua state is terminated and that's
+  -- how we can collect coverage of cove executed there
+  --]]
+    if woker_name then
+      luacov.shutdown()
+    end
+  end
+}
+
+debug.setmetatable(__GLOBAL_COVERAGE_WATCHDOG.pool, mt)

test/functional/util/merge_coveralls.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+import argparse
+import json
+import os
+import sys
+import codecs
+
+import requests
+
+# Python 2/3 compatibility
+if sys.version_info.major > 2:
+    xrange = range
+
+# install path to repository mapping
+# if path mapped to None, it means that the file should be ignored (i.e. test file/helper)
+# first matched path counts.
+# terminating slash should be added for directories
+path_mapping = [
+    ("${install-dir}/share/rspamd/lib/fun.lua", None),
+    ("${install-dir}/share/rspamd/lib/", "lualib/"),
+    ("${install-dir}/share/rspamd/rules/" , "rules/"),
+    ("${install-dir}/share/rspamd/lib/torch/" , None),
+    ("${build-dir}/CMakeFiles/", None),
+    ("${build-dir}/contrib/", None),
+    ("${build-dir}/test", None),
+    ("${project-root}/test/lua/", None),
+    ("${project-root}/test/", None),
+    ("${project-root}/clang-plugin/", None),
+    ("${project-root}/CMakeFiles/", None),
+    ("${project-root}/contrib/", None),
+    ("${project-root}/", ""),
+    ("contrib/", None),
+    ("CMakeFiles/", None),
+]
+
+parser = argparse.ArgumentParser(description='')
+parser.add_argument('--input', type=str, required=True, nargs='+', help='input files')
+parser.add_argument('--output', type=str, required=True, help='output file)')
+parser.add_argument('--root', type=str, required=False, default="/rspamd/src/github.com/rspamd/rspamd", help='repository root)')
+parser.add_argument('--install-dir', type=str, required=False, default="/rspamd/install", help='install root)')
+parser.add_argument('--build-dir', type=str, required=False, default="/rspamd/build", help='build root)')
+parser.add_argument('--token', type=str, help='If present, the file will be uploaded to coveralls)')
+
+def merge_coverage_vectors(c1, c2):
+    assert(len(c1) == len(c2))
+
+    for i in range(0, len(c1)):
+        if c1[i] is None and c2[i] is None:
+            pass
+        elif type(c1[i]) is int and c2[i] is None:
+            pass
+        elif c1[i] is None and type(c2[i]) is int:
+            c1[i] = c2[i]
+        elif type(c1[i]) is int and type(c2[i]) is int:
+            c1[i] += c2[i]
+        else:
+            raise RuntimeError("bad element types at %d: %s, %s", i, type(c1[i]), type(c1[i]))
+
+    return c1
+
+
+def normalize_name(name):
+    orig_name = name
+    name = os.path.normpath(name)
+    if not os.path.isabs(name):
+        name = os.path.abspath(repository_root + "/" + name)
+    for k in path_mapping:
+        if name.startswith(k[0]):
+            if k[1] is None:
+                return None
+            else:
+                name = k[1] + name[len(k[0]):]
+                break
+    return name
+
+def merge(files, j1):
+    for sf in j1['source_files']:
+        name = normalize_name(sf['name'])
+        if name is None:
+            continue
+        if name in files:
+            files[name]['coverage'] = merge_coverage_vectors(files[name]['coverage'], sf['coverage'])
+        else:
+            sf['name'] = name
+            files[name] = sf
+            if not ('source' in sf):
+                path = "%s/%s" % (repository_root, sf['name'])
+                if os.path.isfile(path):
+                    with open(path) as f:
+                        files[name]['source'] = f.read()
+
+    return files
+
+def prepare_path_mapping():
+    for i in range(0, len(path_mapping)):
+        new_key = path_mapping[i][0].replace("${install-dir}", install_dir)
+        new_key = new_key.replace("${project-root}", repository_root)
+        new_key = new_key.replace("${build-dir}", build_dir)
+
+        path_mapping[i] = (new_key, path_mapping[i][1])
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+
+    repository_root = os.path.abspath(os.path.expanduser(args.root))
+    install_dir = os.path.normpath(os.path.expanduser(args.install_dir))
+    build_dir = os.path.normpath(os.path.expanduser(args.build_dir))
+
+    prepare_path_mapping()
+
+    with codecs.open(args.input[0], 'r', encoding='utf-8') as fh:
+        j1 = json.load(fh)
+
+    files = merge({}, j1)
+    for i in range(1, len(args.input)):
+        with codecs.open(args.input[i], 'r', encoding='utf-8') as fh:
+            j2 = json.load(fh)
+
+        files = merge(files, j2)
+
+        if 'git' not in j1 and 'git' in j2:
+            j1['git'] = j2['git']
+        if 'service_name' not in j1 and 'service_name' in j2:
+            j1['service_name'] = j2['service_name']
+        if 'service_job_id' not in j1 and 'service_job_id' in j2:
+            j1['service_job_id'] = j2['service_job_id']
+
+        if not j1['service_job_id'] and 'CIRCLE_BUILD_NUM' in os.environ:
+            j1['service_job_id'] = os.environ['CIRCLE_BUILD_NUM']
+        elif not j1['service_job_id'] and 'DRONE_PREV_BUILD_NUMBER' in os.environ:
+            j1['service_job_id'] = os.environ['DRONE_PREV_BUILD_NUMBER']
+
+        if 'CIRCLECI' in os.environ and os.environ['CIRCLECI']:
+            j1['service_name'] = 'circleci'
+        elif 'DRONE' in os.environ and os.environ['DRONE']:
+            j1['service_name'] = 'drone'
+
+    j1['source_files'] = list(files.values())
+
+    with open(args.output, 'w') as f:
+        f.write(json.dumps(j1))
+
+    if args.token:
+        j1['repo_token'] = args.token
+        print("sending data to coveralls...")
+        r = requests.post('https://coveralls.io/api/v1/jobs', files={"json_file": json.dumps(j1)})
+        response = json.loads(r.text)
+        print("uploaded %s\nmessage:%s" % (response['url'], response['message']))
+
+    # post https://coveralls.io/api/v1/jobs
+    # print args
+
+

test/lua/unit/lua_util.extract_specific_urls.lua

@@ -1,3 +1,6 @@
+
+local msg
+
 context("Lua util - extract_specific_urls", function()
   local util  = require 'lua_util'
   local mpool = require "rspamd_mempool"
@@ -43,6 +46,7 @@ context("Lua util - extract_specific_urls", function()
     {expect = url_list, filter = nil, limit = 9999, need_emails = true, prefix = 'p'},
     {expect = {}, filter = (function() return false end), limit = 9999, need_emails = true, prefix = 'p'},
     {expect = {"domain4.co.net", "test.com"}, filter = nil, limit = 2, need_emails = true, prefix = 'p'},
+    {expect = {"domain4.co.net", "test.com", "domain3.org"}, filter = nil, limit = 3, need_emails = true, prefix = 'p'},
     {
       expect = {"gov.co.net", "tesco.co.net", "domain1.co.net", "domain2.co.net", "domain3.co.net", "domain4.co.net"},
       filter = (function(s) return s:get_host():sub(-4) == ".net" end),
@@ -79,17 +83,16 @@ context("Lua util - extract_specific_urls", function()
 
   local pool = mpool.create()
 
-  for i,c in ipairs(cases) do
-
-    local function prepare_url_list(c)
-      return fun.totable(fun.map(
+  local function prepare_url_list(list)
+    return fun.totable(fun.map(
     function (u) return url.create(pool, u) end,
-    c.input or url_list
+    list or url_list
     ))
   end
 
+  for i,c in ipairs(cases) do
     test("extract_specific_urls, backward compatibility case #" .. i, function()
-      task_object.urls = prepare_url_list(c)
+      task_object.urls = prepare_url_list(c.input)
       if (c.esld_limit) then
         -- not awailable in deprecated version
         return
@@ -106,7 +109,7 @@ context("Lua util - extract_specific_urls", function()
     end)
 
     test("extract_specific_urls " .. i, function()
-      task_object.urls = prepare_url_list(c)
+      task_object.urls = prepare_url_list(c.input)
 
       local actual = util.extract_specific_urls({
         task = task_object,
@@ -127,6 +130,20 @@ context("Lua util - extract_specific_urls", function()
     end)
   end
 
+  test("extract_specific_urls, another case", function()
+    task_object.urls = prepare_url_list {"abc.net", "abc.com", "abc.net", "abc.za.org"}
+    local actual = util.extract_specific_urls(task_object, 3, true)
+
+    local actual_result = prepare_actual_result(actual)
+    table.sort(actual_result)
+    --[[
+      local s = logger.slog("%1 =?= %2", c.expect, actual_result)
+      print(s) --]]
+
+    assert_rspamd_table_eq({actual = actual_result, expect = {"abc.com", "abc.net", "abc.za.org"}})
+  end)
+
+
 --[[ ******************* kinda functional *************************************** ]]
   local test_dir = string.gsub(debug.getinfo(1).source, "^@(.+/)[^/]+$", "%1")
   local tld_file = string.format('%s/%s', test_dir, "test_tld.dat")
@@ -156,7 +173,36 @@ context("Lua util - extract_specific_urls", function()
     local cfg = rspamd_util.config_from_ucl(config, "INIT_URL,INIT_LIBS,INIT_SYMCACHE,INIT_VALIDATE,INIT_PRELOAD_MAPS")
     assert_not_nil(cfg)
 
-    local msg = [[
+    local expect = {"example.net", "domain.com"}
+    local res,task = rspamd_task.load_from_string(msg, rspamd_config)
+
+    if not res then
+      assert_true(false, "failed to load message")
+    end
+
+    if not task:process_message() then
+      assert_true(false, "failed to process message")
+    end
+
+    local actual = util.extract_specific_urls({
+      task = task,
+      limit = 2,
+      esld_limit = 2,
+    })
+
+    local actual_result = prepare_actual_result(actual)
+
+    --[[
+      local s = logger.slog("case[%1] %2 =?= %3", i, expect, actual_result)
+      print(s) --]]
+
+    assert_equal("domain.com", actual_result[1], "checking that first url is the one with highest suspiciousness level")
+
+  end)
+end)
+
+--[=========[ *******************  message  ******************* ]=========]
+msg = [[
 From: <>
 To: <nobody@example.com>
 Subject: test
@@ -187,30 +233,3 @@ Content-Type: text/html; charset="utf-8"
 <a href="http://domain.com">http://example.net/</a>
 </html>
 ]]
-    local expect = {"example.net", "domain.com"}
-    local res,task = rspamd_task.load_from_string(msg, rspamd_config)
-
-    if not res then
-      assert_true(false, "failed to load message")
-    end
-
-    if not task:process_message() then
-      assert_true(false, "failed to process message")
-    end
-
-    local actual = util.extract_specific_urls({
-      task = task,
-      limit = 2,
-      esld_limit = 2,
-    })
-
-    local actual_result = prepare_actual_result(actual)
-
-    --[[
-      local s = logger.slog("case[%1] %2 =?= %3", i, expect, actual_result)
-      print(s) --]]
-
-    assert_equal("domain.com", actual_result[1], "checking that first url is the one with highest suspiciousness level")
-
-  end)
-end)