Merge remote-tracking branch 'upstream/master' into documentation

* upstream/master: (90 commits)
  [Fix] Plug memory leak in proxy
  [Feature] Do not print garbadge in --compact output
  [Fix] Fix encrypted proxy requests
  [Fix] Do not delete uninitialized events
  [Feature] Add protection against open files limit and accepting sockets
  [Fix] Another fix for redis timeouts
  [Fix] Fix order of initialization
  [Feature] Use file lock in logger to avoid deadlocks
  [Fix] Fix errors handling in the proxy
  [Fix] More fixes for redis refcounts
  [Fix] Initialize parser scripts properly
  [Fix] Try to fix issue in redis stats backend when task is closed
  [Fix] Fix usage of rdns reply structure
  [Fix] Fix symbol name for spf soft fail
  [Fix] Fix setting path for lua
  [Doc] Update regexp module documentation
  [Minor] Fix names
  [Fix] Add missing types
  [Feature] Implement braced regexp quantifiers
  [Fix] Implement new automata to skip empty lines for dkim signing
  ...

9 files changed, 416 insertions, 95 deletions

diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c
index f774ac126..ebbc29d61 100644
--- a/src/libserver/cfg_rcl.c
+++ b/src/libserver/cfg_rcl.c
@@ -2668,7 +2668,6 @@ rspamd_rcl_parse_struct_keypair (rspamd_mempool_t *pool,
 	struct rspamd_rcl_struct_parser *pd = ud;
 	struct rspamd_cryptobox_keypair **target, *kp;
 
-
 	target = (struct rspamd_cryptobox_keypair **)(((gchar *)pd->user_struct) +
 			pd->offset);
 	if (obj->type == UCL_OBJECT) {
@@ -2698,6 +2697,49 @@ rspamd_rcl_parse_struct_keypair (rspamd_mempool_t *pool,
 	return TRUE;
 }
 
+gboolean
+rspamd_rcl_parse_struct_pubkey (rspamd_mempool_t *pool,
+	const ucl_object_t *obj,
+	gpointer ud,
+	struct rspamd_rcl_section *section,
+	GError **err)
+{
+	struct rspamd_rcl_struct_parser *pd = ud;
+	struct rspamd_cryptobox_pubkey **target, *pk;
+	gsize len;
+	const gchar *str;
+
+	target = (struct rspamd_cryptobox_pubkey **)(((gchar *)pd->user_struct) +
+			pd->offset);
+	if (obj->type == UCL_STRING) {
+		str = ucl_object_tolstring (obj, &len);
+		pk = rspamd_pubkey_from_base32 (str, len, RSPAMD_KEYPAIR_KEX,
+				RSPAMD_CRYPTOBOX_MODE_25519);
+
+		if (pk != NULL) {
+			*target = pk;
+		}
+		else {
+			g_set_error (err,
+					CFG_RCL_ERROR,
+					EINVAL,
+					"cannot load the pubkey specified: %s",
+					ucl_object_key (obj));
+			return FALSE;
+		}
+	}
+	else {
+		g_set_error (err,
+				CFG_RCL_ERROR,
+				EINVAL,
+				"no sane pubkey found in the element: %s",
+				ucl_object_key (obj));
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
 static void
 rspamd_rcl_insert_string_list_item (gpointer *target, rspamd_mempool_t *pool,
 		const gchar *src, gboolean is_hash)
diff --git a/src/libserver/cfg_rcl.h b/src/libserver/cfg_rcl.h
index ee0a1b526..1a27b056f 100644
--- a/src/libserver/cfg_rcl.h
+++ b/src/libserver/cfg_rcl.h
@@ -278,6 +278,21 @@ gboolean rspamd_rcl_parse_struct_keypair (rspamd_mempool_t *pool,
 	GError **err);
 
 /**
+ * Parse a pubkey field of a structure
+ * @param cfg config pointer
+ * @param obj object to parse
+ * @param ud struct_parser structure (flags mean the exact structure used)
+ * @param section the current section
+ * @param err error pointer
+ * @return TRUE if a value has been successfully parsed
+ */
+gboolean rspamd_rcl_parse_struct_pubkey (rspamd_mempool_t *pool,
+	const ucl_object_t *obj,
+	gpointer ud,
+	struct rspamd_rcl_section *section,
+	GError **err);
+
+/**
  * Parse a inet addr field of a structure
  * @param cfg config pointer
  * @param obj object to parse
diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c
index 5052960f0..1f65733ba 100644
--- a/src/libserver/dkim.c
+++ b/src/libserver/dkim.c
@@ -1260,6 +1260,147 @@ rspamd_dkim_simple_body_step (rspamd_dkim_context_t *ctx,
 	return (len != 0);
 }
 
+static const gchar *
+rspamd_dkim_skip_empty_lines (const gchar *start, const gchar *end,
+		guint type, gboolean *need_crlf)
+{
+	const gchar *p = end - 1, *t;
+	enum {
+		init = 0,
+		init_2,
+		got_cr,
+		got_lf,
+		got_crlf,
+		test_spaces,
+	} state = init;
+	guint skip = 0;
+
+	while (p >= start + 2) {
+		switch (state) {
+		case init:
+			if (*p == '\r') {
+				state = got_cr;
+			}
+			else if (*p == '\n') {
+				state = got_lf;
+			}
+			else if (type == DKIM_CANON_RELAXED && *p == ' ') {
+				skip = 0;
+				state = test_spaces;
+			}
+			else {
+				if (type == DKIM_CANON_SIMPLE) {
+					*need_crlf = TRUE;
+				}
+
+				goto end;
+			}
+			break;
+		case init_2:
+			if (*p == '\r') {
+				state = got_cr;
+			}
+			else if (*p == '\n') {
+				state = got_lf;
+			}
+			else if (type == DKIM_CANON_RELAXED && *p == ' ') {
+				skip = 0;
+				state = test_spaces;
+			}
+			else {
+				goto end;
+			}
+			break;
+		case got_cr:
+			if (*(p - 1) == '\r') {
+				p --;
+				state = got_cr;
+			}
+			else if (*(p - 1) == '\n') {
+				if ((*p - 2) == '\r') {
+					/* \r\n\r -> we know about one line */
+					p -= 1;
+					state = got_crlf;
+				}
+				else {
+					/* \n\r -> we know about one line */
+					p -= 1;
+					state = got_lf;
+				}
+			}
+			else if (type == DKIM_CANON_RELAXED && *(p - 1) == ' ') {
+				skip = 1;
+				state = test_spaces;
+			}
+			else {
+				goto end;
+			}
+			break;
+		case got_lf:
+			if (*(p - 1) == '\r') {
+				state = got_crlf;
+			}
+			else if (*(p - 1) == '\n') {
+				/* We know about one line */
+				p --;
+				state = got_lf;
+			}
+			else if (type == DKIM_CANON_RELAXED && *(p - 1) == ' ') {
+				skip = 1;
+				state = test_spaces;
+			}
+			else {
+				goto end;
+			}
+			break;
+		case got_crlf:
+			if (p > start - 2) {
+				if (*(p - 3) == '\r') {
+					p -= 2;
+					state = got_cr;
+				}
+				else if (*(p - 3) == '\n') {
+					p -= 2;
+					state = got_lf;
+				}
+				else if (type == DKIM_CANON_RELAXED && *(p - 3) == ' ') {
+					skip = 2;
+					state = test_spaces;
+				}
+				else {
+					goto end;
+				}
+			}
+			else {
+				goto end;
+			}
+			break;
+		case test_spaces:
+			t = p - skip;
+
+			while (t > start - 2 && *t == ' ') {
+				t --;
+			}
+
+			if (*t == '\r') {
+				p = t;
+				state = got_cr;
+			}
+			else if (*t == '\n') {
+				p = t;
+				state = got_lf;
+			}
+			else {
+				goto end;
+			}
+			break;
+		}
+	}
+
+end:
+	return p;
+}
+
 static gboolean
 rspamd_dkim_canonize_body (rspamd_dkim_context_t *ctx,
 	const gchar *start,
@@ -1267,6 +1408,7 @@ rspamd_dkim_canonize_body (rspamd_dkim_context_t *ctx,
 {
 	const gchar *p;
 	guint remain = ctx->len ? ctx->len : (guint)(end - start);
+	gboolean need_crlf = FALSE;
 
 	if (start == NULL) {
 		/* Empty body */
@@ -1279,22 +1421,9 @@ rspamd_dkim_canonize_body (rspamd_dkim_context_t *ctx,
 	}
 	else {
 		/* Strip extra ending CRLF */
-		p = end - 1;
-		while (p >= start + 2) {
-			if (*p == '\n' && *(p - 1) == '\r' && *(p - 2) == '\n') {
-				p -= 2;
-			}
-			else if (*p == '\n' && *(p - 1) == '\n') {
-				p--;
-			}
-			else if (*p == '\r' && *(p - 1) == '\r') {
-				p--;
-			}
-			else {
-				break;
-			}
-		}
+		p = rspamd_dkim_skip_empty_lines (start, end, ctx->body_canon_type, &need_crlf);
 		end = p + 1;
+
 		if (end == start) {
 			/* Empty body */
 			if (ctx->body_canon_type == DKIM_CANON_SIMPLE) {
@@ -1308,7 +1437,15 @@ rspamd_dkim_canonize_body (rspamd_dkim_context_t *ctx,
 			if (ctx->body_canon_type == DKIM_CANON_SIMPLE) {
 				/* Simple canonization */
 				while (rspamd_dkim_simple_body_step (ctx, ctx->body_hash,
-						&start, end - start, &remain)) ;
+						&start, end - start, &remain));
+
+				if (need_crlf) {
+					start = "\r\n";
+					end = start + 2;
+					remain = 2;
+					rspamd_dkim_simple_body_step (ctx, ctx->body_hash,
+							&start, end - start, &remain);
+				}
 			}
 			else {
 				while (rspamd_dkim_relaxed_body_step (ctx, ctx->body_hash,
diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c
index 26e595e5f..e58e8a546 100644
--- a/src/libserver/fuzzy_backend.c
+++ b/src/libserver/fuzzy_backend.c
@@ -32,6 +32,7 @@ struct rspamd_fuzzy_backend {
 
 static const gdouble sql_sleep_time = 0.1;
 static const guint max_retries = 10;
+static const guint32 flags_mask = (1U << 31);
 
 #define msg_err_fuzzy_backend(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
         backend->pool->tag.tagname, backend->pool->tag.uid, \
@@ -52,17 +53,21 @@ static const guint max_retries = 10;
 
 static const char *create_tables_sql =
 		"BEGIN;"
-		"CREATE TABLE digests("
-		"id INTEGER PRIMARY KEY,"
-		"flag INTEGER NOT NULL,"
-		"digest TEXT NOT NULL,"
-		"value INTEGER,"
-		"time INTEGER);"
-		"CREATE TABLE shingles("
-		"value INTEGER NOT NULL,"
-		"number INTEGER NOT NULL,"
-		"digest_id INTEGER REFERENCES digests(id) ON DELETE CASCADE "
-		"ON UPDATE CASCADE);"
+		"CREATE TABLE IF NOT EXISTS digests("
+		"	id INTEGER PRIMARY KEY,"
+		"	flag INTEGER NOT NULL,"
+		"	digest TEXT NOT NULL,"
+		"	value INTEGER,"
+		"	time INTEGER);"
+		"CREATE TABLE IF NOT EXISTS shingles("
+		"	value INTEGER NOT NULL,"
+		"	number INTEGER NOT NULL,"
+		"	digest_id INTEGER REFERENCES digests(id) ON DELETE CASCADE "
+		"	ON UPDATE CASCADE);"
+		"CREATE TABLE IF NOT EXISTS sources("
+		"	name TEXT UNIQUE,"
+		"	version INTEGER,"
+		"	last INTEGER);"
 		"CREATE UNIQUE INDEX IF NOT EXISTS d ON digests(digest);"
 		"CREATE INDEX IF NOT EXISTS t ON digests(time);"
 		"CREATE INDEX IF NOT EXISTS dgst_id ON shingles(digest_id);"
@@ -93,7 +98,9 @@ enum rspamd_fuzzy_statement_idx {
 	RSPAMD_FUZZY_BACKEND_EXPIRE,
 	RSPAMD_FUZZY_BACKEND_VACUUM,
 	RSPAMD_FUZZY_BACKEND_DELETE_ORPHANED,
+	RSPAMD_FUZZY_BACKEND_ADD_SOURCE,
 	RSPAMD_FUZZY_BACKEND_VERSION,
+	RSPAMD_FUZZY_BACKEND_SET_VERSION,
 	RSPAMD_FUZZY_BACKEND_MAX
 };
 static struct rspamd_fuzzy_stmts {
@@ -214,12 +221,26 @@ static struct rspamd_fuzzy_stmts {
 		.result = SQLITE_DONE
 	},
 	{
+		.idx = RSPAMD_FUZZY_BACKEND_ADD_SOURCE,
+		.sql = "INSERT OR IGNORE INTO sources(name, version, last) VALUES (?1, ?2, ?3);",
+		.args = "TII",
+		.stmt = NULL,
+		.result = SQLITE_DONE
+	},
+	{
 		.idx = RSPAMD_FUZZY_BACKEND_VERSION,
-		.sql = "PRAGMA user_version;",
-		.args = "",
+		.sql = "SELECT version FROM sources WHERE name=?1;",
+		.args = "T",
 		.stmt = NULL,
 		.result = SQLITE_ROW
 	},
+	{
+		.idx = RSPAMD_FUZZY_BACKEND_SET_VERSION,
+		.sql = "UPDATE sources SET version=?1, last=?2 WHERE name=?3;",
+		.args = "IIT",
+		.stmt = NULL,
+		.result = SQLITE_DONE
+	},
 };
 
 static GQuark
@@ -410,7 +431,7 @@ rspamd_fuzzy_backend_open_db (const gchar *path, GError **err)
 	bk->expired = 0;
 	bk->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "fuzzy_backend");
 	bk->db = rspamd_sqlite3_open_or_create (bk->pool, bk->path,
-			create_tables_sql, err);
+			create_tables_sql, 1, err);
 
 	if (bk->db == NULL) {
 		rspamd_fuzzy_backend_close (bk);
@@ -506,6 +527,10 @@ rspamd_fuzzy_backend_check (struct rspamd_fuzzy_backend *backend,
 			rep.prob = 1.0;
 			rep.flag = sqlite3_column_int (
 					prepared_stmts[RSPAMD_FUZZY_BACKEND_CHECK].stmt, 2);
+
+			if (!(rep.flag & flags_mask) && rep.flag > 0) {
+				rep.flag = (1U << (rep.flag - 1)) | flags_mask;
+			}
 		}
 	}
 	else if (cmd->shingles_count > 0) {
@@ -589,6 +614,10 @@ rspamd_fuzzy_backend_check (struct rspamd_fuzzy_backend *backend,
 						rep.flag = sqlite3_column_int (
 								prepared_stmts[RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID].stmt,
 								3);
+
+						if (!(rep.flag & flags_mask) && rep.flag > 0) {
+							rep.flag = (1U << (rep.flag - 1)) | flags_mask;
+						}
 					}
 				}
 			}
@@ -610,7 +639,8 @@ rspamd_fuzzy_backend_check (struct rspamd_fuzzy_backend *backend,
 }
 
 gboolean
-rspamd_fuzzy_backend_prepare_update (struct rspamd_fuzzy_backend *backend)
+rspamd_fuzzy_backend_prepare_update (struct rspamd_fuzzy_backend *backend,
+		const gchar *source)
 {
 	gint rc;
 
@@ -643,6 +673,11 @@ rspamd_fuzzy_backend_add (struct rspamd_fuzzy_backend *backend,
 		return FALSE;
 	}
 
+	if (cmd->flag > 31 || cmd->flag == 0) {
+		msg_err_fuzzy_backend ("flag more than 31 is no longer supported");
+		return FALSE;
+	}
+
 	rc = rspamd_fuzzy_backend_run_stmt (backend, FALSE,
 			RSPAMD_FUZZY_BACKEND_CHECK,
 			cmd->digest);
@@ -654,7 +689,7 @@ rspamd_fuzzy_backend_add (struct rspamd_fuzzy_backend *backend,
 				2);
 		rspamd_fuzzy_backend_cleanup_stmt (backend, RSPAMD_FUZZY_BACKEND_CHECK);
 
-		if (flag == cmd->flag) {
+		if (flag & (1U << (cmd->flag - 1))) {
 			/* We need to increase weight */
 			rc = rspamd_fuzzy_backend_run_stmt (backend, TRUE,
 					RSPAMD_FUZZY_BACKEND_UPDATE,
@@ -669,11 +704,28 @@ rspamd_fuzzy_backend_add (struct rspamd_fuzzy_backend *backend,
 		}
 		else {
 			/* We need to relearn actually */
+			if (flag & flags_mask) {
+				/* This is already new format */
+				flag |= (1U << (cmd->flag - 1));
+			}
+			else {
+				/* Convert to the new format */
+				if (flag > 31 || flag == 0) {
+					msg_warn_fuzzy_backend ("storage had flag more than 31, remove "
+							"it");
+					flag = cmd->flag | flags_mask;
+				}
+				else {
+					flag = (1U << (flag - 1)) | (1U << (cmd->flag - 1)) | flags_mask;
+				}
+			}
+
 			rc = rspamd_fuzzy_backend_run_stmt (backend, TRUE,
 					RSPAMD_FUZZY_BACKEND_UPDATE_FLAG,
 					(gint64) cmd->value,
-					(gint64) cmd->flag,
+					(gint64) flag,
 					cmd->digest);
+
 			if (rc != SQLITE_OK) {
 				msg_warn_fuzzy_backend ("cannot update hash to %d -> "
 						"%*xs: %s", (gint) cmd->flag,
@@ -686,7 +738,7 @@ rspamd_fuzzy_backend_add (struct rspamd_fuzzy_backend *backend,
 		rspamd_fuzzy_backend_cleanup_stmt (backend, RSPAMD_FUZZY_BACKEND_CHECK);
 		rc = rspamd_fuzzy_backend_run_stmt (backend, FALSE,
 				RSPAMD_FUZZY_BACKEND_INSERT,
-				(gint) cmd->flag,
+				(gint) (1U << (cmd->flag - 1)),
 				cmd->digest,
 				(gint64) cmd->value,
 				(gint64) timestamp);
@@ -729,42 +781,47 @@ rspamd_fuzzy_backend_add (struct rspamd_fuzzy_backend *backend,
 }
 
 gboolean
-rspamd_fuzzy_backend_finish_update (struct rspamd_fuzzy_backend *backend)
+rspamd_fuzzy_backend_finish_update (struct rspamd_fuzzy_backend *backend,
+		const gchar *source)
 {
 	gint rc, wal_frames, wal_checkpointed, ver;
-	gint64 version = 0;
-	gchar version_buf[128];
+
+	/* Get and update version */
+	ver = rspamd_fuzzy_backend_version (backend, source);
+	++ver;
 
 	rc = rspamd_fuzzy_backend_run_stmt (backend, TRUE,
-			RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT);
+				RSPAMD_FUZZY_BACKEND_SET_VERSION,
+				(gint64)ver, (gint64)time (NULL), source);
 
-	if (rc != SQLITE_OK) {
-		msg_warn_fuzzy_backend ("cannot commit updates: %s",
-				sqlite3_errmsg (backend->db));
-		rspamd_fuzzy_backend_run_stmt (backend, TRUE,
-				RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK);
-		return FALSE;
-	}
-	else {
-		if (!rspamd_sqlite3_sync (backend->db, &wal_frames, &wal_checkpointed)) {
-			msg_warn_fuzzy_backend ("cannot commit checkpoint: %s",
+	if (rc == SQLITE_OK) {
+		rc = rspamd_fuzzy_backend_run_stmt (backend, TRUE,
+				RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT);
+
+		if (rc != SQLITE_OK) {
+			msg_warn_fuzzy_backend ("cannot commit updates: %s",
 					sqlite3_errmsg (backend->db));
+			rspamd_fuzzy_backend_run_stmt (backend, TRUE,
+					RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK);
+			return FALSE;
 		}
-		else if (wal_checkpointed > 0) {
-			msg_info_fuzzy_backend ("total number of frames in the wal file: "
-					"%d, checkpointed: %d", wal_frames, wal_checkpointed);
+		else {
+			if (!rspamd_sqlite3_sync (backend->db, &wal_frames, &wal_checkpointed)) {
+				msg_warn_fuzzy_backend ("cannot commit checkpoint: %s",
+						sqlite3_errmsg (backend->db));
+			}
+			else if (wal_checkpointed > 0) {
+				msg_info_fuzzy_backend ("total number of frames in the wal file: "
+						"%d, checkpointed: %d", wal_frames, wal_checkpointed);
+			}
 		}
 	}
-
-	/* Get and update version */
-	ver = rspamd_fuzzy_backend_version (backend);
-	++ver;
-	rspamd_snprintf (version_buf, sizeof (version_buf), "PRAGMA user_version=%d;",
-			ver);
-
-	if (sqlite3_exec (backend->db, version_buf, NULL, NULL, NULL) != SQLITE_OK) {
-			msg_err_fuzzy_backend ("cannot set database version to %L: %s",
-					version, sqlite3_errmsg (backend->db));
+	else {
+		msg_warn_fuzzy_backend ("cannot update version for %s: %s", source,
+				sqlite3_errmsg (backend->db));
+		rspamd_fuzzy_backend_run_stmt (backend, TRUE,
+				RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK);
+		return FALSE;
 	}
 
 	return TRUE;
@@ -774,16 +831,61 @@ gboolean
 rspamd_fuzzy_backend_del (struct rspamd_fuzzy_backend *backend,
 		const struct rspamd_fuzzy_cmd *cmd)
 {
-	int rc;
+	int rc = -1;
+	guint32 flag;
 
 	if (backend == NULL) {
 		return FALSE;
 	}
 
-	rc = rspamd_fuzzy_backend_run_stmt (backend, TRUE,
-			RSPAMD_FUZZY_BACKEND_DELETE,
+	rc = rspamd_fuzzy_backend_run_stmt (backend, FALSE,
+			RSPAMD_FUZZY_BACKEND_CHECK,
 			cmd->digest);
 
+	if (rc == SQLITE_OK) {
+		/* Check flag */
+		flag = sqlite3_column_int64 (
+				prepared_stmts[RSPAMD_FUZZY_BACKEND_CHECK].stmt,
+				2);
+		rspamd_fuzzy_backend_cleanup_stmt (backend, RSPAMD_FUZZY_BACKEND_CHECK);
+
+		if (!(flag & flags_mask)) {
+			flag = (1U << (flag - 1)) | flags_mask;
+		}
+
+		if (flag & (1U << (cmd->flag - 1))) {
+			flag &= ~(1U << (cmd->flag - 1));
+
+			if (flag == 0) {
+				/* It is the last flag, so delete hash completely */
+				rc = rspamd_fuzzy_backend_run_stmt (backend, TRUE,
+							RSPAMD_FUZZY_BACKEND_DELETE,
+							cmd->digest);
+			}
+			else {
+				/* We need to delete specific flag */
+				rc = rspamd_fuzzy_backend_run_stmt (backend, TRUE,
+						RSPAMD_FUZZY_BACKEND_UPDATE_FLAG,
+						(gint64) cmd->value,
+						(gint64) flag,
+						cmd->digest);
+				if (rc != SQLITE_OK) {
+					msg_warn_fuzzy_backend ("cannot update hash to %d -> "
+							"%*xs: %s", (gint) cmd->flag,
+							(gint) sizeof (cmd->digest), cmd->digest,
+							sqlite3_errmsg (backend->db));
+				}
+			}
+		}
+		else {
+			/* The hash has a wrong flag, ignoring */
+		}
+	}
+	else {
+		/* Hash is missing */
+		rspamd_fuzzy_backend_cleanup_stmt (backend, RSPAMD_FUZZY_BACKEND_CHECK);
+	}
+
 	return (rc == SQLITE_OK);
 }
 
@@ -976,13 +1078,14 @@ rspamd_fuzzy_backend_count (struct rspamd_fuzzy_backend *backend)
 }
 
 gint
-rspamd_fuzzy_backend_version (struct rspamd_fuzzy_backend *backend)
+rspamd_fuzzy_backend_version (struct rspamd_fuzzy_backend *backend,
+		const gchar *source)
 {
-	gint ret = 0;
+	gint ret = -1;
 
 	if (backend) {
 		if (rspamd_fuzzy_backend_run_stmt (backend, FALSE,
-				RSPAMD_FUZZY_BACKEND_VERSION) == SQLITE_OK) {
+				RSPAMD_FUZZY_BACKEND_VERSION, source) == SQLITE_OK) {
 			ret = sqlite3_column_int64 (
 					prepared_stmts[RSPAMD_FUZZY_BACKEND_VERSION].stmt, 0);
 		}
diff --git a/src/libserver/fuzzy_backend.h b/src/libserver/fuzzy_backend.h
index bcd199d1a..91a613f2a 100644
--- a/src/libserver/fuzzy_backend.h
+++ b/src/libserver/fuzzy_backend.h
@@ -46,7 +46,8 @@ struct rspamd_fuzzy_reply rspamd_fuzzy_backend_check (
 /**
  * Prepare storage for updates (by starting transaction)
  */
-gboolean rspamd_fuzzy_backend_prepare_update (struct rspamd_fuzzy_backend *backend);
+gboolean rspamd_fuzzy_backend_prepare_update (struct rspamd_fuzzy_backend *backend,
+		const gchar *source);
 
 /**
  * Add digest to the database
@@ -72,7 +73,8 @@ gboolean rspamd_fuzzy_backend_del (
 /**
  * Commit updates to storage
  */
-gboolean rspamd_fuzzy_backend_finish_update (struct rspamd_fuzzy_backend *backend);
+gboolean rspamd_fuzzy_backend_finish_update (struct rspamd_fuzzy_backend *backend,
+		const gchar *source);
 
 /**
  * Sync storage
@@ -90,7 +92,7 @@ gboolean rspamd_fuzzy_backend_sync (struct rspamd_fuzzy_backend *backend,
 void rspamd_fuzzy_backend_close (struct rspamd_fuzzy_backend *backend);
 
 gsize rspamd_fuzzy_backend_count (struct rspamd_fuzzy_backend *backend);
-gint rspamd_fuzzy_backend_version (struct rspamd_fuzzy_backend *backend);
+gint rspamd_fuzzy_backend_version (struct rspamd_fuzzy_backend *backend, const gchar *source);
 gsize rspamd_fuzzy_backend_expired (struct rspamd_fuzzy_backend *backend);
 
 const gchar * rspamd_fuzzy_backend_id (struct rspamd_fuzzy_backend *backend);
diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c
index d314d3fdc..a48de05dc 100644
--- a/src/libserver/protocol.c
+++ b/src/libserver/protocol.c
@@ -873,9 +873,8 @@ rspamd_metric_result_ucl (struct rspamd_task *task,
 	return obj;
 }
 
-static void
-rspamd_ucl_torspamc_output (struct rspamd_task *task,
-	ucl_object_t *top,
+void
+rspamd_ucl_torspamc_output (const ucl_object_t *top,
 	rspamd_fstring_t **out)
 {
 	const ucl_object_t *metric, *score,
@@ -927,12 +926,15 @@ rspamd_ucl_torspamc_output (struct rspamd_task *task,
 		}
 	}
 
-	rspamd_printf_fstring (out, "Message-ID: %s\r\n", task->message_id);
+	elt = ucl_object_lookup (top, "message-id");
+	if (elt != NULL) {
+		rspamd_printf_fstring (out, "Message-ID: %s\r\n",
+				ucl_object_tostring (elt));
+	}
 }
 
 static void
-rspamd_ucl_tospamc_output (struct rspamd_task *task,
-	ucl_object_t *top,
+rspamd_ucl_tospamc_output (const ucl_object_t *top,
 	rspamd_fstring_t **out)
 {
 	const ucl_object_t *metric, *score,
@@ -1055,10 +1057,10 @@ rspamd_protocol_http_reply (struct rspamd_http_message *msg,
 	}
 	else {
 		if (RSPAMD_TASK_IS_SPAMC (task)) {
-			rspamd_ucl_tospamc_output (task, top, &msg->body);
+			rspamd_ucl_tospamc_output (top, &msg->body);
 		}
 		else {
-			rspamd_ucl_torspamc_output (task, top, &msg->body);
+			rspamd_ucl_torspamc_output (top, &msg->body);
 		}
 	}
 
diff --git a/src/libserver/protocol.h b/src/libserver/protocol.h
index 3c8383565..1f7acbab2 100644
--- a/src/libserver/protocol.h
+++ b/src/libserver/protocol.h
@@ -82,5 +82,13 @@ ucl_object_t * rspamd_protocol_write_ucl (struct rspamd_task *task);
  */
 void rspamd_protocol_write_reply (struct rspamd_task *task);
 
+/**
+ * Convert rspamd output to legacy protocol reply
+ * @param task
+ * @param top
+ * @param out
+ */
+void rspamd_ucl_torspamc_output (const ucl_object_t *top,
+	rspamd_fstring_t **out);
 
 #endif
diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c
index 3e308415d..332486cdd 100644
--- a/src/libserver/re_cache.c
+++ b/src/libserver/re_cache.c
@@ -746,7 +746,7 @@ rspamd_re_cache_exec_re (struct rspamd_task *task,
 				re_class->type_data,
 				is_strong);
 
-		if (headerlist) {
+		if (headerlist && headerlist->len > 0) {
 			scvec = g_malloc (sizeof (*scvec) * headerlist->len);
 			lenvec = g_malloc (sizeof (*lenvec) * headerlist->len);
 
@@ -795,7 +795,7 @@ rspamd_re_cache_exec_re (struct rspamd_task *task,
 				re_class->type_data,
 				is_strong);
 
-		if (headerlist) {
+		if (headerlist && headerlist->len > 0) {
 			scvec = g_malloc (sizeof (*scvec) * headerlist->len);
 			lenvec = g_malloc (sizeof (*lenvec) * headerlist->len);
 
@@ -1494,7 +1494,8 @@ rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache,
 			 * crc - 8 bytes checksum
 			 * <hyperscan blob>
 			 */
-			crc = XXH64 (hs_serialized, serialized_len, 0xdeadbabe);
+			crc = rspamd_cryptobox_fast_hash_specific (RSPAMD_CRYPTOBOX_XXHASH64,
+					hs_serialized, serialized_len, 0xdeadbabe);
 
 			if (cache->vectorized_hyperscan) {
 				iov[0].iov_base = (void *) rspamd_hs_magic_vector;
diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c
index 740c4cd7a..bf596a343 100644
--- a/src/libserver/worker_util.c
+++ b/src/libserver/worker_util.c
@@ -248,9 +248,9 @@ rspamd_prepare_worker (struct rspamd_worker *worker, const char *name,
 	void (*accept_handler)(int, short, void *))
 {
 	struct event_base *ev_base;
-	struct event *accept_event;
+	struct event *accept_events;
 	GList *cur;
-	gint listen_socket;
+	struct rspamd_worker_listen_socket *ls;
 
 #ifdef WITH_PROFILER
 	extern void _start (void), etext (void);
@@ -271,17 +271,20 @@ rspamd_prepare_worker (struct rspamd_worker *worker, const char *name,
 	/* Accept all sockets */
 	if (accept_handler) {
 		cur = worker->cf->listen_socks;
+
 		while (cur) {
-			listen_socket = GPOINTER_TO_INT (cur->data);
-			if (listen_socket != -1) {
-				accept_event = g_slice_alloc0 (sizeof (struct event));
-				event_set (accept_event, listen_socket, EV_READ | EV_PERSIST,
+			ls = cur->data;
+
+			if (ls->fd != -1) {
+				accept_events = g_slice_alloc0 (sizeof (struct event) * 2);
+				event_set (&accept_events[0], ls->fd, EV_READ | EV_PERSIST,
 						accept_handler, worker);
-				event_base_set (ev_base, accept_event);
-				event_add (accept_event, NULL);
+				event_base_set (ev_base, &accept_events[0]);
+				event_add (&accept_events[0], NULL);
 				worker->accept_events = g_list_prepend (worker->accept_events,
-						accept_event);
+						accept_events);
 			}
+
 			cur = g_list_next (cur);
 		}
 	}
@@ -293,7 +296,7 @@ void
 rspamd_worker_stop_accept (struct rspamd_worker *worker)
 {
 	GList *cur;
-	struct event *event;
+	struct event *events;
 	GHashTableIter it;
 	struct rspamd_worker_signal_handler *sigh;
 	gpointer k, v;
@@ -302,10 +305,18 @@ rspamd_worker_stop_accept (struct rspamd_worker *worker)
 	/* Remove all events */
 	cur = worker->accept_events;
 	while (cur) {
-		event = cur->data;
-		event_del (event);
+		events = cur->data;
+
+		if (event_get_base (&events[0])) {
+			event_del (&events[0]);
+		}
+
+		if (event_get_base (&events[1])) {
+			event_del (&events[1]);
+		}
+
 		cur = g_list_next (cur);
-		g_slice_free1 (sizeof (struct event), event);
+		g_slice_free1 (sizeof (struct event) * 2, events);
 	}
 
 	if (worker->accept_events != NULL) {