Merge remote-tracking branch 'upstream/master'

CMakeLists.txt

@@ -331,8 +331,19 @@ CHECK_SYMBOL_EXISTS(I_SETSIG "sys/types.h;sys/ioctl.h" HAVE_SETSIG)
 CHECK_SYMBOL_EXISTS(O_ASYNC "sys/types.h;sys/fcntl.h" HAVE_OASYNC)
 CHECK_SYMBOL_EXISTS(O_NOFOLLOW "sys/types.h;sys/fcntl.h" HAVE_ONOFOLLOW)
 CHECK_SYMBOL_EXISTS(O_CLOEXEC "sys/types.h;sys/fcntl.h" HAVE_OCLOEXEC)
+
+# OpenSSL specific stuff
 LIST(APPEND CMAKE_REQUIRED_INCLUDES "${LIBSSL_INCLUDE}")
+IF(LIBCRYPT_LIBRARY_PATH)
+	SET(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};-L${LIBCRYPT_LIBRARY_PATH};${LIBCRYPT_LIBRARY}")
+	SET(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};-L${LIBSSL_LIBRARY_PATH};${LIBSSL_LIBRARY}")
+ELSE()
+	SET(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};-lcrypt;-lssl")
+ENDIF()
+
 CHECK_SYMBOL_EXISTS(SSL_set_tlsext_host_name "openssl/ssl.h" HAVE_SSL_TLSEXT_HOSTNAME)
+CHECK_SYMBOL_EXISTS(FIPS_mode "openssl/crypto.h" HAVE_FIPS_MODE)
+
 CHECK_SYMBOL_EXISTS(dirfd "sys/types.h;unistd.h;dirent.h" HAVE_DIRFD)
 CHECK_SYMBOL_EXISTS(fpathconf "sys/types.h;unistd.h" HAVE_FPATHCONF)
 CHECK_SYMBOL_EXISTS(sigaltstack "signal.h" HAVE_SIGALTSTACK)

config.h.in

@@ -32,6 +32,7 @@
 #cmakedefine HAVE_FCNTL_H        1
 #cmakedefine HAVE_FDATASYNC      1
 #cmakedefine HAVE_FETCH_H        1
+#cmakedefine HAVE_FIPS_MODE      1
 #cmakedefine HAVE_FLOCK          1
 #cmakedefine HAVE_FPATHCONF      1
 #cmakedefine HAVE_GETPAGESIZE    1

lualib/lua_maps_expressions.lua

@@ -78,7 +78,7 @@ local function process_func(elt, task)
 
   local res = elt.expr:process(process_atom)
 
-  if res then
+  if res > 0 then
     return res,matched
   end
 

src/hs_helper.c

@@ -32,8 +32,8 @@ worker_t hs_helper_worker = {
 		"hs_helper",                /* Name */
 		init_hs_helper,             /* Init function */
 		start_hs_helper,            /* Start function */
-		RSPAMD_WORKER_UNIQUE|RSPAMD_WORKER_KILLABLE|RSPAMD_WORKER_ALWAYS_START,
-		RSPAMD_WORKER_SOCKET_NONE,  /* No socket */
+		RSPAMD_WORKER_UNIQUE|RSPAMD_WORKER_KILLABLE|RSPAMD_WORKER_ALWAYS_START|RSPAMD_WORKER_NO_TERMINATE_DELAY,
+		RSPAMD_WORKER_SOCKET_NONE,
 		RSPAMD_WORKER_VER           /* Version info */
 };
 

src/libmime/lang_detection.c

@@ -1484,13 +1484,15 @@ rspamd_language_detector_unicode_scripts (struct rspamd_task *task,
 static inline void
 rspamd_language_detector_set_language (struct rspamd_task *task,
 									   struct rspamd_mime_text_part *part,
-									   const gchar *code)
+									   const gchar *code,
+									   struct rspamd_language_elt *elt)
 {
 	struct rspamd_lang_detector_res *r;
 
 	r = rspamd_mempool_alloc0 (task->task_pool, sizeof (*r));
 	r->prob = 1.0;
 	r->lang = code;
+	r->elt = elt;
 
 	if (part->languages == NULL) {
 		part->languages = g_ptr_array_sized_new (1);
@@ -1515,7 +1517,7 @@ rspamd_language_detector_try_uniscript (struct rspamd_task *task,
 				msg_debug_lang_det ("set language based on unicode script %s",
 						unicode_langs[i].lang);
 				rspamd_language_detector_set_language (task, part,
-						unicode_langs[i].lang);
+						unicode_langs[i].lang, NULL);
 
 				return TRUE;
 			}
@@ -1533,7 +1535,7 @@ rspamd_language_detector_try_uniscript (struct rspamd_task *task,
 					msg_debug_lang_det ("set language based on unicode script %s",
 							unicode_langs[i].lang);
 					rspamd_language_detector_set_language (task, part,
-							unicode_langs[i].lang);
+							unicode_langs[i].lang, NULL);
 
 					return TRUE;
 				}
@@ -1545,7 +1547,7 @@ rspamd_language_detector_try_uniscript (struct rspamd_task *task,
 		msg_debug_lang_det ("guess chinese based on CJK characters: %d chinese, %d special",
 				nchinese, nspecial);
 		rspamd_language_detector_set_language (task, part,
-				"zh-CN");
+				"zh-CN", NULL);
 
 		return TRUE;
 	}
@@ -1680,8 +1682,7 @@ rspamd_language_detector_try_stop_words (struct rspamd_task *task,
 	if (kh_size (cbdata.res) > 0) {
 		gint cur_matches;
 		double max_rate = G_MINDOUBLE;
-		const gchar *sel = NULL;
-		struct rspamd_language_elt *cur_lang;
+		struct rspamd_language_elt *cur_lang, *sel = NULL;
 
 		kh_foreach (cbdata.res, cur_lang, cur_matches, {
 			if (cur_matches < stop_words_threshold) {
@@ -1692,7 +1693,7 @@ rspamd_language_detector_try_stop_words (struct rspamd_task *task,
 
 			if (rate > max_rate) {
 				max_rate = rate;
-				sel = cur_lang->name;
+				sel = cur_lang;
 			}
 			msg_debug_lang_det ("found %d stop words from %s: %3f rate",
 					cur_matches, cur_lang->name, rate);
@@ -1700,9 +1701,9 @@ rspamd_language_detector_try_stop_words (struct rspamd_task *task,
 
 		if (max_rate > 0 && sel) {
 			msg_debug_lang_det ("set language based on stop words script %s, %.3f found",
-					sel, max_rate);
+					sel->name, max_rate);
 			rspamd_language_detector_set_language (task, part,
-					sel);
+					sel->name, sel);
 
 			ret = TRUE;
 		}
@@ -1761,17 +1762,17 @@ rspamd_language_detector_detect (struct rspamd_task *task,
 					(int)default_short_text_limit);
 			switch (cat) {
 			case RSPAMD_LANGUAGE_CYRILLIC:
-				rspamd_language_detector_set_language (task, part, "ru");
+				rspamd_language_detector_set_language (task, part, "ru", NULL);
 				break;
 			case RSPAMD_LANGUAGE_DEVANAGARI:
-				rspamd_language_detector_set_language (task, part, "hi");
+				rspamd_language_detector_set_language (task, part, "hi", NULL);
 				break;
 			case RSPAMD_LANGUAGE_ARAB:
-				rspamd_language_detector_set_language (task, part, "ar");
+				rspamd_language_detector_set_language (task, part, "ar", NULL);
 				break;
 			default:
 			case RSPAMD_LANGUAGE_LATIN:
-				rspamd_language_detector_set_language (task, part, "en");
+				rspamd_language_detector_set_language (task, part, "en", NULL);
 				break;
 			}
 			msg_debug_lang_det ("set %s language based on symbols category",
@@ -1792,7 +1793,7 @@ rspamd_language_detector_detect (struct rspamd_task *task,
 
 			if (r == rs_detect_none) {
 				msg_debug_lang_det ("no trigramms found, fallback to english");
-				rspamd_language_detector_set_language (task, part, "en");
+				rspamd_language_detector_set_language (task, part, "en", NULL);
 			} else if (r == rs_detect_multiple) {
 				/* Check our guess */
 
@@ -1873,7 +1874,7 @@ rspamd_language_detector_detect (struct rspamd_task *task,
 			ret = TRUE;
 		}
 		else if (part->languages == NULL) {
-			rspamd_language_detector_set_language (task, part, "en");
+			rspamd_language_detector_set_language (task, part, "en", NULL);
 		}
 
 		kh_destroy (rspamd_candidates_hash, candidates);

src/libmime/mime_parser.c

@@ -707,6 +707,11 @@ rspamd_mime_process_multipart_node (struct rspamd_task *task,
 					npart->raw_headers_str,
 					npart->raw_headers_len,
 					FALSE);
+
+			/* Preserve the natural order */
+			if (npart->headers_order) {
+				LL_REVERSE2 (npart->headers_order, ord_next);
+			}
 		}
 
 		hdr = rspamd_message_get_header_from_hash (npart->raw_headers,
@@ -1265,6 +1270,11 @@ rspamd_mime_parse_message (struct rspamd_task *task,
 						TRUE);
 				npart->raw_headers = rspamd_message_headers_ref (
 						MESSAGE_FIELD (task, raw_headers));
+
+				/* Preserve the natural order */
+				if (MESSAGE_FIELD (task, headers_order)) {
+					LL_REVERSE2 (MESSAGE_FIELD (task, headers_order), ord_next);
+				}
 			}
 
 			hdr = rspamd_message_get_header_from_hash (
@@ -1290,6 +1300,11 @@ rspamd_mime_parse_message (struct rspamd_task *task,
 							TRUE);
 					npart->raw_headers = rspamd_message_headers_ref (
 							MESSAGE_FIELD (task, raw_headers));
+
+					/* Preserve the natural order */
+					if (MESSAGE_FIELD (task, headers_order)) {
+						LL_REVERSE2 (MESSAGE_FIELD (task, headers_order), ord_next);
+					}
 				}
 
 				hdr = rspamd_message_get_header_from_hash (
@@ -1341,6 +1356,11 @@ rspamd_mime_parse_message (struct rspamd_task *task,
 						npart->raw_headers_str,
 						npart->raw_headers_len,
 						FALSE);
+
+				/* Preserve the natural order */
+				if (npart->headers_order) {
+					LL_REVERSE2 (npart->headers_order, ord_next);
+				}
 			}
 
 			hdr = rspamd_message_get_header_from_hash (npart->raw_headers,

src/libserver/cfg_file.h

@@ -227,7 +227,6 @@ struct rspamd_worker_conf {
 	guint64 rlimit_maxcore;                         /**< maximum core file size								*/
 	GHashTable *params;                             /**< params for worker									*/
 	GQueue *active_workers;                         /**< linked list of spawned workers						*/
-	gboolean has_socket;                            /**< whether we should make listening socket in main process */
 	gpointer *ctx;                                  /**< worker's context									*/
 	ucl_object_t *options;                          /**< other worker's options								*/
 	struct rspamd_worker_lua_script *scripts;       /**< registered lua scripts								*/

src/libserver/dkim.c

@@ -2347,6 +2347,21 @@ rspamd_dkim_check_bh_cached (struct rspamd_dkim_common_ctx *ctx,
 	return res;
 }
 
+static const char *
+rspamd_dkim_type_to_string (enum rspamd_dkim_type t)
+{
+	switch (t) {
+	case RSPAMD_DKIM_NORMAL:
+		return "dkim";
+	case RSPAMD_DKIM_ARC_SIG:
+		return "arc_sig";
+	case RSPAMD_DKIM_ARC_SEAL:
+	default:
+		return "arc_seal";
+
+	}
+}
+
 /**
  * Check task for dkim context using dkim key
  * @param ctx dkim verify context
@@ -2445,8 +2460,9 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx,
 		/* Check bh field */
 		if (memcmp (ctx->bh, cached_bh->digest_normal, ctx->bhlen) != 0) {
 			msg_info_dkim (
-					"dkim: bh value mismatch: got %*Bs, expected %*Bs; "
+					"%s: bh value mismatch: got %*Bs, expected %*Bs; "
 					"body length %d->%d; d=%s; s=%s",
+					rspamd_dkim_type_to_string (ctx->common.type),
 					(gint)dlen, cached_bh->digest_normal,
 					(gint)dlen, ctx->bh,
 					(gint)(body_end - body_start), ctx->common.body_canonicalised,
@@ -2581,8 +2597,9 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx,
 			res->fail_reason = "rsa verify failed";
 
 			msg_info_dkim (
-					"dkim: RSA verification failure: got %*Bs, expected %*Bs; "
+					"%s: RSA verification failure: got %*Bs, expected %*Bs; "
 					"body length %d->%d; headers length %d; d=%s; s=%s",
+					rspamd_dkim_type_to_string (ctx->common.type),
 					(gint)dlen, raw_digest,
 					(gint)dlen, ctx->b,
 					(gint)(body_end - body_start), ctx->common.body_canonicalised,
@@ -2594,8 +2611,9 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx,
 		if (ECDSA_verify (nid, raw_digest, dlen, ctx->b, ctx->blen,
 				key->key.key_ecdsa) != 1) {
 			msg_info_dkim (
-					"dkim: ECDSA verification failure: got %*Bs, expected %*Bs; "
+					"%s: ECDSA verification failure: got %*Bs, expected %*Bs; "
 					"body length %d->%d; headers length %d; d=%s; s=%s",
+					rspamd_dkim_type_to_string (ctx->common.type),
 					(gint)dlen, raw_digest,
 					(gint)dlen, ctx->b,
 					(gint)(body_end - body_start), ctx->common.body_canonicalised,
@@ -2610,8 +2628,9 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx,
 		if (!rspamd_cryptobox_verify (ctx->b, ctx->blen, raw_digest, dlen,
 				key->key.key_eddsa, RSPAMD_CRYPTOBOX_MODE_25519)) {
 			msg_info_dkim (
-					"dkim: EDDSA verification failure: got %*Bs, expected %*Bs; "
+					"%s: EDDSA verification failure: got %*Bs, expected %*Bs; "
 					"body length %d->%d; headers length %d; d=%s; s=%s",
+					rspamd_dkim_type_to_string (ctx->common.type),
 					(gint)dlen, raw_digest,
 					(gint)dlen, ctx->b,
 					(gint)(body_end - body_start), ctx->common.body_canonicalised,

src/libserver/worker_util.c

@@ -258,8 +258,13 @@ rspamd_worker_usr2_handler (struct rspamd_worker_signal_handler *sigh, void *arg
 		static ev_timer shutdown_ev, shutdown_check_ev;
 		ev_tstamp shutdown_ts;
 
-		shutdown_ts = MAX (SOFT_SHUTDOWN_TIME,
-				sigh->worker->srv->cfg->task_timeout * 2.0);
+		if (sigh->worker->flags & RSPAMD_WORKER_NO_TERMINATE_DELAY) {
+			shutdown_ts = 0.0;
+		}
+		else {
+			shutdown_ts = MAX (SOFT_SHUTDOWN_TIME,
+					sigh->worker->srv->cfg->task_timeout * 2.0);
+		}
 
 		rspamd_worker_ignore_signal (sigh);
 		sigh->worker->state = rspamd_worker_state_terminating;
@@ -277,11 +282,14 @@ rspamd_worker_usr2_handler (struct rspamd_worker_signal_handler *sigh, void *arg
 				shutdown_ts, 0.0);
 		ev_timer_start (sigh->event_loop, &shutdown_ev);
 
-		/* This timer checks if we are ready to die and is called frequently */
-		shutdown_check_ev.data = sigh->worker;
-		ev_timer_init (&shutdown_check_ev, rspamd_worker_shutdown_check,
-				0.5, 0.5);
-		ev_timer_start (sigh->event_loop, &shutdown_check_ev);
+		if (!(sigh->worker->flags & RSPAMD_WORKER_NO_TERMINATE_DELAY)) {
+			/* This timer checks if we are ready to die and is called frequently */
+			shutdown_check_ev.data = sigh->worker;
+			ev_timer_init (&shutdown_check_ev, rspamd_worker_shutdown_check,
+					0.5, 0.5);
+			ev_timer_start (sigh->event_loop, &shutdown_check_ev);
+		}
+
 		rspamd_worker_stop_accept (sigh->worker);
 	}
 
@@ -311,8 +319,13 @@ rspamd_worker_term_handler (struct rspamd_worker_signal_handler *sigh, void *arg
 		static ev_timer shutdown_ev, shutdown_check_ev;
 		ev_tstamp shutdown_ts;
 
-		shutdown_ts = MAX (SOFT_SHUTDOWN_TIME,
-				sigh->worker->srv->cfg->task_timeout * 2.0);
+		if (sigh->worker->flags & RSPAMD_WORKER_NO_TERMINATE_DELAY) {
+			shutdown_ts = 0.0;
+		}
+		else {
+			shutdown_ts = MAX (SOFT_SHUTDOWN_TIME,
+					sigh->worker->srv->cfg->task_timeout * 2.0);
+		}
 
 		rspamd_worker_ignore_signal (sigh);
 		sigh->worker->state = rspamd_worker_state_terminating;
@@ -334,13 +347,16 @@ rspamd_worker_term_handler (struct rspamd_worker_signal_handler *sigh, void *arg
 					shutdown_ts, 0.0);
 			ev_timer_start (sigh->event_loop, &shutdown_ev);
 
-			/* This timer checks if we are ready to die and is called frequently */
-			shutdown_check_ev.data = sigh->worker;
-			ev_timer_init (&shutdown_check_ev, rspamd_worker_shutdown_check,
-					0.5, 0.5);
-			ev_timer_start (sigh->event_loop, &shutdown_check_ev);
+			if (!(sigh->worker->flags & RSPAMD_WORKER_NO_TERMINATE_DELAY)) {
+				/* This timer checks if we are ready to die and is called frequently */
+				shutdown_check_ev.data = sigh->worker;
+				ev_timer_init (&shutdown_check_ev, rspamd_worker_shutdown_check,
+						0.5, 0.5);
+				ev_timer_start (sigh->event_loop, &shutdown_check_ev);
+			}
 		}
 		else {
+			/* Flag to die has been already set */
 			ev_break (sigh->event_loop, EVBREAK_ALL);
 		}
 	}

src/libutil/util.c

@@ -2484,6 +2484,7 @@ rspamd_config_libs (struct rspamd_external_libs_ctx *ctx,
 		}
 
 		if (cfg->fips_mode) {
+#ifdef HAVE_FIPS_MODE
 			int mode = FIPS_mode ();
 			unsigned long err = (unsigned long)-1;
 
@@ -2505,6 +2506,9 @@ rspamd_config_libs (struct rspamd_external_libs_ctx *ctx,
 			else {
 				msg_info_config ("OpenSSL FIPS mode is enabled");
 			}
+#else
+			msg_warn_config ("SSL FIPS mode is enabled but not supported by OpenSSL library!");
+#endif
 		}
 
 		if (cfg->ssl_ca_path) {

src/lua/lua_mimepart.c

@@ -2011,7 +2011,7 @@ lua_mimepart_headers_foreach (lua_State *L)
 		if (part->headers_order) {
 			hdr = part->headers_order;
 
-			LL_FOREACH (hdr, cur) {
+			LL_FOREACH2 (hdr, cur, ord_next) {
 				if (re && re->re) {
 					if (!rspamd_regexp_match (re->re, cur->name,
 							strlen (cur->name),FALSE)) {

src/lua/lua_task.c

@@ -6269,7 +6269,7 @@ lua_task_headers_foreach (lua_State *L)
 			if (MESSAGE_FIELD (task, headers_order)) {
 				hdr = MESSAGE_FIELD (task, headers_order);
 
-				LL_FOREACH (hdr, cur) {
+				LL_FOREACH2 (hdr, cur, ord_next) {
 					if (re && re->re) {
 						if (!rspamd_regexp_match (re->re, cur->name,
 								strlen (cur->name), FALSE)) {

src/rspamd.h

@@ -64,6 +64,7 @@ enum rspamd_worker_flags {
 	RSPAMD_WORKER_ALWAYS_START = (1 << 4),
 	RSPAMD_WORKER_SCANNER = (1 << 5),
 	RSPAMD_WORKER_CONTROLLER = (1 << 6),
+	RSPAMD_WORKER_NO_TERMINATE_DELAY = (1 << 7),
 };
 
 struct rspamd_worker_accept_event {
@@ -107,7 +108,7 @@ struct rspamd_worker {
 	struct rspamd_worker_accept_event *accept_events; /**< socket events				*/
 	struct rspamd_worker_conf *cf;  /**< worker config data								*/
 	gpointer ctx;                   /**< worker's specific data							*/
-	enum rspamd_worker_flags flags; /**< worker's flags									*/
+	gint flags;                     /**< worker's flags (enum rspamd_worker_flags)		*/
 	gint control_pipe[2];           /**< control pipe. [0] is used by main process,
 	                                                   [1] is used by a worker			*/
 	gint srv_pipe[2];               /**< used by workers to request something from the