Merge pull request #4691 from rspamd/vstakhov-gtube-patterns

[Feature] Reiterate on gtube patterns

4 files changed, 77 insertions, 31 deletions

diff --git a/src/libmime/message.c b/src/libmime/message.c
index 327b546e7..3acc935e6 100644
--- a/src/libmime/message.c
+++ b/src/libmime/message.c
@@ -1,11 +1,11 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
+/*
+ * Copyright 2023 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -638,7 +638,7 @@ rspamd_multipattern_gtube_cb(struct rspamd_multipattern *mp,
 	struct rspamd_task *task = (struct rspamd_task *) context;
 
 	if (strnum > 0) {
-		if (task->cfg->enable_test_patterns) {
+		if (task->cfg->gtube_patterns_policy == RSPAMD_GTUBE_ALL) {
 			return strnum + 1;
 		}
 
@@ -654,9 +654,10 @@ rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part)
 	static const gsize max_check_size = 8 * 1024;
 	gint ret;
 	enum rspamd_action_type act = METRIC_ACTION_NOACTION;
+	enum rspamd_gtube_patterns_policy policy = task->cfg ? task->cfg->gtube_patterns_policy : RSPAMD_GTUBE_REJECT;
 	g_assert(part != NULL);
 
-	if (gtube_matcher == NULL) {
+	if (gtube_matcher == NULL && policy != RSPAMD_GTUBE_DISABLED) {
 		gtube_matcher = rspamd_multipattern_create(RSPAMD_MULTIPATTERN_DEFAULT);
 
 		rspamd_multipattern_add_pattern(gtube_matcher,
@@ -683,7 +684,8 @@ rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part)
 	}
 
 	if (part->utf_content.len >= sizeof(gtube_pattern_reject) &&
-		part->utf_content.len <= max_check_size) {
+		part->utf_content.len <= max_check_size &&
+		policy != RSPAMD_GTUBE_DISABLED) {
 		if ((ret = rspamd_multipattern_lookup(gtube_matcher, part->utf_content.begin,
 											  part->utf_content.len,
 											  rspamd_multipattern_gtube_cb, task, NULL)) > 0) {
@@ -693,15 +695,12 @@ rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part)
 				act = METRIC_ACTION_REJECT;
 				break;
 			case 2:
-				g_assert(task->cfg->enable_test_patterns);
 				act = METRIC_ACTION_ADD_HEADER;
 				break;
 			case 3:
-				g_assert(task->cfg->enable_test_patterns);
 				act = METRIC_ACTION_REWRITE_SUBJECT;
 				break;
 			case 4:
-				g_assert(task->cfg->enable_test_patterns);
 				act = METRIC_ACTION_NOACTION;
 				break;
 			}
diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h
index 22f754938..4cb87d974 100644
--- a/src/libserver/cfg_file.h
+++ b/src/libserver/cfg_file.h
@@ -312,6 +312,12 @@ enum rspamd_config_settings_policy {
 	RSPAMD_SETTINGS_POLICY_IMPLICIT_DENY = 2,
 };
 
+enum rspamd_gtube_patterns_policy {
+	RSPAMD_GTUBE_DISABLED = 0, /* Disabled */
+	RSPAMD_GTUBE_REJECT,       /* Reject message with GTUBE pattern */
+	RSPAMD_GTUBE_ALL           /* Check all GTUBE like patterns */
+};
+
 struct rspamd_config_settings_elt {
 	guint32 id;
 	enum rspamd_config_settings_policy policy;
@@ -344,22 +350,22 @@ struct rspamd_config {
 	GHashTable *groups;     /**< groups of symbols								*/
 	void *actions;          /**< all actions of the metric (opaque type)		*/
 
-	gboolean one_shot_mode;              /**< rules add only one symbol							*/
-	gboolean check_text_attachements;    /**< check text attachements as text					*/
-	gboolean check_all_filters;          /**< check all filters									*/
-	gboolean allow_raw_input;            /**< scan messages with invalid mime					*/
-	gboolean disable_hyperscan;          /**< disable hyperscan usage							*/
-	gboolean vectorized_hyperscan;       /**< use vectorized hyperscan matching					*/
-	gboolean enable_shutdown_workaround; /**< enable workaround for legacy SA clients (exim)		*/
-	gboolean ignore_received;            /**< Ignore data from the first received header			*/
-	gboolean enable_sessions_cache;      /**< Enable session cache for debug						*/
-	gboolean enable_experimental;        /**< Enable experimental plugins						*/
-	gboolean disable_pcre_jit;           /**< Disable pcre JIT									*/
-	gboolean own_lua_state;              /**< True if we have created lua_state internally		*/
-	gboolean soft_reject_on_timeout;     /**< If true emit soft reject on task timeout (if not reject) */
-	gboolean public_groups_only;         /**< Output merely public groups everywhere				*/
-	gboolean enable_test_patterns;       /**< Enable test patterns								*/
-	gboolean enable_css_parser;          /**< Enable css parsing in HTML							*/
+	gboolean one_shot_mode;                                  /**< rules add only one symbol							*/
+	gboolean check_text_attachements;                        /**< check text attachements as text					*/
+	gboolean check_all_filters;                              /**< check all filters									*/
+	gboolean allow_raw_input;                                /**< scan messages with invalid mime					*/
+	gboolean disable_hyperscan;                              /**< disable hyperscan usage							*/
+	gboolean vectorized_hyperscan;                           /**< use vectorized hyperscan matching					*/
+	gboolean enable_shutdown_workaround;                     /**< enable workaround for legacy SA clients (exim)		*/
+	gboolean ignore_received;                                /**< Ignore data from the first received header			*/
+	gboolean enable_sessions_cache;                          /**< Enable session cache for debug						*/
+	gboolean enable_experimental;                            /**< Enable experimental plugins						*/
+	gboolean disable_pcre_jit;                               /**< Disable pcre JIT									*/
+	gboolean own_lua_state;                                  /**< True if we have created lua_state internally		*/
+	gboolean soft_reject_on_timeout;                         /**< If true emit soft reject on task timeout (if not reject) */
+	gboolean public_groups_only;                             /**< Output merely public groups everywhere				*/
+	enum rspamd_gtube_patterns_policy gtube_patterns_policy; /**< Enable test patterns								*/
+	gboolean enable_css_parser;                              /**< Enable css parsing in HTML							*/
 
 	gsize max_cores_size;       /**< maximum size occupied by rspamd core files			*/
 	gsize max_cores_count;      /**< maximum number of core files						*/
diff --git a/src/libserver/cfg_rcl.cxx b/src/libserver/cfg_rcl.cxx
index 3f6ca2c56..b43014a1c 100644
--- a/src/libserver/cfg_rcl.cxx
+++ b/src/libserver/cfg_rcl.cxx
@@ -31,6 +31,7 @@
 
 #include <string>
 #include <filesystem>
+#include <algorithm>// for std::transform
 #include <memory>
 #include "contrib/ankerl/unordered_dense.h"
 #include "fmt/core.h"
@@ -347,6 +348,41 @@ rspamd_rcl_options_handler(rspamd_mempool_t *pool, const ucl_object_t *obj,
 		}
 	}
 
+	const auto *gtube_patterns = ucl_object_lookup(obj, "gtube_patterns");
+	if (gtube_patterns != nullptr && ucl_object_type(gtube_patterns) == UCL_STRING) {
+		auto gtube_st = std::string{ucl_object_tostring(gtube_patterns)};
+		std::transform(gtube_st.begin(), gtube_st.end(), gtube_st.begin(), [](const auto c) -> int {
+			if (c <= 'Z' && c >= 'A')
+				return c - ('Z' - 'z');
+			return c;
+		});
+
+
+		if (gtube_st == "all") {
+			cfg->gtube_patterns_policy = RSPAMD_GTUBE_ALL;
+		}
+		else if (gtube_st == "reject") {
+			cfg->gtube_patterns_policy = RSPAMD_GTUBE_REJECT;
+		}
+		else if (gtube_st == "disabled" || gtube_st == "disable") {
+			cfg->gtube_patterns_policy = RSPAMD_GTUBE_DISABLED;
+		}
+		else {
+			g_set_error(err,
+						CFG_RCL_ERROR,
+						EINVAL,
+						"invalid GTUBE patterns policy: %s",
+						gtube_st.c_str());
+			return FALSE;
+		}
+	}
+	else if (auto *enable_test_patterns = ucl_object_lookup(obj, "enable_test_patterns"); enable_test_patterns != nullptr) {
+		/* Legacy setting */
+		if (!!ucl_object_toboolean(enable_test_patterns)) {
+			cfg->gtube_patterns_policy = RSPAMD_GTUBE_ALL;
+		}
+	}
+
 	if (rspamd_rcl_section_parse_defaults(cfg,
 										  *section, cfg->cfg_pool, obj,
 										  cfg, err)) {
@@ -1877,12 +1913,6 @@ rspamd_rcl_config_init(struct rspamd_config *cfg, GHashTable *skip_sections)
 									   0,
 									   "Output merely public groups everywhere");
 		rspamd_rcl_add_default_handler(sub,
-									   "enable_test_patterns",
-									   rspamd_rcl_parse_struct_boolean,
-									   G_STRUCT_OFFSET(struct rspamd_config, enable_test_patterns),
-									   0,
-									   "Enable test GTUBE like patterns (not for production!)");
-		rspamd_rcl_add_default_handler(sub,
 									   "enable_css_parser",
 									   rspamd_rcl_parse_struct_boolean,
 									   G_STRUCT_OFFSET(struct rspamd_config, enable_css_parser),
@@ -2160,6 +2190,16 @@ rspamd_rcl_config_init(struct rspamd_config *cfg, GHashTable *skip_sections)
 									   0,
 									   "Events backend to use: kqueue, epoll, select, poll or auto (default: auto)");
 
+		rspamd_rcl_add_doc_by_path(cfg,
+								   "options",
+								   "Swtich mode of gtube patterns: disable, reject, all",
+								   "gtube_patterns",
+								   UCL_STRING,
+								   nullptr,
+								   0,
+								   "reject",
+								   0);
+
 		/* Neighbours configuration */
 		rspamd_rcl_add_section_doc(&top, sub, "neighbours", "name",
 								   rspamd_rcl_neighbours_handler,
diff --git a/src/libserver/cfg_utils.cxx b/src/libserver/cfg_utils.cxx
index 416d52e05..e22cbe9e3 100644
--- a/src/libserver/cfg_utils.cxx
+++ b/src/libserver/cfg_utils.cxx
@@ -286,6 +286,7 @@ rspamd_config_new(enum rspamd_config_init_flags flags)
 	cfg->max_recipients = 1024;
 	cfg->max_blas_threads = 1;
 	cfg->max_opts_len = 4096;
+	cfg->gtube_patterns_policy = RSPAMD_GTUBE_REJECT;
 
 	/* Default log line */
 	cfg->log_format_str = rspamd_mempool_strdup(cfg->cfg_pool,