From c203493c90484c9fba6102338a63c36e62511718 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 8 Nov 2023 14:34:33 +0000 Subject: [Feature] Reiterate on gtube patterns --- src/libmime/message.c | 16 +++++++--------- src/libserver/cfg_file.h | 38 ++++++++++++++++++++++---------------- src/libserver/cfg_rcl.cxx | 35 +++++++++++++++++++++++++++++------ src/libserver/cfg_utils.cxx | 1 + 4 files changed, 59 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/src/libmime/message.c b/src/libmime/message.c index 327b546e7..0c11f5075 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -1,11 +1,11 @@ -/*- - * Copyright 2016 Vsevolod Stakhov +/* + * Copyright 2023 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -638,7 +638,7 @@ rspamd_multipattern_gtube_cb(struct rspamd_multipattern *mp, struct rspamd_task *task = (struct rspamd_task *) context; if (strnum > 0) { - if (task->cfg->enable_test_patterns) { + if (task->cfg->gtube_patterns_policy == RSPAMD_GTUBE_ALL) { return strnum + 1; } @@ -656,7 +656,7 @@ rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part) enum rspamd_action_type act = METRIC_ACTION_NOACTION; g_assert(part != NULL); - if (gtube_matcher == NULL) { + if (gtube_matcher == NULL && task->cfg->gtube_patterns_policy != RSPAMD_GTUBE_DISABLED) { gtube_matcher = rspamd_multipattern_create(RSPAMD_MULTIPATTERN_DEFAULT); rspamd_multipattern_add_pattern(gtube_matcher, @@ -683,7 +683,8 @@ rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part) } if (part->utf_content.len >= sizeof(gtube_pattern_reject) && - part->utf_content.len <= max_check_size) { + part->utf_content.len <= max_check_size && + task->cfg->gtube_patterns_policy != RSPAMD_GTUBE_DISABLED) { if ((ret = rspamd_multipattern_lookup(gtube_matcher, part->utf_content.begin, part->utf_content.len, rspamd_multipattern_gtube_cb, task, NULL)) > 0) { @@ -693,15 +694,12 @@ rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part) act = METRIC_ACTION_REJECT; break; case 2: - g_assert(task->cfg->enable_test_patterns); act = METRIC_ACTION_ADD_HEADER; break; case 3: - g_assert(task->cfg->enable_test_patterns); act = METRIC_ACTION_REWRITE_SUBJECT; break; case 4: - g_assert(task->cfg->enable_test_patterns); act = METRIC_ACTION_NOACTION; break; } diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index 22f754938..4cb87d974 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -312,6 +312,12 @@ enum rspamd_config_settings_policy { RSPAMD_SETTINGS_POLICY_IMPLICIT_DENY = 2, }; +enum rspamd_gtube_patterns_policy { + RSPAMD_GTUBE_DISABLED = 0, /* Disabled */ + RSPAMD_GTUBE_REJECT, /* Reject message with GTUBE pattern */ + RSPAMD_GTUBE_ALL /* Check all GTUBE like patterns */ +}; + struct rspamd_config_settings_elt { guint32 id; enum rspamd_config_settings_policy policy; @@ -344,22 +350,22 @@ struct rspamd_config { GHashTable *groups; /**< groups of symbols */ void *actions; /**< all actions of the metric (opaque type) */ - gboolean one_shot_mode; /**< rules add only one symbol */ - gboolean check_text_attachements; /**< check text attachements as text */ - gboolean check_all_filters; /**< check all filters */ - gboolean allow_raw_input; /**< scan messages with invalid mime */ - gboolean disable_hyperscan; /**< disable hyperscan usage */ - gboolean vectorized_hyperscan; /**< use vectorized hyperscan matching */ - gboolean enable_shutdown_workaround; /**< enable workaround for legacy SA clients (exim) */ - gboolean ignore_received; /**< Ignore data from the first received header */ - gboolean enable_sessions_cache; /**< Enable session cache for debug */ - gboolean enable_experimental; /**< Enable experimental plugins */ - gboolean disable_pcre_jit; /**< Disable pcre JIT */ - gboolean own_lua_state; /**< True if we have created lua_state internally */ - gboolean soft_reject_on_timeout; /**< If true emit soft reject on task timeout (if not reject) */ - gboolean public_groups_only; /**< Output merely public groups everywhere */ - gboolean enable_test_patterns; /**< Enable test patterns */ - gboolean enable_css_parser; /**< Enable css parsing in HTML */ + gboolean one_shot_mode; /**< rules add only one symbol */ + gboolean check_text_attachements; /**< check text attachements as text */ + gboolean check_all_filters; /**< check all filters */ + gboolean allow_raw_input; /**< scan messages with invalid mime */ + gboolean disable_hyperscan; /**< disable hyperscan usage */ + gboolean vectorized_hyperscan; /**< use vectorized hyperscan matching */ + gboolean enable_shutdown_workaround; /**< enable workaround for legacy SA clients (exim) */ + gboolean ignore_received; /**< Ignore data from the first received header */ + gboolean enable_sessions_cache; /**< Enable session cache for debug */ + gboolean enable_experimental; /**< Enable experimental plugins */ + gboolean disable_pcre_jit; /**< Disable pcre JIT */ + gboolean own_lua_state; /**< True if we have created lua_state internally */ + gboolean soft_reject_on_timeout; /**< If true emit soft reject on task timeout (if not reject) */ + gboolean public_groups_only; /**< Output merely public groups everywhere */ + enum rspamd_gtube_patterns_policy gtube_patterns_policy; /**< Enable test patterns */ + gboolean enable_css_parser; /**< Enable css parsing in HTML */ gsize max_cores_size; /**< maximum size occupied by rspamd core files */ gsize max_cores_count; /**< maximum number of core files */ diff --git a/src/libserver/cfg_rcl.cxx b/src/libserver/cfg_rcl.cxx index 3f6ca2c56..fbfcbcd34 100644 --- a/src/libserver/cfg_rcl.cxx +++ b/src/libserver/cfg_rcl.cxx @@ -347,6 +347,35 @@ rspamd_rcl_options_handler(rspamd_mempool_t *pool, const ucl_object_t *obj, } } + const auto *gtube_patterns = ucl_object_lookup(obj, "gtube_patterns"); + if (gtube_patterns != nullptr && ucl_object_type(gtube_patterns) == UCL_STRING) { + const auto *gtube_st = ucl_object_tostring(gtube_patterns); + + if (g_ascii_strcasecmp(gtube_st, "all") == 0) { + cfg->gtube_patterns_policy = RSPAMD_GTUBE_ALL; + } + else if (g_ascii_strcasecmp(gtube_st, "reject") == 0) { + cfg->gtube_patterns_policy = RSPAMD_GTUBE_REJECT; + } + else if (g_ascii_strcasecmp(gtube_st, "disable") == 0) { + cfg->gtube_patterns_policy = RSPAMD_GTUBE_DISABLED; + } + else { + g_set_error(err, + CFG_RCL_ERROR, + EINVAL, + "invalid GTUBE patterns policy: %s", + gtube_st); + return FALSE; + } + } + else if (auto *enable_test_patterns = ucl_object_lookup(obj, "enable_test_patterns"); enable_test_patterns != nullptr) { + /* Legacy setting */ + if (!!ucl_object_toboolean(enable_test_patterns)) { + cfg->gtube_patterns_policy = RSPAMD_GTUBE_ALL; + } + } + if (rspamd_rcl_section_parse_defaults(cfg, *section, cfg->cfg_pool, obj, cfg, err)) { @@ -1876,12 +1905,6 @@ rspamd_rcl_config_init(struct rspamd_config *cfg, GHashTable *skip_sections) G_STRUCT_OFFSET(struct rspamd_config, public_groups_only), 0, "Output merely public groups everywhere"); - rspamd_rcl_add_default_handler(sub, - "enable_test_patterns", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET(struct rspamd_config, enable_test_patterns), - 0, - "Enable test GTUBE like patterns (not for production!)"); rspamd_rcl_add_default_handler(sub, "enable_css_parser", rspamd_rcl_parse_struct_boolean, diff --git a/src/libserver/cfg_utils.cxx b/src/libserver/cfg_utils.cxx index 416d52e05..e22cbe9e3 100644 --- a/src/libserver/cfg_utils.cxx +++ b/src/libserver/cfg_utils.cxx @@ -286,6 +286,7 @@ rspamd_config_new(enum rspamd_config_init_flags flags) cfg->max_recipients = 1024; cfg->max_blas_threads = 1; cfg->max_opts_len = 4096; + cfg->gtube_patterns_policy = RSPAMD_GTUBE_REJECT; /* Default log line */ cfg->log_format_str = rspamd_mempool_strdup(cfg->cfg_pool, -- cgit v1.2.3 From e6d7103b9e5496e9628df7b1367c326133b13809 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 8 Nov 2023 14:51:46 +0000 Subject: [Minor] Fix unit tests --- src/libmime/message.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/libmime/message.c b/src/libmime/message.c index 0c11f5075..3acc935e6 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -654,9 +654,10 @@ rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part) static const gsize max_check_size = 8 * 1024; gint ret; enum rspamd_action_type act = METRIC_ACTION_NOACTION; + enum rspamd_gtube_patterns_policy policy = task->cfg ? task->cfg->gtube_patterns_policy : RSPAMD_GTUBE_REJECT; g_assert(part != NULL); - if (gtube_matcher == NULL && task->cfg->gtube_patterns_policy != RSPAMD_GTUBE_DISABLED) { + if (gtube_matcher == NULL && policy != RSPAMD_GTUBE_DISABLED) { gtube_matcher = rspamd_multipattern_create(RSPAMD_MULTIPATTERN_DEFAULT); rspamd_multipattern_add_pattern(gtube_matcher, @@ -684,7 +685,7 @@ rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part) if (part->utf_content.len >= sizeof(gtube_pattern_reject) && part->utf_content.len <= max_check_size && - task->cfg->gtube_patterns_policy != RSPAMD_GTUBE_DISABLED) { + policy != RSPAMD_GTUBE_DISABLED) { if ((ret = rspamd_multipattern_lookup(gtube_matcher, part->utf_content.begin, part->utf_content.len, rspamd_multipattern_gtube_cb, task, NULL)) > 0) { -- cgit v1.2.3 From 3e228133bb945d5c1a3a381365b3c3a3d552098b Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Thu, 9 Nov 2023 13:45:36 +0000 Subject: [Minor] Document `gtube_patterns` --- src/libserver/cfg_rcl.cxx | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/libserver/cfg_rcl.cxx b/src/libserver/cfg_rcl.cxx index fbfcbcd34..b43014a1c 100644 --- a/src/libserver/cfg_rcl.cxx +++ b/src/libserver/cfg_rcl.cxx @@ -31,6 +31,7 @@ #include #include +#include // for std::transform #include #include "contrib/ankerl/unordered_dense.h" #include "fmt/core.h" @@ -349,15 +350,21 @@ rspamd_rcl_options_handler(rspamd_mempool_t *pool, const ucl_object_t *obj, const auto *gtube_patterns = ucl_object_lookup(obj, "gtube_patterns"); if (gtube_patterns != nullptr && ucl_object_type(gtube_patterns) == UCL_STRING) { - const auto *gtube_st = ucl_object_tostring(gtube_patterns); + auto gtube_st = std::string{ucl_object_tostring(gtube_patterns)}; + std::transform(gtube_st.begin(), gtube_st.end(), gtube_st.begin(), [](const auto c) -> int { + if (c <= 'Z' && c >= 'A') + return c - ('Z' - 'z'); + return c; + }); + - if (g_ascii_strcasecmp(gtube_st, "all") == 0) { + if (gtube_st == "all") { cfg->gtube_patterns_policy = RSPAMD_GTUBE_ALL; } - else if (g_ascii_strcasecmp(gtube_st, "reject") == 0) { + else if (gtube_st == "reject") { cfg->gtube_patterns_policy = RSPAMD_GTUBE_REJECT; } - else if (g_ascii_strcasecmp(gtube_st, "disable") == 0) { + else if (gtube_st == "disabled" || gtube_st == "disable") { cfg->gtube_patterns_policy = RSPAMD_GTUBE_DISABLED; } else { @@ -365,7 +372,7 @@ rspamd_rcl_options_handler(rspamd_mempool_t *pool, const ucl_object_t *obj, CFG_RCL_ERROR, EINVAL, "invalid GTUBE patterns policy: %s", - gtube_st); + gtube_st.c_str()); return FALSE; } } @@ -2183,6 +2190,16 @@ rspamd_rcl_config_init(struct rspamd_config *cfg, GHashTable *skip_sections) 0, "Events backend to use: kqueue, epoll, select, poll or auto (default: auto)"); + rspamd_rcl_add_doc_by_path(cfg, + "options", + "Swtich mode of gtube patterns: disable, reject, all", + "gtube_patterns", + UCL_STRING, + nullptr, + 0, + "reject", + 0); + /* Neighbours configuration */ rspamd_rcl_add_section_doc(&top, sub, "neighbours", "name", rspamd_rcl_neighbours_handler, -- cgit v1.2.3