- /*-
- * Copyright 2016 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- #include "config.h"
- #include "mem_pool.h"
- #include "scan_result.h"
- #include "rspamd.h"
- #include "message.h"
- #include "lua/lua_common.h"
- #include "libserver/cfg_file_private.h"
- #include "libmime/scan_result_private.h"
- #include "contrib/fastutf8/fastutf8.h"
- #include <math.h>
- #include "contrib/uthash/utlist.h"
-
- #define msg_debug_metric(...) rspamd_conditional_debug_fast (NULL, NULL, \
- rspamd_metric_log_id, "metric", task->task_pool->tag.uid, \
- G_STRFUNC, \
- __VA_ARGS__)
-
- INIT_LOG_MODULE(metric)
-
- /* Average symbols count to optimize hash allocation */
- static struct rspamd_counter_data symbols_count;
-
- static void
- rspamd_scan_result_dtor (gpointer d)
- {
- struct rspamd_scan_result *r = (struct rspamd_scan_result *)d;
- struct rspamd_symbol_result sres;
-
- rspamd_set_counter_ema (&symbols_count, kh_size (r->symbols), 0.5);
-
- kh_foreach_value (r->symbols, sres, {
- if (sres.options) {
- kh_destroy (rspamd_options_hash, sres.options);
- }
- });
- kh_destroy (rspamd_symbols_hash, r->symbols);
- kh_destroy (rspamd_symbols_group_hash, r->sym_groups);
- }
-
- struct rspamd_scan_result *
- rspamd_create_metric_result (struct rspamd_task *task)
- {
- struct rspamd_scan_result *metric_res;
- guint i;
-
- metric_res = task->result;
-
- if (metric_res != NULL) {
- return metric_res;
- }
-
- metric_res = rspamd_mempool_alloc0 (task->task_pool,
- sizeof (struct rspamd_scan_result));
- metric_res->symbols = kh_init (rspamd_symbols_hash);
- metric_res->sym_groups = kh_init (rspamd_symbols_group_hash);
-
- /* Optimize allocation */
- kh_resize (rspamd_symbols_group_hash, metric_res->sym_groups, 4);
-
- if (symbols_count.mean > 4) {
- kh_resize (rspamd_symbols_hash, metric_res->symbols, symbols_count.mean);
- }
- else {
- kh_resize (rspamd_symbols_hash, metric_res->symbols, 4);
- }
-
- if (task->cfg) {
- struct rspamd_action *act, *tmp;
-
- metric_res->actions_limits = rspamd_mempool_alloc0 (task->task_pool,
- sizeof (struct rspamd_action_result) * HASH_COUNT (task->cfg->actions));
- i = 0;
-
- HASH_ITER (hh, task->cfg->actions, act, tmp) {
- if (!(act->flags & RSPAMD_ACTION_NO_THRESHOLD)) {
- metric_res->actions_limits[i].cur_limit = act->threshold;
- }
- metric_res->actions_limits[i].action = act;
-
- i ++;
- }
-
- metric_res->nactions = i;
- }
-
- rspamd_mempool_add_destructor (task->task_pool,
- rspamd_scan_result_dtor,
- metric_res);
-
- return metric_res;
- }
-
- static inline int
- rspamd_pr_sort (const struct rspamd_passthrough_result *pra,
- const struct rspamd_passthrough_result *prb)
- {
- return prb->priority - pra->priority;
- }
-
- void
- rspamd_add_passthrough_result (struct rspamd_task *task,
- struct rspamd_action *action,
- guint priority,
- double target_score,
- const gchar *message,
- const gchar *module,
- guint flags)
- {
- struct rspamd_scan_result *metric_res;
- struct rspamd_passthrough_result *pr;
-
- metric_res = task->result;
-
- pr = rspamd_mempool_alloc (task->task_pool, sizeof (*pr));
- pr->action = action;
- pr->priority = priority;
- pr->message = message;
- pr->module = module;
- pr->target_score = target_score;
- pr->flags = flags;
-
- DL_APPEND (metric_res->passthrough_result, pr);
- DL_SORT (metric_res->passthrough_result, rspamd_pr_sort);
-
- if (!isnan (target_score)) {
-
- msg_info_task ("<%s>: set pre-result to '%s' %s(%.2f): '%s' from %s(%d)",
- MESSAGE_FIELD_CHECK (task, message_id), action->name,
- flags & RSPAMD_PASSTHROUGH_LEAST ? "*least " : "",
- target_score,
- message, module, priority);
- }
- else {
- msg_info_task ("<%s>: set pre-result to '%s' %s(no score): '%s' from %s(%d)",
- MESSAGE_FIELD_CHECK (task, message_id), action->name,
- flags & RSPAMD_PASSTHROUGH_LEAST ? "*least " : "",
- message, module, priority);
- }
- }
-
- static inline gdouble
- rspamd_check_group_score (struct rspamd_task *task,
- const gchar *symbol,
- struct rspamd_symbols_group *gr,
- gdouble *group_score,
- gdouble w)
- {
- if (gr != NULL && group_score && gr->max_score > 0.0 && w > 0.0) {
- if (*group_score >= gr->max_score && w > 0) {
- msg_info_task ("maximum group score %.2f for group %s has been reached,"
- " ignoring symbol %s with weight %.2f", gr->max_score,
- gr->name, symbol, w);
- return NAN;
- }
- else if (*group_score + w > gr->max_score) {
- w = gr->max_score - *group_score;
- }
- }
-
- return w;
- }
-
- #ifndef DBL_EPSILON
- #define DBL_EPSILON 2.2204460492503131e-16
- #endif
-
- static struct rspamd_symbol_result *
- insert_metric_result (struct rspamd_task *task,
- const gchar *symbol,
- double weight,
- const gchar *opt,
- enum rspamd_symbol_insert_flags flags)
- {
- struct rspamd_scan_result *metric_res;
- struct rspamd_symbol_result *s = NULL;
- gdouble final_score, *gr_score = NULL, next_gf = 1.0, diff;
- struct rspamd_symbol *sdef;
- struct rspamd_symbols_group *gr = NULL;
- const ucl_object_t *mobj, *sobj;
- gint max_shots, ret;
- guint i;
- khiter_t k;
- gboolean single = !!(flags & RSPAMD_SYMBOL_INSERT_SINGLE);
- gchar *sym_cpy;
-
- metric_res = task->result;
-
- if (!isfinite (weight)) {
- msg_warn_task ("detected %s score for symbol %s, replace it with zero",
- isnan (weight) ? "NaN" : "infinity", symbol);
- weight = 0.0;
- }
-
- msg_debug_metric ("want to insert symbol %s, initial weight %.2f",
- symbol, weight);
-
- sdef = g_hash_table_lookup (task->cfg->symbols, symbol);
- if (sdef == NULL) {
- if (flags & RSPAMD_SYMBOL_INSERT_ENFORCE) {
- final_score = 1.0 * weight; /* Enforce static weight to 1.0 */
- }
- else {
- final_score = 0.0;
- }
-
- msg_debug_metric ("no symbol definition for %s; final multiplier %.2f",
- symbol, final_score);
- }
- else {
- if (sdef->cache_item) {
- /* Check if we can insert this symbol at all */
- if (!rspamd_symcache_is_item_allowed (task, sdef->cache_item, FALSE)) {
- msg_debug_metric ("symbol %s is not allowed to be inserted due to settings",
- symbol);
- return NULL;
- }
- }
-
- final_score = (*sdef->weight_ptr) * weight;
-
- PTR_ARRAY_FOREACH (sdef->groups, i, gr) {
- k = kh_get (rspamd_symbols_group_hash, metric_res->sym_groups, gr);
-
- if (k == kh_end (metric_res->sym_groups)) {
- k = kh_put (rspamd_symbols_group_hash, metric_res->sym_groups,
- gr, &ret);
- kh_value (metric_res->sym_groups, k) = 0;
- }
- }
-
- msg_debug_metric ("metric multiplier for %s is %.2f",
- symbol, *sdef->weight_ptr);
- }
-
- if (task->settings) {
- gdouble corr;
- mobj = ucl_object_lookup (task->settings, "scores");
-
- if (!mobj) {
- /* Legacy */
- mobj = task->settings;
- }
- else {
- msg_debug_metric ("found scores in the settings");
- }
-
- sobj = ucl_object_lookup (mobj, symbol);
- if (sobj != NULL && ucl_object_todouble_safe (sobj, &corr)) {
- msg_debug_metric ("settings: changed weight of symbol %s from %.2f "
- "to %.2f * %.2f",
- symbol, final_score, corr, weight);
- final_score = corr * weight;
- }
- }
-
- k = kh_get (rspamd_symbols_hash, metric_res->symbols, symbol);
- if (k != kh_end (metric_res->symbols)) {
- /* Existing metric score */
- s = &kh_value (metric_res->symbols, k);
- if (single) {
- max_shots = 1;
- }
- else {
- if (sdef) {
- max_shots = sdef->nshots;
- }
- else {
- max_shots = task->cfg->default_max_shots;
- }
- }
-
- msg_debug_metric ("nshots: %d for symbol %s", max_shots, symbol);
-
- if (!single && (max_shots > 0 && (s->nshots >= max_shots))) {
- single = TRUE;
- }
-
- s->nshots ++;
-
- if (opt) {
- rspamd_task_add_result_option (task, s, opt, strlen (opt));
- }
-
- /* Adjust diff */
- if (!single) {
- diff = final_score;
- msg_debug_metric ("symbol %s can be inserted multiple times: %.2f weight",
- symbol, diff);
- }
- else {
- if (fabs (s->score) < fabs (final_score) &&
- signbit (s->score) == signbit (final_score)) {
- /* Replace less significant weight with a more significant one */
- diff = final_score - s->score;
- msg_debug_metric ("symbol %s can be inserted single time;"
- " weight adjusted %.2f + %.2f",
- symbol, s->score, diff);
- }
- else {
- diff = 0;
- }
- }
-
- if (diff) {
- /* Handle grow factor */
- if (metric_res->grow_factor && diff > 0) {
- diff *= metric_res->grow_factor;
- next_gf *= task->cfg->grow_factor;
- }
- else if (diff > 0) {
- next_gf = task->cfg->grow_factor;
- }
-
- msg_debug_metric ("adjust grow factor to %.2f for symbol %s (%.2f final)",
- next_gf, symbol, diff);
-
- if (sdef) {
- PTR_ARRAY_FOREACH (sdef->groups, i, gr) {
- gdouble cur_diff;
-
- k = kh_get (rspamd_symbols_group_hash,
- metric_res->sym_groups, gr);
- g_assert (k != kh_end (metric_res->sym_groups));
- gr_score = &kh_value (metric_res->sym_groups, k);
- cur_diff = rspamd_check_group_score (task, symbol, gr,
- gr_score, diff);
-
- if (isnan (cur_diff)) {
- /* Limit reached, do not add result */
- msg_debug_metric (
- "group limit %.2f is reached for %s when inserting symbol %s;"
- " drop score %.2f",
- *gr_score, gr->name, symbol, diff);
-
- diff = NAN;
- break;
- } else if (gr_score) {
- *gr_score += cur_diff;
-
- if (cur_diff < diff) {
- /* Reduce */
- msg_debug_metric (
- "group limit %.2f is reached for %s when inserting symbol %s;"
- " reduce score %.2f - %.2f",
- *gr_score, gr->name, symbol, diff, cur_diff);
- diff = cur_diff;
- }
- }
- }
- }
-
- if (!isnan (diff)) {
- metric_res->score += diff;
- metric_res->grow_factor = next_gf;
-
- if (single) {
- msg_debug_metric ("final score for single symbol %s = %.2f; %.2f diff",
- symbol, final_score, diff);
- s->score = final_score;
- } else {
- msg_debug_metric ("increase final score for multiple symbol %s += %.2f = %.2f",
- symbol, s->score, diff);
- s->score += diff;
- }
- }
- }
- }
- else {
- /* New result */
- sym_cpy = rspamd_mempool_strdup (task->task_pool, symbol);
- k = kh_put (rspamd_symbols_hash, metric_res->symbols,
- sym_cpy, &ret);
- g_assert (ret > 0);
- s = &kh_value (metric_res->symbols, k);
- memset (s, 0, sizeof (*s));
-
- /* Handle grow factor */
- if (metric_res->grow_factor && final_score > 0) {
- final_score *= metric_res->grow_factor;
- next_gf *= task->cfg->grow_factor;
- }
- else if (final_score > 0) {
- next_gf = task->cfg->grow_factor;
- }
-
- msg_debug_metric ("adjust grow factor to %.2f for symbol %s (%.2f final)",
- next_gf, symbol, final_score);
-
- s->name = sym_cpy;
- s->sym = sdef;
- s->nshots = 1;
-
- if (sdef) {
- /* Check group limits */
- PTR_ARRAY_FOREACH (sdef->groups, i, gr) {
- gdouble cur_score;
-
- k = kh_get (rspamd_symbols_group_hash, metric_res->sym_groups, gr);
- g_assert (k != kh_end (metric_res->sym_groups));
- gr_score = &kh_value (metric_res->sym_groups, k);
- cur_score = rspamd_check_group_score (task, symbol, gr,
- gr_score, final_score);
-
- if (isnan (cur_score)) {
- /* Limit reached, do not add result */
- msg_debug_metric (
- "group limit %.2f is reached for %s when inserting symbol %s;"
- " drop score %.2f",
- *gr_score, gr->name, symbol, final_score);
- final_score = NAN;
- break;
- } else if (gr_score) {
- *gr_score += cur_score;
-
- if (cur_score < final_score) {
- /* Reduce */
- msg_debug_metric (
- "group limit %.2f is reached for %s when inserting symbol %s;"
- " reduce score %.2f - %.2f",
- *gr_score, gr->name, symbol, final_score, cur_score);
- final_score = cur_score;
- }
- }
- }
- }
-
- if (!isnan (final_score)) {
- const double epsilon = DBL_EPSILON;
-
- metric_res->score += final_score;
- metric_res->grow_factor = next_gf;
- s->score = final_score;
-
- if (final_score > epsilon) {
- metric_res->npositive ++;
- metric_res->positive_score += final_score;
- }
- else if (final_score < -epsilon) {
- metric_res->nnegative ++;
- metric_res->negative_score += fabs (final_score);
- }
- }
- else {
- s->score = 0;
- }
-
- if (opt) {
- rspamd_task_add_result_option (task, s, opt, strlen (opt));
- }
- }
-
- msg_debug_metric ("final insertion for symbol %s, score %.2f, factor: %f",
- symbol,
- s->score,
- final_score);
-
- return s;
- }
-
- struct rspamd_symbol_result *
- rspamd_task_insert_result_full (struct rspamd_task *task,
- const gchar *symbol,
- double weight,
- const gchar *opt,
- enum rspamd_symbol_insert_flags flags)
- {
- struct rspamd_symbol_result *s = NULL;
-
- if (task->processed_stages & (RSPAMD_TASK_STAGE_IDEMPOTENT >> 1)) {
- msg_err_task ("cannot insert symbol %s on idempotent phase",
- symbol);
-
- return NULL;
- }
-
- /* Insert symbol to default metric */
- s = insert_metric_result (task,
- symbol,
- weight,
- opt,
- flags);
-
- /* Process cache item */
- if (s && task->cfg->cache && s->sym) {
- rspamd_symcache_inc_frequency (task->cfg->cache, s->sym->cache_item);
- }
-
- return s;
- }
-
- static gchar *
- rspamd_task_option_safe_copy (struct rspamd_task *task,
- const gchar *val,
- gsize vlen,
- gsize *outlen)
- {
- const gchar *p, *end;
-
- p = val;
- end = val + vlen;
- vlen = 0; /* Reuse */
-
- while (p < end) {
- if (*p & 0x80) {
- UChar32 uc;
- gint off = 0;
-
- U8_NEXT (p, off, end - p, uc);
-
- if (uc > 0) {
- if (u_isprint (uc)) {
- vlen += off;
- }
- else {
- /* We will replace it with 0xFFFD */
- vlen += MAX (off, 3);
- }
- }
- else {
- vlen += MAX (off, 3);
- }
-
- p += off;
- }
- else if (!g_ascii_isprint (*p)) {
- /* Another 0xFFFD */
- vlen += 3;
- p ++;
- }
- else {
- p ++;
- vlen ++;
- }
- }
-
- gchar *dest, *d;
-
- dest = rspamd_mempool_alloc (task->task_pool, vlen + 1);
- d = dest;
- p = val;
-
- while (p < end) {
- if (*p & 0x80) {
- UChar32 uc;
- gint off = 0;
-
- U8_NEXT (p, off, end - p, uc);
-
- if (uc > 0) {
- if (u_isprint (uc)) {
- memcpy (d, p, off);
- d += off;
- }
- else {
- /* We will replace it with 0xFFFD */
- *d++ = '\357';
- *d++ = '\277';
- *d++ = '\275';
- }
- }
- else {
- *d++ = '\357';
- *d++ = '\277';
- *d++ = '\275';
- }
-
- p += off;
- }
- else if (!g_ascii_isprint (*p)) {
- /* Another 0xFFFD */
- *d++ = '\357';
- *d++ = '\277';
- *d++ = '\275';
- p ++;
- }
- else {
- *d++ = *p++;
- }
- }
-
- *d = '\0';
- *(outlen) = d - dest;
-
- return dest;
- }
-
- gboolean
- rspamd_task_add_result_option (struct rspamd_task *task,
- struct rspamd_symbol_result *s,
- const gchar *val,
- gsize vlen)
- {
- struct rspamd_symbol_option *opt, srch;
- gboolean ret = FALSE;
- gchar *opt_cpy = NULL;
- gsize cpy_len;
- khiter_t k;
- gint r;
-
- if (s && val) {
- if (s->opts_len < 0) {
- /* Cannot add more options, give up */
- msg_debug_task ("cannot add more options to symbol %s when adding option %s",
- s->name, val);
- return FALSE;
- }
-
- if (!s->options) {
- s->options = kh_init (rspamd_options_hash);
- }
-
- if (vlen + s->opts_len > task->cfg->max_opts_len) {
- /* Add truncated option */
- msg_info_task ("cannot add more options to symbol %s when adding option %s",
- s->name, val);
- val = "...";
- vlen = 3;
- s->opts_len = -1;
- }
-
- if (!(s->sym && (s->sym->flags & RSPAMD_SYMBOL_FLAG_ONEPARAM)) &&
- kh_size (s->options) < task->cfg->default_max_shots) {
- opt_cpy = rspamd_task_option_safe_copy (task, val, vlen, &cpy_len);
- /* Append new options */
- srch.option = (gchar *)opt_cpy;
- srch.optlen = cpy_len;
- k = kh_get (rspamd_options_hash, s->options, &srch);
-
- if (k == kh_end (s->options)) {
- opt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*opt));
- opt->optlen = cpy_len;
- opt->option = opt_cpy;
-
- kh_put (rspamd_options_hash, s->options, opt, &r);
- DL_APPEND (s->opts_head, opt);
-
- ret = TRUE;
- }
- }
- else {
- /* Skip addition */
- ret = FALSE;
- }
-
- if (ret && s->opts_len >= 0) {
- s->opts_len += vlen;
- }
- }
- else if (!val) {
- ret = TRUE;
- }
-
- return ret;
- }
-
- struct rspamd_action*
- rspamd_check_action_metric (struct rspamd_task *task)
- {
- struct rspamd_action_result *action_lim,
- *noaction = NULL;
- struct rspamd_action *selected_action = NULL, *least_action = NULL;
- struct rspamd_passthrough_result *pr;
- double max_score = -(G_MAXDOUBLE), sc;
- int i;
- struct rspamd_scan_result *mres = task->result;
- gboolean seen_least = FALSE;
-
- if (mres->passthrough_result != NULL) {
- DL_FOREACH (mres->passthrough_result, pr) {
- if (!seen_least || !(pr->flags & RSPAMD_PASSTHROUGH_LEAST)) {
- sc = pr->target_score;
- selected_action = pr->action;
-
- if (!(pr->flags & RSPAMD_PASSTHROUGH_LEAST)) {
- if (!isnan (sc)) {
- if (pr->action->action_type == METRIC_ACTION_NOACTION) {
- mres->score = MIN (sc, mres->score);
- }
- else {
- mres->score = sc;
- }
- }
-
- return selected_action;
- }
- else {
- seen_least = true;
- least_action = selected_action;
-
- if (isnan (sc)) {
-
- if (selected_action->flags & RSPAMD_ACTION_NO_THRESHOLD) {
- /*
- * In this case, we have a passthrough action that
- * is `least` action, however, there is no threshold
- * on it.
- *
- * Hence, we imply the following logic:
- *
- * - we leave score unchanged
- * - we apply passthrough no threshold action unless
- * score based action *is not* reject, otherwise
- * we apply reject action
- */
- }
- else {
- sc = selected_action->threshold;
- max_score = sc;
- }
- }
- else {
- max_score = sc;
- }
- }
- }
- }
- }
- /* We are not certain about the results during processing */
-
- /*
- * Select result by score
- */
- for (i = mres->nactions - 1; i >= 0; i--) {
- action_lim = &mres->actions_limits[i];
- sc = action_lim->cur_limit;
-
- if (action_lim->action->action_type == METRIC_ACTION_NOACTION) {
- noaction = action_lim;
- }
-
- if (isnan (sc) ||
- (action_lim->action->flags & (RSPAMD_ACTION_NO_THRESHOLD|RSPAMD_ACTION_HAM))) {
- continue;
- }
-
- if (mres->score >= sc && sc > max_score) {
- selected_action = action_lim->action;
- max_score = sc;
- }
- }
-
- if (selected_action == NULL) {
- selected_action = noaction->action;
- }
-
- if (selected_action) {
-
- if (seen_least) {
-
- if (least_action->flags & RSPAMD_ACTION_NO_THRESHOLD) {
- if (selected_action->action_type != METRIC_ACTION_REJECT &&
- selected_action->action_type != METRIC_ACTION_DISCARD) {
- /* Override score based action with least action */
- selected_action = least_action;
- }
- }
- else {
- /* Adjust score if needed */
- mres->score = MAX (max_score, mres->score);
- }
- }
-
- return selected_action;
- }
-
- return noaction->action;
- }
-
- struct rspamd_symbol_result*
- rspamd_task_find_symbol_result (struct rspamd_task *task, const char *sym)
- {
- struct rspamd_symbol_result *res = NULL;
- khiter_t k;
-
-
- if (task->result) {
- k = kh_get (rspamd_symbols_hash, task->result->symbols, sym);
-
- if (k != kh_end (task->result->symbols)) {
- res = &kh_value (task->result->symbols, k);
- }
- }
-
- return res;
- }
-
- void
- rspamd_task_symbol_result_foreach (struct rspamd_task *task,
- GHFunc func,
- gpointer ud)
- {
- const gchar *kk;
- struct rspamd_symbol_result res;
-
- if (func && task->result) {
- kh_foreach (task->result->symbols, kk, res, {
- func ((gpointer)kk, (gpointer)&res, ud);
- });
- }
- }
|