From 661f634db96c2c049439b5bd7afae5af232c6ae7 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 26 Apr 2017 17:13:10 +0100 Subject: [PATCH] [Rework] Remove multiple metrics support from Rspamd --- src/libmime/filter.c | 11 ++-- src/libmime/filter.h | 4 +- src/libmime/message.c | 2 +- src/libserver/composites.c | 9 +-- src/libserver/protocol.c | 25 +++++---- src/libserver/roll_history.c | 3 +- src/libserver/symbols_cache.c | 26 +++------ src/libserver/task.c | 9 +-- src/libserver/task.h | 4 +- src/libstat/stat_process.c | 4 +- src/lua/lua_task.c | 103 ++++++++++------------------------ 11 files changed, 71 insertions(+), 129 deletions(-) diff --git a/src/libmime/filter.c b/src/libmime/filter.c index 4a1b8bcbc..b16018507 100644 --- a/src/libmime/filter.c +++ b/src/libmime/filter.c @@ -26,19 +26,20 @@ #define COMMON_PART_FACTOR 95 struct rspamd_metric_result * -rspamd_create_metric_result (struct rspamd_task *task, const gchar *name) +rspamd_create_metric_result (struct rspamd_task *task) { struct rspamd_metric_result *metric_res; struct rspamd_metric *metric; guint i; - metric_res = g_hash_table_lookup (task->results, name); + metric_res = task->result; if (metric_res != NULL) { return metric_res; } - metric = g_hash_table_lookup (task->cfg->metrics, name); + metric = task->cfg->default_metric; + if (metric == NULL) { return NULL; } @@ -64,8 +65,6 @@ rspamd_create_metric_result (struct rspamd_task *task, const gchar *name) } metric_res->action = METRIC_ACTION_MAX; - g_hash_table_insert (task->results, (gpointer) metric->name, - metric_res); return metric_res; } @@ -108,7 +107,7 @@ insert_metric_result (struct rspamd_task *task, const ucl_object_t *mobj, *sobj; gint max_shots; - metric_res = rspamd_create_metric_result (task, metric->name); + metric_res = rspamd_create_metric_result (task); sdef = g_hash_table_lookup (metric->symbols, symbol); if (sdef == NULL) { diff --git a/src/libmime/filter.h b/src/libmime/filter.h index 3d8fa38a8..49904962f 100644 --- a/src/libmime/filter.h +++ b/src/libmime/filter.h @@ -47,11 +47,9 @@ struct rspamd_metric_result { /** * Create or return existing result for the specified metric name * @param task task object - * @param name name of metric * @return metric result or NULL if metric `name` has not been found */ -struct rspamd_metric_result * rspamd_create_metric_result (struct rspamd_task *task, - const gchar *name); +struct rspamd_metric_result * rspamd_create_metric_result (struct rspamd_task *task); /** * Insert a result to task diff --git a/src/libmime/message.c b/src/libmime/message.c index 0094afc6b..15483f089 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -544,7 +544,7 @@ rspamd_message_process_text_part (struct rspamd_task *task, if (rspamd_check_gtube (task, text_part)) { struct rspamd_metric_result *mres; - mres = rspamd_create_metric_result (task, DEFAULT_METRIC); + mres = rspamd_create_metric_result (task); if (mres != NULL) { mres->score = rspamd_task_get_required_score (task, mres); diff --git a/src/libserver/composites.c b/src/libserver/composites.c index bf12799aa..89281b117 100644 --- a/src/libserver/composites.c +++ b/src/libserver/composites.c @@ -366,12 +366,11 @@ composites_remove_symbols (gpointer key, gpointer value, gpointer data) } static void -composites_metric_callback (gpointer key, gpointer value, gpointer data) +composites_metric_callback (struct rspamd_metric_result *metric_res, + struct rspamd_task *task) { - struct rspamd_task *task = (struct rspamd_task *)data; struct composites_data *cd = rspamd_mempool_alloc (task->task_pool, sizeof (struct composites_data)); - struct rspamd_metric_result *metric_res = (struct rspamd_metric_result *)value; cd->task = task; cd->metric_res = (struct rspamd_metric_result *)metric_res; @@ -394,7 +393,9 @@ composites_metric_callback (gpointer key, gpointer value, gpointer data) void rspamd_make_composites (struct rspamd_task *task) { - g_hash_table_foreach (task->results, composites_metric_callback, task); + if (task->result) { + composites_metric_callback (task->result, task); + } } diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c index 4acf0202d..5a0a52de9 100644 --- a/src/libserver/protocol.c +++ b/src/libserver/protocol.c @@ -29,6 +29,12 @@ * described below */ #define MSG_CMD_CHECK "check" + +/* + * Modern check version + */ +#define MSG_CMD_CHECK_V2 "checkv2" + /* * Check if message is spam or not, and return score plus list * of symbols hit @@ -168,7 +174,10 @@ rspamd_protocol_handle_url (struct rspamd_task *task, case 'c': case 'C': /* check */ - if (g_ascii_strncasecmp (p, MSG_CMD_CHECK, pathlen) == 0) { + if (g_ascii_strncasecmp (p, MSG_CMD_CHECK_V2, pathlen) == 0) { + task->cmd = CMD_CHECK_V2; + } + else if (g_ascii_strncasecmp (p, MSG_CMD_CHECK, pathlen) == 0) { task->cmd = CMD_CHECK; } else { @@ -1116,13 +1125,8 @@ rspamd_protocol_write_ucl (struct rspamd_task *task, } if (flags & RSPAMD_PROTOCOL_METRICS) { - g_hash_table_iter_init (&hiter, task->results); - /* Convert results to an ucl object */ - while (g_hash_table_iter_next (&hiter, &h, &v)) { - metric_res = (struct rspamd_metric_result *)v; - obj = rspamd_metric_result_ucl (task, metric_res); - ucl_object_insert_key (top, obj, h, 0, false); - } + obj = rspamd_metric_result_ucl (task, task->result); + ucl_object_insert_key (top, obj, DEFAULT_METRIC, 0, false); } if (flags & RSPAMD_PROTOCOL_MESSAGES) { @@ -1328,7 +1332,8 @@ rspamd_protocol_http_reply (struct rspamd_http_message *msg, end: if (!(task->flags & RSPAMD_TASK_FLAG_NO_STAT)) { /* Update stat for default metric */ - metric_res = g_hash_table_lookup (task->results, DEFAULT_METRIC); + metric_res = task->result; + if (metric_res != NULL) { if (metric_res->action != METRIC_ACTION_MAX) { @@ -1515,7 +1520,7 @@ rspamd_protocol_write_log_pipe (struct rspamd_worker_ctx *ctx, if (lp->fd != -1) { switch (lp->type) { case RSPAMD_LOG_PIPE_SYMBOLS: - mres = g_hash_table_lookup (task->results, DEFAULT_METRIC); + mres = task->result; if (mres) { n = g_hash_table_size (mres->symbols); diff --git a/src/libserver/roll_history.c b/src/libserver/roll_history.c index ea79b590b..01aa9f88a 100644 --- a/src/libserver/roll_history.c +++ b/src/libserver/roll_history.c @@ -143,7 +143,8 @@ rspamd_roll_history_update (struct roll_history *history, } /* Get default metric */ - metric_res = g_hash_table_lookup (task->results, DEFAULT_METRIC); + metric_res = task->result; + if (metric_res == NULL) { row->symbols[0] = '\0'; row->action = METRIC_ACTION_NOACTION; diff --git a/src/libserver/symbols_cache.c b/src/libserver/symbols_cache.c index b0f63134a..51addf04f 100644 --- a/src/libserver/symbols_cache.c +++ b/src/libserver/symbols_cache.c @@ -1158,34 +1158,22 @@ rspamd_symbols_cache_metric_limit (struct rspamd_task *task, struct cache_savepoint *cp) { struct rspamd_metric_result *res; - GList *cur; - struct rspamd_metric *metric; double ms; if (task->flags & RSPAMD_TASK_FLAG_PASS_ALL) { return FALSE; } - cur = task->cfg->metrics_list; - if (cp->lim == 0.0) { - /* - * Look for metric that has the maximum reject score - */ - while (cur) { - metric = cur->data; - res = g_hash_table_lookup (task->results, metric->name); + res = task->result; - if (res) { - ms = rspamd_task_get_required_score (task, res); + if (res) { + ms = rspamd_task_get_required_score (task, res); - if (!isnan (ms) && cp->lim < ms) { - cp->rs = res; - cp->lim = ms; - } + if (!isnan (ms) && cp->lim < ms) { + cp->rs = res; + cp->lim = ms; } - - cur = g_list_next (cur); } } @@ -1493,7 +1481,7 @@ rspamd_symbols_cache_make_checkpoint (struct rspamd_task *task, checkpoint->pass = RSPAMD_CACHE_PASS_INIT; task->checkpoint = checkpoint; - rspamd_create_metric_result (task, DEFAULT_METRIC); + task->result = rspamd_create_metric_result (task); return checkpoint; } diff --git a/src/libserver/task.c b/src/libserver/task.c index 95e682eed..4a8b35b9c 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -83,11 +83,6 @@ rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg) new_task->task_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "task"); - new_task->results = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); - rspamd_mempool_add_destructor (new_task->task_pool, - (rspamd_mempool_destruct_t) g_hash_table_unref, - new_task->results); - new_task->raw_headers = g_hash_table_new_full (rspamd_strcase_hash, rspamd_strcase_equal, NULL, rspamd_ptr_array_free_hard); new_task->headers_order = g_queue_new (); @@ -987,7 +982,7 @@ rspamd_task_log_metric_res (struct rspamd_task *task, GPtrArray *sorted_symbols; guint i, j; - mres = g_hash_table_lookup (task->results, DEFAULT_METRIC); + mres = task->result; if (mres != NULL) { switch (lf->type) { @@ -1415,7 +1410,7 @@ rspamd_task_get_required_score (struct rspamd_task *task, struct rspamd_metric_r guint i; if (m == NULL) { - m = g_hash_table_lookup (task->results, DEFAULT_METRIC); + m = task->result; if (m == NULL) { return NAN; diff --git a/src/libserver/task.h b/src/libserver/task.h index e892e776f..549c49989 100644 --- a/src/libserver/task.h +++ b/src/libserver/task.h @@ -32,6 +32,7 @@ enum rspamd_command { CMD_SKIP, CMD_PING, CMD_PROCESS, + CMD_CHECK_V2, CMD_OTHER }; @@ -158,8 +159,7 @@ struct rspamd_task { GHashTable *emails; /**< list of parsed emails */ GHashTable *raw_headers; /**< list of raw headers */ GQueue *headers_order; /**< order of raw headers */ - GHashTable *results; /**< hash table of metric_result indexed by - * metric's name */ + struct rspamd_metric_result *result; /**< Metric result */ GHashTable *lua_cache; /**< cache of lua objects */ GPtrArray *tokens; /**< statistics tokens */ diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c index f5c19e664..74bf89d5f 100644 --- a/src/libstat/stat_process.c +++ b/src/libstat/stat_process.c @@ -1074,7 +1074,7 @@ rspamd_stat_check_autolearn (struct rspamd_task *task) * - We learn spam if action is ACTION_REJECT * - We learn ham if score is less than zero */ - mres = g_hash_table_lookup (task->results, DEFAULT_METRIC); + mres = task->result; if (mres) { @@ -1117,7 +1117,7 @@ rspamd_stat_check_autolearn (struct rspamd_task *task) spam_score = t; } - mres = g_hash_table_lookup (task->results, DEFAULT_METRIC); + mres = task->result; if (mres) { if (mres->score >= spam_score) { diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index 1e3a9ce39..01ca936dc 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -1207,7 +1207,7 @@ lua_task_set_pre_result (lua_State * L) if (action < METRIC_ACTION_MAX && action >= METRIC_ACTION_REJECT) { /* We also need to set the default metric to that result */ - mres = rspamd_create_metric_result (task, DEFAULT_METRIC); + mres = rspamd_create_metric_result (task); if (mres != NULL) { mres->score = mres->metric->actions[action].score; mres->action = action; @@ -2804,7 +2804,6 @@ lua_task_get_archives (lua_State *L) static inline gboolean lua_push_symbol_result (lua_State *L, struct rspamd_task *task, - struct rspamd_metric *metric, const gchar *symbol, struct rspamd_symbol_result *symbol_result, gboolean add_metric, @@ -2816,7 +2815,8 @@ lua_push_symbol_result (lua_State *L, gint j = 1, e = 4; if (!symbol_result) { - metric_res = g_hash_table_lookup (task->results, metric->name); + metric_res = task->result; + if (metric_res) { s = g_hash_table_lookup (metric_res->symbols, symbol); } @@ -2835,11 +2835,6 @@ lua_push_symbol_result (lua_State *L, lua_createtable (L, 0, e); - if (add_metric) { - lua_pushstring (L, "metric"); - lua_pushstring (L, metric->name); - lua_settable (L, -3); - } if (add_name) { lua_pushstring (L, "name"); lua_pushstring (L, symbol); @@ -2883,45 +2878,21 @@ lua_task_get_symbol (lua_State *L) { struct rspamd_task *task = lua_check_task (L, 1); const gchar *symbol; - struct rspamd_metric *metric; - GList *cur = NULL, *metric_list; gboolean found = FALSE; gint i = 1; symbol = luaL_checkstring (L, 2); if (task && symbol) { - metric_list = g_hash_table_lookup (task->cfg->metrics_symbols, symbol); - if (metric_list) { - lua_createtable (L, 1, 0); - cur = metric_list; - } - else { - metric = task->cfg->default_metric; - } + lua_createtable (L, 1, 0); - if (!cur && metric) { - lua_createtable (L, 1, 0); - - if ((found = lua_push_symbol_result (L, task, metric, symbol, - NULL, TRUE, FALSE))) { - lua_rawseti (L, -2, i++); - } - else { - /* Pop table */ - lua_pop (L, 1); - } + if ((found = lua_push_symbol_result (L, task, symbol, + NULL, TRUE, FALSE))) { + lua_rawseti (L, -2, i++); } else { - while (cur) { - metric = cur->data; - if (lua_push_symbol_result (L, task, metric, symbol, - NULL, TRUE, FALSE)) { - lua_rawseti (L, -2, i++); - found = TRUE; - } - cur = g_list_next (cur); - } + /* Pop table */ + lua_pop (L, 1); } } else { @@ -2946,7 +2917,7 @@ lua_task_has_symbol (lua_State *L) symbol = luaL_checkstring (L, 2); if (task && symbol) { - mres = g_hash_table_lookup (task->results, DEFAULT_METRIC); + mres = task->result; if (mres) { found = g_hash_table_lookup (mres->symbols, symbol) != NULL; @@ -2972,7 +2943,7 @@ lua_task_get_symbols (lua_State *L) struct rspamd_symbol_result *s; if (task) { - mres = g_hash_table_lookup (task->results, DEFAULT_METRIC); + mres = task->result; if (mres) { lua_createtable (L, g_hash_table_size (mres->symbols), 0); @@ -3004,7 +2975,6 @@ static gint lua_task_get_symbols_all (lua_State *L) { struct rspamd_task *task = lua_check_task (L, 1); - struct rspamd_metric *metric; struct rspamd_metric_result *mres; GHashTableIter it; gpointer k, v; @@ -3012,15 +2982,15 @@ lua_task_get_symbols_all (lua_State *L) gint i = 1; if (task) { - metric = task->cfg->default_metric; - mres = g_hash_table_lookup (task->results, DEFAULT_METRIC); + mres = task->result; + if (mres) { found = TRUE; lua_createtable (L, g_hash_table_size (mres->symbols), 0); g_hash_table_iter_init (&it, mres->symbols); while (g_hash_table_iter_next (&it, &k, &v)) { - lua_push_symbol_result (L, task, metric, k, v, FALSE, TRUE); + lua_push_symbol_result (L, task, k, v, FALSE, TRUE); lua_rawseti (L, -2, i++); } } @@ -3048,7 +3018,7 @@ lua_task_get_symbols_numeric (lua_State *L) struct rspamd_symbol_result *s; if (task) { - mres = g_hash_table_lookup (task->results, DEFAULT_METRIC); + mres = task->result; if (mres) { lua_createtable (L, g_hash_table_size (mres->symbols), 0); @@ -3511,10 +3481,10 @@ lua_task_set_settings (lua_State *L) if (act) { /* Adjust desired actions */ - mres = g_hash_table_lookup (task->results, DEFAULT_METRIC); + mres = task->result; if (mres == NULL) { - mres = rspamd_create_metric_result (task, DEFAULT_METRIC); + mres = rspamd_create_metric_result (task); } for (i = 0; i < METRIC_ACTION_MAX; i++) { @@ -3805,15 +3775,11 @@ static gint lua_task_get_metric_score (lua_State *L) { struct rspamd_task *task = lua_check_task (L, 1); - const gchar *metric_name; gdouble rs; struct rspamd_metric_result *metric_res; - metric_name = luaL_checkstring (L, 2); - - if (task && metric_name) { - if ((metric_res = - g_hash_table_lookup (task->results, metric_name)) != NULL) { + if (task) { + if ((metric_res = task->result) != NULL) { lua_createtable (L, 2, 0); lua_pushnumber (L, isnan (metric_res->score) ? 0.0 : metric_res->score); rs = rspamd_task_get_required_score (task, metric_res); @@ -3836,19 +3802,11 @@ static gint lua_task_get_metric_action (lua_State *L) { struct rspamd_task *task = lua_check_task (L, 1); - const gchar *metric_name; struct rspamd_metric_result *metric_res; enum rspamd_metric_action action; - metric_name = luaL_checkstring (L, 2); - - if (metric_name == NULL) { - metric_name = DEFAULT_METRIC; - } - - if (task && metric_name) { - if ((metric_res = - g_hash_table_lookup (task->results, metric_name)) != NULL) { + if (task) { + if ((metric_res = task->result) != NULL) { action = rspamd_check_action_metric (task, metric_res); lua_pushstring (L, rspamd_action_to_str (action)); } @@ -3867,20 +3825,18 @@ static gint lua_task_set_metric_score (lua_State *L) { struct rspamd_task *task = lua_check_task (L, 1); - const gchar *metric_name; struct rspamd_metric_result *metric_res; gdouble nscore; - metric_name = luaL_checkstring (L, 2); - nscore = luaL_checknumber (L, 3); - - if (metric_name == NULL) { - metric_name = DEFAULT_METRIC; + if (lua_isnumber (L, 2)) { + nscore = luaL_checknumber (L, 2); + } + else { + nscore = luaL_checknumber (L, 3); } - if (task && metric_name) { - if ((metric_res = - g_hash_table_lookup (task->results, metric_name)) != NULL) { + if (task) { + if ((metric_res = task->result) != NULL) { msg_debug_task ("set metric score from %.2f to %.2f", metric_res->score, nscore); metric_res->score = nscore; @@ -3914,8 +3870,7 @@ lua_task_set_metric_action (lua_State *L) action_name = luaL_checkstring (L, 3); if (task && metric_name && action_name) { - if ((metric_res = - g_hash_table_lookup (task->results, metric_name)) != NULL) { + if ((metric_res = task->result) != NULL) { if (rspamd_action_from_str (action_name, &action)) { metric_res->action = action; -- 2.39.5