From: Vsevolod Stakhov Date: Sat, 20 May 2023 14:22:43 +0000 (+0100) Subject: [Feature] Add controller endpoint to get fuzzy hashes from messages X-Git-Tag: 3.6~111 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=cac66961924d22e2454db745ccef118a08ea6184;p=rspamd.git [Feature] Add controller endpoint to get fuzzy hashes from messages Sample usage: ``` curl -XPOST 'http://localhost:11334/plugins/fuzzy/hashes?flag=1' --data-binary '@-' < file ``` Sample output: ```json { "hashes": { "local": [ "24b6e7de2f489778d828c827079c48bacb086f816d0a7acabbe42e8d0da703b89b913176ad67eefaf5b54fa59f5e0ecfc7015846c4043fcfb0c7a4ed7a235025", "72789777cbec926f4143de4c08c87acc3fbf3b909b5c39f1edcf82ed12e2d8bc2f56be8d68ee681feccf44ca04e3eca5b8ec039cb84a0d40e22258c370a10cbb" ], "rspamd.com": [ "24b6e7de2f489778d828c827079c48bacb086f816d0a7acabbe42e8d0da703b89b913176ad67eefaf5b54fa59f5e0ecfc7015846c4043fcfb0c7a4ed7a235025", "72789777cbec926f4143de4c08c87acc3fbf3b909b5c39f1edcf82ed12e2d8bc2f56be8d68ee681feccf44ca04e3eca5b8ec039cb84a0d40e22258c370a10cbb" ], }, "success": true } ``` Issue: #4489 --- diff --git a/rules/controller/fuzzy.lua b/rules/controller/fuzzy.lua new file mode 100644 index 000000000..7e4c96fe1 --- /dev/null +++ b/rules/controller/fuzzy.lua @@ -0,0 +1,46 @@ +--[[ +Copyright (c) 2023, Vsevolod Stakhov + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local function handle_gen_fuzzy(task, conn, req_params) + if type(rspamd_plugins.fuzzy_check) == 'table' then + local ret, hashes + task:process_message() + if req_params.rule then + ret,hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, req_params.rule) + elseif req_params.flag then + ret,hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, tonumber(req_params.flag)) + else + conn:send_error(404, 'missing rule or flag') + return + end + + if ret then + conn:send_ucl({success = true, hashes = hashes}) + else + conn:send_error(500, 'cannot generate hashes') + end + else + conn:send_error(404, 'fuzzy_check is not enabled') + end +end + +return { + hashes = { + handler = handle_gen_fuzzy, + need_task = true, + enable = false + }, +} \ No newline at end of file diff --git a/rules/controller/init.lua b/rules/controller/init.lua index 90bb137c4..9d60200f8 100644 --- a/rules/controller/init.lua +++ b/rules/controller/init.lua @@ -28,6 +28,7 @@ local controller_plugin_paths = { maps = dofile(local_rules .. "/controller/maps.lua"), neural = dofile(local_rules .. "/controller/neural.lua"), selectors = dofile(local_rules .. "/controller/selectors.lua"), + fuzzy = dofile(local_rules .. "/controller/fuzzy.lua"), } if rspamd_util.file_exists(local_conf .. '/controller.lua') then diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 842094c54..fd8e56cce 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -194,6 +194,7 @@ static gint fuzzy_attach_controller (struct module_ctx *ctx, static gint fuzzy_lua_learn_handler (lua_State *L); static gint fuzzy_lua_unlearn_handler (lua_State *L); static gint fuzzy_lua_gen_hashes_handler (lua_State *L); +static gint fuzzy_lua_hex_hashes_handler (lua_State *L); module_t fuzzy_check_module = { "fuzzy_check", @@ -1217,6 +1218,9 @@ fuzzy_check_module_config (struct rspamd_config *cfg, bool validate) lua_pushstring (L, "gen_hashes"); lua_pushcfunction (L, fuzzy_lua_gen_hashes_handler); lua_settable (L, -3); + lua_pushstring (L, "hex_hashes"); + lua_pushcfunction (L, fuzzy_lua_hex_hashes_handler); + lua_settable (L, -3); /* Finish fuzzy_check key */ lua_settable (L, -3); } @@ -4058,6 +4062,107 @@ fuzzy_lua_gen_hashes_handler (lua_State *L) return 1; } +static gint +fuzzy_lua_hex_hashes_handler (lua_State *L) +{ + struct rspamd_task *task = lua_check_task (L, 1); + + if (task == NULL) { + return luaL_error(L, "invalid arguments"); + } + + guint flag = 0, weight = 1, send_flags = 0; + const gchar *symbol; + struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context (task->cfg); + struct fuzzy_rule *rule; + GPtrArray *commands; + gint i; + + if (lua_type (L, 2) == LUA_TNUMBER) { + flag = lua_tonumber (L, 2); + } + else if (lua_type (L, 2) == LUA_TSTRING) { + struct fuzzy_rule *rule; + GHashTableIter it; + gpointer k, v; + struct fuzzy_mapping *map; + + symbol = lua_tostring (L, 2); + + PTR_ARRAY_FOREACH (fuzzy_module_ctx->fuzzy_rules, i, rule) { + if (flag != 0) { + break; + } + + g_hash_table_iter_init (&it, rule->mappings); + + while (g_hash_table_iter_next (&it, &k, &v)) { + map = v; + + if (g_ascii_strcasecmp (symbol, map->symbol) == 0) { + flag = map->fuzzy_flag; + break; + } + } + } + } + + if (flag == 0) { + return luaL_error (L, "bad flag"); + } + + lua_createtable (L, 0, fuzzy_module_ctx->fuzzy_rules->len); + + PTR_ARRAY_FOREACH (fuzzy_module_ctx->fuzzy_rules, i, rule) { + /* Check for flag */ + if (g_hash_table_lookup (rule->mappings, + GINT_TO_POINTER (flag)) == NULL) { + msg_debug_task ("skip rule %s as it has no flag %d defined" + " false", rule->name, flag); + continue; + } + + commands = fuzzy_generate_commands (task, rule, FUZZY_CHECK, flag, + weight, send_flags); + + lua_pushstring (L, rule->name); + + if (commands != NULL) { + lua_createtable (L, commands->len, 0); + /* + * We have all commands cached, so we can just read their cached value to + * get hex hashes + */ + struct rspamd_mime_part *mp; + gint j, part_idx = 1; + + PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), j, mp) { + struct rspamd_cached_shingles *cached; + + cached = fuzzy_cmd_get_cached(rule, task, mp); + + if (cached) { + gchar hexbuf[rspamd_cryptobox_HASHBYTES * 2 + 1]; + gint r = rspamd_encode_hex_buf (cached->digest, sizeof(cached->digest), hexbuf, + sizeof (hexbuf)); + lua_pushlstring (L, hexbuf, r); + lua_rawseti(L, -2, part_idx++); + } + } + + g_ptr_array_free (commands, TRUE); + } + else { + lua_pushnil(L); + } + + /* res[rule->name] = {hex_hash1, ..., hex_hashn} */ + lua_settable(L, -3); + } + + return 1; +} + static gboolean fuzzy_add_handler (struct rspamd_http_connection_entry *conn_ent, struct rspamd_http_message *msg, struct module_ctx *ctx)