[Feature] Add controller endpoint to get fuzzy hashes from messages

Sample usage:

```
curl -XPOST 'http://localhost:11334/plugins/fuzzy/hashes?flag=1' --data-binary '@-' < file
```

Sample output:
```json
{
  "hashes": {
    "local": [
      "24b6e7de2f489778d828c827079c48bacb086f816d0a7acabbe42e8d0da703b89b913176ad67eefaf5b54fa59f5e0ecfc7015846c4043fcfb0c7a4ed7a235025",
      "72789777cbec926f4143de4c08c87acc3fbf3b909b5c39f1edcf82ed12e2d8bc2f56be8d68ee681feccf44ca04e3eca5b8ec039cb84a0d40e22258c370a10cbb"
    ],
    "rspamd.com": [
      "24b6e7de2f489778d828c827079c48bacb086f816d0a7acabbe42e8d0da703b89b913176ad67eefaf5b54fa59f5e0ecfc7015846c4043fcfb0c7a4ed7a235025",
      "72789777cbec926f4143de4c08c87acc3fbf3b909b5c39f1edcf82ed12e2d8bc2f56be8d68ee681feccf44ca04e3eca5b8ec039cb84a0d40e22258c370a10cbb"
    ],
  },
  "success": true
}
```

Issue: #4489

diff --git a/rules/controller/fuzzy.lua b/rules/controller/fuzzy.lua
new file mode 100644 (file)
index 0000000..7e4c96f
--- /dev/null
+++ b/rules/controller/fuzzy.lua
@@ -0,0 +1,46 @@
+--[[
+Copyright (c) 2023, Vsevolod Stakhov <vsevolod@rspamd.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local function handle_gen_fuzzy(task, conn, req_params)
+  if type(rspamd_plugins.fuzzy_check) == 'table' then
+    local ret, hashes
+    task:process_message()
+    if req_params.rule then
+      ret,hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, req_params.rule)
+    elseif req_params.flag then
+      ret,hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, tonumber(req_params.flag))
+    else
+      conn:send_error(404, 'missing rule or flag')
+      return
+    end
+
+    if ret then
+      conn:send_ucl({success = true, hashes = hashes})
+    else
+      conn:send_error(500, 'cannot generate hashes')
+    end
+  else
+    conn:send_error(404, 'fuzzy_check is not enabled')
+  end
+end
+
+return {
+  hashes = {
+    handler = handle_gen_fuzzy,
+    need_task = true,
+    enable = false
+  },
+}
\ No newline at end of file

diff --git a/rules/controller/init.lua b/rules/controller/init.lua
index 90bb137c49d5f9f0b9fd9cedcfea655813f264a8..9d60200f8774b9a666b7235e9207b674fb2909e6 100644 (file)
--- a/rules/controller/init.lua
+++ b/rules/controller/init.lua
@@ -28,6 +28,7 @@ local controller_plugin_paths = {
   maps = dofile(local_rules .. "/controller/maps.lua"),
   neural = dofile(local_rules .. "/controller/neural.lua"),
   selectors = dofile(local_rules .. "/controller/selectors.lua"),
+  fuzzy = dofile(local_rules .. "/controller/fuzzy.lua"),
 }
 
 if rspamd_util.file_exists(local_conf .. '/controller.lua') then

diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index 842094c54d6253c5f218d1eb55218e836fe975d1..fd8e56cce40d7457852ac3cf965f2c1648f617e2 100644 (file)
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -194,6 +194,7 @@ static gint fuzzy_attach_controller (struct module_ctx *ctx,
 static gint fuzzy_lua_learn_handler (lua_State *L);
 static gint fuzzy_lua_unlearn_handler (lua_State *L);
 static gint fuzzy_lua_gen_hashes_handler (lua_State *L);
+static gint fuzzy_lua_hex_hashes_handler (lua_State *L);
 
 module_t fuzzy_check_module = {
                "fuzzy_check",
@@ -1217,6 +1218,9 @@ fuzzy_check_module_config (struct rspamd_config *cfg, bool validate)
                lua_pushstring (L, "gen_hashes");
                lua_pushcfunction (L, fuzzy_lua_gen_hashes_handler);
                lua_settable (L, -3);
+               lua_pushstring (L, "hex_hashes");
+               lua_pushcfunction (L, fuzzy_lua_hex_hashes_handler);
+               lua_settable (L, -3);
                /* Finish fuzzy_check key */
                lua_settable (L, -3);
        }
@@ -4058,6 +4062,107 @@ fuzzy_lua_gen_hashes_handler (lua_State *L)
        return 1;
 }
 
+static gint
+fuzzy_lua_hex_hashes_handler (lua_State *L)
+{
+       struct rspamd_task *task = lua_check_task (L, 1);
+
+       if (task == NULL) {
+               return luaL_error(L, "invalid arguments");
+       }
+
+       guint flag = 0, weight = 1, send_flags = 0;
+       const gchar *symbol;
+       struct fuzzy_ctx *fuzzy_module_ctx = fuzzy_get_context (task->cfg);
+       struct fuzzy_rule *rule;
+       GPtrArray *commands;
+       gint i;
+
+       if (lua_type (L, 2) == LUA_TNUMBER) {
+               flag = lua_tonumber (L, 2);
+       }
+       else if (lua_type (L, 2) == LUA_TSTRING) {
+               struct fuzzy_rule *rule;
+               GHashTableIter it;
+               gpointer k, v;
+               struct fuzzy_mapping *map;
+
+               symbol = lua_tostring (L, 2);
+
+               PTR_ARRAY_FOREACH (fuzzy_module_ctx->fuzzy_rules, i, rule) {
+                       if (flag != 0) {
+                               break;
+                       }
+
+                       g_hash_table_iter_init (&it, rule->mappings);
+
+                       while (g_hash_table_iter_next (&it, &k, &v)) {
+                               map = v;
+
+                               if (g_ascii_strcasecmp (symbol, map->symbol) == 0) {
+                                       flag = map->fuzzy_flag;
+                                       break;
+                               }
+                       }
+               }
+       }
+
+       if (flag == 0) {
+               return luaL_error (L, "bad flag");
+       }
+
+       lua_createtable (L, 0, fuzzy_module_ctx->fuzzy_rules->len);
+
+       PTR_ARRAY_FOREACH (fuzzy_module_ctx->fuzzy_rules, i, rule) {
+               /* Check for flag */
+               if (g_hash_table_lookup (rule->mappings,
+                       GINT_TO_POINTER (flag)) == NULL) {
+                       msg_debug_task ("skip rule %s as it has no flag %d defined"
+                                                  " false", rule->name, flag);
+                       continue;
+               }
+
+               commands = fuzzy_generate_commands (task, rule, FUZZY_CHECK, flag,
+                       weight, send_flags);
+
+               lua_pushstring (L, rule->name);
+
+               if (commands != NULL) {
+                       lua_createtable (L, commands->len, 0);
+                       /*
+                        * We have all commands cached, so we can just read their cached value to
+                        * get hex hashes
+                        */
+                       struct rspamd_mime_part *mp;
+                       gint j, part_idx = 1;
+
+                       PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), j, mp) {
+                               struct rspamd_cached_shingles *cached;
+
+                               cached = fuzzy_cmd_get_cached(rule, task, mp);
+
+                               if (cached) {
+                                       gchar hexbuf[rspamd_cryptobox_HASHBYTES * 2 + 1];
+                                       gint r = rspamd_encode_hex_buf (cached->digest, sizeof(cached->digest), hexbuf,
+                                                       sizeof (hexbuf));
+                                       lua_pushlstring (L, hexbuf, r);
+                                       lua_rawseti(L, -2, part_idx++);
+                               }
+                       }
+
+                       g_ptr_array_free (commands, TRUE);
+               }
+               else {
+                       lua_pushnil(L);
+               }
+
+               /* res[rule->name] = {hex_hash1, ..., hex_hashn} */
+               lua_settable(L, -3);
+       }
+
+       return 1;
+}
+
 static gboolean
 fuzzy_add_handler (struct rspamd_http_connection_entry *conn_ent,
        struct rspamd_http_message *msg, struct module_ctx *ctx)