From 7e6bec3c411cf368f37c1701141a5f65895b741a Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@rspamd.com>
Date: Wed, 11 Dec 2024 13:55:03 +0000
Subject: [Rework] Multimap should use only distinct text parts for content
 matching

Issue: #5248
---
 lualib/lua_mime.lua | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

(limited to 'lualib')

diff --git a/lualib/lua_mime.lua b/lualib/lua_mime.lua
index 1135f2b63..f68758ec9 100644
--- a/lualib/lua_mime.lua
+++ b/lualib/lua_mime.lua
@@ -956,6 +956,49 @@ exports.get_displayed_text_part = function(task)
   return html_part or text_part
 end
 
+--[[[
+-- @function lua_mime.get_distinct_text_parts(task)
+-- Returns the list of parts that are visible or have a distinct content
+-- @param {task} task Rspamd task object
+-- @return array of {text_part} a selected part
+--]]
+exports.get_distinct_text_parts = function(task)
+  local text_parts = task:get_text_parts()
+  if not text_parts then
+    return {}
+  end
+
+  local text_part_idx
+
+  local distance = task:get_mempool():get_variable('parts_distance', 'double')
+  if not distance then
+    return text_parts
+  end
+  distance = tonumber(distance)
+
+  if distance > 0.5 then
+    -- Parts are distinct
+    return text_parts
+  end
+
+  -- First pass: categorize parts
+  for i, part in ipairs(text_parts) do
+    local mp = part:get_mimepart()
+    if not mp:is_attachment() then
+      if not part:is_html() then
+        -- Found text part that is similar to html part
+        text_part_idx = i
+      end
+    end
+  end
+
+  if text_part_idx then
+    table.remove(text_parts, text_part_idx)
+  end
+
+  return text_parts
+end
+
 --[[[
 -- @function lua_mime.anonymize_message(task, settings)
 -- Anonymizes message content by replacing sensitive data
-- 
cgit v1.2.3