1 files changed, 51 insertions, 18 deletions

diff --git a/lualib/lua_util.lua b/lualib/lua_util.lua
index 8f24b2871..842f079b2 100644
--- a/lualib/lua_util.lua
+++ b/lualib/lua_util.lua
@@ -84,9 +84,17 @@ end
 exports.rspamd_str_split = rspamd_str_split
 exports.str_split = rspamd_str_split
 
-exports.rspamd_str_trim = function(s)
+local function rspamd_str_trim(s)
   return match(ptrim, s)
 end
+exports.rspamd_str_trim = rspamd_str_trim
+--[[[
+-- @function lua_util.str_trim(text)
+-- Returns a string with no trailing and leading spaces
+-- @param {string} text input text
+-- @return {string} string with no trailing and leading spaces
+--]]
+exports.str_trim = rspamd_str_trim
 
 --[[[
 -- @function lua_util.round(number, decimalPlaces)
@@ -640,8 +648,9 @@ exports.filter_specific_urls = function (urls, params)
     if params.prefix then
       cache_key = params.prefix
     else
-      cache_key = string.format('sp_urls_%d%s', params.limit,
-          tostring(params.need_emails or false))
+      cache_key = string.format('sp_urls_%d%s%s', params.limit,
+          tostring(params.need_emails or false),
+          tostring(params.need_images or false))
     end
     local cached = params.task:cache_get(cache_key)
 
@@ -654,14 +663,6 @@ exports.filter_specific_urls = function (urls, params)
 
   if params.filter then urls = fun.totable(fun.filter(params.filter, urls)) end
 
-  if #urls <= params.limit and #urls <= params.esld_limit then
-    if params.task and not params.no_cache then
-      params.task:cache_set(cache_key, urls)
-    end
-
-    return urls
-  end
-
   -- Filter by tld:
   local tlds = {}
   local eslds = {}
@@ -701,6 +702,16 @@ exports.filter_specific_urls = function (urls, params)
       end
     end
 
+    if flags.image then
+      if not params.need_images then
+        -- Ignore url
+        return
+      else
+        -- Penalise images in urls
+        priority = 0
+      end
+    end
+
     local esld = u:get_tld()
     local str_hash = tostring(u)
 
@@ -843,6 +854,7 @@ end
 - - filter <callback> (default = nil)
 - - prefix <string> cache prefix (default = nil)
 - - ignore_redirected <bool> (default = false)
+- - need_images <bool> (default = false)
 -- }
 -- Apply heuristic in extracting of urls from task, this function
 -- tries its best to extract specific number of urls from a task based on
@@ -854,6 +866,7 @@ exports.extract_specific_urls = function(params_or_task, lim, need_emails, filte
     limit = 9999,
     esld_limit = 9999,
     need_emails = false,
+    need_images = false,
     filter = nil,
     prefix = nil,
     ignore_ip = false,
@@ -875,10 +888,10 @@ exports.extract_specific_urls = function(params_or_task, lim, need_emails, filte
     }
   end
   for k,v in pairs(default_params) do
-    if not params[k] then params[k] = v end
+    if type(params[k]) == 'nil' and v ~= nil then params[k] = v end
   end
 
-  local urls = params.task:get_urls(params.need_emails)
+  local urls = params.task:get_urls(params.need_emails, params.need_images)
 
   return exports.filter_specific_urls(urls, params)
 end
@@ -965,6 +978,14 @@ exports.init_debug_logging = function(config)
   end
 end
 
+exports.enable_debug_logging = function()
+  unconditional_debug = true
+end
+
+exports.disable_debug_logging = function()
+  unconditional_debug = false
+end
+
 --[[[
 -- @function lua_util.debugm(module, [log_object], format, ...)
 -- Performs fast debug log for a specific module
@@ -1207,26 +1228,38 @@ end
 
 ---[[[
 -- @function lua_util.table_digest(t)
--- Returns hash of all values if t[1] is string or all keys otherwise
+-- Returns hash of all values if t[1] is string or all keys/values otherwise
 -- @param {table} t input array or map
 -- @return {string} base32 representation of blake2b hash of all strings
 --]]]
-exports.table_digest = function(t)
+local function table_digest(t)
   local cr = require "rspamd_cryptobox_hash"
   local h = cr.create()
 
   if t[1] then
     for _,e in ipairs(t) do
-      h:update(tostring(e))
+      if type(e) == 'table' then
+        h:update(table_digest(e))
+      else
+        h:update(tostring(e))
+      end
     end
   else
-    for k,_ in pairs(t) do
-      h:update(k)
+    for k,v in pairs(t) do
+      h:update(tostring(k))
+
+      if type(v) == 'string' then
+        h:update(v)
+      elseif type(v) == 'table' then
+        h:update(table_digest(v))
+      end
     end
   end
  return h:base32()
 end
 
+exports.table_digest = table_digest
+
 ---[[[
 -- @function lua_util.toboolean(v)
 -- Converts a string or a number to boolean