Merge pull request #1455 from fatalbanana/tr

[Minor] URL tags: fix restoring of tags

1 files changed, 50 insertions, 22 deletions

diff --git a/src/plugins/lua/url_tags.lua b/src/plugins/lua/url_tags.lua
index 4718d23d2..8cdc22a6c 100644
--- a/src/plugins/lua/url_tags.lua
+++ b/src/plugins/lua/url_tags.lua
@@ -25,6 +25,8 @@ local settings = {
   expire = 3600, -- 1 hour
   -- prefix for redis keys
   key_prefix = 'Ut.',
+  -- tags in this list are not persisted
+  ignore_tags = {},
 }
 
 local rspamd_logger = require "rspamd_logger"
@@ -174,11 +176,13 @@ local function tags_save(task)
         tags[tld] = {}
       end
       for ut, utv in pairs(utags) do
-        if not tags[tld][ut] then
-          tags[tld][ut] = {}
-        end
-        for _, e in ipairs(utv) do
-          tags[tld][ut][e] = true
+        if not settings.ignore_tags[ut] then
+          if not tags[tld][ut] then
+            tags[tld][ut] = {}
+          end
+          for _, e in ipairs(utv) do
+            tags[tld][ut][e] = true
+          end
         end
       end
     end
@@ -228,7 +232,7 @@ local function tags_save(task)
     for tag, mtags in pairs(domtags) do
       local tmp2 = {}
       for k in pairs(mtags) do
-        table.insert(tmp2, k)
+        table.insert(tmp2, tostring(rspamd_util.encode_base32(k)))
       end
       tmp[tag] = tmp2
     end
@@ -283,22 +287,26 @@ local function tags_restore(task)
     for i = 1, d_len do
       if type(data[i]) == 'string' then
         local tld = tld_reverse[i]
-        for time, tag, meta in string.gmatch(data[i], '(%d+)|([^|]+)|(.+)') do
-          if (time + settings.expire) > now then
-            local metatags = {}
-            for m in string.gmatch(meta, '[^,]+') do
-              table.insert(metatags, m)
-            end
-            for _, idx in ipairs(tlds[tld]) do
-              if not tracking[tld] then
-                tracking[tld] = {}
-              end
-              if not tracking[tld][tag] then
-                tracking[tld][tag] = {}
-              end
-              for _, ttag in ipairs(metatags) do
-                urls[idx]:add_tag(tag, ttag, mpool)
-                tracking[tld][tag][ttag] = true
+        for goo in string.gmatch(data[i], '[^/]+') do
+          for time, tag, meta in string.gmatch(goo, '(%d+)|([^|]+)|(.+)') do
+            if not settings.ignore_tags[tag] then
+              if (time + settings.expire) > now then
+                local metatags = {}
+                for m in string.gmatch(meta, '[^,]+') do
+                  table.insert(metatags, m)
+                end
+                for _, idx in ipairs(tlds[tld]) do
+                  if not tracking[tld] then
+                    tracking[tld] = {}
+                  end
+                  if not tracking[tld][tag] then
+                    tracking[tld][tag] = {}
+                  end
+                  for _, ttag in ipairs(metatags) do
+                    urls[idx]:add_tag(tag, tostring(rspamd_util.decode_base32(ttag)), mpool)
+                    tracking[tld][tag][ttag] = true
+                  end
+                end
               end
             end
           end
@@ -344,6 +352,26 @@ end
 for k, v in pairs(opts) do
   settings[k] = v
 end
+local function list_to_hash(list)
+  if type(list) == 'table' then
+    if list[1] then
+      local h = {}
+      for _, e in ipairs(list) do
+        h[e] = true
+      end
+      return h
+    else
+      return list
+    end
+  elseif type(list) == 'string' then
+    local h = {}
+    h[list] = true
+    return h
+  else
+    return {}
+  end
+end
+settings.ignore_tags = list_to_hash(settings.ignore_tags)
 
 rspamd_config:add_on_load(function(cfg, ev_base, worker)
   if not (worker:get_name() == 'normal' and worker:get_index() == 0) then return end