From 2d0ee54a1295245b8c207da554b91ff2e17b1d13 Mon Sep 17 00:00:00 2001 From: Andrew Lewis Date: Fri, 24 Feb 2017 12:06:09 +0200 Subject: [Minor] URL tags: fix restoring of tags --- src/plugins/lua/url_tags.lua | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/plugins/lua/url_tags.lua b/src/plugins/lua/url_tags.lua index 4718d23d2..346e5a11a 100644 --- a/src/plugins/lua/url_tags.lua +++ b/src/plugins/lua/url_tags.lua @@ -283,22 +283,24 @@ local function tags_restore(task) for i = 1, d_len do if type(data[i]) == 'string' then local tld = tld_reverse[i] - for time, tag, meta in string.gmatch(data[i], '(%d+)|([^|]+)|(.+)') do - if (time + settings.expire) > now then - local metatags = {} - for m in string.gmatch(meta, '[^,]+') do - table.insert(metatags, m) - end - for _, idx in ipairs(tlds[tld]) do - if not tracking[tld] then - tracking[tld] = {} - end - if not tracking[tld][tag] then - tracking[tld][tag] = {} + for goo in string.gmatch(data[i], '[^/]+') do + for time, tag, meta in string.gmatch(goo, '(%d+)|([^|]+)|(.+)') do + if (time + settings.expire) > now then + local metatags = {} + for m in string.gmatch(meta, '[^,]+') do + table.insert(metatags, m) end - for _, ttag in ipairs(metatags) do - urls[idx]:add_tag(tag, ttag, mpool) - tracking[tld][tag][ttag] = true + for _, idx in ipairs(tlds[tld]) do + if not tracking[tld] then + tracking[tld] = {} + end + if not tracking[tld][tag] then + tracking[tld][tag] = {} + end + for _, ttag in ipairs(metatags) do + urls[idx]:add_tag(tag, ttag, mpool) + tracking[tld][tag][ttag] = true + end end end end -- cgit v1.2.3 From 4049a334514f42d09048014be96c42e3865b1b5a Mon Sep 17 00:00:00 2001 From: Andrew Lewis Date: Fri, 24 Feb 2017 12:53:37 +0200 Subject: [Minor] URL tags: support not persisting certain tags --- src/plugins/lua/url_tags.lua | 64 +++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/plugins/lua/url_tags.lua b/src/plugins/lua/url_tags.lua index 346e5a11a..774386e66 100644 --- a/src/plugins/lua/url_tags.lua +++ b/src/plugins/lua/url_tags.lua @@ -25,6 +25,8 @@ local settings = { expire = 3600, -- 1 hour -- prefix for redis keys key_prefix = 'Ut.', + -- tags in this list are not persisted + ignore_tags = {}, } local rspamd_logger = require "rspamd_logger" @@ -174,11 +176,13 @@ local function tags_save(task) tags[tld] = {} end for ut, utv in pairs(utags) do - if not tags[tld][ut] then - tags[tld][ut] = {} - end - for _, e in ipairs(utv) do - tags[tld][ut][e] = true + if not settings.ignore_tags[ut] then + if not tags[tld][ut] then + tags[tld][ut] = {} + end + for _, e in ipairs(utv) do + tags[tld][ut][e] = true + end end end end @@ -285,21 +289,23 @@ local function tags_restore(task) local tld = tld_reverse[i] for goo in string.gmatch(data[i], '[^/]+') do for time, tag, meta in string.gmatch(goo, '(%d+)|([^|]+)|(.+)') do - if (time + settings.expire) > now then - local metatags = {} - for m in string.gmatch(meta, '[^,]+') do - table.insert(metatags, m) - end - for _, idx in ipairs(tlds[tld]) do - if not tracking[tld] then - tracking[tld] = {} - end - if not tracking[tld][tag] then - tracking[tld][tag] = {} + if not settings.ignore_tags[tag] then + if (time + settings.expire) > now then + local metatags = {} + for m in string.gmatch(meta, '[^,]+') do + table.insert(metatags, m) end - for _, ttag in ipairs(metatags) do - urls[idx]:add_tag(tag, ttag, mpool) - tracking[tld][tag][ttag] = true + for _, idx in ipairs(tlds[tld]) do + if not tracking[tld] then + tracking[tld] = {} + end + if not tracking[tld][tag] then + tracking[tld][tag] = {} + end + for _, ttag in ipairs(metatags) do + urls[idx]:add_tag(tag, ttag, mpool) + tracking[tld][tag][ttag] = true + end end end end @@ -346,6 +352,26 @@ end for k, v in pairs(opts) do settings[k] = v end +local function list_to_hash(list) + if type(list) == 'table' then + if list[1] then + local h = {} + for _, e in ipairs(list) do + h[e] = true + end + return h + else + return list + end + elseif type(list) == 'string' then + local h = {} + h[list] = true + return h + else + return {} + end +end +settings.ignore_tags = list_to_hash(settings.ignore_tags) rspamd_config:add_on_load(function(cfg, ev_base, worker) if not (worker:get_name() == 'normal' and worker:get_index() == 0) then return end -- cgit v1.2.3 From 4082b890a71c6dbf56f57db1962aba9abe42e70a Mon Sep 17 00:00:00 2001 From: Andrew Lewis Date: Fri, 24 Feb 2017 13:19:46 +0200 Subject: [Minor] URL tags: use base32 encoding --- src/plugins/lua/url_tags.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/plugins/lua/url_tags.lua b/src/plugins/lua/url_tags.lua index 774386e66..8cdc22a6c 100644 --- a/src/plugins/lua/url_tags.lua +++ b/src/plugins/lua/url_tags.lua @@ -232,7 +232,7 @@ local function tags_save(task) for tag, mtags in pairs(domtags) do local tmp2 = {} for k in pairs(mtags) do - table.insert(tmp2, k) + table.insert(tmp2, tostring(rspamd_util.encode_base32(k))) end tmp[tag] = tmp2 end @@ -303,7 +303,7 @@ local function tags_restore(task) tracking[tld][tag] = {} end for _, ttag in ipairs(metatags) do - urls[idx]:add_tag(tag, ttag, mpool) + urls[idx]:add_tag(tag, tostring(rspamd_util.decode_base32(ttag)), mpool) tracking[tld][tag][ttag] = true end end -- cgit v1.2.3