From: Vsevolod Stakhov Date: Wed, 21 Aug 2019 14:40:49 +0000 (+0100) Subject: [Rework] Remove deprecated plugins X-Git-Tag: 2.0~366 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=e971c539fa763d3f632735ec59f1d5e50685467f;p=rspamd.git [Rework] Remove deprecated plugins --- diff --git a/src/plugins/lua/url_reputation.lua b/src/plugins/lua/url_reputation.lua deleted file mode 100644 index e7d35697d..000000000 --- a/src/plugins/lua/url_reputation.lua +++ /dev/null @@ -1,384 +0,0 @@ ---[[ -Copyright (c) 2016-2017, Vsevolod Stakhov -Copyright (c) 2017, Andrew Lewis - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -if confighelp then - return -end - --- A plugin that restores/persists URL tags & calculates reputation - -local E = {} -local N = 'url_reputation' - -local whitelist, redis_params, redis_incr_script_id -local settings = { - expire = 86400, -- 1 day - key_prefix = 'Ur.', - symbols = { - white = 'URL_REPUTATION_WHITE', - black = 'URL_REPUTATION_BLACK', - grey = 'URL_REPUTATION_GREY', - neutral = 'URL_REPUTATION_NEUTRAL', - }, - foreign_symbols = { - dmarc = 'DMARC_POLICY_ALLOW', - dkim = 'R_DKIM_ALLOW', - spf = 'R_SPF_ALLOW', - }, - ignore_surbl = { - URIBL_BLOCKED = true, - DBL_PROHIBIT = true, - SURBL_BLOCKED = true, - }, - -- how many messages to score reputation - threshold = 5, - -- set reputation for only so many TLDs - update_limit = 1, - -- query dynamic reputation for up to so many TLDs - query_limit = 100, - -- try find most relevant URL - relevance = true, -} - -local scale = { - 'white', -- 1 - 'neutral', -- 2 - 'grey', -- 3 - 'black', -- 4 -} - -local rspamd_logger = require "rspamd_logger" -local rspamd_util = require "rspamd_util" -local lua_util = require "lua_util" -local rspamd_redis = require "lua_redis" - -local redis_incr_script = [[ -for _, k in ipairs(KEYS) do - redis.call('INCR', k) -end -]] - --- Function to load the script -local function load_scripts(cfg, ev_base) - redis_incr_script_id = rspamd_redis.add_redis_script(redis_incr_script, redis_params) -end - --- Calculates URL reputation -local function url_reputation_check(task) - - local tags = {} - local tlds = {} - local tld_count = 0 - local reputation = 2 - local which - local confidence - - -- Insert symbol - local function insert_results() - if which and confidence then - task:insert_result(settings.symbols[scale[reputation]], confidence, which) - end - end - - -- Calculate reputation - local function dynamic_reputation() - - local subset = {} - local keys = {} - - -- Spit out log if INCR fails - local function redis_incr_cb(err) - if err then - rspamd_logger.errx(task, 'couldnt increment reputation: %s', err) - if string.match(err, 'NOSCRIPT') then - load_scripts(rspamd_config, task:get_ev_base()) - end - end - end - - local function rep_get_cb(err, data) - -- Abort if we couldn't query redis for reputation info - if err then - rspamd_logger.errx(task, 'couldnt get dynamic reputation: %s', err) - return - end - - -- Try find worst reputation domain and set reputation accordingly - local i, x, highest = 1, 1, 0 - while(data[i]) do - if type(data[i]) == 'string' then - local scores = {} - scores.total = tonumber(data[i]) - if scores.total >= settings.threshold then - local highest_k - scores.white = tonumber(data[i+1]) - scores.black = tonumber(data[i+2]) - scores.grey = tonumber(data[i+3]) - scores.neutral = tonumber(data[i+4]) - for k, v in pairs(scores) do - if (v > highest) then - highest_k = k - highest = v - end - end - if highest_k == 'black' then - reputation = 4 - which = subset[x] - confidence = scores.black / scores.total - elseif highest_k == 'grey' and reputation ~= 4 then - reputation = 3 - which = subset[x] - confidence = scores.grey / scores.total - elseif highest_k == 'white' and reputation == 2 then - reputation = 1 - which = subset[x] - confidence = scores.white / scores.total - elseif highest_k == 'neutral' and reputation <= 2 then - reputation = 2 - which = subset[x] - confidence = scores.neutral / scores.total - end - end - end - i = i + 5 - x = x + 1 - end - local rk - if which then - -- Update reputation for guilty domain only - rk = { - settings.key_prefix .. which .. '_total', - settings.key_prefix .. which .. '_' .. scale[reputation], - } - else - -- No reputation found, pick some URLs - local most_relevant - if tld_count == 1 then - most_relevant = next(tlds) - end - if settings.relevance then - if not most_relevant then - local dmarc = ((task:get_symbol(settings.foreign_symbols['dmarc']) or E)[1] or E).options - local dkim = ((task:get_symbol(settings.foreign_symbols['dkim']) or E)[1] or E).options - local spf = task:get_symbol(settings.foreign_symbols['spf']) - local hostname = task:get_hostname() - if hostname then - hostname = rspamd_util.get_tld(hostname) - end - if spf then - local from = task:get_from(1) - if ((from or E)[1] or E).domain then - spf = rspamd_util.get_tld(from[1]['domain']) - else - local helo = task:get_helo() - if helo then - spf = rspamd_util.get_tld(helo) - end - end - end - for _, t in ipairs(tlds) do - if t == dmarc then - most_relevant = t - break - elseif t == dkim then - most_relevant = t - break - elseif t == spf then - most_relevant = t - break - elseif t == hostname then - most_relevant = t - break - end - end - if not most_relevant and reputation >= 3 then - -- no authenticated domain, count surbl tags - local max_surbl_guilt - for dom, tag in pairs(tags) do - local guilt = 0 - local stags = tag['surbl'] - if stags then - for k in pairs(stags) do - if not settings.ignore_surbl[k] then - guilt = guilt + 1 - end - end - if guilt > 1 then - if not most_relevant then - most_relevant = dom - max_surbl_guilt = guilt - elseif guilt > max_surbl_guilt then - most_relevant = dom - max_surbl_guilt = guilt - end - end - end - end - end - end - end - - rk = {} - local added = 0 - if most_relevant then - tlds = {most_relevant} - which = most_relevant - end - for t in pairs(tlds) do - if settings.update_limit and added > settings.update_limit then - rspamd_logger.warnx(task, 'Not updating reputation on all TLDs') - break - end - table.insert(rk, settings.key_prefix .. t .. '_total') - table.insert(rk, settings.key_prefix .. t .. '_' .. scale[reputation]) - added = added + 1 - end - end - if rk[2] then - local ret = rspamd_redis.exec_redis_script(redis_incr_script_id, - {task = task, is_write = true}, - redis_incr_cb, - rk) - if not ret then - rspamd_logger.errx(task, 'couldnt schedule increment') - end - end - insert_results() - end - - local action = task:get_metric_action('default') - if action == 'reject' then - reputation = 4 - elseif action == 'add header' then - reputation = 3 - elseif action == 'no action' or action == 'greylist' then - local score = task:get_metric_score('default')[1] - if score < 0 then - reputation = 1 - end - end - - local added = 0 - for k in pairs(tlds) do - if settings.query_limit and added >= settings.query_limit then - rspamd_logger.warnx(task, 'not querying reputation for all TLDs') - break - end - if (not whitelist) or (not whitelist:get_key(k)) then - added = added + 1 - table.insert(subset, k) - table.insert(keys, settings.key_prefix .. k .. '_total') - table.insert(keys, settings.key_prefix .. k .. '_white') - table.insert(keys, settings.key_prefix .. k .. '_black') - table.insert(keys, settings.key_prefix .. k .. '_grey') - table.insert(keys, settings.key_prefix .. k .. '_neutral') - end - end - - local key = keys[1] - if key then - rspamd_redis_make_request(task, - redis_params, - key, - false, -- is write - rep_get_cb, --callback - 'MGET', -- command - keys - ) - end - end - - -- Figure out what tags are present for each URL - for _, url in ipairs(task:get_urls(false)) do - local tld = url:get_tld() - if not tlds[tld] then - tlds[tld] = true - tld_count = tld_count + 1 - end - local utags = url:get_tags() - if next(utags) then - local dom = url:get_tld() - if not tags[dom] then - tags[dom] = {} - end - for ut, utv in pairs(utags) do - if tags[dom][ut] then - for _, e in ipairs(utv) do - table.insert(tags[dom][ut], e) - end - else - tags[dom][ut] = utv - end - end - end - end - if next(tlds) then - dynamic_reputation() - end -end - -if not lua_util.check_experimental(N) then - return -end - -local opts = rspamd_config:get_all_opt(N) -if not opts then return end -redis_params = rspamd_parse_redis_server(N) -if not redis_params then - rspamd_logger.warnx(rspamd_config, 'no servers are specified, disabling module') - lua_util.disable_module(N, "redis") - return -end -for k, v in pairs(opts) do - if k == 'ignore_surbl' then - if type(v) == 'table' then - if next(v) ~= 1 then - settings[k] = v - else - settings[k] = {} - for _, n in ipairs(v) do - settings[k][n] = true - end - end - end - else - settings[k] = v - end -end -if settings.threshold < 1 then - rspamd_logger.errx(rspamd_config, 'threshold should be >= 1, disabling module') - lua_util.disable_module(N, "config") - return -end - -whitelist = rspamd_map_add(N, 'whitelist', 'map', 'URL reputation whitelist') -rspamd_config:add_on_load(function(cfg, ev_base, worker) - load_scripts(cfg, ev_base) -end) -local id = rspamd_config:register_symbol({ - name = 'URL_REPUTATION_CHECK', - type = 'postfilter', - callback = url_reputation_check, - priority = 10 -}) -for _, v in pairs(settings.symbols) do - rspamd_config:register_symbol({ - name = v, - parent = id, - type = 'virtual' - }) -end diff --git a/src/plugins/lua/url_tags.lua b/src/plugins/lua/url_tags.lua deleted file mode 100644 index c0f7ffa74..000000000 --- a/src/plugins/lua/url_tags.lua +++ /dev/null @@ -1,293 +0,0 @@ ---[[ -Copyright (c) 2016-2017, Vsevolod Stakhov -Copyright (c) 2017, Andrew Lewis - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -if confighelp then - return -end - --- A plugin that restores/persists URL tags - -local N = 'url_tags' - -local redis_params, redis_set_script_id -local settings = { - -- lifetime for tags - expire = 3600, -- 1 hour - -- prefix for redis keys - key_prefix = 'Ut.', - -- tags in this list are not persisted - ignore_tags = {}, -} - -local rspamd_logger = require "rspamd_logger" -local rspamd_util = require "rspamd_util" -local lua_util = require "lua_util" -local lua_redis = require "lua_redis" -local ucl = require "ucl" - --- Tags are stored in format: [timestamp]|[tag1],[timestamp]|[tag2] -local redis_set_script_head = 'local expiry = ' -local redis_set_script_tail = [[ -local now = math.floor(table.remove(ARGV)) -local res = redis.call('MGET', unpack(KEYS)) -local data = {} -for i = 1, #res do - local which = KEYS[i] - if type(res[i]) == 'string' then - data[which] = {} - for goo in string.gmatch(res[i], '[^/]+') do - local metatags = {} - local time, tag, meta = string.match(goo, '(%d+)|([^|]+)|(.+)') - if (time + expiry) > now then - for m in string.gmatch(meta, '[^,]+') do - metatags[m] = true - end - data[which][tag] = {time, metatags} - end - end - end - for goo in string.gmatch(ARGV[i], '[^/]+') do - local metatags = {} - if not data[which] then - data[which] = {} - end - local tag, meta = string.match(goo, '([^|]+)|(.+)') - for m in string.gmatch(meta, '[^,]+') do - metatags[m] = true - end - data[which][tag] = {now, metatags} - end - local tmp2 = {} - for k, v in pairs(data[which]) do - local meta_list = {} - for kk in pairs(v[2]) do - table.insert(meta_list, kk) - end - table.insert(tmp2, v[1] .. '|' .. k .. '|' .. table.concat(meta_list, ',')) - end - redis.call('SETEX', which, expiry, table.concat(tmp2, '/')) -end -]] - --- Function to load the script -local function load_scripts(cfg, ev_base) - local set_script = - redis_set_script_head .. - settings.expire .. - '\n' .. - redis_set_script_tail - redis_set_script_id = lua_redis.add_redis_script(set_script, redis_params) -end - --- Saves tags to redis -local function tags_save(task) - - local tags = {} - -- Figure out what tags are present for each TLD - for _, url in ipairs(task:get_urls(false)) do - local utags = url:get_tags() - if next(utags) then - local tld = url:get_tld() - if not tags[tld] then - tags[tld] = {} - end - for ut, utv in pairs(utags) do - if not settings.ignore_tags[ut] then - if not tags[tld][ut] then - tags[tld][ut] = {} - end - for _, e in ipairs(utv) do - tags[tld][ut][e] = true - end - end - end - end - end - if not next(tags) then - return - end - - -- Don't populate old tags - local old_tags = task:get_mempool():get_variable('urltags') - if old_tags then - local parser = ucl.parser() - local res, err = parser:parse_string(old_tags) - if not res then - rspamd_logger.errx(task, 'Parser error: %s', err) - return - end - local obj = parser:get_object() - for dom, domtags in pairs(obj) do - if tags[dom] then - for tag, mtags in pairs(domtags) do - for mtag in pairs(mtags) do - tags[dom][tag][mtag] = nil - end - if not next(tags[dom][tag]) then - tags[dom][tag] = nil - end - end - if not next(tags[dom]) then - tags[dom] = nil - end - end - end - end - - -- Abort if no tags remaining - if not next(tags) then - return - end - - -- Prepare arguments to send to Redis - local redis_keys = {} - local redis_args = {} - local tmp3 = {} - for dom, domtags in pairs(tags) do - local tmp = {} - for tag, mtags in pairs(domtags) do - local tmp2 = {} - for k in pairs(mtags) do - table.insert(tmp2, tostring(rspamd_util.encode_base32(k))) - end - tmp[tag] = tmp2 - end - tmp3[dom] = tmp - end - for dom, domtags in pairs(tmp3) do - table.insert(redis_keys, settings.key_prefix .. dom) - local tmp4 = {} - for tag, mtags in pairs(domtags) do - table.insert(tmp4, tag .. '|' .. table.concat(mtags, ',')) - end - table.insert(redis_args, table.concat(tmp4, '/')) - end - table.insert(redis_args, rspamd_util.get_time()) - - -- Send query to redis - lua_redis.exec_redis_script( - redis_set_script_id, - {task = task, is_write = true}, - function() end, redis_keys, redis_args) -end - -local function tags_restore(task) - - local urls - local tlds = {} - local tld_reverse = {} - local mpool = task:get_mempool() - - local function redis_get_cb(err, data) - if err then - rspamd_logger.errx(task, 'Redis error: %s', err) - return - end - local d_len = #data - if d_len == 0 then return end - local now = rspamd_util.get_time() - local tracking = {} - for i = 1, d_len do - if type(data[i]) == 'string' then - local tld = tld_reverse[i] - for goo in string.gmatch(data[i], '[^/]+') do - for time, tag, meta in string.gmatch(goo, '(%d+)|([^|]+)|(.+)') do - if not settings.ignore_tags[tag] then - if (time + settings.expire) > now then - local metatags = {} - for m in string.gmatch(meta, '[^,]+') do - table.insert(metatags, m) - end - for _, idx in ipairs(tlds[tld]) do - if not tracking[tld] then - tracking[tld] = {} - end - if not tracking[tld][tag] then - tracking[tld][tag] = {} - end - for _, ttag in ipairs(metatags) do - urls[idx]:add_tag(tag, tostring(rspamd_util.decode_base32(ttag)), mpool) - tracking[tld][tag][ttag] = true - end - end - end - end - end - end - end - end - mpool:set_variable('urltags', ucl.to_format(tracking, 'ucl')) - end - - urls = task:get_urls(false) - for idx = 1, #urls do - local tld = urls[idx]:get_tld() - tld_reverse[idx] = tld - if not tlds[tld] then - tlds[tld] = {} - end - table.insert(tlds[tld], idx) - end - local first = next(tlds) - if first then - local keys = {} - for x in pairs(tlds) do - table.insert(keys, settings.key_prefix .. x) - end - rspamd_redis_make_request(task, - redis_params, - first, - false, -- is write - redis_get_cb, --callback - 'MGET', -- command - keys - ) - end -end - -if not lua_util.check_experimental(N) then - return -end - -local opts = rspamd_config:get_all_opt(N) -if not opts then return end -redis_params = rspamd_parse_redis_server(N) -if not redis_params then - lua_util.disable_module(N, "redis") - rspamd_logger.warnx(rspamd_config, 'no servers are specified, disabling module') - return -end -for k, v in pairs(opts) do - settings[k] = v -end -settings.ignore_tags = lua_util.list_to_hash(settings.ignore_tags) - -rspamd_config:add_on_load(function(cfg, ev_base, worker) - load_scripts(cfg, ev_base) -end) -rspamd_config:register_symbol({ - name = 'URL_TAGS_SAVE', - type = 'postfilter', - callback = tags_save, - priority = 10 -}) -rspamd_config:register_symbol({ - name = 'URL_TAGS_RESTORE', - type = 'prefilter', - callback = tags_restore, - priority = 5 -})