From: Vsevolod Stakhov Date: Wed, 17 Jul 2019 16:00:28 +0000 (+0100) Subject: [Slashing] Remove ip_score module X-Git-Tag: 2.0~555 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=a7675359a1f7dffa91430d69c17a0a0a364c8fc2;p=rspamd.git [Slashing] Remove ip_score module --- diff --git a/conf/modules.d/ip_score.conf b/conf/modules.d/ip_score.conf deleted file mode 100644 index 065c6ca2e..000000000 --- a/conf/modules.d/ip_score.conf +++ /dev/null @@ -1,26 +0,0 @@ -# Please don't modify this file as your changes might be overwritten with -# the next update. -# -# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine -# parameters defined on the top level -# -# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add -# parameters defined on the top level -# -# For specific modules or configuration you can also modify -# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults -# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults -# -# See https://rspamd.com/doc/tutorials/writing_rules.html for details - -ip_score { -# servers = "localhost"; -# threshold = 100; -# reject_score = 3; -# no_action_score = -2; -# add_header_score = 1; -# whitelist = "file:///ip_map"; - .include(try=true,priority=5) "${DBDIR}/dynamic/ip_score.conf" - .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/ip_score.conf" - .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/ip_score.conf" -} diff --git a/src/plugins/lua/ip_score.lua b/src/plugins/lua/ip_score.lua deleted file mode 100644 index 9416f4e88..000000000 --- a/src/plugins/lua/ip_score.lua +++ /dev/null @@ -1,426 +0,0 @@ ---[[ -Copyright (c) 2011-2015, Vsevolod Stakhov - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -if confighelp then - return -end - --- IP score is a module that set ip score of specific ip, asn, country -local rspamd_logger = require "rspamd_logger" -local rspamd_util = require "rspamd_util" -local lua_util = require "lua_util" - --- Default settings -local redis_params = nil -local whitelist = nil -local asn_cc_whitelist = nil -local check_authed = false -local check_local = false -local M = "ip_score" -local N = M - -local options = { - actions = { -- how each action is treated in scoring - ['reject'] = 1.0, - ['add header'] = 0.25, - ['rewrite subject'] = 0.25, - ['no action'] = 1.0 - }, - scores = { -- how each component is evaluated - ['asn'] = 0.4, - ['country'] = 0.01, - ['ipnet'] = 0.5, - ['ip'] = 1.0 - }, - symbol = 'IP_SCORE', -- symbol to be inserted - hash = 'ip_score', -- hash table in redis used for storing scores - asn_prefix = 'a:', -- prefix for ASN hashes - country_prefix = 'c:', -- prefix for country hashes - ipnet_prefix = 'n:', -- prefix for ipnet hashes - servers = '', -- list of servers - lower_bound = 10, -- minimum number of messages to be scored - metric = 'default', - old_shard = false, - min_score = nil, - max_score = nil, - score_divisor = 1, -} - -local function ip_score_hash_key(asn, country, ipnet, ip) - if options.old_shard then - if country then - return country - elseif asn then - return asn - elseif ipnet then - return ipnet - else - return ip:to_string() - end - end - - -- Better sharding - return string.format('%s:%s:%s:%s', asn, country, ipnet, - ip:to_string()) -end - -local function ip_score_get_task_vars(task) - local pool = task:get_mempool() - local asn, country, ipnet - if pool:get_variable("asn") then - asn = pool:get_variable("asn") - end - if pool:get_variable("country") then - country = pool:get_variable("country") - end - if pool:get_variable("ipnet") then - ipnet = pool:get_variable("ipnet") - end - - return asn, country, ipnet -end - -local function normalize_score(sc, total, mult) - if total < options['lower_bound'] then - return 0 - end - return mult * rspamd_util.tanh(2.718281 * sc / total) -end - --- Set score based on metric's action -local ip_score_set = function(task) - if lua_util.is_rspamc_or_controller(task) then return end - local function new_score_set(score, old_score, old_total) - local new_total - if old_total == -1 or old_total ~= old_total then - new_total = 1 - else - new_total = old_total + 1 - end - - if score ~= score then - score = 0 - end - - return old_score + score, new_total - end - - local score_set_cb = function(err) - if err then - rspamd_logger.infox(task, 'got error while IP score changing: %1', err) - end - end - - local ip = task:get_from_ip() - if not check_authed and task:get_user() then - return - end - local action = task:get_metric_action(options['metric']) - if not ip or not ip:is_valid() then - return - end - if not check_local and ip:is_local() then - return - end - - local pool = task:get_mempool() - local asn, country, ipnet = ip_score_get_task_vars(task) - - if not pool:has_variable('ip_score') or not asn or not country or not ipnet then - return - end - - local asn_score,total_asn, - country_score,total_country, - ipnet_score,total_ipnet, - ip_score, total_ip = pool:get_variable('ip_score', - 'double,double,double,double,double,double,double,double') - lua_util.debugm(M, task, - 'raw scores: asn: %s, total_asn: %s, country: %s, ' .. - 'total_country: %s, ipnet: %s, total_ipnet: %s, ip:%s, total_ip: %s', - asn_score, total_asn, - country_score, total_country, - ipnet_score, total_ipnet, - ip_score, total_ip) - - local score_mult = 0 - if options['actions'][action] then - score_mult = options['actions'][action] - end - local score = task:get_metric_score(options['metric'])[1] - if action == 'no action' and score > 0 then - score_mult = 0 - end - - score = score_mult * rspamd_util.tanh (2.718281 * (score/options['score_divisor'])) - - local hkey = ip_score_hash_key(asn, country, ipnet, ip) - - asn_score,total_asn = new_score_set(score, asn_score, total_asn) - country_score,total_country = new_score_set(score, country_score, total_country) - ipnet_score,total_ipnet = new_score_set(score, ipnet_score, total_ipnet) - ip_score,total_ip = new_score_set(score, ip_score, total_ip) - lua_util.debugm(M, task, - 'processed scores: asn: %s, total_asn: %s, country: %s, total_country: %s,' .. - ' ipnet: %s, total_ipnet: %s, ip:%s, total_ip: %s', - asn_score,total_asn, - country_score,total_country, - ipnet_score,total_ipnet, - ip_score, total_ip) - local redis_args = {options['hash'], - options['asn_prefix'] .. asn, string.format('%f|%d', asn_score, total_asn), - options['country_prefix'] .. country, string.format('%f|%d', country_score, total_country), - options['ipnet_prefix'] .. ipnet, string.format('%f|%d', ipnet_score, total_ipnet), - ip:to_string(), string.format('%f|%d', ip_score, total_ip)} - - local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - hkey, -- hash key - true, -- is write - score_set_cb, --callback - 'HMSET', -- command - redis_args -- arguments - ) - if not ret then - rspamd_logger.errx(task, 'error connecting to redis') - return - end - - -- Now insert final result - asn_score = normalize_score(asn_score, total_asn, options['scores']['asn']) - country_score = normalize_score(country_score, total_country, - options['scores']['country']) - ipnet_score = normalize_score(ipnet_score, total_ipnet, - options['scores']['ipnet']) - ip_score = normalize_score(ip_score, total_ip, options['scores']['ip']) - - local total_score = 0.0 - local description_t = {} - - if ip_score ~= 0 then - total_score = total_score + ip_score - table.insert(description_t, - 'ip: ' .. '(' .. string.format('%.2f', ip_score * 10) .. ')') - end - if ipnet_score ~= 0 then - total_score = total_score + ipnet_score - table.insert(description_t, - 'ipnet: ' .. ipnet .. '(' .. string.format('%.2f', ipnet_score * 10) .. ')') - end - if asn_score ~= 0 then - total_score = total_score + asn_score - table.insert(description_t, - 'asn: ' .. asn .. '(' .. string.format('%.2f', asn_score * 10) .. ')') - end - if country_score ~= 0 then - total_score = total_score + country_score - table.insert(description_t, - 'country: ' .. country .. '(' .. string.format('%.2f', country_score * 10) .. ')') - end - - if options['max_score'] and (total_score*10) > options['max_score'] then - total_score = options['max_score']/10 - end - if options['min_score'] and (total_score*10) < options['min_score'] then - total_score = options['min_score']/10 - end - - if total_score ~= 0 then - task:insert_result(options['symbol'], total_score, table.concat(description_t, ', ')) - end -end - --- Check score for ip in keystorage -local ip_score_check = function(task) - local asn, country, ipnet = ip_score_get_task_vars(task) - local ip = task:get_from_ip() - - local ip_score_redis_cb = function(err, data) - if err then - rspamd_logger.errx(task, 'Redis error: %s', err) - -- XXX: upstreams - end - local function calculate_score(score) - local parts = lua_util.rspamd_str_split(score, '|') - local rep = tonumber(parts[1]) - local total = tonumber(parts[2]) - - if rep ~= rep then rep = 0 end - if total ~= total then total = 0 end - - return rep, total - end - - if err then - -- Key is not found or error occurred - return - elseif data then - -- Scores and total number of messages per bucket - local asn_score,total_asn, - country_score,total_country, - ipnet_score,total_ipnet, - ip_score, total_ip = 0, -1, 0, -1, 0, -1, 0, -1 - if data[1] and type(data[1]) ~= 'userdata' then - asn_score,total_asn = calculate_score(data[1]) - end - if data[2] and type(data[2]) ~= 'userdata' then - country_score,total_country = calculate_score(data[2]) - end - if data[3] and type(data[3]) ~= 'userdata' then - ipnet_score,total_ipnet = calculate_score(data[3]) - end - if data[4] and type(data[4]) ~= 'userdata' then - ip_score,total_ip = calculate_score(data[4]) - end - -- Save everything for the post filter - task:get_mempool():set_variable('ip_score', asn_score,total_asn, - country_score,total_country, - ipnet_score,total_ipnet, - ip_score, total_ip) - end - end - - local function create_get_command() - local cmd = 'HMGET' - - local args = {options['hash']} - - if asn then - table.insert(args, options['asn_prefix'] .. asn) - else - -- fake arg - table.insert(args, options['asn_prefix']) - end - if country then - table.insert(args, options['country_prefix'] .. country) - else - -- fake arg - table.insert(args, options['country_prefix']) - end - if ipnet then - table.insert(args, options['ipnet_prefix'] .. ipnet) - else - -- fake arg - table.insert(args, options['ipnet_prefix']) - end - - table.insert(args, ip:to_string()) - - return cmd, args - end - - if task:get_user() and not check_authed then - rspamd_logger.infox(task, "skip IP Score for authorized users") - return - end - if ip and ip:is_local() and not check_local then - rspamd_logger.infox(task, "skip IP Score for local networks") - return - end - if ip:is_valid() then - -- Check IP whitelist - if whitelist then - if whitelist:get_key(task:get_from_ip()) then - -- Address is whitelisted - return - end - end - -- Check ASN & country whitelist - if asn_cc_whitelist then - if asn_cc_whitelist:get_key(country) then - return - end - if asn_cc_whitelist:get_key(asn) then - return - end - end - - local cmd, args = create_get_command() - - local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - ip_score_hash_key(asn, country, ipnet, ip), -- hash key - false, -- is write - ip_score_redis_cb, --callback - cmd, -- command - args -- arguments - ) - if not ret then - rspamd_logger.errx(task, 'error connecting to redis') - end - end -end - -local function try_opts(where) - local ret = false - local opts = rspamd_config:get_all_opt(where) - if type(opts) == 'table' then - if type(opts['check_local']) == 'boolean' then - check_local = opts['check_local'] - ret = true - end - if type(opts['check_authed']) == 'boolean' then - check_authed = opts['check_authed'] - ret = true - end - end - - return ret -end - -if not try_opts(N) then try_opts('options') end - --- Configuration options -local configure_ip_score_module = function() - local opts = rspamd_config:get_all_opt(N) - - if not opts then return end - for k,v in pairs(opts) do - options[k] = v - end - redis_params = rspamd_parse_redis_server('ip_score') - if not redis_params then - rspamd_logger.infox(rspamd_config, 'no servers are specified') - return - end - asn_cc_whitelist = rspamd_map_add('ip_score', 'asn_cc_whitelist', 'map', - 'IP score whitelisted ASNs/countries') - whitelist = rspamd_map_add('ip_score', 'whitelist', 'radix', - 'IP score whitelisted ips') - return true -end - - -if not configure_ip_score_module() then return end -if redis_params then - -- Register ip_score module - rspamd_config:register_symbol({ - name = 'IPSCORE_SAVE', - type = 'postfilter,nostat', - priority = 5, - callback = ip_score_set, - flags = 'empty', - }) - rspamd_config:register_symbol({ - name = options['symbol'], - callback = ip_score_check, - group = 'reputation', - score = 2.0, - flags = 'empty', - }) -else - lua_util.disable_module(N, "redis") -end