123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539 |
- --[[
- Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
- Copyright (c) 2016, Alexey Savelyev <info@homeweb.ru>
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ]]--
-
- --[[
- Example domains whitelist config:
- greylist {
- # Search "example.com" and "mail.example.com" for "mx.out.mail.example.com":
- whitelist_domains_url = [
- "$LOCAL_CONFDIR/local.d/maps.d/greylist-whitelist-domains.inc",
- "${CONFDIR}/maps.d/maillist.inc",
- "${CONFDIR}/maps.d/redirectors.inc",
- "${CONFDIR}/maps.d/dmarc_whitelist.inc",
- "${CONFDIR}/maps.d/spf_dkim_whitelist.inc",
- "${CONFDIR}/maps.d/surbl-whitelist.inc",
- "https://maps.rspamd.com/freemail/free.txt.zst"
- ];
- }
- Example config for exim users:
- greylist {
- action = "greylist";
- }
- --]]
-
- if confighelp then
- rspamd_config:add_example(nil, 'greylist',
- "Performs adaptive greylisting using Redis",
- [[
- greylist {
- # Buckets expire (1 day by default)
- expire = 1d;
- # Greylisting timeout
- timeout = 5m;
- # Redis prefix
- key_prefix = 'rg';
- # Use body hash up to this value of bytes for greylisting
- max_data_len = 10k;
- # Default greylisting message
- message = 'Try again later';
- # Append symbol on greylisting
- symbol = 'GREYLIST';
- # Default action change (for Exim use `greylist`)
- action = 'soft reject';
- # Skip greylisting if one of the following symbols has been found
- whitelist_symbols = [];
- # Mask bits for ipv4
- ipv4_mask = 19;
- # Mask bits for ipv6
- ipv6_mask = 64;
- # Tell when greylisting is expired (appended to `message`)
- report_time = false;
- # Greylist local messages
- check_local = false;
- # Greylist messages from authenticated users
- check_authed = false;
- }
- ]])
- return
- end
-
- -- A plugin that implements greylisting using redis
-
- local redis_params
- local whitelisted_ip
- local whitelist_domains_map
- local toint = math.ifloor or math.floor
- local settings = {
- expire = 86400, -- 1 day by default
- timeout = 300, -- 5 minutes by default
- key_prefix = 'rg', -- default hash name
- max_data_len = 10240, -- default data limit to hash
- message = 'Try again later', -- default greylisted message
- symbol = 'GREYLIST',
- action = 'soft reject', -- default greylisted action
- whitelist_symbols = {}, -- whitelist when specific symbols have been found
- ipv4_mask = 19, -- Mask bits for ipv4
- ipv6_mask = 64, -- Mask bits for ipv6
- report_time = false, -- Tell when greylisting is expired (appended to `message`)
- check_local = false,
- check_authed = false,
- }
-
- local rspamd_logger = require "rspamd_logger"
- local rspamd_util = require "rspamd_util"
- local lua_redis = require "lua_redis"
- local lua_util = require "lua_util"
- local fun = require "fun"
- local hash = require "rspamd_cryptobox_hash"
- local rspamd_lua_utils = require "lua_util"
- local lua_map = require "lua_maps"
- local N = "greylist"
-
- local function data_key(task)
- local cached = task:get_mempool():get_variable("grey_bodyhash")
- if cached then
- return cached
- end
-
- local body = task:get_rawbody()
-
- if not body then
- return nil
- end
-
- local len = body:len()
- if len > settings['max_data_len'] then
- len = settings['max_data_len']
- end
-
- local h = hash.create()
- h:update(body, len)
-
- local b32 = settings['key_prefix'] .. 'b' .. h:base32():sub(1, 20)
- task:get_mempool():set_variable("grey_bodyhash", b32)
- return b32
- end
-
- local function envelope_key(task)
- local cached = task:get_mempool():get_variable("grey_metahash")
- if cached then
- return cached
- end
-
- local from = task:get_from('smtp')
- local h = hash.create()
-
- local addr = '<>'
- if from and from[1] then
- addr = from[1]['addr']
- end
-
- h:update(addr)
- local rcpt = task:get_recipients('smtp')
- if rcpt then
- table.sort(rcpt, function(r1, r2)
- return r1['addr'] < r2['addr']
- end)
-
- fun.each(function(r)
- h:update(r['addr'])
- end, rcpt)
- end
-
- local ip = task:get_ip()
-
- if ip and ip:is_valid() then
- local s
- if ip:get_version() == 4 then
- s = tostring(ip:apply_mask(settings['ipv4_mask']))
- else
- s = tostring(ip:apply_mask(settings['ipv6_mask']))
- end
- h:update(s)
- end
-
- local b32 = settings['key_prefix'] .. 'm' .. h:base32():sub(1, 20)
- task:get_mempool():set_variable("grey_metahash", b32)
- return b32
- end
-
- -- Returns pair of booleans: found,greylisted
- local function check_time(task, tm, type, now)
- local t = tonumber(tm)
-
- if not t then
- rspamd_logger.errx(task, 'not a valid number: %s', tm)
- return false, false
- end
-
- if now - t < settings['timeout'] then
- return true, true
- else
- -- We just set variable to pass when in post-filter stage
- task:get_mempool():set_variable("grey_whitelisted", type)
-
- return true, false
- end
- end
-
- local function greylist_message(task, end_time, why)
- task:insert_result(settings['symbol'], 0.0, 'greylisted', end_time, why)
-
- if not settings.check_local and rspamd_lua_utils.is_rspamc_or_controller(task) then
- return
- end
-
- if settings.message_func then
- task:set_pre_result(settings['action'],
- settings.message_func(task, end_time), N)
- else
- local message = settings['message']
- if settings.report_time then
- message = string.format("%s: %s", message, end_time)
- end
- task:set_pre_result(settings['action'], message, N)
- end
-
- task:set_flag('greylisted')
- end
-
- local function greylist_check(task)
- local ip = task:get_ip()
-
- if ((not settings.check_authed and task:get_user()) or
- (not settings.check_local and ip and ip:is_local())) then
- rspamd_logger.infox(task, "skip greylisting for local networks and/or authorized users");
- return
- end
-
- if ip and ip:is_valid() and whitelisted_ip then
- if whitelisted_ip:get_key(ip) then
- -- Do not check whitelisted ip
- rspamd_logger.infox(task, 'skip greylisting for whitelisted IP')
- return
- end
- end
-
- local body_key = data_key(task)
- local meta_key = envelope_key(task)
- local hash_key = body_key .. meta_key
-
- local function redis_get_cb(err, data)
- local ret_body = false
- local greylisted_body = false
- local ret_meta = false
- local greylisted_meta = false
-
- if data then
- local end_time_body, end_time_meta
- local now = rspamd_util.get_time()
-
- if data[1] and type(data[1]) ~= 'userdata' then
- local tm = tonumber(data[1]) or now
- ret_body, greylisted_body = check_time(task, data[1], 'body', now)
- if greylisted_body then
- end_time_body = tm + settings['timeout']
- task:get_mempool():set_variable("grey_greylisted_body",
- rspamd_util.time_to_string(end_time_body))
- end
- end
-
- if data[2] and type(data[2]) ~= 'userdata' then
- if not ret_body or greylisted_body then
- local tm = tonumber(data[2]) or now
- ret_meta, greylisted_meta = check_time(task, data[2], 'meta', now)
-
- if greylisted_meta then
- end_time_meta = tm + settings['timeout']
- task:get_mempool():set_variable("grey_greylisted_meta",
- rspamd_util.time_to_string(end_time_meta))
- end
- end
- end
-
- local how
- local end_time_str
-
- if not ret_body and not ret_meta then
- -- no record found
- task:get_mempool():set_variable("grey_greylisted", 'true')
- elseif greylisted_body and greylisted_meta then
- end_time_str = rspamd_util.time_to_string(
- math.min(end_time_body, end_time_meta))
- how = 'meta and body'
- elseif greylisted_body then
- end_time_str = rspamd_util.time_to_string(end_time_body)
- how = 'body only'
- elseif greylisted_meta then
- end_time_str = rspamd_util.time_to_string(end_time_meta)
- how = 'meta only'
- end
-
- if how and end_time_str then
- rspamd_logger.infox(task, 'greylisted until "%s" (%s)',
- end_time_str, how)
- greylist_message(task, end_time_str, 'too early')
- end
- elseif err then
- rspamd_logger.errx(task, 'got error while getting greylisting keys: %1', err)
- return
- end
- end
-
- local ret = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- hash_key, -- hash key
- false, -- is write
- redis_get_cb, --callback
- 'MGET', -- command
- { body_key, meta_key } -- arguments
- )
- if not ret then
- rspamd_logger.errx(task, 'cannot make redis request to check results')
- end
- end
-
- local function greylist_set(task)
- local action = task:get_metric_action()
- local ip = task:get_ip()
-
- -- Don't do anything if pre-result has been already set
- if task:has_pre_result() then
- return
- end
-
- -- Check whitelist_symbols
- for _, sym in ipairs(settings.whitelist_symbols) do
- if task:has_symbol(sym) then
- rspamd_logger.infox(task, 'skip greylisting as we have found symbol %s', sym)
- if action == 'greylist' then
- -- We are going to accept message
- rspamd_logger.infox(task, 'downgrading metric action from "greylist" to "no action"')
- task:disable_action('greylist')
- end
- return
- end
- end
-
- -- We need to update this on each scan, as it can vary per settings or be redefined dynamically
- local greylist_min_score = settings.greylist_min_score or task:get_metric_threshold('greylist')
- if greylist_min_score then
- local score = task:get_metric_score()[1]
- if score < greylist_min_score then
- rspamd_logger.infox(task, 'Score too low - skip greylisting')
- if action == 'greylist' then
- -- We are going to accept message
- rspamd_logger.infox(task, 'Downgrading metric action from "greylist" to "no action"')
- task:disable_action('greylist')
- end
- return
- end
- end
-
- if ((not settings.check_authed and task:get_user()) or
- (not settings.check_local and ip and ip:is_local())) then
- if action == 'greylist' then
- -- We are going to accept message
- rspamd_logger.infox(task, 'Downgrading metric action from "greylist" to "no action"')
- task:disable_action('greylist')
- end
- return
- end
-
- if ip and ip:is_valid() and whitelisted_ip then
- if whitelisted_ip:get_key(ip) then
- if action == 'greylist' then
- -- We are going to accept message
- rspamd_logger.infox(task, 'Downgrading metric action from "greylist" to "no action"')
- task:disable_action('greylist')
- end
- return
- end
- end
-
- local is_whitelisted = task:get_mempool():get_variable("grey_whitelisted")
- local do_greylisting = task:get_mempool():get_variable("grey_greylisted")
- local do_greylisting_required = task:get_mempool():get_variable("grey_greylisted_required")
-
- -- Third and second level domains whitelist
- if not is_whitelisted and whitelist_domains_map then
- local hostname = task:get_hostname()
- if hostname then
- local domain = rspamd_util.get_tld(hostname)
- if whitelist_domains_map:get_key(hostname) or (domain and whitelist_domains_map:get_key(domain)) then
- is_whitelisted = 'meta'
- rspamd_logger.infox(task, 'skip greylisting for whitelisted domain')
- end
- end
- end
-
- if action == 'reject' or
- not do_greylisting_required and action == 'no action' then
- return
- end
- local body_key = data_key(task)
- local meta_key = envelope_key(task)
- local upstream, ret, conn
- local hash_key = body_key .. meta_key
-
- local function redis_set_cb(err)
- if err then
- rspamd_logger.errx(task, 'got error %s when setting greylisting record on server %s',
- err, upstream:get_addr())
- end
- end
-
- local is_rspamc = rspamd_lua_utils.is_rspamc_or_controller(task)
-
- if is_whitelisted then
- if action == 'greylist' then
- -- We are going to accept message
- rspamd_logger.infox(task, 'Downgrading metric action from "greylist" to "no action"')
- task:disable_action('greylist')
- end
-
- task:insert_result(settings['symbol'], 0.0, 'pass', is_whitelisted)
- rspamd_logger.infox(task, 'greylisting pass (%s) until %s',
- is_whitelisted,
- rspamd_util.time_to_string(rspamd_util.get_time() + settings['expire']))
-
- if not settings.check_local and is_rspamc then
- return
- end
-
- ret, conn, upstream = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- hash_key, -- hash key
- true, -- is write
- redis_set_cb, --callback
- 'EXPIRE', -- command
- { body_key, tostring(toint(settings['expire'])) } -- arguments
- )
- -- Update greylisting record expire
- if ret then
- conn:add_cmd('EXPIRE', {
- meta_key, tostring(toint(settings['expire']))
- })
- else
- rspamd_logger.errx(task, 'got error while connecting to redis')
- end
- elseif do_greylisting or do_greylisting_required then
- if not settings.check_local and is_rspamc then
- return
- end
- local t = tostring(toint(rspamd_util.get_time()))
- local end_time = rspamd_util.time_to_string(t + settings['timeout'])
- rspamd_logger.infox(task, 'greylisted until "%s", new record', end_time)
- greylist_message(task, end_time, 'new record')
- -- Create new record
- ret, conn, upstream = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- hash_key, -- hash key
- true, -- is write
- redis_set_cb, --callback
- 'SETEX', -- command
- { body_key, tostring(toint(settings['expire'])), t } -- arguments
- )
-
- if ret then
- conn:add_cmd('SETEX', {
- meta_key, tostring(toint(settings['expire'])), t
- })
- else
- rspamd_logger.errx(task, 'got error while connecting to redis')
- end
- else
- if action ~= 'no action' and action ~= 'reject' then
- local grey_res = task:get_mempool():get_variable("grey_greylisted_body")
-
- if grey_res then
- -- We need to delay message, hence set a temporary result
- rspamd_logger.infox(task, 'greylisting delayed until "%s": body', grey_res)
- greylist_message(task, grey_res, 'body')
- else
- grey_res = task:get_mempool():get_variable("grey_greylisted_meta")
- if grey_res then
- greylist_message(task, grey_res, 'meta')
- end
- end
- else
- task:insert_result(settings['symbol'], 0.0, 'greylisted', 'passed')
- end
- end
- end
-
- local opts = rspamd_config:get_all_opt('greylist')
- if opts then
- if opts['message_func'] then
- settings.message_func = assert(load(opts['message_func']))()
- end
-
- for k, v in pairs(opts) do
- if k ~= 'message_func' then
- settings[k] = v
- end
- end
-
- local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N,
- false, false)
- settings.check_local = auth_and_local_conf[1]
- settings.check_authed = auth_and_local_conf[2]
-
- if settings['greylist_min_score'] then
- settings['greylist_min_score'] = tonumber(settings['greylist_min_score'])
- end
-
- whitelisted_ip = lua_map.rspamd_map_add(N, 'whitelisted_ip', 'radix',
- 'Greylist whitelist ip map')
- whitelist_domains_map = lua_map.rspamd_map_add(N, 'whitelist_domains_url',
- 'map', 'Greylist whitelist domains map')
-
- redis_params = lua_redis.parse_redis_server(N)
- if not redis_params then
- rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
- rspamd_lua_utils.disable_module(N, "redis")
- else
- lua_redis.register_prefix(settings.key_prefix .. 'b[a-z0-9]{20}', N,
- 'Greylisting elements (body hashes)"', {
- type = 'string',
- })
- lua_redis.register_prefix(settings.key_prefix .. 'm[a-z0-9]{20}', N,
- 'Greylisting elements (meta hashes)"', {
- type = 'string',
- })
- rspamd_config:register_symbol({
- name = 'GREYLIST_SAVE',
- type = 'postfilter',
- callback = greylist_set,
- priority = lua_util.symbols_priorities.medium,
- augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
- })
- local id = rspamd_config:register_symbol({
- name = 'GREYLIST_CHECK',
- type = 'prefilter',
- callback = greylist_check,
- priority = lua_util.symbols_priorities.medium,
- augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }
- })
- rspamd_config:register_symbol({
- name = settings.symbol,
- type = 'virtual',
- parent = id,
- score = 0,
- })
- end
- end
|