From 643ca8d0dab13ff0502a85aae859c1733030bd7b Mon Sep 17 00:00:00 2001 From: Andrew Lewis Date: Wed, 17 Aug 2016 11:33:06 +0200 Subject: [PATCH] [Feature] ASN module; support matching ASN/country in multimap - Also removed stuff from ip_score - Updated tests --- src/plugins/lua/asn.lua | 100 +++++++++++++++++++++++ src/plugins/lua/ip_score.lua | 41 ---------- src/plugins/lua/multimap.lua | 14 +++- test/functional/cases/102_multimap.robot | 18 ++++ test/functional/configs/multimap.conf | 10 +++ 5 files changed, 141 insertions(+), 42 deletions(-) create mode 100644 src/plugins/lua/asn.lua diff --git a/src/plugins/lua/asn.lua b/src/plugins/lua/asn.lua new file mode 100644 index 000000000..28cd431b9 --- /dev/null +++ b/src/plugins/lua/asn.lua @@ -0,0 +1,100 @@ +--[[ +Copyright (c) 2011-2016, Vsevolod Stakhov +Copyright (c) 2016, Andrew Lewis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local rspamd_logger = require "rspamd_logger" +local rspamd_regexp = require "rspamd_regexp" + +local options = { + provider_type = 'cymru', + provider_info = { + ip4 = 'origin.asn.cymru.com', + ip6 = 'origin6.asn.cymru.com', + }, + symbol = 'ASN', +} + +local cymru_re = rspamd_regexp.create_cached("[\\|\\s]") + +local function asn_check(task) + + local function asn_set(asn, ipnet, country) + local descr_t = {} + if asn then + task:get_mempool():set_variable("asn", asn) + table.insert(descr_t, "asn:" .. asn) + end + if ipnet then + task:get_mempool():set_variable("ipnet", ipnet) + table.insert(descr_t, "ipnet:" .. ipnet) + end + if country then + task:get_mempool():set_variable("country", country) + table.insert(descr_t, "country:" .. country) + end + if options['symbol'] then + task:insert_result(options['symbol'], 0.0, table.concat(descr_t, ', ')) + end + end + + local asn_check_func = {} + function asn_check_func.cymru(ip) + local function cymru_dns_cb(resolver, to_resolve, results, err, key) + if not (results and results[1]) then return end + local parts = cymru_re:split(results[1]) + -- "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8 + asn_set(parts[1], parts[2], parts[3]) + end + local dnsbl = options['provider_info']['ip' .. ip:get_version()] + local req_name = rspamd_logger.slog("%1.%2", + table.concat(ip:inversed_str_octets(), '.'), dnsbl) + task:get_resolver():resolve_txt(task:get_session(), task:get_mempool(), + req_name, cymru_dns_cb) + end + + local ip = task:get_from_ip() + if not (ip and ip:is_valid()) then return end + asn_check_func[options['provider_type']](ip) +end + +-- Configuration options +local configure_asn_module = function() + local opts = rspamd_config:get_all_opt('asn') + if opts then + for k,v in pairs(opts) do + options[k] = v + end + end + if options['provider_type'] == 'cymru' then + if not options['provider_info'] and options['provider_info']['ip4'] and + options['provider_info']['ip6'] then + rspamd_logger.errx("Missing required provider_info for cymru") + return false + end + else + rspamd_logger.errx("Unknown provider_type: %s", options['provider_type']) + return false + end + return true +end + +if configure_asn_module() then + rspamd_config:register_symbol({ + name = 'ASN_CHECK', + type = 'prefilter', + callback = asn_check, + }) +end diff --git a/src/plugins/lua/ip_score.lua b/src/plugins/lua/ip_score.lua index f3f4800b2..ee561f91e 100644 --- a/src/plugins/lua/ip_score.lua +++ b/src/plugins/lua/ip_score.lua @@ -28,8 +28,6 @@ local whitelist = nil local asn_cc_whitelist = nil local options = { - asn_provider = 'origin.asn.cymru.com', -- provider for ASN data - asn6_provider = 'origin6.asn.cymru.com', -- provider for ASN data actions = { -- how each action is treated in scoring ['reject'] = 1.0, ['add header'] = 0.25, @@ -56,38 +54,6 @@ local options = { local asn_re = rspamd_regexp.create_cached("[\\|\\s]") -local function asn_check(task) - local ip = task:get_from_ip() - - local function asn_dns_cb(resolver, to_resolve, results, err, key) - if results and results[1] then - local parts = asn_re:split(results[1]) - -- "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8 - if parts[1] then - task:get_mempool():set_variable("asn", parts[1]) - end - if parts[2] then - task:get_mempool():set_variable("ipnet", parts[2]) - end - if parts[3] then - task:get_mempool():set_variable("country", parts[3]) - end - end - end - - if ip and ip:is_valid() then - local asn_provider = 'asn_provider' - if ip:get_version() == 6 then - asn_provider = 'asn6_provider' - end - local req_name = rspamd_logger.slog("%1.%2", - table.concat(ip:inversed_str_octets(), '.'), options[asn_provider]) - - task:get_resolver():resolve_txt(task:get_session(), task:get_mempool(), - req_name, asn_dns_cb) - end -end - local function ip_score_hash_key(asn, country, ipnet, ip) -- We use the most common attribute as hashing key if country then @@ -364,13 +330,6 @@ end configure_ip_score_module() if redis_params then -- Register ip_score module - if options['asn_provider'] then - rspamd_config:register_symbol({ - name = 'ASN_CHECK', - type = 'prefilter', - callback = asn_check, - }) - end rspamd_config:register_symbol({ name = 'IPSCORE_SAVE', type = 'postfilter', diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 31cd01d50..a4577676a 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -509,6 +509,16 @@ local function multimap_callback(task, rule) if hostname and hostname ~= 'unknown' then match_hostname(rule, hostname) end + elseif rt == 'asn' then + local asn = task:get_mempool():get_variable('asn') + if asn then + match_rule(rule, asn) + end + elseif rt == 'country' then + local country = task:get_mempool():get_variable('country') + if country then + match_rule(rule, country) + end end end @@ -595,7 +605,9 @@ local function add_multimap_rule(key, newrule) or newrule['type'] == 'filename' or newrule['type'] == 'url' or newrule['type'] == 'content' - or newrule['type'] == 'hostname' then + or newrule['type'] == 'hostname' + or newrule['type'] == 'asn' + or newrule['type'] == 'country' then if newrule['regexp'] then newrule['hash'] = rspamd_config:add_map ({ url = newrule['map'], diff --git a/test/functional/cases/102_multimap.robot b/test/functional/cases/102_multimap.robot index 364787b15..de3595378 100644 --- a/test/functional/cases/102_multimap.robot +++ b/test/functional/cases/102_multimap.robot @@ -210,6 +210,24 @@ MAP - REDIS - URL NOFILTER - MISS ${result} = Scan Message With Rspamc ${URL4} Check Rspamc ${result} REDIS_URL_NOFILTER inverse=1 +MAP - REDIS - ASN - HIT + Redis HSET asn 15169 ${EMPTY} + ${result} = Scan Message With Rspamc ${MESSAGE} -i 8.8.8.8 + Check Rspamc ${result} REDIS_ASN + +MAP - REDIS - ASN - MISS + ${result} = Scan Message With Rspamc ${MESSAGE} -i 46.228.47.114 + Check Rspamc ${result} REDIS_ASN inverse=1 + +MAP - REDIS - CC - HIT + Redis HSET cc US ${EMPTY} + ${result} = Scan Message With Rspamc ${MESSAGE} -i 8.8.8.8 + Check Rspamc ${result} REDIS_COUNTRY + +MAP - REDIS - ASN - MISS + ${result} = Scan Message With Rspamc ${MESSAGE} -i 46.228.47.114 + Check Rspamc ${result} REDIS_COUNTRY inverse=1 + *** Keywords *** Multimap Setup ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/multimap.conf diff --git a/test/functional/configs/multimap.conf b/test/functional/configs/multimap.conf index 96258c15a..3823914cf 100644 --- a/test/functional/configs/multimap.conf +++ b/test/functional/configs/multimap.conf @@ -1,3 +1,5 @@ +asn { +} redis { servers = "${REDIS_ADDR}:${REDIS_PORT}"; } @@ -96,4 +98,12 @@ multimap { type = "url"; map = "redis://urlnofilter"; } + REDIS_COUNTRY { + type = "country"; + map = "redis://cc"; + } + REDIS_ASN { + type = "asn"; + map = "redis://asn"; + } } -- 2.39.5