From 08caf16449ad25e1aa6d659908713c4dcfae2dec Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Tue, 24 Dec 2013 18:24:54 +0000 Subject: [PATCH] Add hfilter - an advanced set of helo and hostname checks. Submitted by: AL --- conf/lua/hfilter.lua | 246 +++++++++++++++++++++++++++++++++++++++++++ conf/lua/rspamd.lua | 4 + conf/metrics.conf | 18 ++++ 3 files changed, 268 insertions(+) create mode 100644 conf/lua/hfilter.lua diff --git a/conf/lua/hfilter.lua b/conf/lua/hfilter.lua new file mode 100644 index 000000000..daf544745 --- /dev/null +++ b/conf/lua/hfilter.lua @@ -0,0 +1,246 @@ +--[[ +Rating for checks_hellohost and checks_hello: +5 - very hard +4 - hard +3 - meduim +2 - low +1 - very low +--]] + +--Checks for HELO and Hostname +local checks_hellohost = { +['[.-]dynamic[.-]'] = 4, ['dynamic[.-][0-9]'] = 4, ['[0-9][.-]?dynamic'] = 4, +['[.-]dyn[.-]'] = 4, ['dyn[.-][0-9]'] = 4, ['[0-9][.-]?dyn'] = 4, +['[.-]clients?[.-]'] = 4, ['clients?[.-][0-9]'] = 4, ['[0-9][.-]?clients?'] = 4, +['[.-]dynip[.-]'] = 4, ['dynip[.-][0-9]'] = 4, ['[0-9][.-]?dynip'] = 4, +['[.-]broadband[.-]'] = 4, ['broadband[.-][0-9]'] = 4, ['[0-9][.-]?broadband'] = 4, +['[.-]broad[.-]'] = 4, ['broad[.-][0-9]'] = 4, ['[0-9][.-]?broad'] = 4, +['[.-]bredband[.-]'] = 4, ['bredband[.-][0-9]'] = 4, ['[0-9][.-]?bredband'] = 4, +['[.-]nat[.-]'] = 4, ['nat[.-][0-9]'] = 4, ['[0-9][.-]?nat'] = 4, +['[.-]pptp[.-]'] = 4, ['pptp[.-][0-9]'] = 4, ['[0-9][.-]?pptp'] = 4, +['[.-]pppoe[.-]'] = 4, ['pppoe[.-][0-9]'] = 4, ['[0-9][.-]?pppoe'] = 4, +['[.-]ppp[.-]'] = 4, ['ppp[.-][0-9]'] = 4, ['[0-9][.-]?ppp'] = 4, +['[.-][a|x]?dsl[.-]'] = 3, ['[a|x]?dsl[.-]?[0-9]'] = 3, ['[0-9][.-]?[a|x]?dsl'] = 3, +['[.-][a|x]?dsl-dynamic[.-]'] = 4, ['[a|x]?dsl-dynamic[.-]?[0-9]'] = 4, ['[0-9][.-]?[a|x]?dsl-dynamic'] = 4, +['[.-][a|x]?dsl-line[.-]'] = 3, ['[a|x]?dsl-line[.-]?[0-9]'] = 3, ['[0-9][.-]?[a|x]?dsl-line'] = 3, +['[.-]dhcp[.-]'] = 4, ['dhcp[.-][0-9]'] = 4, ['[0-9][.-]?dhcp'] = 4, +['[.-]catv[.-]'] = 4, ['catv[.-][0-9]'] = 4, ['[0-9][.-]?catv'] = 4, +['[.-]wifi[.-]'] = 4, ['wifi[.-][0-9]'] = 4, ['[0-9][.-]?wifi'] = 4, +['[.-]unused-addr[.-]'] = 5, ['unused-addr[.-][0-9]'] = 5, ['[0-9][.-]?unused-addr'] = 5, +['[.-]dial-?up[.-]'] = 4, ['dial-?up[.-][0-9]'] = 4, ['[0-9][.-]?dial-?up'] = 4, +['[.-]gprs[.-]'] = 4, ['gprs[.-][0-9]'] = 4, ['[0-9][.-]?gprs'] = 4, +['[.-]cdma[.-]'] = 4, ['cdma[.-][0-9]'] = 4, ['[0-9][.-]?cdma'] = 4, +['[.-]homeuser[.-]'] = 4, ['homeuser[.-][0-9]'] = 4, ['[0-9][.-]?homeuser'] = 4, +['[.-]in-?addr[.-]'] = 3, ['in-?addr[.-][0-9]'] = 3, ['[0-9][.-]?in-?addr'] = 3, +['[.-]pool[.-]'] = 3, ['pool[.-][0-9]'] = 3, ['[0-9][.-]?pool'] = 3, +['[.-]cable[.-]'] = 5, ['cable[.-][0-9]'] = 5, ['[0-9][.-]?cable'] = 5, +['[.-]host[.-]'] = 3, ['host[.-][0-9]'] = 3, ['[0-9][.-]?host'] = 3, +['[.-]customers[.-]'] = 2, ['customers[.-][0-9]'] = 2, ['[0-9][.-]?customers'] = 2 +} + +--Checks for HELO only +local checks_hello = { +['localhost$'] = 5, +['^(dsl)?(device|speedtouch)\\.lan$'] = 5, +['\\.(lan|local|home|localdomain|intra|in-addr.arpa|priv|online|user|veloxzon)$'] = 5, +['^\\[*127\.'] = 5, ['^\\[*10\\.'] = 5, ['^\\[*172\\.16\\.'] = 5, ['^\\[*192\\.168\\.'] = 5, +--bareip +['^\[*\\d+[x.-]\\d+[x.-]\\d+[x.-]\\d+\\]*$'] = 5 +} + +-- +local function trim1(s) + return (s:gsub("^%s*(.-)%s*$", "%1")) +end + +-- +local function check_regexp(str, regexp_text) + local re = regexp.get_cached(regexp_text) + if not re then re = regexp.create(regexp_text, 'i') end + if re:match(str) then return true end +return false +end + +-- +local function hfilter(task) + local recvh = task:get_received_headers() + + if table.maxn(recvh) == 0 then + return false + end + + --IP-- + local ip = false + local rip = task:get_from_ip() + if rip then + ip = rip:to_string() + if ip and ip == '0.0.0.0' then + ip = false + end + end + + --HOSTNAME-- + local r = recvh[1] + local hostname = false + local hostname_lower = false + if r['real_hostname'] and ( r['real_hostname'] ~= 'unknown' or not check_regexp(r['real_hostname'], '^\\d+\\.\\d+\\.\\d+\\.\\d+$') ) then + hostname = r['real_hostname'] + hostname_lower = string.lower(hostname) + end + + --HELO-- + local helo = task:get_helo() + local helo_lower = false + if helo then + helo_lower = string.lower(helo) + else + helo = false + helo_lower = false + end + + -- Check's HELO + local checks_hello_found = false + if helo then + -- Regexp check HELO + for regexp,weight in pairs(checks_hello) do + if check_regexp(helo_lower, regexp) then + task:insert_result('HFILTER_HELO' .. weight, 1.0) + checks_hello_found = true + break + end + end + if not checks_hello_found then + local checks_hello_found = false + for regexp,weight in pairs(checks_hellohost) do + if check_regexp(helo_lower, regexp) then + task:insert_result('HFILTER_HELO' .. weight, 1.0) + break + end + end + end + + -------- + local function hfilter_heloip_cb_mx_a(resolver, to_resolve, results, err) + task:inc_dns_req() + if not results then + task:insert_result('HFILTER_HELO_NORESOLVE_MX', 1.0) + elseif ip then + for _,result in pairs(results) do + local helo_ip = result:to_string() + if helo_ip == ip then + --task:insert_result('HFILTER_CHECK_HELO_IP_TRUE_MX', 0.0) + return true + end + end + task:insert_result('HFILTER_CHECK_HELO_IP_MX', 1.0) + end + end + -- + local function hfilter_heloip_cb_mx(resolver, to_resolve, results, err) + task:inc_dns_req() + if not results then + task:insert_result('HFILTER_HELO_NORESOLVE_A_OR_MX', 1.0) + else + for _,mx in pairs(results) do + if mx['name'] then + task:get_resolver():resolve_a(task:get_session(), task:get_mempool(), mx['name'], hfilter_heloip_cb_mx_a) + end + end + end + end + -- + local function hfilter_heloip_cb_a(resolver, to_resolve, results, err) + task:inc_dns_req() + if not results then + task:get_resolver():resolve_mx(task:get_session(), task:get_mempool(), helo_lower, hfilter_heloip_cb_mx) + elseif ip then + for _,result in pairs(results) do + local helo_ip = result:to_string() + if helo_ip == ip then + --task:insert_result('HFILTER_CHECK_HELO_IP_TRUE_A', 0.0) + return true + end + end + task:insert_result('HFILTER_CHECK_HELO_IP_A', 1.0) + end + end + -------- + + --FQDN check HELO + if check_regexp(helo, '(?=^.{4,255}$)(^((?!-)[a-zA-Z0-9-]{1,63}(? 0 and parts[1]:get_content() then + local part_text = trim1(parts[1]:get_content()) + local total_part_len = string.len(part_text) + if total_part_len > 0 then + local urls = task:get_urls() + if urls then + local total_url_len = 0 + for _,url in ipairs(urls) do + total_url_len = total_url_len + string.len(url:get_text()) + end + + if total_url_len > 0 then + if total_url_len + 7 > total_part_len then + task:insert_result('HFILTER_URL_ONLY', 1.00) + else + if not string.find(part_text, "\n") then + task:insert_result('HFILTER_URL_ONELINE', 1.00) + end + end + end + end + end + end + end + + return false +end + +rspamd_config:register_symbols(hfilter, 1.0, +'HFILTER_HELO_1', 'HFILTER_HELO_2', 'HFILTER_HELO_3', 'HFILTER_HELO_4', 'HFILTER_HELO_5', +'HFILTER_HELO_NORESOLVE_MX', 'HFILTER_CHECK_HELO_IP_MX', 'HFILTER_HELO_NORESOLVE_A_OR_MX', +'HFILTER_CHECK_HELO_IP_A', 'HFILTER_HELO_NOT_FQDN', +'HFILTER_HOSTNAME_1', 'HFILTER_HOSTNAME_2', 'HFILTER_HOSTNAME_3', 'HFILTER_HOSTNAME_4', 'HFILTER_HOSTNAME_5', +'HFILTER_HOSTNAME_NOPTR', +'HFILTER_URL_ONLY', +'HFILTER_URL_ONELINE'); diff --git a/conf/lua/rspamd.lua b/conf/lua/rspamd.lua index 16b99cb01..f51715934 100644 --- a/conf/lua/rspamd.lua +++ b/conf/lua/rspamd.lua @@ -97,6 +97,10 @@ local function file_exists(filename) end end +if file_exists('hfilter.lua') then + dofile('hfilter.lua') +end + if file_exists('rspamd.local.lua') then dofile('rspamd.local.lua') end diff --git a/conf/metrics.conf b/conf/metrics.conf index 4db1e45a3..0489f90e6 100644 --- a/conf/metrics.conf +++ b/conf/metrics.conf @@ -698,4 +698,22 @@ metric { description = "Helo host FQDN check"; name = "HELO_NOT_FQDN"; } +# hfilter symbols + symbol { weight = 1.00; name = "HFILTER_HELO_1"; description = "Helo host checks (very low)"; } + symbol { weight = 2.00; name = "HFILTER_HELO_2"; description = "Helo host checks (low)"; } + symbol { weight = 3.00; name = "HFILTER_HELO_3"; description = "Helo host checks (medium)"; } + symbol { weight = 3.50; name = "HFILTER_HELO_4"; description = "Helo host checks (hard)"; } + symbol { weight = 4.00; name = "HFILTER_HELO_5"; description = "Helo host checks (very hard)"; } + symbol { weight = 1.00; name = "HFILTER_HOSTNAME_1"; description = "Hostname checks (very low)"; } + symbol { weight = 2.00; name = "HFILTER_HOSTNAME_2"; description = "Hostname checks (low)"; } + symbol { weight = 3.00; name = "HFILTER_HOSTNAME_3"; description = "Hostname checks (medium)"; } + symbol { weight = 3.50; name = "HFILTER_HOSTNAME_4"; description = "Hostname checks (hard)"; } + symbol { weight = 4.00; name = "HFILTER_HOSTNAME_5"; description = "Hostname checks (very hard)"; } + symbol { weight = 2.50; name = "HFILTER_CHECK_HELO_IP_A"; description = "Helo A IP != hostname IP"; } + symbol { weight = 2.00; name = "HFILTER_HELO_NORESOLVE_A_OR_MX"; description = "Helo no resolve to A or MX"; } + symbol { weight = 2.00; name = "HFILTER_CHECK_HELO_IP_MX"; description = "Helo MX IP != hostname IP"; } + symbol { weight = 2.00; name = "HFILTER_HELO_NORESOLVE_MX"; description = "MX found in Helo and no resolve"; } + symbol { weight = 4.00; name = "HFILTER_HOSTNAME_NOPTR"; description = "No PTR for IP"; } + symbol { weight = 3.50; name = "HFILTER_URL_ONLY"; description = "URL only in body"; } + symbol { weight = 2.00; name = "HFILTER_URL_ONELINE"; description = "One line URL and text in body"; } } -- 2.39.5