1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321 |
- --[[
- Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ]]--
-
- if confighelp then
- return
- end
-
- -- Multimap is rspamd module designed to define and operate with different maps
-
- local rules = {}
- local rspamd_logger = require "rspamd_logger"
- local rspamd_util = require "rspamd_util"
- local rspamd_regexp = require "rspamd_regexp"
- local rspamd_expression = require "rspamd_expression"
- local rspamd_ip = require "rspamd_ip"
- local lua_util = require "lua_util"
- local lua_selectors = require "lua_selectors"
- local lua_maps = require "lua_maps"
- local redis_params
- local fun = require "fun"
- local N = 'multimap'
-
- local value_types = {
- ip = {
- get_value = function(ip) return ip:to_string() end,
- },
- from = {
- get_value = function(val) return val end,
- },
- helo = {
- get_value = function(val) return val end,
- },
- header = {
- get_value = function(val) return val end,
- },
- rcpt = {
- get_value = function(val) return val end,
- },
- user = {
- get_value = function(val) return val end,
- },
- url = {
- get_value = function(val) return val end,
- },
- dnsbl = {
- get_value = function(ip) return ip:to_string() end,
- },
- filename = {
- get_value = function(val) return val end,
- },
- content = {
- get_value = function() return nil end,
- },
- hostname = {
- get_value = function(val) return val end,
- },
- asn = {
- get_value = function(val) return val end,
- },
- country = {
- get_value = function(val) return val end,
- },
- received = {
- get_value = function(val) return val end,
- },
- mempool = {
- get_value = function(val) return val end,
- },
- selector = {
- get_value = function(val) return val end,
- },
- symbol_options = {
- get_value = function(val) return val end,
- },
- }
-
- local function ip_to_rbl(ip, rbl)
- return table.concat(ip:inversed_str_octets(), ".") .. '.' .. rbl
- end
-
- local function apply_hostname_filter(task, filter, hostname, r)
- if filter == 'tld' then
- local tld = rspamd_util.get_tld(hostname)
- return tld
- elseif filter == 'top' then
- local tld = rspamd_util.get_tld(hostname)
- return tld:match('[^.]*$') or tld
- else
- if not r['re_filter'] then
- local pat = string.match(filter, 'tld:regexp:(.+)')
- if not pat then
- rspamd_logger.errx(task, 'bad search filter: %s', filter)
- return
- end
- r['re_filter'] = rspamd_regexp.create_cached(pat)
- if not r['re_filter'] then
- rspamd_logger.errx(task, 'couldnt create regex: %s', pat)
- return
- end
- end
- local tld = rspamd_util.get_tld(hostname)
- local res = r['re_filter']:search(tld)
- if res then
- return res[1]
- else
- return nil
- end
- end
- end
-
- local function apply_url_filter(task, filter, url, r)
- if not filter then
- return url:get_host()
- end
-
- if filter == 'tld' then
- return url:get_tld()
- elseif filter == 'top' then
- local tld = url:get_tld()
- return tld:match('[^.]*$') or tld
- elseif filter == 'full' then
- return url:get_text()
- elseif filter == 'is_phished' then
- if url:is_phished() then
- return url:get_host()
- else
- return nil
- end
- elseif filter == 'is_redirected' then
- if url:is_redirected() then
- return url:get_host()
- else
- return nil
- end
- elseif filter == 'is_obscured' then
- if url:is_obscured() then
- return url:get_host()
- else
- return nil
- end
- elseif filter == 'path' then
- return url:get_path()
- elseif filter == 'query' then
- return url:get_query()
- elseif string.find(filter, 'tag:') then
- local tags = url:get_tags()
- local want_tag = string.match(filter, 'tag:(.*)')
- for _, t in ipairs(tags) do
- if t == want_tag then
- return url:get_host()
- end
- end
- return nil
- elseif string.find(filter, 'tld:regexp:') then
- if not r['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
- if type and pat then
- r['re_filter'] = rspamd_regexp.create_cached(pat)
- end
- end
-
- if not r['re_filter'] then
- rspamd_logger.errx(task, 'bad search filter: %s', filter)
- else
- local results = r['re_filter']:search(url:get_tld())
- if results then
- return results[1]
- else
- return nil
- end
- end
- elseif string.find(filter, 'full:regexp:') then
- if not r['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
- if type and pat then
- r['re_filter'] = rspamd_regexp.create_cached(pat)
- end
- end
-
- if not r['re_filter'] then
- rspamd_logger.errx(task, 'bad search filter: %s', filter)
- else
- local results = r['re_filter']:search(url:get_text())
- if results then
- return results[1]
- else
- return nil
- end
- end
- elseif string.find(filter, 'regexp:') then
- if not r['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
- if type and pat then
- r['re_filter'] = rspamd_regexp.create_cached(pat)
- end
- end
-
- if not r['re_filter'] then
- rspamd_logger.errx(task, 'bad search filter: %s', filter)
- else
- local results = r['re_filter']:search(url:get_host())
- if results then
- return results[1]
- else
- return nil
- end
- end
- elseif string.find(filter, '^template:') then
- if not r['template'] then
- r['template'] = string.match(filter, '^template:(.+)')
- end
-
- if r['template'] then
- return lua_util.template(r['template'], url:to_table())
- end
- end
-
- return url:get_host()
- end
-
- local function apply_addr_filter(task, filter, input, rule)
- if filter == 'email:addr' or filter == 'email' then
- local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
- if addr and addr[1] then
- return fun.totable(fun.map(function(a) return a.addr end, addr))
- end
- elseif filter == 'email:user' then
- local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
- if addr and addr[1] then
- return fun.totable(fun.map(function(a) return a.user end, addr))
- end
- elseif filter == 'email:domain' then
- local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
- if addr and addr[1] then
- return fun.totable(fun.map(function(a) return a.domain end, addr))
- end
- elseif filter == 'email:domain:tld' then
- local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
- if addr and addr[1] then
- return fun.totable(fun.map(function(a) return rspamd_util.get_tld(a.domain) end, addr))
- end
- elseif filter == 'email:name' then
- local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
- if addr and addr[1] then
- return fun.totable(fun.map(function(a) return a.name end, addr))
- end
- elseif filter == 'ip_addr' then
- local ip_addr = rspamd_ip.from_string(input)
-
- if ip_addr and ip_addr:is_valid() then
- return ip_addr
- end
- else
- -- regexp case
- if not rule['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
- if type and pat then
- rule['re_filter'] = rspamd_regexp.create_cached(pat)
- end
- end
-
- if not rule['re_filter'] then
- rspamd_logger.errx(task, 'bad search filter: %s', filter)
- else
- local results = rule['re_filter']:search(input)
- if results then
- return results[1]
- end
- end
- end
-
- return input
- end
- local function apply_filename_filter(task, filter, fn, r)
- if filter == 'extension' or filter == 'ext' then
- return string.match(fn, '%.([^.]+)$')
- elseif string.find(filter, 'regexp:') then
- if not r['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
- if type and pat then
- r['re_filter'] = rspamd_regexp.create_cached(pat)
- end
- end
-
- if not r['re_filter'] then
- rspamd_logger.errx(task, 'bad search filter: %s', filter)
- else
- local results = r['re_filter']:search(fn)
- if results then
- return results[1]
- else
- return nil
- end
- end
- end
-
- return fn
- end
-
- local function apply_regexp_filter(task, filter, fn, r)
- if string.find(filter, 'regexp:') then
- if not r['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
- if type and pat then
- r['re_filter'] = rspamd_regexp.create_cached(pat)
- end
- end
-
- if not r['re_filter'] then
- rspamd_logger.errx(task, 'bad search filter: %s', filter)
- else
- local results = r['re_filter']:search(fn, false, true)
- if results then
- return results[1][2]
- else
- return nil
- end
- end
- end
-
- return fn
- end
-
- local function apply_content_filter(task, filter)
- if filter == 'body' then
- return {task:get_rawbody()}
- elseif filter == 'full' then
- return {task:get_content()}
- elseif filter == 'headers' then
- return {task:get_raw_headers()}
- elseif filter == 'text' then
- local ret = {}
- for _,p in ipairs(task:get_text_parts()) do
- table.insert(ret, p:get_content())
- end
- return ret
- elseif filter == 'rawtext' then
- local ret = {}
- for _,p in ipairs(task:get_text_parts()) do
- table.insert(ret, p:get_raw_content())
- end
- return ret
- elseif filter == 'oneline' then
- local ret = {}
- for _,p in ipairs(task:get_text_parts()) do
- table.insert(ret, p:get_content_oneline())
- end
- return ret
- else
- rspamd_logger.errx(task, 'bad search filter: %s', filter)
- end
-
- return {}
- end
-
- local multimap_filters = {
- from = apply_addr_filter,
- rcpt = apply_addr_filter,
- helo = apply_hostname_filter,
- symbol_options = apply_regexp_filter,
- header = apply_addr_filter,
- url = apply_url_filter,
- filename = apply_filename_filter,
- mempool = apply_regexp_filter,
- selector = apply_regexp_filter,
- hostname = apply_hostname_filter,
- --content = apply_content_filter, -- Content filters are special :(
- }
-
- local multimap_grammar
-
- local function multimap_query_redis(key, task, value, callback)
- local cmd = 'HGET'
- if type(value) == 'userdata' and value.class == 'rspamd{ip}' then
- cmd = 'HMGET'
- end
-
- local srch = {key}
-
- -- Insert all ips for some mask :(
- if type(value) == 'userdata' and value.class == 'rspamd{ip}' then
- srch[#srch + 1] = tostring(value)
- -- IPv6 case
- local maxbits = 128
- local minbits = 64
- if value:get_version() == 4 then
- maxbits = 32
- minbits = 8
- end
- for i=maxbits,minbits,-1 do
- local nip = value:apply_mask(i):tostring() .. "/" .. i
- srch[#srch + 1] = nip
- end
- else
- srch[#srch + 1] = value
- end
-
- local function redis_map_cb(err, data)
- lua_util.debugm(N, task, 'got reply from Redis when trying to get key %s: err=%s, data=%s',
- key, err, data)
- if not err and type(data) ~= 'userdata' then
- callback(data)
- end
- end
-
- return rspamd_redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- false, -- is write
- redis_map_cb, --callback
- cmd, -- command
- srch -- arguments
- )
- end
-
- local function multimap_callback(task, rule)
- local pre_filter = rule['prefilter']
-
- local function match_element(r, value, callback)
- if not value then
- return false
- end
-
- lua_util.debugm(N, task, 'check value %s for multimap %s', value,
- rule.symbol)
-
- local ret = false
-
- if r.redis_key then
- -- Deal with hash name here: it can be either plain string or a selector
- if type(r.redis_key) == 'string' then
- ret = multimap_query_redis(r.redis_key, task, value, callback)
- else
- -- Here we have a selector
- local results = r.redis_key(task)
-
- -- Here we need to spill this function into multiple queries
- if type(results) == 'table' then
- for _,res in ipairs(results) do
- ret = multimap_query_redis(res, task, value, callback)
-
- if not ret then
- break
- end
- end
- else
- ret = multimap_query_redis(results, task, value, callback)
- end
- end
-
- return ret
- elseif r.radix then
- ret = r.radix:get_key(value)
- elseif r.hash then
- if type(value) == 'userdata' then
- if value.class == 'rspamd{ip}' then
- value = value:tostring()
- end
- end
- ret = r.hash:get_key(value)
- end
-
- lua_util.debugm(N, task, 'found return "%s" for multimap %s', ret,
- rule.symbol)
-
- if ret then
- if type(ret) == 'table' then
- for _,elt in ipairs(ret) do
- callback(elt)
- end
-
- ret = true
- else
- callback(ret)
- end
- end
-
- return ret
- end
-
- -- Parse result in form: <symbol>:<score>|<symbol>|<score>
- local function parse_ret(parse_rule, p_ret)
- if p_ret and type(p_ret) == 'string' then
- local lpeg = require "lpeg"
-
- if not multimap_grammar then
- local number = {}
-
- local digit = lpeg.R("09")
- number.integer =
- (lpeg.S("+-") ^ -1) *
- (digit ^ 1)
-
- -- Matches: .6, .899, .9999873
- number.fractional =
- (lpeg.P(".") ) *
- (digit ^ 1)
-
- -- Matches: 55.97, -90.8, .9
- number.decimal =
- (number.integer * -- Integer
- (number.fractional ^ -1)) + -- Fractional
- (lpeg.S("+-") * number.fractional) -- Completely fractional number
-
- local sym_start = lpeg.R("az", "AZ") + lpeg.S("_")
- local sym_elt = sym_start + lpeg.R("09")
- local symbol = sym_start * sym_elt ^0
- local symbol_cap = lpeg.Cg(symbol, 'symbol')
- local score_cap = lpeg.Cg(number.decimal, 'score')
- local symscore_cap = (symbol_cap * lpeg.P(":") * score_cap)
- local grammar = symscore_cap + symbol_cap + score_cap
- multimap_grammar = lpeg.Ct(grammar)
- end
- local tbl = multimap_grammar:match(p_ret)
-
- if tbl then
- local sym
- local score = 1.0
-
- if tbl['symbol'] then
- sym = tbl['symbol']
- end
- if tbl['score'] then
- score = tbl['score']
- end
-
- return true,sym,score
- else
- if p_ret ~= '' then
- rspamd_logger.infox(task, '%s: cannot parse string "%s"',
- parse_rule.symbol, p_ret)
- end
-
- return true,nil,1.0
- end
- elseif type(p_ret) == 'boolean' then
- return p_ret,nil,1.0
- end
-
- return false,nil,0.0
- end
-
- local function insert_results(result, opt)
- local _,symbol,score = parse_ret(rule, result)
- local forced = false
- if symbol then
- if rule.symbols_set then
- if not rule.symbols_set[symbol] then
- rspamd_logger.infox(task, 'symbol %s is not registered for map %s, ' ..
- 'replace it with just %s',
- symbol, rule.symbol, rule.symbol)
- symbol = rule.symbol
- end
- elseif rule.disable_multisymbol then
- symbol = rule.symbol
- if type(opt) == 'table' then
- table.insert(opt, result)
- elseif type(opt) ~= nil then
- opt = {opt,result}
- else
- opt = {result}
- end
- else
- forced = true
- end
- else
- symbol = rule.symbol
- end
-
- if opt then
- if type(opt) == 'table' then
- task:insert_result(forced, symbol, score, fun.totable(fun.map(tostring, opt)))
- else
- task:insert_result(forced, symbol, score, tostring(opt))
- end
-
- else
- task:insert_result(forced, symbol, score)
- end
-
- if pre_filter then
- local message = rule.message
- if rule.message_func then
- message = rule.message_func(task, rule.symbol, opt)
- end
- if message then
- task:set_pre_result(rule.action, message, N)
- else
- task:set_pre_result(rule.action, 'Matched map: ' .. rule.symbol, N)
- end
- end
- end
-
- -- Match a single value for against a single rule
- local function match_rule(r, value)
- local function rule_callback(result)
- if result then
- if type(result) == 'table' then
- for _,rs in ipairs(result) do
- if type(rs) ~= 'userdata' then
- rule_callback(rs)
- end
- end
- return
- end
- local opt = value_types[r['type']].get_value(value)
- insert_results(result, opt)
- end
- end
-
- if r.filter or r.type == 'url' then
- local fn = multimap_filters[r.type]
-
- if fn then
-
- local filtered_value = fn(task, r.filter, value, r)
- lua_util.debugm(N, task, 'apply filter %s for rule %s: %s -> %s',
- r.filter, r.symbol, value, filtered_value)
- value = filtered_value
- end
- end
-
- if type(value) == 'table' then
- fun.each(function(elt) match_element(r, elt, rule_callback) end, value)
- else
- match_element(r, value, rule_callback)
- end
- end
-
- -- Match list of values according to the field
- local function match_list(r, ls, fields)
- if ls then
- if fields then
- fun.each(function(e)
- local match = e[fields[1]]
- if match then
- if fields[2] then
- match = fields[2](match)
- end
- match_rule(r, match)
- end
- end, ls)
- else
- fun.each(function(e) match_rule(r, e) end, ls)
- end
- end
- end
-
- local function match_addr(r, addr)
- match_list(r, addr, {'addr'})
-
- if not r.filter then
- match_list(r, addr, {'domain'})
- match_list(r, addr, {'user'})
- end
- end
-
- local function match_url(r, url)
- match_rule(r, url)
- end
-
- local function match_hostname(r, hostname)
- match_rule(r, hostname)
- end
-
- local function match_filename(r, fn)
- match_rule(r, fn)
- end
-
- local function match_received_header(r, pos, total, h)
- local use_tld = false
- local filter = r['filter'] or 'real_ip'
- if filter:match('^tld:') then
- filter = filter:sub(5)
- use_tld = true
- end
- local v = h[filter]
- if v then
- local min_pos = tonumber(r['min_pos'])
- local max_pos = tonumber(r['max_pos'])
- if min_pos then
- if min_pos < 0 then
- if min_pos == -1 then
- if (pos ~= total) then
- return
- end
- else
- if pos <= (total - (min_pos*-1)) then
- return
- end
- end
- elseif pos < min_pos then
- return
- end
- end
- if max_pos then
- if max_pos < -1 then
- if (total - (max_pos*-1)) >= pos then
- return
- end
- elseif max_pos > 0 then
- if pos > max_pos then
- return
- end
- end
- end
- local match_flags = r['flags']
- local nmatch_flags = r['nflags']
- if match_flags or nmatch_flags then
- local got_flags = h['flags']
- if match_flags then
- for _, flag in ipairs(match_flags) do
- if not got_flags[flag] then return end
- end
- end
- if nmatch_flags then
- for _, flag in ipairs(nmatch_flags) do
- if got_flags[flag] then return end
- end
- end
- end
- if filter == 'real_ip' or filter == 'from_ip' then
- if type(v) == 'string' then
- v = rspamd_ip.from_string(v)
- end
- if v and v:is_valid() then
- match_rule(r, v)
- end
- else
- if use_tld and type(v) == 'string' then
- v = rspamd_util.get_tld(v)
- end
- match_rule(r, v)
- end
- end
- end
-
- local function match_content(r)
- local data
-
- if r['filter'] then
- data = apply_content_filter(task, r['filter'], r)
- else
- data = {task:get_content()}
- end
-
- for _,v in ipairs(data) do
- match_rule(r, v)
- end
- end
-
- if rule.expression and not rule.combined then
- local res,trace = rule['expression']:process_traced(task)
-
- if not res or res == 0 then
- lua_util.debugm(N, task, 'condition is false for %s',
- rule.symbol)
- return
- else
- lua_util.debugm(N, task, 'condition is true for %s: %s',
- rule.symbol,
- trace)
- end
- end
-
- local process_rule_funcs = {
- ip = function()
- local ip = task:get_from_ip()
- if ip and ip:is_valid() then
- match_rule(rule, ip)
- end
- end,
- dnsbl = function()
- local ip = task:get_from_ip()
- if ip and ip:is_valid() then
- local to_resolve = ip_to_rbl(ip, rule['map'])
- local function dns_cb(_, _, results, err)
- lua_util.debugm(N, rspamd_config,
- 'resolve() finished: results=%1, err=%2, to_resolve=%3',
- results, err, to_resolve)
-
- if err and
- (err ~= 'requested record is not found' and
- err ~= 'no records with this name') then
- rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, results)
- elseif results then
- task:insert_result(rule['symbol'], 1, rule['map'])
- if pre_filter then
- task:set_pre_result(rule['action'],
- 'Matched map: ' .. rule['symbol'], N)
- end
- end
- end
-
- task:get_resolver():resolve_a({
- task= task,
- name = to_resolve,
- callback = dns_cb,
- forced = true
- })
- end
- end,
- header = function()
- if type(rule['header']) == 'table' then
- for _,rh in ipairs(rule['header']) do
- local hv = task:get_header_full(rh)
- match_list(rule, hv, {'decoded'})
- end
- else
- local hv = task:get_header_full(rule['header'])
- match_list(rule, hv, {'decoded'})
- end
- end,
- rcpt = function()
- if task:has_recipients('smtp') then
- local rcpts = task:get_recipients('smtp')
- match_addr(rule, rcpts)
- elseif task:has_recipients('mime') then
- local rcpts = task:get_recipients('mime')
- match_addr(rule, rcpts)
- end
- end,
- from = function()
- if task:has_from('smtp') then
- local from = task:get_from('smtp')
- match_addr(rule, from)
- elseif task:has_from('mime') then
- local from = task:get_from('mime')
- match_addr(rule, from)
- end
- end,
- helo = function()
- local helo = task:get_helo()
- if helo then
- match_hostname(rule, helo)
- end
- end,
- url = function()
- if task:has_urls() then
- local msg_urls = task:get_urls()
-
- for _,url in ipairs(msg_urls) do
- match_url(rule, url)
- end
- end
- end,
- user = function()
- local user = task:get_user()
- if user then
- match_rule(rule, user)
- end
- end,
- filename = function()
- local parts = task:get_parts()
-
- local function filter_parts(p)
- return p:is_attachment() or (not p:is_text()) and (not p:is_multipart())
- end
-
- local function filter_archive(p)
- local ext = p:get_detected_ext()
- local det_type = 'unknown'
-
- if ext then
- local lua_magic_types = require "lua_magic/types"
- local det_t = lua_magic_types[ext]
-
- if det_t then
- det_type = det_t.type
- end
- end
-
- return p:is_archive() and det_type == 'archive' and not rule.skip_archives
- end
-
- for _,p in fun.iter(fun.filter(filter_parts, parts)) do
- if filter_archive(p) then
- local fnames = p:get_archive():get_files(1000)
-
- for _,fn in ipairs(fnames) do
- match_filename(rule, fn)
- end
- end
-
- local fn = p:get_filename()
- if fn then
- match_filename(rule, fn)
- end
- -- Also deal with detected content type
- if not rule.skip_detected then
- local ext = p:get_detected_ext()
-
- if ext then
- local fake_fname = string.format('detected.%s', ext)
- lua_util.debugm(N, task, 'detected filename %s',
- fake_fname)
- match_filename(rule, fake_fname)
- end
- end
- end
- end,
-
- content = function()
- match_content(rule)
- end,
- hostname = function()
- local hostname = task:get_hostname()
- if hostname then
- match_hostname(rule, hostname)
- end
- end,
- asn = function()
- local asn = task:get_mempool():get_variable('asn')
- if asn then
- match_rule(rule, asn)
- end
- end,
- country = function()
- local country = task:get_mempool():get_variable('country')
- if country then
- match_rule(rule, country)
- end
- end,
- mempool = function()
- local var = task:get_mempool():get_variable(rule['variable'])
- if var then
- match_rule(rule, var)
- end
- end,
- symbol_options = function()
- local sym = task:get_symbol(rule['target_symbol'])
- if sym and sym[1].options then
- for _, o in ipairs(sym[1].options) do
- match_rule(rule, o)
- end
- end
- end,
- received = function()
- local hdrs = task:get_received_headers()
- if hdrs and hdrs[1] then
- if not rule['artificial'] then
- hdrs = fun.filter(function(h)
- return not h['flags']['artificial']
- end, hdrs):totable()
- end
- for pos, h in ipairs(hdrs) do
- match_received_header(rule, pos, #hdrs, h)
- end
- end
- end,
- selector = function()
- local elts = rule.selector(task)
-
- if elts then
- if type(elts) == 'table' then
- for _,elt in ipairs(elts) do
- match_rule(rule, elt)
- end
- else
- match_rule(rule, elts)
- end
- end
- end,
- combined = function()
- local ret,trace = rule.combined:process(task)
- if ret and ret ~= 0 then
- for n,t in pairs(trace) do
- insert_results(t.value, string.format("%s=%s",
- n, t.matched))
- end
- end
- end,
- }
-
- local rt = rule.type
- local process_func = process_rule_funcs[rt]
- if process_func then
- process_func()
- else
- rspamd_logger.errx(task, 'Unrecognised rule type: %s', rt)
- end
- end
-
- local function gen_multimap_callback(rule)
- return function(task)
- multimap_callback(task, rule)
- end
- end
-
- local function add_multimap_rule(key, newrule)
- local ret = false
-
- local function multimap_load_hash(rule)
- if rule['regexp'] then
- if rule['multi'] then
- rule.hash = lua_maps.map_add_from_ucl(rule.map, 'regexp_multi',
- rule.description)
- else
- rule.hash = lua_maps.map_add_from_ucl(rule.map, 'regexp',
- rule.description)
- end
- elseif rule['glob'] then
- if rule['multi'] then
- rule.hash = lua_maps.map_add_from_ucl(rule.map, 'glob_multi',
- rule.description)
- else
- rule.hash = lua_maps.map_add_from_ucl(rule.map, 'glob',
- rule.description)
- end
- else
- rule.hash = lua_maps.map_add_from_ucl(rule.map, 'hash',
- rule.description)
- end
- end
-
- local known_generic_types = {
- header = true,
- rcpt = true,
- from = true,
- helo = true,
- symbol_options = true,
- filename = true,
- url = true,
- user = true,
- content = true,
- hostname = true,
- asn = true,
- country = true,
- mempool = true,
- selector = true,
- combined = true
- }
-
- if newrule['message_func'] then
- newrule['message_func'] = assert(load(newrule['message_func']))()
- end
- if newrule['url'] and not newrule['map'] then
- newrule['map'] = newrule['url']
- end
- if not (newrule.map or newrule.rules) then
- rspamd_logger.errx(rspamd_config, 'incomplete rule, missing map')
- return nil
- end
- if not newrule['symbol'] and key then
- newrule['symbol'] = key
- elseif not newrule['symbol'] then
- rspamd_logger.errx(rspamd_config, 'incomplete rule, missing symbol')
- return nil
- end
- if not newrule['description'] then
- newrule['description'] = string.format('multimap, type %s: %s', newrule['type'],
- newrule['symbol'])
- end
- if newrule['type'] == 'mempool' and not newrule['variable'] then
- rspamd_logger.errx(rspamd_config, 'mempool map requires variable')
- return nil
- end
- if newrule['type'] == 'selector' then
- if not newrule['selector'] then
- rspamd_logger.errx(rspamd_config, 'selector map requires selector definition')
- return nil
- else
- local selector = lua_selectors.create_selector_closure(
- rspamd_config, newrule['selector'], newrule['delimiter'] or "")
-
- if not selector then
- rspamd_logger.errx(rspamd_config, 'selector map has invalid selector: "%s", symbol: %s',
- newrule['selector'], newrule['symbol'])
- return nil
- end
-
- newrule.selector = selector
- end
- end
- if type(newrule['map']) == 'string' and
- string.find(newrule['map'], '^redis://.*$') then
- if not redis_params then
- rspamd_logger.infox(rspamd_config, 'no redis servers are specified, ' ..
- 'cannot add redis map %s: %s', newrule['symbol'], newrule['map'])
- return nil
- end
-
- newrule['redis_key'] = string.match(newrule['map'], '^redis://(.*)$')
-
- if newrule['redis_key'] then
- ret = true
- end
- elseif type(newrule['map']) == 'string' and
- string.find(newrule['map'], '^redis%+selector://.*$') then
- if not redis_params then
- rspamd_logger.infox(rspamd_config, 'no redis servers are specified, ' ..
- 'cannot add redis map %s: %s', newrule['symbol'], newrule['map'])
- return nil
- end
-
- local selector_str = string.match(newrule['map'], '^redis%+selector://(.*)$')
- local selector = lua_selectors.create_selector_closure(
- rspamd_config, selector_str, newrule['delimiter'] or "")
-
- if not selector then
- rspamd_logger.errx(rspamd_config, 'redis selector map has invalid selector: "%s", symbol: %s',
- selector_str, newrule['symbol'])
- return nil
- end
-
- newrule['redis_key'] = selector
- ret = true
- elseif newrule.type == 'combined' then
- local lua_maps_expressions = require "lua_maps_expressions"
- newrule.combined = lua_maps_expressions.create(rspamd_config,
- {
- rules = newrule.rules,
- expression = newrule.expression
- }, N, 'Combined map for ' .. newrule.symbol)
- if not newrule.combined then
- rspamd_logger.errx(rspamd_config, 'cannot add combined map for %s', newrule.symbol)
- else
- ret = true
- end
- else
- if newrule['type'] == 'ip' then
- newrule['radix'] = lua_maps.map_add_from_ucl(newrule.map, 'radix',
- newrule.description)
- if newrule['radix'] then
- ret = true
- else
- rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
- newrule['map'])
- end
- elseif newrule['type'] == 'received' then
- if type(newrule['flags']) == 'table' and newrule['flags'][1] then
- newrule['flags'] = newrule['flags']
- elseif type(newrule['flags']) == 'string' then
- newrule['flags'] = {newrule['flags']}
- end
- if type(newrule['nflags']) == 'table' and newrule['nflags'][1] then
- newrule['nflags'] = newrule['nflags']
- elseif type(newrule['nflags']) == 'string' then
- newrule['nflags'] = {newrule['nflags']}
- end
- local filter = newrule['filter'] or 'real_ip'
- if filter == 'real_ip' or filter == 'from_ip' then
- newrule['radix'] = lua_maps.map_add_from_ucl(newrule.map, 'radix',
- newrule.description)
- if newrule['radix'] then
- ret = true
- end
- else
- multimap_load_hash(newrule)
-
- if newrule['hash'] then
- ret = true
- else
- rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
- newrule['map'])
- end
- end
- elseif known_generic_types[newrule.type] then
-
- if newrule.filter == 'ip_addr' then
- newrule['radix'] = lua_maps.map_add_from_ucl(newrule.map, 'radix',
- newrule.description)
- elseif not newrule.combined then
- multimap_load_hash(newrule)
- end
-
- if newrule.hash or newrule.radix then
- ret = true
- else
- rspamd_logger.warnx(rspamd_config, 'Cannot add rule: map doesn\'t exists: %1',
- newrule['map'])
- end
- elseif newrule['type'] == 'dnsbl' then
- ret = true
- end
- end
-
- if newrule['action'] then
- newrule['prefilter'] = true
- else
- newrule['prefilter'] = false
- end
-
- if ret then
- if newrule['type'] == 'symbol_options' then
- rspamd_config:register_dependency(newrule['symbol'], newrule['target_symbol'])
- end
- if newrule['require_symbols'] and not newrule['prefilter'] then
- local atoms = {}
-
- local function parse_atom(str)
- local atom = table.concat(fun.totable(fun.take_while(function(c)
- if string.find(', \t()><+!|&\n', c) then
- return false
- end
- return true
- end, fun.iter(str))), '')
- table.insert(atoms, atom)
- return atom
- end
-
- local function process_atom(atom, task)
- local f_ret = task:has_symbol(atom)
- lua_util.debugm(N, rspamd_config, 'check for symbol %s: %s', atom, f_ret)
-
- if f_ret then
- return 1
- end
-
- return 0
- end
-
- local expression = rspamd_expression.create(newrule['require_symbols'],
- {parse_atom, process_atom}, rspamd_config:get_mempool())
- if expression then
- newrule['expression'] = expression
-
- fun.each(function(v)
- lua_util.debugm(N, rspamd_config, 'add dependency %s -> %s',
- newrule['symbol'], v)
- rspamd_config:register_dependency(newrule['symbol'], v)
- end, atoms)
- end
- end
- return newrule
- end
-
- return nil
- end
-
- -- Registration
- local opts = rspamd_config:get_all_opt(N)
- if opts and type(opts) == 'table' then
- redis_params = rspamd_parse_redis_server(N)
- for k,m in pairs(opts) do
- if type(m) == 'table' and m['type'] then
- local rule = add_multimap_rule(k, m)
- if not rule then
- rspamd_logger.errx(rspamd_config, 'cannot add rule: "'..k..'"')
- else
- rspamd_logger.infox(rspamd_config, 'added multimap rule: %s (%s)',
- k, rule.type)
- table.insert(rules, rule)
- end
- end
- end
- -- add fake symbol to check all maps inside a single callback
- fun.each(function(rule)
- local id = rspamd_config:register_symbol({
- type = 'normal',
- name = rule['symbol'],
- callback = gen_multimap_callback(rule),
- })
- if rule['symbols'] then
- -- Find allowed symbols by this map
- rule['symbols_set'] = {}
- fun.each(function(s)
- rspamd_config:register_symbol({
- type = 'virtual',
- name = s,
- parent = id,
- score = tonumber(rule.score or "0") or 0, -- Default score
- })
- rule['symbols_set'][s] = 1
- end, rule['symbols'])
- end
- if not rule.score then
- rspamd_logger.infox(rspamd_config, 'set default score 0 for multimap rule %s', rule.symbol)
- rule.score = 0
- end
- if rule.score then
- -- Register metric symbol
- rule.name = rule.symbol
- rule.description = rule.description or 'multimap symbol'
- rule.group = rule.group or N
-
- local tmp_flags
- tmp_flags = rule.flags
-
- if rule.type == 'received' and rule.flags then
- -- XXX: hack to allow received flags/nflags
- -- See issue #3526 on GH
- rule.flags = nil
- end
-
- -- XXX: for combined maps we use trace, so flags must include one_shot to avoid scores multiplication
- if rule.combined and not rule.flags then
- rule.flags = 'one_shot'
- end
- rspamd_config:set_metric_symbol(rule)
- rule.flags = tmp_flags
- end
- end, fun.filter(function(r) return not r['prefilter'] end, rules))
-
- -- prefilter symbols
- fun.each(function(rule)
- rspamd_config:register_symbol({
- type = 'prefilter',
- name = rule['symbol'],
- score = tonumber(rule.score or "0") or 0,
- callback = gen_multimap_callback(rule),
- })
- end, fun.filter(function(r) return r['prefilter'] end, rules))
-
- if #rules == 0 then
- lua_util.disable_module(N, "config")
- end
- end
|