OPTION(ENABLE_LUA_TRACE "Trace all Lua C API invocations [default: OFF]" OFF) | OPTION(ENABLE_LUA_TRACE "Trace all Lua C API invocations [default: OFF]" OFF) | ||||
OPTION(ENABLE_LUA_REPL "Enables Lua repl (requires C++11 compiler) [default: ON]" ON) | OPTION(ENABLE_LUA_REPL "Enables Lua repl (requires C++11 compiler) [default: ON]" ON) | ||||
OPTION(SYSTEM_ZSTD "Use system zstd instead of bundled one [default: OFF]" OFF) | OPTION(SYSTEM_ZSTD "Use system zstd instead of bundled one [default: OFF]" OFF) | ||||
OPTION(SYSTEM_FMT "Use system fmt instead of bundled one [defalut: OFF]" OFF) | |||||
OPTION(SYSTEM_FMT "Use system fmt instead of bundled one [default: OFF]" OFF) | |||||
OPTION(SYSTEM_DOCTEST "Use system doctest instead of bundled one [default: OFF]" OFF) | OPTION(SYSTEM_DOCTEST "Use system doctest instead of bundled one [default: OFF]" OFF) | ||||
############################# INCLUDE SECTION ############################################# | ############################# INCLUDE SECTION ############################################# |
.include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/mime_types_group.conf" | .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/mime_types_group.conf" | ||||
} | } | ||||
# Used to limit maximium score | |||||
# Used to limit maximum score | |||||
group "excessqp" { | group "excessqp" { | ||||
max_score = 2.4; | max_score = 2.4; | ||||
.include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/excessqp_group.conf" | .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/excessqp_group.conf" |
} | } | ||||
} | } | ||||
if ($m->{'stucts'}) { | |||||
if ( scalar(@{ $m->{'stucts'} }) > 0 ) { | |||||
print "\n## Stucts\n\nThe module `$mname` defines the following stucts.\n\n"; | |||||
if ($m->{'structs'}) { | |||||
if ( scalar(@{ $m->{'structs'} }) > 0 ) { | |||||
print "\n## Structs\n\nThe module `$mname` defines the following structs.\n\n"; | |||||
foreach ( @{ $m->{'stucts'} } ) { | |||||
print_stuct_markdown( "Stuct", $_->{'name'}, $_ ); | |||||
foreach ( @{ $m->{'structs'} } ) { | |||||
print_struct_markdown( "Struct", $_->{'name'}, $_ ); | |||||
print "\nBack to [module description](#$m->{'id'}).\n\n"; | print "\nBack to [module description](#$m->{'id'}).\n\n"; | ||||
count = 1; | count = 1; | ||||
# Password for normal commands (use rspamadm pw) | # Password for normal commands (use rspamadm pw) | ||||
password = "$2$anydoddx67ggcs74owybhcwqsq3z67q4$udympbo8pfcfqkeiiuj7gegabk5jpt8edmhseujhar9ooyuzig5b"; | password = "$2$anydoddx67ggcs74owybhcwqsq3z67q4$udympbo8pfcfqkeiiuj7gegabk5jpt8edmhseujhar9ooyuzig5b"; | ||||
# Password for privilleged commands (use rspamadm pw) | |||||
# Password for privileged commands (use rspamadm pw) | |||||
enable_password = "$2$nx6sqkxtewx9c5s3hxjmabaxdcr46pk9$45qajkbyqx77abapiqugpjpsojj38zcqn7xnp3ekqyu674koux4b"; | enable_password = "$2$nx6sqkxtewx9c5s3hxjmabaxdcr46pk9$45qajkbyqx77abapiqugpjpsojj38zcqn7xnp3ekqyu674koux4b"; | ||||
# Path to webiu static files | # Path to webiu static files | ||||
static_dir = "${WWWDIR}"; | static_dir = "${WWWDIR}"; |
<div class="row"> | <div class="row"> | ||||
<div class="col-lg-6"> | <div class="col-lg-6"> | ||||
<div class="card bg-light shadow card-body card p-2"> | <div class="card bg-light shadow card-body card p-2"> | ||||
<p>Learn Bayessian classifier:</p> | |||||
<p>Learn Bayesian classifier:</p> | |||||
<form class="form-inline"> | <form class="form-inline"> | ||||
<div class="form-group"> | <div class="form-group"> | ||||
<div class="btn-group"> | <div class="btn-group"> |
end | end | ||||
``` | ``` | ||||
- Never name a parameter `arg`, this will take precendence over the `arg` object that is given to every function scope in older versions of Lua. | |||||
- Never name a parameter `arg`, this will take precedence over the `arg` object that is given to every function scope in older versions of Lua. | |||||
```lua | ```lua | ||||
-- bad | -- bad |
local actions_set = lua_util.list_to_hash(actions_defs) | local actions_set = lua_util.list_to_hash(actions_defs) | ||||
-- Now check actions section for garbadge | |||||
-- Now check actions section for garbage | |||||
actions_set['unknown_weight'] = true | actions_set['unknown_weight'] = true | ||||
actions_set['grow_factor'] = true | actions_set['grow_factor'] = true | ||||
actions_set['subject'] = true | actions_set['subject'] = true |
local extracted_js = maybe_extract_object_stream(js, pdf, task) | local extracted_js = maybe_extract_object_stream(js, pdf, task) | ||||
if not extracted_js then | if not extracted_js then | ||||
lua_util.debugm(N, task, 'invalid type for javascript from %s:%s: %s', | |||||
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s', | |||||
obj.major, obj.minor, js) | obj.major, obj.minor, js) | ||||
else | else | ||||
js = extracted_js | js = extracted_js | ||||
lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s', | lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s', | ||||
obj.major, obj.minor, obj.js.data) | obj.major, obj.minor, obj.js.data) | ||||
else | else | ||||
lua_util.debugm(N, task, 'invalid type for javascript from %s:%s: %s', | |||||
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s', | |||||
obj.major, obj.minor, js) | obj.major, obj.minor, js) | ||||
end | end | ||||
elseif obj.dict.F then | elseif obj.dict.F then | ||||
if obj.dict.S and obj.dict.JS then | if obj.dict.S and obj.dict.JS then | ||||
obj.type = 'Javascript' | obj.type = 'Javascript' | ||||
lua_util.debugm(N, task, 'implicit type for Javascript object %s:%s', | |||||
lua_util.debugm(N, task, 'implicit type for JavaScript object %s:%s', | |||||
obj.major, obj.minor) | obj.major, obj.minor) | ||||
else | else | ||||
lua_util.debugm(N, task, 'no type for %s:%s', | lua_util.debugm(N, task, 'no type for %s:%s', | ||||
local extracted_js = maybe_extract_object_stream(js, pdf, task) | local extracted_js = maybe_extract_object_stream(js, pdf, task) | ||||
if not extracted_js then | if not extracted_js then | ||||
lua_util.debugm(N, task, 'invalid type for javascript from %s:%s: %s', | |||||
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s', | |||||
obj.major, obj.minor, js) | obj.major, obj.minor, js) | ||||
else | else | ||||
js = extracted_js | js = extracted_js | ||||
lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s', | lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s', | ||||
obj.major, obj.minor, obj.js.data) | obj.major, obj.minor, obj.js.data) | ||||
else | else | ||||
lua_util.debugm(N, task, 'invalid type for javascript from %s:%s: %s', | |||||
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s', | |||||
obj.major, obj.minor, js) | obj.major, obj.minor, js) | ||||
end | end | ||||
end | end | ||||
pdf_object.openaction.object.major, pdf_object.openaction.object.minor) | pdf_object.openaction.object.major, pdf_object.openaction.object.minor) | ||||
table.insert(pdf_output.fuzzy_hashes, pdf_object.openaction.bin_hash) | table.insert(pdf_output.fuzzy_hashes, pdf_object.openaction.bin_hash) | ||||
else | else | ||||
lua_util.debugm(N, task, "pdf: skip fuzzy hash from Javascript: %s, too short: %s", | |||||
lua_util.debugm(N, task, "pdf: skip fuzzy hash from JavaScript: %s, too short: %s", | |||||
pdf_object.openaction.hash, #pdf_object.openaction.data) | pdf_object.openaction.hash, #pdf_object.openaction.data) | ||||
end | end | ||||
end | end | ||||
-- All hashes | -- All hashes | ||||
for h,sc in pairs(pdf_object.scripts) do | for h,sc in pairs(pdf_object.scripts) do | ||||
if config.min_js_fuzzy and #sc.data >= config.min_js_fuzzy then | if config.min_js_fuzzy and #sc.data >= config.min_js_fuzzy then | ||||
lua_util.debugm(N, task, "pdf: add fuzzy hash from Javascript: %s; size = %s; object: %s:%s", | |||||
lua_util.debugm(N, task, "pdf: add fuzzy hash from JavaScript: %s; size = %s; object: %s:%s", | |||||
sc.hash, | sc.hash, | ||||
#sc.data, | #sc.data, | ||||
sc.object.major, sc.object.minor) | sc.object.major, sc.object.minor) | ||||
table.insert(pdf_output.fuzzy_hashes, h) | table.insert(pdf_output.fuzzy_hashes, h) | ||||
else | else | ||||
lua_util.debugm(N, task, "pdf: skip fuzzy hash from Javascript: %s, too short: %s", | |||||
lua_util.debugm(N, task, "pdf: skip fuzzy hash from JavaScript: %s, too short: %s", | |||||
sc.hash, #sc.data) | sc.hash, #sc.data) | ||||
end | end | ||||
end | end |
local id = part:get_id() | local id = part:get_id() | ||||
lua_util.debugm(N, task, 'check binary part %s: %s', id, ct) | lua_util.debugm(N, task, 'check binary part %s: %s', id, ct) | ||||
-- For bad mime mime parts we implicitly enable fuzzy check | |||||
-- For bad mime parts we implicitly enable fuzzy check | |||||
local mime_trace = (task:get_symbol('MIME_TRACE') or {})[1] | local mime_trace = (task:get_symbol('MIME_TRACE') or {})[1] | ||||
local opts = {} | local opts = {} | ||||
0, -- ascii characters rate | 0, -- ascii characters rate | ||||
0, -- non-ascii characters rate | 0, -- non-ascii characters rate | ||||
0, -- capital characters rate | 0, -- capital characters rate | ||||
0, -- numeric cahracters | |||||
0, -- numeric characters | |||||
} | } | ||||
for _,p in ipairs(tp) do | for _,p in ipairs(tp) do | ||||
local stats = p:get_stats() | local stats = p:get_stats() | ||||
'ascii_characters_rate', | 'ascii_characters_rate', | ||||
'non_ascii_characters_rate', | 'non_ascii_characters_rate', | ||||
'capital_characters_rate', | 'capital_characters_rate', | ||||
'numeric_cahracters' | |||||
'numeric_characters' | |||||
}, | }, | ||||
description = [[Functions for words data matching: | description = [[Functions for words data matching: | ||||
- average length of the words | - average length of the words |
for _,m in ipairs(result) do | for _,m in ipairs(result) do | ||||
local master = flatten_redis_table(m) | local master = flatten_redis_table(m) | ||||
-- Wrap IPv6-adresses in brackets | |||||
-- Wrap IPv6-addresses in brackets | |||||
if (master.ip:match(":")) then | if (master.ip:match(":")) then | ||||
master.ip = "["..master.ip.."]" | master.ip = "["..master.ip.."]" | ||||
end | end | ||||
lutil.debugm(N, rspamd_config, | lutil.debugm(N, rspamd_config, | ||||
'found slave for master %s with ip %s and port %s', | 'found slave for master %s with ip %s and port %s', | ||||
v.name, slave.ip, slave.port) | v.name, slave.ip, slave.port) | ||||
-- Wrap IPv6-adresses in brackets | |||||
-- Wrap IPv6-addresses in brackets | |||||
if (slave.ip:match(":")) then | if (slave.ip:match(":")) then | ||||
slave.ip = "["..slave.ip.."]" | slave.ip = "["..slave.ip.."]" | ||||
end | end |
lua_util.debugm(rule.name, task, '%s: extension matched: |%s|%s|', rule.log_prefix, ext, ext2) | lua_util.debugm(rule.name, task, '%s: extension matched: |%s|%s|', rule.log_prefix, ext, ext2) | ||||
return true | return true | ||||
elseif match_filter(task, rule, fname, rule.mime_parts_filter_regex, 'regex') then | elseif match_filter(task, rule, fname, rule.mime_parts_filter_regex, 'regex') then | ||||
lua_util.debugm(rule.name, task, '%s: filname regex matched', rule.log_prefix) | |||||
lua_util.debugm(rule.name, task, '%s: filename regex matched', rule.log_prefix) | |||||
return true | return true | ||||
end | end | ||||
end | end |
rule['symbol'], rule['type']) | rule['symbol'], rule['type']) | ||||
end | end | ||||
else | else | ||||
-- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occured | |||||
-- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occurred | |||||
-- see http://www.f-prot.com/support/helpfiles/unix/appendix_c.html for more detail | -- see http://www.f-prot.com/support/helpfiles/unix/appendix_c.html for more detail | ||||
local vname = string.match(data, '^[1-3] <[%w%s]-: (.-)>') | local vname = string.match(data, '^[1-3] <[%w%s]-: (.-)>') | ||||
if not vname then | if not vname then |
rspamd_logger.warnx(task, 'Message is oversized (REJ 4): %s', data) | rspamd_logger.warnx(task, 'Message is oversized (REJ 4): %s', data) | ||||
common.yield_result(task, rule, 'SAVDI: Message oversized (REJ 4)', | common.yield_result(task, rule, 'SAVDI: Message oversized (REJ 4)', | ||||
0.0, 'fail', maybe_part) | 0.0, 'fail', maybe_part) | ||||
-- excplicitly set REJ1 message when SAVDIreports a protocol error | |||||
-- explicitly set REJ1 message when SAVDIreports a protocol error | |||||
elseif string.find(data, 'REJ 1') then | elseif string.find(data, 'REJ 1') then | ||||
rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data) | rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data) | ||||
common.yield_result(task, rule, 'SAVDI: Protocol error (REJ 1)', | common.yield_result(task, rule, 'SAVDI: Protocol error (REJ 1)', |
--[[[ | --[[[ | ||||
-- @module virustotal | -- @module virustotal | ||||
-- This module contains Virustotal integration support | |||||
-- https://www.virustotal.com/ | -- https://www.virustotal.com/ | ||||
--]] | --]] | ||||
return rspamd_text.fromtable(selectors, delimiter) | return rspamd_text.fromtable(selectors, delimiter) | ||||
end | end | ||||
else | else | ||||
-- We need to do a spill on each table selector and make a cortezian product | |||||
-- We need to do a spill on each table selector and make a cortesian product | |||||
-- e.g. s:tbl:s -> s:telt1:s + s:telt2:s ... | -- e.g. s:tbl:s -> s:telt1:s + s:telt2:s ... | ||||
local tbl = {} | local tbl = {} | ||||
local res = {} | local res = {} |
-- symbol_ham = YYY | -- symbol_ham = YYY | ||||
-- db_spam = XXX.sqlite | -- db_spam = XXX.sqlite | ||||
-- db_ham = YYY.sqlite | -- db_ham = YYY.sqlite | ||||
-- learn_cache = ZZZ.sqlite | |||||
-- per_user = true/false | -- per_user = true/false | ||||
-- label = str | -- label = str | ||||
-- } | -- } |
end | end | ||||
--[[[ | --[[[ | ||||
-- @function lua_util.str_split(text, delimiter) | |||||
-- Splits text into a numeric table by delimiter | |||||
-- @param {string} text delimited text | |||||
-- @param {string} delimiter the delimiter | |||||
-- @return {table} numeric table containing string parts | -- @return {table} numeric table containing string parts | ||||
--]] | --]] | ||||
}, | }, | ||||
uncertain = { | uncertain = { | ||||
can_learn = false, | can_learn = false, | ||||
description = 'not certainity in verdict' | |||||
description = 'not certainty in verdict' | |||||
} | } | ||||
} | } | ||||
if munging_opts.munge_map_condition then | if munging_opts.munge_map_condition then | ||||
local accepted,trace = munging_opts.munge_map_condition:process(task) | local accepted,trace = munging_opts.munge_map_condition:process(task) | ||||
if not accepted then | if not accepted then | ||||
lua_util.debugm(task, 'skip munging, maps condition not satisified: (%s)', | |||||
lua_util.debugm(task, 'skip munging, maps condition not satisfied: (%s)', | |||||
trace) | trace) | ||||
-- Excepted | -- Excepted | ||||
return | return | ||||
-- Used to check dmarc record, check elements and produce dmarc policy processed | -- Used to check dmarc record, check elements and produce dmarc policy processed | ||||
-- result. | -- result. | ||||
-- Returns: | -- Returns: | ||||
-- false,false - record is garbage | |||||
-- false,error_message - record is invalid | -- false,error_message - record is invalid | ||||
-- true,policy_table - record is valid and parsed | -- true,policy_table - record is valid and parsed | ||||
]] | ]] | ||||
end | end | ||||
result.raw_elts = elts | result.raw_elts = elts | ||||
else | else | ||||
return false,false -- Ignore garbadge | |||||
return false,false -- Ignore garbage | |||||
end | end | ||||
return true, result | return true, result |
if nspam <= train_opts.max_trains then | if nspam <= train_opts.max_trains then | ||||
if train_opts.spam_skip_prob then | if train_opts.spam_skip_prob then | ||||
if coin <= train_opts.spam_skip_prob then | if coin <= train_opts.spam_skip_prob then | ||||
rspamd_logger.infox(task, 'skip %s sample probabilisticaly; probability %s (%s skip chance)', learn_type, | |||||
rspamd_logger.infox(task, 'skip %s sample probabilistically; probability %s (%s skip chance)', learn_type, | |||||
coin, train_opts.spam_skip_prob) | coin, train_opts.spam_skip_prob) | ||||
return false | return false | ||||
end | end | ||||
if nham <= train_opts.max_trains then | if nham <= train_opts.max_trains then | ||||
if train_opts.ham_skip_prob then | if train_opts.ham_skip_prob then | ||||
if coin <= train_opts.ham_skip_prob then | if coin <= train_opts.ham_skip_prob then | ||||
rspamd_logger.infox(task, 'skip %s sample probabilisticaly; probability %s (%s skip chance)', learn_type, | |||||
rspamd_logger.infox(task, 'skip %s sample probabilistically; probability %s (%s skip chance)', learn_type, | |||||
coin, train_opts.ham_skip_prob) | coin, train_opts.ham_skip_prob) | ||||
return false | return false | ||||
end | end | ||||
else | else | ||||
local inputs, outputs = {}, {} | local inputs, outputs = {}, {} | ||||
-- Used to show sparsed vectors in a convenient format (for debugging only) | |||||
-- Used to show parsed vectors in a convenient format (for debugging only) | |||||
local function debug_vec(t) | local function debug_vec(t) | ||||
local ret = {} | local ret = {} | ||||
for i,v in ipairs(t) do | for i,v in ipairs(t) do |
end | end | ||||
local function get_excluded_symbols(known_symbols, correlations, seen_total) | local function get_excluded_symbols(known_symbols, correlations, seen_total) | ||||
-- Walk results once to collect all symbols & count ocurrences | |||||
-- Walk results once to collect all symbols & count occurrences | |||||
local remove = {} | local remove = {} | ||||
local known_symbols_list = {} | local known_symbols_list = {} |
return fname | return fname | ||||
end | end | ||||
local function output_dot(opts, nodes, adjastency) | |||||
local function output_dot(opts, nodes, adjacency) | |||||
rspamd_logger.messagex("digraph rspamd {") | rspamd_logger.messagex("digraph rspamd {") | ||||
for k,node in pairs(nodes) do | for k,node in pairs(nodes) do | ||||
local attrs = {"shape=box"} | local attrs = {"shape=box"} | ||||
table.concat(attrs, ',')) | table.concat(attrs, ',')) | ||||
end | end | ||||
end | end | ||||
for _,adj in ipairs(adjastency) do | |||||
for _,adj in ipairs(adjacency) do | |||||
local attrs = {} | local attrs = {} | ||||
local skip = false | local skip = false | ||||
local function load_config_traced(opts) | local function load_config_traced(opts) | ||||
local glob_traces = {} | local glob_traces = {} | ||||
local adjastency = {} | |||||
local adjacency = {} | |||||
local nodes = {} | local nodes = {} | ||||
local function maybe_match_glob(file) | local function maybe_match_glob(file) | ||||
end | end | ||||
local function add_dep(from, node, args) | local function add_dep(from, node, args) | ||||
adjastency[#adjastency + 1] = { | |||||
adjacency[#adjacency + 1] = { | |||||
from = from, | from = from, | ||||
to = node, | to = node, | ||||
args = args | args = args | ||||
os.exit(1) | os.exit(1) | ||||
end | end | ||||
output_dot(opts, nodes, adjastency) | |||||
output_dot(opts, nodes, adjacency) | |||||
end | end | ||||
end | end | ||||
if not symbol_spam or not symbol_ham then | if not symbol_spam or not symbol_ham then | ||||
printf("Calssifier has no symbols defined") | |||||
printf("Classifier has no symbols defined") | |||||
return | return | ||||
end | end | ||||
end | end | ||||
if opts.words then | if opts.words then | ||||
local howw = opts['words_format'] or 'stem' | |||||
local how_words = opts['words_format'] or 'stem' | |||||
table.insert(out_elts[fname], 'meta_words: ' .. | table.insert(out_elts[fname], 'meta_words: ' .. | ||||
print_words(task:get_meta_words(howw), howw == 'full')) | |||||
print_words(task:get_meta_words(how_words), how_words == 'full')) | |||||
end | end | ||||
if opts.text or opts.html then | if opts.text or opts.html then | ||||
end | end | ||||
if opts.words then | if opts.words then | ||||
local howw = opts['words_format'] or 'stem' | |||||
table.insert(out_elts[fname], print_words(part:get_words(howw), | |||||
howw == 'full')) | |||||
local how_words = opts['words_format'] or 'stem' | |||||
table.insert(out_elts[fname], print_words(part:get_words(how_words), | |||||
how_words == 'full')) | |||||
else | else | ||||
table.insert(out_elts[fname], tostring(part:get_content(how))) | table.insert(out_elts[fname], tostring(part:get_content(how))) | ||||
end | end | ||||
end | end | ||||
if opts.words then | if opts.words then | ||||
local howw = opts['words_format'] or 'stem' | |||||
table.insert(out_elts[fname], print_words(part:get_words(howw), | |||||
howw == 'full')) | |||||
local how_words = opts['words_format'] or 'stem' | |||||
table.insert(out_elts[fname], print_words(part:get_words(how_words), | |||||
how_words == 'full')) | |||||
else | else | ||||
if opts.structure then | if opts.structure then | ||||
local hc = part:get_html() | local hc = part:get_html() |
return true, 1.0, 'DSN' | return true, 1.0, 'DSN' | ||||
end | end | ||||
-- Apply heuristics for non-standard bounecs | |||||
-- Apply heuristics for non-standard bounces | |||||
local bounce_sender | local bounce_sender | ||||
local mime_from = task:get_from('mime') | local mime_from = task:get_from('mime') | ||||
if mime_from then | if mime_from then | ||||
-- Look for a message/rfc822(-headers) part inside | -- Look for a message/rfc822(-headers) part inside | ||||
local rfc822_part | local rfc822_part | ||||
parts[10] = nil -- limit numbe of parts to check | |||||
parts[10] = nil -- limit number of parts to check | |||||
for _, p in ipairs(parts) do | for _, p in ipairs(parts) do | ||||
local mime_type, mime_subtype = p:get_type() | local mime_type, mime_subtype = p:get_type() | ||||
if (mime_subtype == 'rfc822' or mime_subtype == 'rfc822-headers') and | if (mime_subtype == 'rfc822' or mime_subtype == 'rfc822-headers') and |
missing_mime = true | missing_mime = true | ||||
end | end | ||||
-- Check presense of MIME specific headers | |||||
-- Check presence of MIME specific headers | |||||
local has_ct_header = task:has_header('Content-Type') | local has_ct_header = task:has_header('Content-Type') | ||||
local has_cte_header = task:has_header('Content-Transfer-Encoding') | local has_cte_header = task:has_header('Content-Transfer-Encoding') | ||||
group = 'headers' | group = 'headers' | ||||
} | } | ||||
-- Mime-OLE is needed but absent (e.g. fake Outlook or fake Exchange) | |||||
local has_msmail_pri = 'header_exists(X-MSMail-Priority)' | local has_msmail_pri = 'header_exists(X-MSMail-Priority)' | ||||
local has_mimeole = 'header_exists(X-MimeOLE)' | local has_mimeole = 'header_exists(X-MimeOLE)' | ||||
local has_squirrelmail_in_mailer = 'X-Mailer=/SquirrelMail\\b/H' | local has_squirrelmail_in_mailer = 'X-Mailer=/SquirrelMail\\b/H' | ||||
reconf['SUBJECT_ENDS_EXCLAIM'] = { | reconf['SUBJECT_ENDS_EXCLAIM'] = { | ||||
re = 'Subject=/!\\s*$/H', | re = 'Subject=/!\\s*$/H', | ||||
description = 'Subject ends with an exclaimation', | |||||
description = 'Subject ends with an exclamation', | |||||
score = 0.0, | score = 0.0, | ||||
group = 'headers' | group = 'headers' | ||||
} | } | ||||
reconf['SUBJECT_HAS_EXCLAIM'] = { | reconf['SUBJECT_HAS_EXCLAIM'] = { | ||||
re = string.format('%s & !%s', 'Subject=/!/H', 'Subject=/!\\s*$/H'), | re = string.format('%s & !%s', 'Subject=/!/H', 'Subject=/!\\s*$/H'), | ||||
description = 'Subject contains an exclaimation', | |||||
description = 'Subject contains an exclamation', | |||||
score = 0.0, | score = 0.0, | ||||
group = 'headers' | group = 'headers' | ||||
} | } | ||||
[[(?:Mozilla )?Thunderbird \d]], | [[(?:Mozilla )?Thunderbird \d]], | ||||
-- Was used by Yahoo Groups in 2000s, no one expected to use this in 2020s | -- Was used by Yahoo Groups in 2000s, no one expected to use this in 2020s | ||||
[[eGroups Message Poster]], | [[eGroups Message Poster]], | ||||
-- Regexp for genuene iOS X-Mailer is below, anything which doesn't match it, | |||||
-- Regexp for genuine iOS X-Mailer is below, anything which doesn't match it, | |||||
-- but starts with 'iPhone Mail' or 'iPad Mail' is likely fake | -- but starts with 'iPhone Mail' or 'iPad Mail' is likely fake | ||||
[[i(?:Phone|Pad) Mail]], | [[i(?:Phone|Pad) Mail]], | ||||
} | } |
gboolean use_ssl; | gboolean use_ssl; | ||||
/* Webui password */ | /* Webui password */ | ||||
gchar *password; | gchar *password; | ||||
/* Privilleged password */ | |||||
/* Privileged password */ | |||||
gchar *enable_password; | gchar *enable_password; | ||||
/* Cached versions of the passwords */ | /* Cached versions of the passwords */ | ||||
rspamd_ftok_t cached_password; | rspamd_ftok_t cached_password; | ||||
g_strstrip (syms[j]); | g_strstrip (syms[j]); | ||||
if (strlen (syms[j]) == 0) { | if (strlen (syms[j]) == 0) { | ||||
/* Empty garbadge */ | |||||
/* Empty garbage */ | |||||
continue; | continue; | ||||
} | } | ||||
} | } | ||||
if (!rspamd_controller_check_password (conn_ent, session, msg, | if (!rspamd_controller_check_password (conn_ent, session, msg, | ||||
cmd->privilleged)) { | |||||
cmd->privileged)) { | |||||
return 0; | return 0; | ||||
} | } | ||||
if (cmd->require_message && (rspamd_http_message_get_body (msg, NULL) == NULL)) { | if (cmd->require_message && (rspamd_http_message_get_body (msg, NULL) == NULL)) { |
p = buf; | p = buf; | ||||
/* | /* | ||||
* Memory layout: n_ext of struct rspamd_fuzzy_cmd_extension | * Memory layout: n_ext of struct rspamd_fuzzy_cmd_extension | ||||
* payload for each extension in a continious data segment | |||||
* payload for each extension in a continuous data segment | |||||
*/ | */ | ||||
storage = g_malloc (n_ext * sizeof (struct rspamd_fuzzy_cmd_extension) + | storage = g_malloc (n_ext * sizeof (struct rspamd_fuzzy_cmd_extension) + | ||||
st_len); | st_len); |
*/ | */ | ||||
/* | /* | ||||
* Sigma function that defines the diagonal connections of a DBG | * Sigma function that defines the diagonal connections of a DBG | ||||
* diagonal front: flip the (g-i)th bit (Inverse Buttferly Graph) | |||||
* diagonal front: flip the (g-i)th bit (Inverse Butterfly Graph) | |||||
* diagonal back: flip the i-(g-1)th bit (Regular Butterfly Graph) | * diagonal back: flip the i-(g-1)th bit (Regular Butterfly Graph) | ||||
*/ | */ | ||||
static uint64_t | static uint64_t |
const gchar *name; /* e.g. "en" or "ru" */ | const gchar *name; /* e.g. "en" or "ru" */ | ||||
gint flags; /* enum rspamd_language_elt_flags */ | gint flags; /* enum rspamd_language_elt_flags */ | ||||
enum rspamd_language_category category; | enum rspamd_language_category category; | ||||
guint trigramms_words; | |||||
guint trigrams_words; | |||||
guint stop_words; | guint stop_words; | ||||
gdouble mean; | gdouble mean; | ||||
gdouble std; | gdouble std; | ||||
guint occurencies; /* total number of parts with this language */ | |||||
guint occurrences; /* total number of parts with this language */ | |||||
}; | }; | ||||
struct rspamd_ngramm_elt { | struct rspamd_ngramm_elt { | ||||
struct rspamd_lang_detector { | struct rspamd_lang_detector { | ||||
GPtrArray *languages; | GPtrArray *languages; | ||||
khash_t(rspamd_trigram_hash) *trigramms[RSPAMD_LANGUAGE_MAX]; /* trigramms frequencies */ | |||||
khash_t(rspamd_trigram_hash) *trigrams[RSPAMD_LANGUAGE_MAX]; /* trigrams frequencies */ | |||||
struct rspamd_stop_word_elt stop_words[RSPAMD_LANGUAGE_MAX]; | struct rspamd_stop_word_elt stop_words[RSPAMD_LANGUAGE_MAX]; | ||||
khash_t(rspamd_stopwords_hash) *stop_words_norm; | khash_t(rspamd_stopwords_hash) *stop_words_norm; | ||||
UConverter *uchar_converter; | UConverter *uchar_converter; | ||||
gsize short_text_limit; | gsize short_text_limit; | ||||
gsize total_occurencies; /* number of all languages found */ | |||||
gsize total_occurrences; /* number of all languages found */ | |||||
ref_entry_t ref; | ref_entry_t ref; | ||||
}; | }; | ||||
return; | return; | ||||
} | } | ||||
else { | else { | ||||
nelt->trigramms_words = ucl_object_toint (ucl_array_find_index (n_words, | |||||
nelt->trigrams_words = ucl_object_toint (ucl_array_find_index (n_words, | |||||
2)); | 2)); | ||||
} | } | ||||
} | } | ||||
nelt->category = cat; | nelt->category = cat; | ||||
htb = d->trigramms[cat]; | |||||
htb = d->trigrams[cat]; | |||||
GPtrArray *ngramms; | GPtrArray *ngramms; | ||||
guint nsym; | guint nsym; | ||||
if (!(nelt->flags & RS_LANGUAGE_LATIN) && | if (!(nelt->flags & RS_LANGUAGE_LATIN) && | ||||
rspamd_language_detector_ucs_is_latin (ucs_elt->s, nsym)) { | rspamd_language_detector_ucs_is_latin (ucs_elt->s, nsym)) { | ||||
ucs_elt->freq = 0; | ucs_elt->freq = 0; | ||||
/* Skip latin ngramm for non-latin language to avoid garbadge */ | |||||
/* Skip latin ngramm for non-latin language to avoid garbage */ | |||||
skipped ++; | skipped ++; | ||||
continue; | continue; | ||||
} | } | ||||
nelt->mean = mean; | nelt->mean = mean; | ||||
nelt->std = std; | nelt->std = std; | ||||
msg_debug_lang_det_cfg ("loaded %s language, %d trigramms, " | |||||
msg_debug_lang_det_cfg ("loaded %s language, %d trigrams, " | |||||
"%d ngramms loaded; " | "%d ngramms loaded; " | ||||
"std=%.2f, mean=%.2f, skipped=%d, loaded=%d, stop_words=%d; " | "std=%.2f, mean=%.2f, skipped=%d, loaded=%d, stop_words=%d; " | ||||
"(%s)", | "(%s)", | ||||
nelt->name, | nelt->name, | ||||
(gint)nelt->trigramms_words, | |||||
(gint)nelt->trigrams_words, | |||||
total, | total, | ||||
std, mean, | std, mean, | ||||
skipped, loaded, nelt->stop_words, | skipped, loaded, nelt->stop_words, | ||||
{ | { | ||||
if (d) { | if (d) { | ||||
for (guint i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { | for (guint i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { | ||||
kh_destroy (rspamd_trigram_hash, d->trigramms[i]); | |||||
kh_destroy (rspamd_trigram_hash, d->trigrams[i]); | |||||
rspamd_multipattern_destroy (d->stop_words[i].mp); | rspamd_multipattern_destroy (d->stop_words[i].mp); | ||||
g_array_free (d->stop_words[i].ranges, TRUE); | g_array_free (d->stop_words[i].ranges, TRUE); | ||||
} | } | ||||
/* Map from ngramm in ucs32 to GPtrArray of rspamd_language_elt */ | /* Map from ngramm in ucs32 to GPtrArray of rspamd_language_elt */ | ||||
for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { | for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { | ||||
ret->trigramms[i] = kh_init (rspamd_trigram_hash); | |||||
ret->trigrams[i] = kh_init (rspamd_trigram_hash); | |||||
#ifdef WITH_HYPERSCAN | #ifdef WITH_HYPERSCAN | ||||
ret->stop_words[i].mp = rspamd_multipattern_create ( | ret->stop_words[i].mp = rspamd_multipattern_create ( | ||||
RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8| | RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8| | ||||
for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { | for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { | ||||
GError *err = NULL; | GError *err = NULL; | ||||
kh_foreach_value (ret->trigramms[i], schain, { | |||||
kh_foreach_value (ret->trigrams[i], schain, { | |||||
chain = &schain; | chain = &schain; | ||||
rspamd_language_detector_process_chain (cfg, chain); | rspamd_language_detector_process_chain (cfg, chain); | ||||
}); | }); | ||||
g_error_free (err); | g_error_free (err); | ||||
} | } | ||||
total += kh_size (ret->trigramms[i]); | |||||
total += kh_size (ret->trigrams[i]); | |||||
} | } | ||||
msg_info_config ("loaded %d languages, " | msg_info_config ("loaded %d languages, " | ||||
"%d trigramms", | |||||
"%d trigrams", | |||||
(gint)ret->languages->len, | (gint)ret->languages->len, | ||||
(gint)total); | (gint)total); | ||||
struct rspamd_lang_detector *d, | struct rspamd_lang_detector *d, | ||||
UChar32 *window, | UChar32 *window, | ||||
khash_t(rspamd_candidates_hash) *candidates, | khash_t(rspamd_candidates_hash) *candidates, | ||||
khash_t(rspamd_trigram_hash) *trigramms) | |||||
khash_t(rspamd_trigram_hash) *trigrams) | |||||
{ | { | ||||
guint i; | guint i; | ||||
gint ret; | gint ret; | ||||
khiter_t k; | khiter_t k; | ||||
gdouble prob; | gdouble prob; | ||||
k = kh_get (rspamd_trigram_hash, trigramms, window); | |||||
if (k != kh_end (trigramms)) { | |||||
chain = &kh_value (trigramms, k); | |||||
k = kh_get (rspamd_trigram_hash, trigrams, window); | |||||
if (k != kh_end (trigrams)) { | |||||
chain = &kh_value (trigrams, k); | |||||
} | } | ||||
if (chain) { | if (chain) { | ||||
struct rspamd_lang_detector *d, | struct rspamd_lang_detector *d, | ||||
rspamd_stat_token_t *tok, | rspamd_stat_token_t *tok, | ||||
khash_t(rspamd_candidates_hash) *candidates, | khash_t(rspamd_candidates_hash) *candidates, | ||||
khash_t(rspamd_trigram_hash) *trigramms) | |||||
khash_t(rspamd_trigram_hash) *trigrams) | |||||
{ | { | ||||
const guint wlen = 3; | const guint wlen = 3; | ||||
UChar32 window[3]; | UChar32 window[3]; | ||||
while ((cur = rspamd_language_detector_next_ngramm (tok, window, wlen, cur)) | while ((cur = rspamd_language_detector_next_ngramm (tok, window, wlen, cur)) | ||||
!= -1) { | != -1) { | ||||
rspamd_language_detector_process_ngramm_full (task, | rspamd_language_detector_process_ngramm_full (task, | ||||
d, window, candidates, trigramms); | |||||
d, window, candidates, trigrams); | |||||
} | } | ||||
} | } | ||||
if (tok->unicode.len >= 3) { | if (tok->unicode.len >= 3) { | ||||
rspamd_language_detector_detect_word (task, d, tok, candidates, | rspamd_language_detector_detect_word (task, d, tok, candidates, | ||||
d->trigramms[cat]); | |||||
d->trigrams[cat]); | |||||
} | } | ||||
} | } | ||||
gdouble adj; | gdouble adj; | ||||
gdouble proba_adjusted, probb_adjusted, freqa, freqb; | gdouble proba_adjusted, probb_adjusted, freqa, freqb; | ||||
if (cbd->d->total_occurencies == 0) { | |||||
if (cbd->d->total_occurrences == 0) { | |||||
return 0; | return 0; | ||||
} | } | ||||
freqa = ((gdouble)canda->elt->occurencies) / | |||||
(gdouble)cbd->d->total_occurencies; | |||||
freqb = ((gdouble)candb->elt->occurencies) / | |||||
(gdouble)cbd->d->total_occurencies; | |||||
freqa = ((gdouble)canda->elt->occurrences) / | |||||
(gdouble)cbd->d->total_occurrences; | |||||
freqb = ((gdouble)candb->elt->occurrences) / | |||||
(gdouble)cbd->d->total_occurrences; | |||||
proba_adjusted = canda->prob; | proba_adjusted = canda->prob; | ||||
probb_adjusted = candb->prob; | probb_adjusted = candb->prob; | ||||
if (!ret) { | if (!ret) { | ||||
if (part->utf_words->len < default_short_text_limit) { | if (part->utf_words->len < default_short_text_limit) { | ||||
r = rs_detect_none; | r = rs_detect_none; | ||||
msg_debug_lang_det ("text is too short for trigramms detection: " | |||||
msg_debug_lang_det ("text is too short for trigrams detection: " | |||||
"%d words; at least %d words required", | "%d words; at least %d words required", | ||||
(int)part->utf_words->len, | (int)part->utf_words->len, | ||||
(int)default_short_text_limit); | (int)default_short_text_limit); | ||||
candidates); | candidates); | ||||
if (r == rs_detect_none) { | if (r == rs_detect_none) { | ||||
msg_debug_lang_det ("no trigramms found, fallback to english"); | |||||
msg_debug_lang_det ("no trigrams found, fallback to english"); | |||||
rspamd_language_detector_set_language (task, part, "en", NULL); | rspamd_language_detector_set_language (task, part, "en", NULL); | ||||
} else if (r == rs_detect_multiple) { | } else if (r == rs_detect_multiple) { | ||||
/* Check our guess */ | /* Check our guess */ | ||||
std = 0.0; | std = 0.0; | ||||
cand_len = 0; | cand_len = 0; | ||||
/* Check distirbution */ | |||||
/* Check distribution */ | |||||
kh_foreach_value (candidates, cand, { | kh_foreach_value (candidates, cand, { | ||||
if (!isnan (cand->prob)) { | if (!isnan (cand->prob)) { | ||||
mean += cand->prob; | mean += cand->prob; | ||||
std /= cand_len; | std /= cand_len; | ||||
} | } | ||||
msg_debug_lang_det ("trigramms checked, %d candidates, %.3f mean, %.4f stddev", | |||||
msg_debug_lang_det ("trigrams checked, %d candidates, %.3f mean, %.4f stddev", | |||||
cand_len, mean, std); | cand_len, mean, std); | ||||
if (cand_len > 0 && std / fabs (mean) < 0.25) { | if (cand_len > 0 && std / fabs (mean) < 0.25) { | ||||
if (result->len > 0 && !frequency_heuristic_applied) { | if (result->len > 0 && !frequency_heuristic_applied) { | ||||
cand = g_ptr_array_index (result, 0); | cand = g_ptr_array_index (result, 0); | ||||
cand->elt->occurencies++; | |||||
d->total_occurencies++; | |||||
cand->elt->occurrences++; | |||||
d->total_occurrences++; | |||||
} | } | ||||
if (part->languages != NULL) { | if (part->languages != NULL) { |
cd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*cd)); | cd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*cd)); | ||||
cd->type = RSPAMD_CT_INLINE; | cd->type = RSPAMD_CT_INLINE; | ||||
/* We can also have content dispositon definitions in Content-Type */ | |||||
/* We can also have content disposition definitions in Content-Type */ | |||||
if (part->ct && part->ct->attrs) { | if (part->ct && part->ct->attrs) { | ||||
RSPAMD_FTOK_ASSIGN (&srch, "name"); | RSPAMD_FTOK_ASSIGN (&srch, "name"); | ||||
found = g_hash_table_lookup (part->ct->attrs, &srch); | found = g_hash_table_lookup (part->ct->attrs, &srch); | ||||
cd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*cd)); | cd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*cd)); | ||||
cd->type = RSPAMD_CT_INLINE; | cd->type = RSPAMD_CT_INLINE; | ||||
/* We can also have content dispositon definitions in Content-Type */ | |||||
/* We can also have content disposition definitions in Content-Type */ | |||||
if (part->ct->attrs) { | if (part->ct->attrs) { | ||||
RSPAMD_FTOK_ASSIGN (&srch, "name"); | RSPAMD_FTOK_ASSIGN (&srch, "name"); | ||||
found = g_hash_table_lookup (part->ct->attrs, &srch); | found = g_hash_table_lookup (part->ct->attrs, &srch); |
if (!seen_ip_in_data) { | if (!seen_ip_in_data) { | ||||
if (rh.real_ip.size() != 0) { | if (rh.real_ip.size() != 0) { | ||||
/* Get anounced hostname (usually helo) */ | |||||
/* Get announced hostname (usually helo) */ | |||||
received_process_rdns(pool, | received_process_rdns(pool, | ||||
rpart.data.as_view(), | rpart.data.as_view(), | ||||
rh.from_hostname); | rh.from_hostname); |
scan_result = task->result; | scan_result = task->result; | ||||
} | } | ||||
/* Find the speicific action config */ | |||||
/* Find the specific action config */ | |||||
struct rspamd_action_config *action_config = NULL; | struct rspamd_action_config *action_config = NULL; | ||||
for (unsigned int i = 0; i < scan_result->nactions; i ++) { | for (unsigned int i = 0; i < scan_result->nactions; i ++) { |
RSPAMD_SYMBOL_FLAG_NORMAL = 0, | RSPAMD_SYMBOL_FLAG_NORMAL = 0, | ||||
RSPAMD_SYMBOL_FLAG_IGNORE_METRIC = (1 << 1), | RSPAMD_SYMBOL_FLAG_IGNORE_METRIC = (1 << 1), | ||||
RSPAMD_SYMBOL_FLAG_ONEPARAM = (1 << 2), | RSPAMD_SYMBOL_FLAG_ONEPARAM = (1 << 2), | ||||
RSPAMD_SYMBOL_FLAG_UNGROUPPED = (1 << 3), | |||||
RSPAMD_SYMBOL_FLAG_UNGROUPED = (1 << 3), | |||||
RSPAMD_SYMBOL_FLAG_DISABLED = (1 << 4), | RSPAMD_SYMBOL_FLAG_DISABLED = (1 << 4), | ||||
RSPAMD_SYMBOL_FLAG_UNSCORED = (1 << 5), | RSPAMD_SYMBOL_FLAG_UNSCORED = (1 << 5), | ||||
}; | }; |
* $RUNDIR - local states directory | * $RUNDIR - local states directory | ||||
* $DBDIR - databases dir | * $DBDIR - databases dir | ||||
* $LOGDIR - logs dir | * $LOGDIR - logs dir | ||||
* $PLUGINSDIR - pluggins dir | |||||
* $PLUGINSDIR - plugins dir | |||||
* $PREFIX - installation prefix | * $PREFIX - installation prefix | ||||
* $VERSION - rspamd version | * $VERSION - rspamd version | ||||
*/ | */ | ||||
/* We have only one statfile */ | /* We have only one statfile */ | ||||
return FALSE; | return FALSE; | ||||
} | } | ||||
/* We have not detected any statfile that has different class, so turn on euristic based on symbol's name */ | |||||
/* We have not detected any statfile that has different class, so turn on heuristic based on symbol's name */ | |||||
has_other = FALSE; | has_other = FALSE; | ||||
cur = cf->statfiles; | cur = cf->statfiles; | ||||
while (cur) { | while (cur) { | ||||
/* Search for symbol group */ | /* Search for symbol group */ | ||||
if (group == NULL) { | if (group == NULL) { | ||||
group = "ungrouped"; | group = "ungrouped"; | ||||
sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPPED; | |||||
sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPED; | |||||
} | } | ||||
else { | else { | ||||
if (strcmp (group, "ungrouped") == 0) { | if (strcmp (group, "ungrouped") == 0) { | ||||
sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPPED; | |||||
sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPED; | |||||
} | } | ||||
} | } | ||||
sym_def->gr = sym_group; | sym_def->gr = sym_group; | ||||
g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); | g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); | ||||
if (!(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPPED)) { | |||||
if (!(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED)) { | |||||
g_ptr_array_add (sym_def->groups, sym_group); | g_ptr_array_add (sym_def->groups, sym_group); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
if (!has_group) { | if (!has_group) { | ||||
/* Non-empty group has a priority over non-groupped one */ | |||||
/* Non-empty group has a priority over non-grouped one */ | |||||
sym_group = g_hash_table_lookup (cfg->groups, group); | sym_group = g_hash_table_lookup (cfg->groups, group); | ||||
if (sym_group == NULL) { | if (sym_group == NULL) { | ||||
sym_group = rspamd_config_new_group (cfg, group); | sym_group = rspamd_config_new_group (cfg, group); | ||||
} | } | ||||
if ((!sym_def->gr) || (sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPPED)) { | |||||
if ((!sym_def->gr) || (sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED)) { | |||||
sym_def->gr = sym_group; | sym_def->gr = sym_group; | ||||
sym_def->flags &= ~RSPAMD_SYMBOL_FLAG_UNGROUPPED; | |||||
sym_def->flags &= ~RSPAMD_SYMBOL_FLAG_UNGROUPED; | |||||
} | } | ||||
g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); | g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); | ||||
sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPPED); | |||||
sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPED); | |||||
g_ptr_array_add (sym_def->groups, sym_group); | g_ptr_array_add (sym_def->groups, sym_group); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
if (!has_group) { | if (!has_group) { | ||||
/* Non-empty group has a priority over non-groupped one */ | |||||
/* Non-empty group has a priority over non-grouped one */ | |||||
sym_group = g_hash_table_lookup (cfg->groups, group); | sym_group = g_hash_table_lookup (cfg->groups, group); | ||||
if (sym_group == NULL) { | if (sym_group == NULL) { | ||||
} | } | ||||
g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); | g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); | ||||
sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPPED); | |||||
sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPED); | |||||
g_ptr_array_add (sym_def->groups, sym_group); | g_ptr_array_add (sym_def->groups, sym_group); | ||||
return TRUE; | return TRUE; |
/** | /** | ||||
* Compile CSS declaration to the html block | * Compile CSS declaration to the html block | ||||
* @param pool used to carry memory requred for html_block | |||||
* @param pool used to carry memory required for html_block | |||||
* @return html block structure | * @return html block structure | ||||
*/ | */ | ||||
auto compile_to_block(rspamd_mempool_t *pool) const -> rspamd::html::html_block *; | auto compile_to_block(rspamd_mempool_t *pool) const -> rspamd::html::html_block *; |
GError **err); | GError **err); | ||||
/** | /** | ||||
* Canonocalise header using relaxed algorithm | |||||
* Canonicalise header using relaxed algorithm | |||||
* @param hname | * @param hname | ||||
* @param hvalue | * @param hvalue | ||||
* @param out | * @param out |
{ | { | ||||
guint ret; | guint ret; | ||||
/* Distirbuted uniformly already */ | |||||
/* Distributed uniformly already */ | |||||
memcpy (&ret, key, sizeof (ret)); | memcpy (&ret, key, sizeof (ret)); | ||||
return ret; | return ret; |
ignore_bad_tag, | ignore_bad_tag, | ||||
tag_end, | tag_end, | ||||
slash_after_value, | slash_after_value, | ||||
slash_in_unqouted_value, | |||||
slash_in_unquoted_value, | |||||
} state; | } state; | ||||
state = static_cast<enum tag_parser_state>(parser_env.cur_state); | state = static_cast<enum tag_parser_state>(parser_env.cur_state); | ||||
*/ | */ | ||||
if (*in == '>') { | if (*in == '>') { | ||||
/* | /* | ||||
* Attribtute name followed by end of tag | |||||
* Attribute name followed by end of tag | |||||
* Should be okay (empty attribute). The rest is handled outside | * Should be okay (empty attribute). The rest is handled outside | ||||
* this automata. | * this automata. | ||||
*/ | */ | ||||
case parse_value: | case parse_value: | ||||
if (*in == '/') { | if (*in == '/') { | ||||
state = slash_in_unqouted_value; | |||||
state = slash_in_unquoted_value; | |||||
} | } | ||||
else if (g_ascii_isspace (*in) || *in == '>' || *in == '"') { | else if (g_ascii_isspace (*in) || *in == '>' || *in == '"') { | ||||
store_component_value(); | store_component_value(); | ||||
state = parse_attr_name; | state = parse_attr_name; | ||||
} | } | ||||
break; | break; | ||||
case slash_in_unqouted_value: | |||||
case slash_in_unquoted_value: | |||||
if (*in == '>') { | if (*in == '>') { | ||||
/* That slash was in fact closing tag slash, wohoo */ | |||||
/* That slash was in fact closing tag slash, woohoo */ | |||||
tag->flags |= FL_CLOSED; | tag->flags |= FL_CLOSED; | ||||
state = tag_end; | state = tag_end; | ||||
store_component_value(); | store_component_value(); |
gsize dlen; | gsize dlen; | ||||
if (visible_part.empty()) { | if (visible_part.empty()) { | ||||
/* No dispalyed url, just some text within <a> tag */ | |||||
/* No displayed url, just some text within <a> tag */ | |||||
return; | return; | ||||
} | } | ||||
if (msg->body_buf.len > 0) { | if (msg->body_buf.len > 0) { | ||||
if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { | if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { | ||||
/* Avoid copying by just maping a shared segment */ | |||||
/* Avoid copying by just mapping a shared segment */ | |||||
new_msg->flags |= RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE; | new_msg->flags |= RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE; | ||||
storage = &new_msg->body_buf.c; | storage = &new_msg->body_buf.c; |
struct upstream_list *uls; | struct upstream_list *uls; | ||||
if (!ctx->ups_ctx) { | if (!ctx->ups_ctx) { | ||||
msg_err ("cannot parse http_proxy %s - upstreams context is udefined", name); | |||||
msg_err ("cannot parse http_proxy %s - upstreams context is undefined", name); | |||||
return; | return; | ||||
} | } | ||||
if (!(rspamd_log->flags & RSPAMD_LOG_FLAG_RSPAMADM)) { | if (!(rspamd_log->flags & RSPAMD_LOG_FLAG_RSPAMADM)) { | ||||
if ((nescaped = rspamd_log_line_need_escape (logbuf, end - logbuf)) != 0) { | if ((nescaped = rspamd_log_line_need_escape (logbuf, end - logbuf)) != 0) { | ||||
gsize unsecaped_len = end - logbuf; | |||||
gchar *logbuf_escaped = g_alloca (unsecaped_len + nescaped * 4); | |||||
gsize unescaped_len = end - logbuf; | |||||
gchar *logbuf_escaped = g_alloca (unescaped_len + nescaped * 4); | |||||
log_line = logbuf_escaped; | log_line = logbuf_escaped; | ||||
end = rspamd_log_line_hex_escape (logbuf, unsecaped_len, | |||||
logbuf_escaped, unsecaped_len + nescaped * 4); | |||||
end = rspamd_log_line_hex_escape (logbuf, unescaped_len, | |||||
logbuf_escaped, unescaped_len + nescaped * 4); | |||||
} | } | ||||
} | } | ||||
/* Now, we do some sanity checks for jittered seconds */ | /* Now, we do some sanity checks for jittered seconds */ | ||||
if (!(how & RSPAMD_MAP_SCHEDULE_INIT)) { | if (!(how & RSPAMD_MAP_SCHEDULE_INIT)) { | ||||
/* Never allow too low interval between timer checks, it is epxensive */ | |||||
/* Never allow too low interval between timer checks, it is expensive */ | |||||
if (jittered_sec < min_timer_interval) { | if (jittered_sec < min_timer_interval) { | ||||
jittered_sec = rspamd_time_jitter (min_timer_interval, 0); | jittered_sec = rspamd_time_jitter (min_timer_interval, 0); | ||||
} | } |
#if !defined(__aarch64__) && !defined(__powerpc64__) | #if !defined(__aarch64__) && !defined(__powerpc64__) | ||||
if (!(map->cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) { | if (!(map->cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) { | ||||
msg_info_map ("disable hyperscan for map %s, ssse3 instructons are not supported by CPU", | |||||
msg_info_map ("disable hyperscan for map %s, ssse3 instructions are not supported by CPU", | |||||
map->name); | map->name); | ||||
return; | return; | ||||
} | } |
sizeof (fl)); | sizeof (fl)); | ||||
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl, | rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl, | ||||
sizeof (fl)); | sizeof (fl)); | ||||
/* Numberic order */ | |||||
/* Numeric order */ | |||||
rspamd_cryptobox_hash_update (re_class->st, (const guchar *)&i, | rspamd_cryptobox_hash_update (re_class->st, (const guchar *)&i, | ||||
sizeof (i)); | sizeof (i)); | ||||
rspamd_cryptobox_hash_update (&st_global, (const guchar *)&i, | rspamd_cryptobox_hash_update (&st_global, (const guchar *)&i, | ||||
rspamd_re_cache_hs_pattern_from_pcre (rspamd_regexp_t *re) | rspamd_re_cache_hs_pattern_from_pcre (rspamd_regexp_t *re) | ||||
{ | { | ||||
/* | /* | ||||
* Workaroung for bug in ragel 7.0.0.11 | |||||
* Workaround for bug in ragel 7.0.0.11 | |||||
* https://github.com/intel/hyperscan/issues/133 | * https://github.com/intel/hyperscan/issues/133 | ||||
*/ | */ | ||||
const gchar *pat = rspamd_regexp_get_pattern (re); | const gchar *pat = rspamd_regexp_get_pattern (re); |
if (o1 == o2) { | if (o1 == o2) { | ||||
/* Heurstic */ | |||||
/* Heuristic */ | |||||
if (i1->priority == i2->priority) { | if (i1->priority == i2->priority) { | ||||
avg_freq = ((gdouble) cache->total_hits / cache->used_items); | avg_freq = ((gdouble) cache->total_hits / cache->used_items); | ||||
avg_weight = (cache->total_weight / cache->used_items); | avg_weight = (cache->total_weight / cache->used_items); | ||||
* one more time | * one more time | ||||
*/ | */ | ||||
msg_debug_cache_task ("postpone finalisation of %s(%d) as there are %d " | msg_debug_cache_task ("postpone finalisation of %s(%d) as there are %d " | ||||
"async events pendning", | |||||
"async events pending", | |||||
item->symbol, item->id, dyn_item->async_events); | item->symbol, item->id, dyn_item->async_events); | ||||
return; | return; |
gboolean exec_only); | gboolean exec_only); | ||||
/** | /** | ||||
* Returns symbcache item flags | |||||
* Returns symcache item flags | |||||
* @param item | * @param item | ||||
* @return | * @return | ||||
*/ | */ | ||||
const struct rspamd_symcache_item_stat * | const struct rspamd_symcache_item_stat * | ||||
rspamd_symcache_item_stat (struct rspamd_symcache_item *item); | rspamd_symcache_item_stat (struct rspamd_symcache_item *item); | ||||
/** | /** | ||||
* Returns if an item is enabled (for virutal it also means that parent should be enabled) | |||||
* Returns if an item is enabled (for virtual it also means that parent should be enabled) | |||||
* @param item | * @param item | ||||
* @return | * @return | ||||
*/ | */ |
struct spf_resolved_element { | struct spf_resolved_element { | ||||
GPtrArray *elts; | GPtrArray *elts; | ||||
gchar *cur_domain; | gchar *cur_domain; | ||||
gboolean redirected; /* Ingnore level, it's redirected */ | |||||
gboolean redirected; /* Ignore level, it's redirected */ | |||||
}; | }; | ||||
struct spf_record { | struct spf_record { | ||||
gboolean ret = FALSE; | gboolean ret = FALSE; | ||||
/* | /* | ||||
* We prefer spf version 1 as other records are mostly likely garbadge | |||||
* We prefer spf version 1 as other records are mostly likely garbage | |||||
* or incorrect records (e.g. spf2 records) | * or incorrect records (e.g. spf2 records) | ||||
*/ | */ | ||||
LL_FOREACH (reply->entries, elt) { | LL_FOREACH (reply->entries, elt) { | ||||
parse_ipv4_mask, | parse_ipv4_mask, | ||||
parse_second_slash, | parse_second_slash, | ||||
parse_ipv6_mask, | parse_ipv6_mask, | ||||
skip_garbadge | |||||
skip_garbage | |||||
} state = 0; | } state = 0; | ||||
const gchar *p = addr->spf_string, *host, *c; | const gchar *p = addr->spf_string, *host, *c; | ||||
gchar *hostbuf; | gchar *hostbuf; | ||||
state = parse_ipv4_mask; | state = parse_ipv4_mask; | ||||
} | } | ||||
else { | else { | ||||
state = skip_garbadge; | |||||
state = skip_garbage; | |||||
} | } | ||||
cur_mask = 0; | cur_mask = 0; | ||||
break; | break; | ||||
} | } | ||||
p++; | p++; | ||||
break; | break; | ||||
case skip_garbadge: | |||||
case skip_garbage: | |||||
p++; | p++; | ||||
break; | break; | ||||
} | } |
enum rspamd_command { | enum rspamd_command { | ||||
CMD_SKIP = 0, | CMD_SKIP = 0, | ||||
CMD_PING, | CMD_PING, | ||||
CMD_CHECK_SPAMC, /* Legacy spamasassin format */ | |||||
CMD_CHECK_SPAMC, /* Legacy spamassassin format */ | |||||
CMD_CHECK_RSPAMC, /* Legacy rspamc format (like SA one) */ | CMD_CHECK_RSPAMC, /* Legacy rspamc format (like SA one) */ | ||||
CMD_CHECK, /* Legacy check - metric json reply */ | CMD_CHECK, /* Legacy check - metric json reply */ | ||||
CMD_CHECK_V2, /* Modern check - symbols in json reply */ | CMD_CHECK_V2, /* Modern check - symbols in json reply */ |
{"schema_encoded", RSPAMD_URL_FLAG_SCHEMAENCODED, -1}, | {"schema_encoded", RSPAMD_URL_FLAG_SCHEMAENCODED, -1}, | ||||
{"path_encoded", RSPAMD_URL_FLAG_PATHENCODED, -1}, | {"path_encoded", RSPAMD_URL_FLAG_PATHENCODED, -1}, | ||||
{"query_encoded", RSPAMD_URL_FLAG_QUERYENCODED, -1}, | {"query_encoded", RSPAMD_URL_FLAG_QUERYENCODED, -1}, | ||||
{"missing_slahes", RSPAMD_URL_FLAG_MISSINGSLASHES, -1}, | |||||
{"missing_slashes", RSPAMD_URL_FLAG_MISSINGSLASHES, -1}, | |||||
{"idn", RSPAMD_URL_FLAG_IDN, -1}, | {"idn", RSPAMD_URL_FLAG_IDN, -1}, | ||||
{"has_port", RSPAMD_URL_FLAG_HAS_PORT, -1}, | {"has_port", RSPAMD_URL_FLAG_HAS_PORT, -1}, | ||||
{"has_user", RSPAMD_URL_FLAG_HAS_USER, -1}, | {"has_user", RSPAMD_URL_FLAG_HAS_USER, -1}, |
* @param pool memory pool | * @param pool memory pool | ||||
* @param task task object | * @param task task object | ||||
* @param part current text part | * @param part current text part | ||||
* @param is_html turn on html euristic | |||||
* @param is_html turn on html heuristic | |||||
*/ | */ | ||||
void rspamd_url_text_extract(rspamd_mempool_t *pool, | void rspamd_url_text_extract(rspamd_mempool_t *pool, | ||||
struct rspamd_task *task, | struct rspamd_task *task, |
static void | static void | ||||
rspamd_worker_drop_priv (struct rspamd_main *rspamd_main) | rspamd_worker_drop_priv (struct rspamd_main *rspamd_main) | ||||
{ | { | ||||
if (rspamd_main->is_privilleged) { | |||||
if (rspamd_main->is_privileged) { | |||||
if (setgid (rspamd_main->workers_gid) == -1) { | if (setgid (rspamd_main->workers_gid) == -1) { | ||||
msg_err_main ("cannot setgid to %d (%s), aborting", | msg_err_main ("cannot setgid to %d (%s), aborting", | ||||
(gint) rspamd_main->workers_gid, | (gint) rspamd_main->workers_gid, |
struct rspamd_custom_controller_command { | struct rspamd_custom_controller_command { | ||||
const gchar *command; | const gchar *command; | ||||
struct module_ctx *ctx; | struct module_ctx *ctx; | ||||
gboolean privilleged; | |||||
gboolean privileged; | |||||
gboolean require_message; | gboolean require_message; | ||||
rspamd_controller_func_t handler; | rspamd_controller_func_t handler; | ||||
}; | }; |
gchar *h; | gchar *h; | ||||
if (rspamd_session_blocked (task->s)) { | if (rspamd_session_blocked (task->s)) { | ||||
return RSPAMD_LEARN_INGORE; | |||||
return RSPAMD_LEARN_IGNORE; | |||||
} | } | ||||
h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); | h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); | ||||
if (h == NULL) { | if (h == NULL) { | ||||
return RSPAMD_LEARN_INGORE; | |||||
return RSPAMD_LEARN_IGNORE; | |||||
} | } | ||||
if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_get, rt, | if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_get, rt, | ||||
gint flag; | gint flag; | ||||
if (rt == NULL || rt->ctx == NULL || rspamd_session_blocked (task->s)) { | if (rt == NULL || rt->ctx == NULL || rspamd_session_blocked (task->s)) { | ||||
return RSPAMD_LEARN_INGORE; | |||||
return RSPAMD_LEARN_IGNORE; | |||||
} | } | ||||
h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); | h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); |
gint64 flag; | gint64 flag; | ||||
if (task->tokens == NULL || task->tokens->len == 0) { | if (task->tokens == NULL || task->tokens->len == 0) { | ||||
return RSPAMD_LEARN_INGORE; | |||||
return RSPAMD_LEARN_IGNORE; | |||||
} | } | ||||
if (ctx != NULL && ctx->db != NULL) { | if (ctx != NULL && ctx->db != NULL) { | ||||
/* Already learned */ | /* Already learned */ | ||||
msg_warn_task ("already seen stat hash: %*bs", | msg_warn_task ("already seen stat hash: %*bs", | ||||
rspamd_cryptobox_HASHBYTES, out); | rspamd_cryptobox_HASHBYTES, out); | ||||
return RSPAMD_LEARN_INGORE; | |||||
return RSPAMD_LEARN_IGNORE; | |||||
} | } | ||||
else { | else { | ||||
/* Need to relearn */ | /* Need to relearn */ | ||||
h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); | h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); | ||||
if (h == NULL) { | if (h == NULL) { | ||||
return RSPAMD_LEARN_INGORE; | |||||
return RSPAMD_LEARN_IGNORE; | |||||
} | } | ||||
flag = !!is_spam ? 1 : 0; | flag = !!is_spam ? 1 : 0; |
typedef enum rspamd_learn_cache_result { | typedef enum rspamd_learn_cache_result { | ||||
RSPAMD_LEARN_OK = 0, | RSPAMD_LEARN_OK = 0, | ||||
RSPAMD_LEARN_UNLEARN, | RSPAMD_LEARN_UNLEARN, | ||||
RSPAMD_LEARN_INGORE | |||||
RSPAMD_LEARN_IGNORE | |||||
} rspamd_learn_t; | } rspamd_learn_t; | ||||
struct rspamd_stat_ctx *rspamd_stat_get_ctx (void); | struct rspamd_stat_ctx *rspamd_stat_get_ctx (void); |
#define RSPAMD_LEARN_OP 1 | #define RSPAMD_LEARN_OP 1 | ||||
#define RSPAMD_UNLEARN_OP 2 | #define RSPAMD_UNLEARN_OP 2 | ||||
static const gdouble similarity_treshold = 80.0; | |||||
static const gdouble similarity_threshold = 80.0; | |||||
static void | static void | ||||
rspamd_stat_tokenize_parts_metadata (struct rspamd_stat_ctx *st_ctx, | rspamd_stat_tokenize_parts_metadata (struct rspamd_stat_ctx *st_ctx, | ||||
} | } | ||||
if (pdiff != NULL && (1.0 - *pdiff) * 100.0 > similarity_treshold) { | |||||
if (pdiff != NULL && (1.0 - *pdiff) * 100.0 > similarity_threshold) { | |||||
msg_debug_bayes ("message has two common parts (%.2f), so skip the last one", | msg_debug_bayes ("message has two common parts (%.2f), so skip the last one", | ||||
*pdiff); | *pdiff); | ||||
break; | break; | ||||
learn_res = cl->cache->check (task, spam, rt); | learn_res = cl->cache->check (task, spam, rt); | ||||
} | } | ||||
if (learn_res == RSPAMD_LEARN_INGORE) { | |||||
if (learn_res == RSPAMD_LEARN_IGNORE) { | |||||
/* Do not learn twice */ | /* Do not learn twice */ | ||||
g_set_error (err, rspamd_stat_quark (), 404, "<%s> has been already " | g_set_error (err, rspamd_stat_quark (), 404, "<%s> has been already " | ||||
"learned as %s, ignore it", MESSAGE_FIELD (task, message_id), | "learned as %s, ignore it", MESSAGE_FIELD (task, message_id), | ||||
} | } | ||||
} | } | ||||
else if (ucl_object_type (obj) == UCL_STRING) { | else if (ucl_object_type (obj) == UCL_STRING) { | ||||
/* Legacy sript */ | |||||
/* Legacy script */ | |||||
lua_script = ucl_object_tostring (obj); | lua_script = ucl_object_tostring (obj); | ||||
if (luaL_dostring (L, lua_script) != 0) { | if (luaL_dostring (L, lua_script) != 0) { |
void rspamd_inet_library_destroy (void); | void rspamd_inet_library_destroy (void); | ||||
/** | /** | ||||
* Create new inet address structure based on the address familiy and opaque init pointer | |||||
* Create new inet address structure based on the address family and opaque init pointer | |||||
* @param af | * @param af | ||||
* @param init | * @param init | ||||
* @return new inet addr | * @return new inet addr |
/** | /** | ||||
* Create new lru hash | * Create new lru hash | ||||
* @param maxsize maximum elements in a hash | * @param maxsize maximum elements in a hash | ||||
* @param maxage maximum age of elemnt | |||||
* @param maxage maximum age of element | |||||
* @param hash_func pointer to hash function | * @param hash_func pointer to hash function | ||||
* @param key_equal_func pointer to function for comparing keys | * @param key_equal_func pointer to function for comparing keys | ||||
* @return new rspamd_hash object | * @return new rspamd_hash object | ||||
/** | /** | ||||
* Create new lru hash | * Create new lru hash | ||||
* @param maxsize maximum elements in a hash | * @param maxsize maximum elements in a hash | ||||
* @param maxage maximum age of elemnt | |||||
* @param maxage maximum age of element | |||||
* @param hash_func pointer to hash function | * @param hash_func pointer to hash function | ||||
* @param key_equal_func pointer to function for comparing keys | * @param key_equal_func pointer to function for comparing keys | ||||
* @return new rspamd_hash object | * @return new rspamd_hash object |
qsort (sz, G_N_ELEMENTS (sz), sizeof (gint), cmp_int); | qsort (sz, G_N_ELEMENTS (sz), sizeof (gint), cmp_int); | ||||
jitter = rspamd_random_uint64_fast () % 10; | jitter = rspamd_random_uint64_fast () % 10; | ||||
/* | /* | ||||
* Take stochaistic quantiles | |||||
* Take stochastic quantiles | |||||
*/ | */ | ||||
sel_pos = sz[50 + jitter]; | sel_pos = sz[50 + jitter]; | ||||
sel_neg = sz[4 + jitter]; | sel_neg = sz[4 + jitter]; |
* Set memory pool variable | * Set memory pool variable | ||||
* @param pool memory pool object | * @param pool memory pool object | ||||
* @param name name of variable | * @param name name of variable | ||||
* @param gpointer value value of variable | |||||
* @param gpointer value of variable | |||||
* @param destructor pointer to function-destructor | * @param destructor pointer to function-destructor | ||||
*/ | */ | ||||
void rspamd_mempool_set_variable (rspamd_mempool_t *pool, | void rspamd_mempool_set_variable (rspamd_mempool_t *pool, |
gdouble float_cookie; /* is it the correct double representation ? */ | gdouble float_cookie; /* is it the correct double representation ? */ | ||||
/* Data Base Structure Definition **** */ | /* Data Base Structure Definition **** */ | ||||
gulong ds_cnt; /* how many different ds provid input to the rrd */ | |||||
gulong ds_cnt; /* how many different ds provide input to the rrd */ | |||||
gulong rra_cnt; /* how many rras will be maintained in the rrd */ | gulong rra_cnt; /* how many rras will be maintained in the rrd */ | ||||
gulong pdp_step; /* pdp interval in seconds */ | gulong pdp_step; /* pdp interval in seconds */ | ||||
* prediction algorithm. */ | * prediction algorithm. */ | ||||
CDP_hw_last_intercept, | CDP_hw_last_intercept, | ||||
/* Last iteration intercept coefficient for the Holt-Winters | /* Last iteration intercept coefficient for the Holt-Winters | ||||
* prediction algorihtm. */ | |||||
* prediction algorithm. */ | |||||
CDP_hw_slope, | CDP_hw_slope, | ||||
/* Current slope coefficient for the Holt-Winters | /* Current slope coefficient for the Holt-Winters | ||||
* prediction algorithm. */ | * prediction algorithm. */ | ||||
CDP_hw_last_slope, | CDP_hw_last_slope, | ||||
/* Last iteration slope coeffient. */ | |||||
/* Last iteration slope coefficient. */ | |||||
CDP_null_count, | CDP_null_count, | ||||
/* Number of sequential Unknown (DNAN) values + 1 preceding | /* Number of sequential Unknown (DNAN) values + 1 preceding | ||||
* the current prediction. | * the current prediction. |
* if needed | * if needed | ||||
* @param input array of `rspamd_fstring_t` | * @param input array of `rspamd_fstring_t` | ||||
* @param key secret key used to generate shingles | * @param key secret key used to generate shingles | ||||
* @param pool pool to allocate shigles array | |||||
* @param pool pool to allocate shingles array | |||||
* @param filter hashes filtering function | * @param filter hashes filtering function | ||||
* @param filterd opaque data for filtering function | * @param filterd opaque data for filtering function | ||||
* @return shingles array | * @return shingles array | ||||
* Generate shingles from the DCT matrix of an image | * Generate shingles from the DCT matrix of an image | ||||
* @param dct discrete cosine transfor matrix (must be 64x64) | * @param dct discrete cosine transfor matrix (must be 64x64) | ||||
* @param key secret key used to generate shingles | * @param key secret key used to generate shingles | ||||
* @param pool pool to allocate shigles array | |||||
* @param pool pool to allocate shingles array | |||||
* @param filter hashes filtering function | * @param filter hashes filtering function | ||||
* @param filterd opaque data for filtering function | * @param filterd opaque data for filtering function | ||||
* @return shingles array | * @return shingles array |
continue; | continue; | ||||
} | } | ||||
else { | else { | ||||
/* Hack, hack, hack, treat =<garbadge> as =<garbadge> */ | |||||
/* Hack, hack, hack, treat =<garbage> as =<garbage> */ | |||||
if (end - o > 1) { | if (end - o > 1) { | ||||
*o++ = '='; | *o++ = '='; | ||||
*o++ = *(p - 1); | *o++ = *(p - 1); |
g_ptr_array_remove_index (ls->alive, upstream->active_idx); | g_ptr_array_remove_index (ls->alive, upstream->active_idx); | ||||
upstream->active_idx = -1; | upstream->active_idx = -1; | ||||
/* We need to update all indicies */ | |||||
/* We need to update all indices */ | |||||
for (i = 0; i < ls->alive->len; i ++) { | for (i = 0; i < ls->alive->len; i ++) { | ||||
cur = g_ptr_array_index (ls->alive, i); | cur = g_ptr_array_index (ls->alive, i); | ||||
cur->active_idx = i; | cur->active_idx = i; |
const gchar *rspamd_upstream_name (struct upstream *up); | const gchar *rspamd_upstream_name (struct upstream *up); | ||||
/** | /** | ||||
* Returns the port of the current addres for the upstream | |||||
* Returns the port of the current address for the upstream | |||||
* @param up | * @param up | ||||
* @return | * @return | ||||
*/ | */ |
* Make a universal socket | * Make a universal socket | ||||
* @param credits host, ip or path to unix socket | * @param credits host, ip or path to unix socket | ||||
* @param port port (used for network sockets) | * @param port port (used for network sockets) | ||||
* @param async make this socket asynced | |||||
* @param async make this socket async | |||||
* @param is_server make this socket as server socket | * @param is_server make this socket as server socket | ||||
* @param try_resolve try name resolution for a socket (BLOCKING) | * @param try_resolve try name resolution for a socket (BLOCKING) | ||||
*/ | */ | ||||
/* | /* | ||||
* Open the PID file and obtain exclusive lock. | * Open the PID file and obtain exclusive lock. | ||||
* We truncate PID file here only to remove old PID immediatelly, | |||||
* We truncate PID file here only to remove old PID immediately, | |||||
* PID file will be truncated again in pidfile_write(), so | * PID file will be truncated again in pidfile_write(), so | ||||
* pidfile_write() can be called multiple times. | * pidfile_write() can be called multiple times. | ||||
*/ | */ |
* @param credits host, ip or path to unix socket | * @param credits host, ip or path to unix socket | ||||
* @param port port (used for network sockets) | * @param port port (used for network sockets) | ||||
* @param type type of socket (SO_STREAM or SO_DGRAM) | * @param type type of socket (SO_STREAM or SO_DGRAM) | ||||
* @param async make this socket asynced | |||||
* @param async make this socket async | |||||
* @param is_server make this socket as server socket | * @param is_server make this socket as server socket | ||||
* @param try_resolve try name resolution for a socket (BLOCKING) | * @param try_resolve try name resolution for a socket (BLOCKING) | ||||
*/ | */ |
/*** | /*** | ||||
* @method rspamd_config:register_dependency(id|name, depname) | * @method rspamd_config:register_dependency(id|name, depname) | ||||
* Create a dependency on symbol identified by name for symbol identified by ID or name. | * Create a dependency on symbol identified by name for symbol identified by ID or name. | ||||
* This affects order of checks only (a symbol is still checked if its dependencys are disabled). | |||||
* This affects order of checks only (a symbol is still checked if its dependencies are disabled). | |||||
* @param {number|string} id id or name of source (numeric id is returned by all register_*_symbol) | * @param {number|string} id id or name of source (numeric id is returned by all register_*_symbol) | ||||
* @param {string} depname dependency name | * @param {string} depname dependency name | ||||
* @example | * @example | ||||
/** | /** | ||||
* @method rspamd_config:get_action(name) | * @method rspamd_config:get_action(name) | ||||
* Gets data for a specific action in config. This function returns number reperesenting action's score | |||||
* Gets data for a specific action in config. This function returns number representing action's score | |||||
* | * | ||||
* @param {string} name name of action | * @param {string} name name of action | ||||
* @return {number} action's score or nil in case of undefined score or action | * @return {number} action's score or nil in case of undefined score or action | ||||
/*** | /*** | ||||
* @method rspamd_config:__newindex(name, callback) | * @method rspamd_config:__newindex(name, callback) | ||||
* This metamethod is called if new indicies are added to the `rspamd_config` object. | |||||
* This metamethod is called if new indices are added to the `rspamd_config` object. | |||||
* Technically, it is the equivalent of @see rspamd_config:register_symbol where `weight` is 1.0. | * Technically, it is the equivalent of @see rspamd_config:register_symbol where `weight` is 1.0. | ||||
* There is also table form invocation that allows to control more things: | * There is also table form invocation that allows to control more things: | ||||
* | * | ||||
* | * | ||||
* - `default`: default option value | * - `default`: default option value | ||||
* - `type`: type of an option (`string`, `number`, `object`, `array` etc) | * - `type`: type of an option (`string`, `number`, `object`, `array` etc) | ||||
* - `reqired`: if an option is required | |||||
* - `required`: if an option is required | |||||
* | * | ||||
* @param {string} path documentation path (e.g. module name) | * @param {string} path documentation path (e.g. module name) | ||||
* @param {string} option name of the option | * @param {string} option name of the option | ||||
} | } | ||||
else | else | ||||
{ | { | ||||
/* Fill in missing fields from lua defintion if they are not set */ | |||||
/* Fill in missing fields from lua definition if they are not set */ | |||||
if (sym->description == NULL) { | if (sym->description == NULL) { | ||||
lua_pushstring (L, "description"); | lua_pushstring (L, "description"); | ||||
lua_gettable (L, -2); | lua_gettable (L, -2); | ||||
} | } | ||||
lua_pop (L, 1); | lua_pop (L, 1); | ||||
if (group) { | if (group) { | ||||
if (sym->flags & RSPAMD_SYMBOL_FLAG_UNGROUPPED) { | |||||
if (sym->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED) { | |||||
/* Unset the "ungrouped" group */ | /* Unset the "ungrouped" group */ | ||||
sym->gr = NULL; | sym->gr = NULL; | ||||
} | } | ||||
/* Add the group. If the symbol was ungrouped, this will | /* Add the group. If the symbol was ungrouped, this will | ||||
* clear RSPAMD_SYMBOL_FLAG_UNGROUPPED from the flags. */ | |||||
* clear RSPAMD_SYMBOL_FLAG_UNGROUPED from the flags. */ | |||||
rspamd_config_add_symbol_group (cfg, name, group); | rspamd_config_add_symbol_group (cfg, name, group); | ||||
} | } | ||||
} | } | ||||
lua_pushboolean (L, true); | lua_pushboolean (L, true); | ||||
lua_settable (L, -3); | lua_settable (L, -3); | ||||
} | } | ||||
if (s->flags & RSPAMD_SYMBOL_FLAG_UNGROUPPED) { | |||||
lua_pushstring (L, "ungroupped"); | |||||
if (s->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED) { | |||||
lua_pushstring (L, "ungrouped"); | |||||
lua_pushboolean (L, true); | lua_pushboolean (L, true); | ||||
lua_settable (L, -3); | lua_settable (L, -3); | ||||
} | } |
* * `mempool` - pool memory pool for storing intermediate data | * * `mempool` - pool memory pool for storing intermediate data | ||||
* * `name` - host name to resolve | * * `name` - host name to resolve | ||||
* * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` | * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` | ||||
* * `forced` - true if needed to override notmal limit for DNS requests | |||||
* * `forced` - true if needed to override normal limit for DNS requests | |||||
* @return {boolean} `true` if DNS request has been scheduled | * @return {boolean} `true` if DNS request has been scheduled | ||||
*/ | */ | ||||
static int | static int | ||||
* * `mempool` - pool memory pool for storing intermediate data | * * `mempool` - pool memory pool for storing intermediate data | ||||
* * `name` - host name to resolve | * * `name` - host name to resolve | ||||
* * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` | * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` | ||||
* * `forced` - true if needed to override notmal limit for DNS requests | |||||
* * `forced` - true if needed to override normal limit for DNS requests | |||||
* @return {boolean} `true` if DNS request has been scheduled | * @return {boolean} `true` if DNS request has been scheduled | ||||
*/ | */ | ||||
static int | static int | ||||
* * `mempool` - pool memory pool for storing intermediate data | * * `mempool` - pool memory pool for storing intermediate data | ||||
* * `name` - host name to resolve | * * `name` - host name to resolve | ||||
* * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` | * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` | ||||
* * `forced` - true if needed to override notmal limit for DNS requests | |||||
* * `forced` - true if needed to override normal limit for DNS requests | |||||
* @return {boolean} `true` if DNS request has been scheduled | * @return {boolean} `true` if DNS request has been scheduled | ||||
*/ | */ | ||||
static int | static int | ||||
* * `mempool` - pool memory pool for storing intermediate data | * * `mempool` - pool memory pool for storing intermediate data | ||||
* * `name` - host name to resolve | * * `name` - host name to resolve | ||||
* * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` | * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` | ||||
* * `forced` - true if needed to override notmal limit for DNS requests | |||||
* * `forced` - true if needed to override normal limit for DNS requests | |||||
* @return {boolean} `true` if DNS request has been scheduled | * @return {boolean} `true` if DNS request has been scheduled | ||||
*/ | */ | ||||
static int | static int | ||||
* * `mempool` - pool memory pool for storing intermediate data | * * `mempool` - pool memory pool for storing intermediate data | ||||
* * `name` - host name to resolve | * * `name` - host name to resolve | ||||
* * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` | * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` | ||||
* * `forced` - true if needed to override notmal limit for DNS requests | |||||
* * `forced` - true if needed to override normal limit for DNS requests | |||||
* @return {boolean} `true` if DNS request has been scheduled | * @return {boolean} `true` if DNS request has been scheduled | ||||
*/ | */ | ||||
static int | static int |
* * By default headers are searched in caseless matter. | * * By default headers are searched in caseless matter. | ||||
* @param {string} name name of header to get | * @param {string} name name of header to get | ||||
* @param {boolean} case_sensitive case sensitiveness flag to search for a header | * @param {boolean} case_sensitive case sensitiveness flag to search for a header | ||||
* @return {number} number of header's occurrencies or 0 if not found | |||||
* @return {number} number of header's occurrences or 0 if not found | |||||
*/ | */ | ||||
LUA_FUNCTION_DEF (mimepart, get_header_count); | LUA_FUNCTION_DEF (mimepart, get_header_count); | ||||
/*** | /*** | ||||
* @method mime_part:get_urls([need_emails|list_protos][, need_images]) | * @method mime_part:get_urls([need_emails|list_protos][, need_images]) | ||||
* Get all URLs found in a mime part. Telephone urls and emails are not included unless explicitly asked in `list_protos` | * Get all URLs found in a mime part. Telephone urls and emails are not included unless explicitly asked in `list_protos` | ||||
* @param {boolean} need_emails if `true` then reutrn also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`) | |||||
* @param {boolean} need_emails if `true` then return also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`) | |||||
* @param {boolean} need_images return urls from images (<img src=...>) as well | * @param {boolean} need_images return urls from images (<img src=...>) as well | ||||
* @return {table rspamd_url} list of all urls found | * @return {table rspamd_url} list of all urls found | ||||
*/ | */ | ||||
lua_pushboolean (L, true); | lua_pushboolean (L, true); | ||||
} | } | ||||
else { | else { | ||||
/* Image or an embeded object */ | |||||
/* Image or an embedded object */ | |||||
lua_pushboolean (L, false); | lua_pushboolean (L, false); | ||||
} | } | ||||
} | } |
struct redisAsyncContext *ac; | struct redisAsyncContext *ac; | ||||
ud = &ctx->async; | ud = &ctx->async; | ||||
msg_debug_lua_redis ("desctructing %p", ctx); | |||||
msg_debug_lua_redis ("destructing %p", ctx); | |||||
if (ud->ctx) { | if (ud->ctx) { | ||||
/*** | /*** | ||||
* @method re:destroy() | * @method re:destroy() | ||||
* Destroy regexp from caches if needed (the pointer is removed by garbadge collector) | |||||
* Destroy regexp from caches if needed (the pointer is removed by garbage collector) | |||||
*/ | */ | ||||
static gint | static gint | ||||
lua_regexp_destroy (lua_State *L) | lua_regexp_destroy (lua_State *L) |
/*** | /*** | ||||
* @method task:remove_result(symbol[, shadow_result]) | * @method task:remove_result(symbol[, shadow_result]) | ||||
* Removes the symbol from a named or unamed/default result | |||||
* Removes the symbol from a named or unnamed/default result | |||||
* @param {string} symbol symbol to remove | * @param {string} symbol symbol to remove | ||||
* @param {string} shadow_result name of shadow result | * @param {string} shadow_result name of shadow result | ||||
* @return {boolean} true if a symbol has been removed | * @return {boolean} true if a symbol has been removed | ||||
/*** | /*** | ||||
* @method task:get_urls([need_emails|list_protos][, need_images]) | * @method task:get_urls([need_emails|list_protos][, need_images]) | ||||
* Get all URLs found in a message. Telephone urls and emails are not included unless explicitly asked in `list_protos` | * Get all URLs found in a message. Telephone urls and emails are not included unless explicitly asked in `list_protos` | ||||
* @param {boolean} need_emails if `true` then reutrn also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`) | |||||
* @param {boolean} need_emails if `true` then return also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`) | |||||
* @param {boolean} need_images return urls from images (<img src=...>) as well | * @param {boolean} need_images return urls from images (<img src=...>) as well | ||||
* @return {table rspamd_url} list of all urls found | * @return {table rspamd_url} list of all urls found | ||||
@example | @example | ||||
* - If both parameters are nil then all urls are included | * - If both parameters are nil then all urls are included | ||||
* @param {table} flags_include included flags | * @param {table} flags_include included flags | ||||
* @param {table} flags_exclude excluded flags | * @param {table} flags_exclude excluded flags | ||||
* @param {table} protocols_mask incude only specific protocols | |||||
* @param {table} protocols_mask include only specific protocols | |||||
* @return {table rspamd_url} list of urls matching conditions | * @return {table rspamd_url} list of urls matching conditions | ||||
*/ | */ | ||||
LUA_FUNCTION_DEF (task, get_urls_filtered); | LUA_FUNCTION_DEF (task, get_urls_filtered); | ||||
/*** | /*** | ||||
* @method task:has_urls([need_emails]) | * @method task:has_urls([need_emails]) | ||||
* Returns 'true' if a task has urls listed | * Returns 'true' if a task has urls listed | ||||
* @param {boolean} need_emails if `true` then reutrn also email urls | |||||
* @param {boolean} need_emails if `true` then return also email urls | |||||
* @return {boolean} true if a task has urls (urls or emails if `need_emails` is true) | * @return {boolean} true if a task has urls (urls or emails if `need_emails` is true) | ||||
*/ | */ | ||||
LUA_FUNCTION_DEF (task, has_urls); | LUA_FUNCTION_DEF (task, has_urls); | ||||
* * By default headers are searched in caseless matter. | * * By default headers are searched in caseless matter. | ||||
* @param {string} name name of header to get | * @param {string} name name of header to get | ||||
* @param {boolean} case_sensitive case sensitiveness flag to search for a header | * @param {boolean} case_sensitive case sensitiveness flag to search for a header | ||||
* @return {number} number of header's occurrencies or 0 if not found | |||||
* @return {number} number of header's occurrences or 0 if not found | |||||
*/ | */ | ||||
LUA_FUNCTION_DEF (task, get_header_count); | LUA_FUNCTION_DEF (task, get_header_count); | ||||
/*** | /*** | ||||
* Order in remove starts from 1, where 0 means 'remove all', and negative value means | * Order in remove starts from 1, where 0 means 'remove all', and negative value means | ||||
* remove from the end | * remove from the end | ||||
* Order in addition means addition from the top: 0 means the most top header, 1 one after, etc | * Order in addition means addition from the top: 0 means the most top header, 1 one after, etc | ||||
* negative order means addtion to the end, e.g. -1 is appending header. | |||||
* negative order means addition to the end, e.g. -1 is appending header. | |||||
* @return {bool} true if header could be modified (always true unless we don't have an unparsed message) | * @return {bool} true if header could be modified (always true unless we don't have an unparsed message) | ||||
*/ | */ | ||||
LUA_FUNCTION_DEF (task, modify_header); | LUA_FUNCTION_DEF (task, modify_header); | ||||
RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT, | RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT, | ||||
"*action=S;message=S;module=S;score=D;priority=i;flags=S;result=S", | "*action=S;message=S;module=S;score=D;priority=i;flags=S;result=S", | ||||
&act_str, &message, &module, &score, &priority, &fl_str, &res_name)) { | &act_str, &message, &module, &score, &priority, &fl_str, &res_name)) { | ||||
gint ret = luaL_error (L, "invald arguments: %s", err->message); | |||||
gint ret = luaL_error (L, "invalid arguments: %s", err->message); | |||||
g_error_free (err); | g_error_free (err); | ||||
return ret; | return ret; | ||||
* UCL itself cannot do it directly. So the trick is to extract the | * UCL itself cannot do it directly. So the trick is to extract the | ||||
* original object, pack it into an array and then insert it back. | * original object, pack it into an array and then insert it back. | ||||
* | * | ||||
* I wish there was a simplier way to do it... | |||||
* I wish there was a simpler way to do it... | |||||
*/ | */ | ||||
const ucl_object_t *add_hdrs = ucl_object_lookup (prev, "add_headers"); | const ucl_object_t *add_hdrs = ucl_object_lookup (prev, "add_headers"); | ||||
const ucl_object_t *nadd_hdrs = ucl_object_lookup (reply, "add_headers"); | const ucl_object_t *nadd_hdrs = ucl_object_lookup (reply, "add_headers"); | ||||
return 1; | return 1; | ||||
} | } | ||||
/* Arvhive methods */ | |||||
/* Archive methods */ | |||||
static gint | static gint | ||||
lua_archive_get_type (lua_State *L) | lua_archive_get_type (lua_State *L) | ||||
{ | { |
* @function rspamd_tcp.connect_sync() | * @function rspamd_tcp.connect_sync() | ||||
* | * | ||||
* Creates pseudo-synchronous TCP connection. | * Creates pseudo-synchronous TCP connection. | ||||
* Each method of the connection requiring IO, becames a yielding point, | |||||
* Each method of the connection requiring IO, becomes a yielding point, | |||||
* i.e. current thread Lua thread is get suspended and resumes as soon as IO is done | * i.e. current thread Lua thread is get suspended and resumes as soon as IO is done | ||||
* | * | ||||
* This class represents low-level API, using of "lua_tcp_sync" module is recommended. | * This class represents low-level API, using of "lua_tcp_sync" module is recommended. |
LUA_FUNCTION_DEF (text, span); | LUA_FUNCTION_DEF (text, span); | ||||
/*** | /*** | ||||
* @method rspamd_text:sub(start[, len]) | * @method rspamd_text:sub(start[, len]) | ||||
* Returns a substrin for lua_text similar to string.sub from Lua | |||||
* Returns a substring for lua_text similar to string.sub from Lua | |||||
* @return {rspamd_text} new rspamd_text with span (must be careful when using with owned texts...) | * @return {rspamd_text} new rspamd_text with span (must be careful when using with owned texts...) | ||||
*/ | */ | ||||
LUA_FUNCTION_DEF (text, sub); | LUA_FUNCTION_DEF (text, sub); | ||||
* @method rspamd_text:base64([line_length, [nline, [fold]]]) | * @method rspamd_text:base64([line_length, [nline, [fold]]]) | ||||
* Returns a text encoded in base64 (new rspamd_text is allocated) | * Returns a text encoded in base64 (new rspamd_text is allocated) | ||||
* | * | ||||
* @param {number} line_length return text splited with newlines up to this attribute | |||||
* @param {number} line_length return text split with newlines up to this attribute | |||||
* @param {string} nline newline type: `cr`, `lf`, `crlf` | * @param {string} nline newline type: `cr`, `lf`, `crlf` | ||||
* @param {boolean} fold use folding when splitting into lines (false by default) | * @param {boolean} fold use folding when splitting into lines (false by default) | ||||
* @return {rspamd_text} new text encoded in base64 | * @return {rspamd_text} new text encoded in base64 |
/** | /** | ||||
* Extracts a thread from the list of available ones. | * Extracts a thread from the list of available ones. | ||||
* It immediately becames running one and should be used to run a Lua script/function straight away. | |||||
* It immediately becomes the running one and should be used to run a Lua script/function straight away. | |||||
* as soon as the code is finished, it should be either returned into list of available threads by | * as soon as the code is finished, it should be either returned into list of available threads by | ||||
* calling lua_thread_pool_return() or terminated by calling lua_thread_pool_terminate_entry() | * calling lua_thread_pool_return() or terminated by calling lua_thread_pool_terminate_entry() | ||||
* if the code finished with error. | * if the code finished with error. |
/*** | /*** | ||||
* @method url:get_count() | * @method url:get_count() | ||||
* Return number of occurrencies for this particular URL | |||||
* @return {number} number of occurrencies | |||||
* Return number of occurrences for this particular URL | |||||
* @return {number} number of occurrences | |||||
*/ | */ | ||||
static gint | static gint | ||||
lua_url_get_count (lua_State *L) | lua_url_get_count (lua_State *L) | ||||
* - `host_encoded`: URL host part is encoded | * - `host_encoded`: URL host part is encoded | ||||
* - `schema_encoded`: URL schema part is encoded | * - `schema_encoded`: URL schema part is encoded | ||||
* - `query_encoded`: URL query part is encoded | * - `query_encoded`: URL query part is encoded | ||||
* - `missing_slahes`: URL has some slashes missing | |||||
* - `missing_slashes`: URL has some slashes missing | |||||
* - `idn`: URL has international characters | * - `idn`: URL has international characters | ||||
* - `has_port`: URL has port | * - `has_port`: URL has port | ||||
* - `has_user`: URL has user part | * - `has_user`: URL has user part |
LUA_FUNCTION_DEF (util, load_rspamd_config); | LUA_FUNCTION_DEF (util, load_rspamd_config); | ||||
/*** | /*** | ||||
* @function util.config_from_ucl(any, string) | * @function util.config_from_ucl(any, string) | ||||
* Load rspamd config from ucl reperesented by any lua table | |||||
* Load rspamd config from ucl represented by any lua table | |||||
* @return {confg} new configuration object suitable for access | * @return {confg} new configuration object suitable for access | ||||
*/ | */ | ||||
LUA_FUNCTION_DEF (util, config_from_ucl); | LUA_FUNCTION_DEF (util, config_from_ucl); | ||||
LUA_FUNCTION_DEF (util, encode_base64); | LUA_FUNCTION_DEF (util, encode_base64); | ||||
/*** | /*** | ||||
* @function util.encode_qp(input[, str_len, [newlines_type]]) | * @function util.encode_qp(input[, str_len, [newlines_type]]) | ||||
* Encodes data in quouted printable breaking lines if needed | |||||
* Encodes data in quoted printable breaking lines if needed | |||||
* @param {text or string} input input data | * @param {text or string} input input data | ||||
* @param {number} str_len optional size of lines or 0 if split is not needed | * @param {number} str_len optional size of lines or 0 if split is not needed | ||||
* @return {rspamd_text} encoded data chunk | * @return {rspamd_text} encoded data chunk | ||||
/*** | /*** | ||||
* @function util.decode_qp(input) | * @function util.decode_qp(input) | ||||
* Decodes data from quouted printable | |||||
* Decodes data from quoted printable | |||||
* @param {text or string} input input data | * @param {text or string} input input data | ||||
* @return {rspamd_text} decoded data chunk | * @return {rspamd_text} decoded data chunk | ||||
*/ | */ | ||||
LUA_FUNCTION_DEF (util, process_message); | LUA_FUNCTION_DEF (util, process_message); | ||||
/*** | /*** | ||||
* @function util.tanh(num) | * @function util.tanh(num) | ||||
* Calculates hyperbolic tanhent of the specified floating point value | |||||
* Calculates hyperbolic tangent of the specified floating point value | |||||
* @param {number} num input number | * @param {number} num input number | ||||
* @return {number} hyperbolic tanhent of the variable | |||||
* @return {number} hyperbolic tangent of the variable | |||||
*/ | */ | ||||
LUA_FUNCTION_DEF (util, tanh); | LUA_FUNCTION_DEF (util, tanh); | ||||
INIT_LOG_MODULE(xmlrpc) | INIT_LOG_MODULE(xmlrpc) | ||||
enum lua_xmlrpc_state { | enum lua_xmlrpc_state { | ||||
read_method_responce = 0, | |||||
read_method_response = 0, | |||||
read_params = 1, | read_params = 1, | ||||
read_param = 2, | read_param = 2, | ||||
read_param_value = 3, | read_param_value = 3, | ||||
msg_debug_xmlrpc ("got start element %s on state %d", name, last_state); | msg_debug_xmlrpc ("got start element %s on state %d", name, last_state); | ||||
switch (ud->parser_state) { | switch (ud->parser_state) { | ||||
case read_method_responce: | |||||
case read_method_response: | |||||
/* Expect tag methodResponse */ | /* Expect tag methodResponse */ | ||||
if (g_ascii_strcasecmp (name, "methodResponse") == 0) { | if (g_ascii_strcasecmp (name, "methodResponse") == 0) { | ||||
ud->parser_state = read_params; | ud->parser_state = read_params; | ||||
msg_debug_xmlrpc ("got end element %s on state %d", name, last_state); | msg_debug_xmlrpc ("got end element %s on state %d", name, last_state); | ||||
switch (ud->parser_state) { | switch (ud->parser_state) { | ||||
case read_method_responce: | |||||
case read_method_response: | |||||
ud->parser_state = error_state; | ud->parser_state = error_state; | ||||
break; | break; | ||||
case read_params: | case read_params: | ||||
if (data != NULL) { | if (data != NULL) { | ||||
ud.L = L; | ud.L = L; | ||||
ud.parser_state = read_method_responce; | |||||
ud.parser_state = read_method_response; | |||||
ud.param_count = 0; | ud.param_count = 0; | ||||
ud.st = g_queue_new (); | ud.st = g_queue_new (); | ||||
* Allowed options: | * Allowed options: | ||||
* - symbol (string): symbol to insert (default: 'R_BAD_CHARSET') | * - symbol (string): symbol to insert (default: 'R_BAD_CHARSET') | ||||
* - threshold (double): value that would be used as threshold in expression characters_changed / total_characters | * - threshold (double): value that would be used as threshold in expression characters_changed / total_characters | ||||
* (e.g. if threshold is 0.1 than charset change should occure more often than in 10 symbols), default: 0.1 | |||||
* (e.g. if threshold is 0.1 than charset change should occur more often than in 10 symbols), default: 0.1 | |||||
*/ | */ | ||||
struct rspamd_custom_controller_command *cmd; | struct rspamd_custom_controller_command *cmd; | ||||
cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd)); | cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd)); | ||||
cmd->privilleged = TRUE; | |||||
cmd->privileged = TRUE; | |||||
cmd->require_message = TRUE; | cmd->require_message = TRUE; | ||||
cmd->handler = fuzzy_add_handler; | cmd->handler = fuzzy_add_handler; | ||||
cmd->ctx = ctx; | cmd->ctx = ctx; | ||||
g_hash_table_insert (commands, "/fuzzyadd", cmd); | g_hash_table_insert (commands, "/fuzzyadd", cmd); | ||||
cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd)); | cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd)); | ||||
cmd->privilleged = TRUE; | |||||
cmd->privileged = TRUE; | |||||
cmd->require_message = TRUE; | cmd->require_message = TRUE; | ||||
cmd->handler = fuzzy_delete_handler; | cmd->handler = fuzzy_delete_handler; | ||||
cmd->ctx = ctx; | cmd->ctx = ctx; | ||||
g_hash_table_insert (commands, "/fuzzydel", cmd); | g_hash_table_insert (commands, "/fuzzydel", cmd); | ||||
cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd)); | cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd)); | ||||
cmd->privilleged = TRUE; | |||||
cmd->privileged = TRUE; | |||||
cmd->require_message = FALSE; | cmd->require_message = FALSE; | ||||
cmd->handler = fuzzy_deletehash_handler; | cmd->handler = fuzzy_deletehash_handler; | ||||
cmd->ctx = ctx; | cmd->ctx = ctx; |
local tokens = {} | local tokens = {} | ||||
-- Tokens occurrences distribution counters | -- Tokens occurrences distribution counters | ||||
local occurr = { | |||||
local occur = { | |||||
ham = {}, | ham = {}, | ||||
spam = {}, | spam = {}, | ||||
total = {} | total = {} | ||||
for k,v in pairs({['ham']=ham, ['spam']=spam, ['total']=total}) do | for k,v in pairs({['ham']=ham, ['spam']=spam, ['total']=total}) do | ||||
if tonumber(v) > 19 then v = 20 end | if tonumber(v) > 19 then v = 20 end | ||||
occurr[k][v] = occurr[k][v] and occurr[k][v] + 1 or 1 | |||||
occur[k][v] = occur[k][v] and occur[k][v] + 1 or 1 | |||||
end | end | ||||
end | end | ||||
end | end | ||||
local occ_distr = {} | local occ_distr = {} | ||||
for _,cl in pairs({'ham', 'spam', 'total'}) do | for _,cl in pairs({'ham', 'spam', 'total'}) do | ||||
local occurr_key = pattern_sha1 .. '_occurrence_' .. cl | |||||
local occur_key = pattern_sha1 .. '_occurrence_' .. cl | |||||
if cursor ~= 0 then | if cursor ~= 0 then | ||||
local n | local n | ||||
for i,v in ipairs(redis.call('HGETALL', occurr_key)) do | |||||
for i,v in ipairs(redis.call('HGETALL', occur_key)) do | |||||
if i % 2 == 1 then | if i % 2 == 1 then | ||||
n = tonumber(v) | n = tonumber(v) | ||||
else | else | ||||
occurr[cl][n] = occurr[cl][n] and occurr[cl][n] + v or v | |||||
occur[cl][n] = occur[cl][n] and occur[cl][n] + v or v | |||||
end | end | ||||
end | end | ||||
local str = '' | local str = '' | ||||
if occurr[cl][0] ~= nil then | |||||
str = '0:' .. occurr[cl][0] .. ',' | |||||
if occur[cl][0] ~= nil then | |||||
str = '0:' .. occur[cl][0] .. ',' | |||||
end | end | ||||
for k,v in ipairs(occurr[cl]) do | |||||
for k,v in ipairs(occur[cl]) do | |||||
if k == 20 then k = '>19' end | if k == 20 then k = '>19' end | ||||
str = str .. k .. ':' .. v .. ',' | str = str .. k .. ':' .. v .. ',' | ||||
end | end | ||||
table.insert(occ_distr, str) | table.insert(occ_distr, str) | ||||
else | else | ||||
redis.call('DEL', occurr_key) | |||||
redis.call('DEL', occur_key) | |||||
end | end | ||||
if next(occurr[cl]) ~= nil then | |||||
redis.call('HMSET', occurr_key, unpack_function(hash2list(occurr[cl]))) | |||||
if next(occur[cl]) ~= nil then | |||||
redis.call('HMSET', occur_key, unpack_function(hash2list(occur[cl]))) | |||||
end | end | ||||
end | end | ||||
-- We can either check BIMI via DNS or check Redis cache | -- We can either check BIMI via DNS or check Redis cache | ||||
-- BIMI check is an external check, so we might prefer Redis to be checked | -- BIMI check is an external check, so we might prefer Redis to be checked | ||||
-- first. On the other hand, DNS request is cheaper and counting low BIMI | -- first. On the other hand, DNS request is cheaper and counting low BIMI | ||||
-- adoptation we would need to have both Redis and DNS request to hit no | |||||
-- adaptation we would need to have both Redis and DNS request to hit no | |||||
-- result. So, it might be better to check DNS first at this stage... | -- result. So, it might be better to check DNS first at this stage... | ||||
check_bimi_dns(task, dmarc_domain_maybe) | check_bimi_dns(task, dmarc_domain_maybe) | ||||
end | end |
max_memory = 50 * 1024 * 1024, -- How many memory should be occupied before sending collection | max_memory = 50 * 1024 * 1024, -- How many memory should be occupied before sending collection | ||||
max_interval = 60, -- Maximum collection interval | max_interval = 60, -- Maximum collection interval | ||||
}, | }, | ||||
collect_garbage = false, -- Peform GC collection after sending the data | |||||
collect_garbage = false, -- Perform GC collection after sending the data | |||||
check_timeout = 10.0, -- Periodic timeout | check_timeout = 10.0, -- Periodic timeout | ||||
timeout = 5.0, | timeout = 5.0, | ||||
bayes_spam_symbols = {'BAYES_SPAM'}, | bayes_spam_symbols = {'BAYES_SPAM'}, |
return false | return false | ||||
end | end | ||||
-- Mailmain 3 allows to disable all List-* headers in settings, but by default it adds them. | |||||
-- Mailman 3 allows to disable all List-* headers in settings, but by default it adds them. | |||||
-- In all other cases all Mailman message should have List-Id header | -- In all other cases all Mailman message should have List-Id header | ||||
if not task:has_header('List-Id') then | if not task:has_header('List-Id') then | ||||
return false | return false |
end | end | ||||
end, fun.filter(function(r) return not r['prefilter'] end, rules)) | end, fun.filter(function(r) return not r['prefilter'] end, rules)) | ||||
-- prefilter symbils | |||||
-- prefilter symbols | |||||
fun.each(function(rule) | fun.each(function(rule) | ||||
rspamd_config:register_symbol({ | rspamd_config:register_symbol({ | ||||
type = 'prefilter', | type = 'prefilter', |
end | end | ||||
else | else | ||||
local to_resolve | local to_resolve | ||||
local orign = req | |||||
local origin = req | |||||
if not resolve_ip then | if not resolve_ip then | ||||
orign = maybe_make_hash(req, rule) | |||||
origin = maybe_make_hash(req, rule) | |||||
to_resolve = string.format('%s.%s', | to_resolve = string.format('%s.%s', | ||||
orign, | |||||
origin, | |||||
rule.rbl) | rule.rbl) | ||||
else | else | ||||
-- First, resolve origin stuff without hashing or anything | -- First, resolve origin stuff without hashing or anything | ||||
to_resolve = orign | |||||
to_resolve = origin | |||||
end | end | ||||
nreq = { | nreq = { |
local mult | local mult | ||||
local how = 'wl' | local how = 'wl' | ||||
-- Can be overriden | |||||
-- Can be overridden | |||||
if rule.blacklist then how = 'bl' end | if rule.blacklist then how = 'bl' end | ||||
local function parse_val(val) | local function parse_val(val) |
return -1; | return -1; | ||||
} | } | ||||
if (main->is_privilleged) { | |||||
if (main->is_privileged) { | |||||
/* Force root user as owner of pid file */ | /* Force root user as owner of pid file */ | ||||
#ifdef HAVE_PIDFILE_FILENO | #ifdef HAVE_PIDFILE_FILENO | ||||
if (fchown (pidfile_fileno (main->pfh), 0, 0) == -1) { | if (fchown (pidfile_fileno (main->pfh), 0, 0) == -1) { | ||||
return 0; | return 0; | ||||
} | } | ||||
/* Detect privilleged mode */ | |||||
/* Detect privileged mode */ | |||||
static void | static void | ||||
detect_priv (struct rspamd_main *rspamd_main) | detect_priv (struct rspamd_main *rspamd_main) | ||||
{ | { | ||||
if (euid == 0) { | if (euid == 0) { | ||||
if (!rspamd_main->cfg->rspamd_user && !is_insecure) { | if (!rspamd_main->cfg->rspamd_user && !is_insecure) { | ||||
msg_err_main ( | msg_err_main ( | ||||
"cannot run rspamd workers as root user, please add -u and -g options to select a proper unprivilleged user or specify --insecure flag"); | |||||
"cannot run rspamd workers as root user, please add -u and -g options to select a proper unprivileged user or specify --insecure flag"); | |||||
exit (EXIT_FAILURE); | exit (EXIT_FAILURE); | ||||
} | } | ||||
else if (is_insecure) { | else if (is_insecure) { | ||||
rspamd_main->is_privilleged = TRUE; | |||||
rspamd_main->is_privileged = TRUE; | |||||
rspamd_main->workers_uid = 0; | rspamd_main->workers_uid = 0; | ||||
rspamd_main->workers_gid = 0; | rspamd_main->workers_gid = 0; | ||||
} | } | ||||
else { | else { | ||||
rspamd_main->is_privilleged = TRUE; | |||||
rspamd_main->is_privileged = TRUE; | |||||
pwd = getpwnam (rspamd_main->cfg->rspamd_user); | pwd = getpwnam (rspamd_main->cfg->rspamd_user); | ||||
if (pwd == NULL) { | if (pwd == NULL) { | ||||
msg_err_main ("user specified does not exists (%s), aborting", | msg_err_main ("user specified does not exists (%s), aborting", | ||||
} | } | ||||
} | } | ||||
else { | else { | ||||
rspamd_main->is_privilleged = FALSE; | |||||
rspamd_main->is_privileged = FALSE; | |||||
rspamd_main->workers_uid = (uid_t)-1; | rspamd_main->workers_uid = (uid_t)-1; | ||||
rspamd_main->workers_gid = (gid_t)-1; | rspamd_main->workers_gid = (gid_t)-1; | ||||
} | } |
rspamd_logger_t *logger; | rspamd_logger_t *logger; | ||||
uid_t workers_uid; /**< worker's uid running to */ | uid_t workers_uid; /**< worker's uid running to */ | ||||
gid_t workers_gid; /**< worker's gid running to */ | gid_t workers_gid; /**< worker's gid running to */ | ||||
gboolean is_privilleged; /**< true if run in privilleged mode */ | |||||
gboolean is_privileged; /**< true if run in privileged mode */ | |||||
gboolean wanna_die; /**< no respawn of processes */ | gboolean wanna_die; /**< no respawn of processes */ | ||||
gboolean cores_throttling; /**< turn off cores when limits are exceeded */ | gboolean cores_throttling; /**< turn off cores when limits are exceeded */ | ||||
struct roll_history *history; /**< rolling history */ | struct roll_history *history; /**< rolling history */ | ||||
*/ | */ | ||||
void register_custom_controller_command (const gchar *name, | void register_custom_controller_command (const gchar *name, | ||||
controller_func_t handler, | controller_func_t handler, | ||||
gboolean privilleged, | |||||
gboolean privileged, | |||||
gboolean require_message); | gboolean require_message); | ||||
#ifdef __cplusplus | #ifdef __cplusplus |
- To make it possible, we explicitly run `umask 0000` in "build" and "functional" stages in .circleci/config.yml | - To make it possible, we explicitly run `umask 0000` in "build" and "functional" stages in .circleci/config.yml | ||||
- After run, we persist coverage data in "coverage.${CIRCLE\_JOB}.dump" during this build flow, see `capture_coverage_data`, | - After run, we persist coverage data in "coverage.${CIRCLE\_JOB}.dump" during this build flow, see `capture_coverage_data`, | ||||
to use it on the final stage. | to use it on the final stage. | ||||
- we user `cpp-coverals` because it is able to save data for coveralls without actually sending it. We send on our own | |||||
- we use `cpp-coveralls` because it is able to save data for coveralls without actually sending it. We send on our own | |||||
along with Lua-coverage. | along with Lua-coverage. | ||||
Lua coverage | Lua coverage | ||||
1. Coverage collecting is initiated and dumped in `test/functional/lua/test_coverage.lua` (there are a lot of comments inside). | 1. Coverage collecting is initiated and dumped in `test/functional/lua/test_coverage.lua` (there are a lot of comments inside). | ||||
This file should be included on the very early stage of test run. Usually it's included via config. | This file should be included on the very early stage of test run. Usually it's included via config. | ||||
2. Coverage is dumped into ${TMPDIR}/%{woker_name}.luacov.stats.out | |||||
2. Coverage is dumped into ${TMPDIR}/%{worker_name}.luacov.stats.out | |||||
3. All worker coverage reports are merged into `lua_coverage_report.json` (see `collect_lua_coverage()`) | 3. All worker coverage reports are merged into `lua_coverage_report.json` (see `collect_lua_coverage()`) | ||||
4. finally, `lua_coverage_report.json` is persisted in build flow (see `functional` stage) | 4. finally, `lua_coverage_report.json` is persisted in build flow (see `functional` stage) | ||||
} | } | ||||
static int | static int | ||||
create_constrainted_split (struct rspamd_cryptobox_segment *seg, int mseg, | |||||
create_constrained_split (struct rspamd_cryptobox_segment *seg, int mseg, | |||||
int constraint, | int constraint, | ||||
guchar *begin, guchar *end) | guchar *begin, guchar *end) | ||||
{ | { | ||||
msg_info ("realistic split of %d chunks encryption: %.0f", cnt, t2 - t1); | msg_info ("realistic split of %d chunks encryption: %.0f", cnt, t2 - t1); | ||||
cnt = create_constrainted_split (seg, max_seg + 1, 32, begin, end); | |||||
cnt = create_constrained_split (seg, max_seg + 1, 32, begin, end); | |||||
t1 = rspamd_get_ticks (TRUE); | t1 = rspamd_get_ticks (TRUE); | ||||
rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac, mode); | rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac, mode); | ||||
t2 = rspamd_get_ticks (TRUE); | t2 = rspamd_get_ticks (TRUE); | ||||
check_result (key, nonce, mac, begin, end); | check_result (key, nonce, mac, begin, end); | ||||
msg_info ("constrainted split of %d chunks encryption: %.0f", cnt, t2 - t1); | |||||
msg_info ("constrained split of %d chunks encryption: %.0f", cnt, t2 - t1); | |||||
for (i = 0; i < random_fuzz_cnt; i ++) { | for (i = 0; i < random_fuzz_cnt; i ++) { | ||||
ms = ottery_rand_range (i % max_seg * 2) + 1; | ms = ottery_rand_range (i % max_seg * 2) + 1; | ||||
} | } | ||||
for (i = 0; i < random_fuzz_cnt; i ++) { | for (i = 0; i < random_fuzz_cnt; i ++) { | ||||
ms = ottery_rand_range (i % max_seg * 10) + 1; | ms = ottery_rand_range (i % max_seg * 10) + 1; | ||||
cnt = create_constrainted_split (seg, ms, i, begin, end); | |||||
cnt = create_constrained_split (seg, ms, i, begin, end); | |||||
t1 = rspamd_get_ticks (TRUE); | t1 = rspamd_get_ticks (TRUE); | ||||
rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac, mode); | rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac, mode); | ||||
t2 = rspamd_get_ticks (TRUE); | t2 = rspamd_get_ticks (TRUE); | ||||
check_result (key, nonce, mac, begin, end); | check_result (key, nonce, mac, begin, end); | ||||
if (i % 1000 == 0) { | if (i % 1000 == 0) { | ||||
msg_info ("constrainted fuzz iterations: %d", i); | |||||
msg_info ("constrained fuzz iterations: %d", i); | |||||
} | } | ||||
} | } | ||||
#include <sys/wait.h> | #include <sys/wait.h> | ||||
#endif | #endif | ||||
#define TEST_BUF "test bufffer" | |||||
#define TEST2_BUF "test bufffertest bufffer" | |||||
#define TEST_BUF "test buffer" | |||||
#define TEST2_BUF "test buffertest buffer" | |||||
void | void | ||||
rspamd_mem_pool_test_func () | rspamd_mem_pool_test_func () |
rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); | rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); | ||||
cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); | cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); | ||||
cfg->libs_ctx = rspamd_init_libs (); | cfg->libs_ctx = rspamd_init_libs (); | ||||
/* More agressive GC, workaround for 'not enough memory' test failures */ | |||||
/* More aggressive GC, workaround for 'not enough memory' test failures */ | |||||
cfg->lua_gc_step *= 2; | cfg->lua_gc_step *= 2; | ||||
rspamd_main->cfg = cfg; | rspamd_main->cfg = cfg; | ||||
cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); | cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); |
warn("{}: no source_files, not a coveralls.io payload?".format(json_file)) | warn("{}: no source_files, not a coveralls.io payload?".format(json_file)) | ||||
return 1 | return 1 | ||||
print("{} ({} soource files)".format(json_file, len(data['source_files']))) | |||||
print("{} ({} source files)".format(json_file, len(data['source_files']))) | |||||
for src_file in sorted(data['source_files'], key=itemgetter('name')): | for src_file in sorted(data['source_files'], key=itemgetter('name')): | ||||
covered_lines = not_skipped_lines = 0 | covered_lines = not_skipped_lines = 0 |