Browse Source

Spelling (#4086)

[Rework] Massive spelling fix from @jsoref
tags/3.2
Josh Soref 2 years ago
parent
commit
2b8e6958f4
No account linked to committer's email address
100 changed files with 263 additions and 255 deletions
  1. 1
    1
      CMakeLists.txt
  2. 1
    1
      conf/groups.conf
  3. 5
    5
      doc/doxydown/doxydown.pl
  4. 1
    1
      interface/README.md
  5. 1
    1
      interface/index.html
  6. 1
    1
      lua_style.md
  7. 1
    1
      lualib/lua_cfg_transform.lua
  8. 8
    8
      lualib/lua_content/pdf.lua
  9. 1
    1
      lualib/lua_fuzzy.lua
  10. 2
    2
      lualib/lua_meta.lua
  11. 2
    2
      lualib/lua_redis.lua
  12. 1
    1
      lualib/lua_scanners/common.lua
  13. 1
    1
      lualib/lua_scanners/fprot.lua
  14. 1
    1
      lualib/lua_scanners/sophos.lua
  15. 1
    0
      lualib/lua_scanners/virustotal.lua
  16. 1
    1
      lualib/lua_selectors/init.lua
  17. 1
    0
      lualib/lua_stat.lua
  18. 4
    0
      lualib/lua_util.lua
  19. 1
    1
      lualib/lua_verdict.lua
  20. 3
    2
      lualib/plugins/dmarc.lua
  21. 3
    3
      lualib/plugins/neural.lua
  22. 1
    1
      lualib/rspamadm/clickhouse.lua
  23. 5
    5
      lualib/rspamadm/configgraph.lua
  24. 1
    1
      lualib/rspamadm/configwizard.lua
  25. 8
    8
      lualib/rspamadm/mime.lua
  26. 2
    2
      rules/bounce.lua
  27. 1
    1
      rules/headers_checks.lua
  28. 4
    3
      rules/regexp/headers.lua
  29. 3
    3
      src/controller.c
  30. 1
    1
      src/fuzzy_storage.c
  31. 1
    1
      src/libcryptobox/catena/catena.c
  32. 32
    32
      src/libmime/lang_detection.c
  33. 2
    2
      src/libmime/mime_parser.c
  34. 1
    1
      src/libmime/received.cxx
  35. 1
    1
      src/libmime/scan_result.c
  36. 1
    1
      src/libserver/cfg_file.h
  37. 11
    11
      src/libserver/cfg_utils.c
  38. 1
    1
      src/libserver/css/css_rule.hxx
  39. 1
    1
      src/libserver/dkim.h
  40. 1
    1
      src/libserver/fuzzy_backend/fuzzy_backend.c
  41. 5
    5
      src/libserver/html/html.cxx
  42. 1
    1
      src/libserver/html/html_url.cxx
  43. 1
    1
      src/libserver/http/http_connection.c
  44. 1
    1
      src/libserver/http/http_context.c
  45. 4
    4
      src/libserver/logger/logger.c
  46. 1
    1
      src/libserver/maps/map.c
  47. 1
    1
      src/libserver/maps/map_helpers.c
  48. 2
    2
      src/libserver/re_cache.c
  49. 2
    2
      src/libserver/rspamd_symcache.c
  50. 2
    2
      src/libserver/rspamd_symcache.h
  51. 5
    5
      src/libserver/spf.c
  52. 1
    1
      src/libserver/task.h
  53. 1
    1
      src/libserver/url.c
  54. 1
    1
      src/libserver/url.h
  55. 1
    1
      src/libserver/worker_util.c
  56. 1
    1
      src/libserver/worker_util.h
  57. 3
    3
      src/libstat/learn_cache/redis_cache.c
  58. 3
    3
      src/libstat/learn_cache/sqlite3_cache.c
  59. 1
    1
      src/libstat/stat_internal.h
  60. 4
    4
      src/libstat/stat_process.c
  61. 1
    1
      src/libutil/addr.h
  62. 2
    2
      src/libutil/hash.h
  63. 1
    1
      src/libutil/mem_pool.c
  64. 1
    1
      src/libutil/mem_pool.h
  65. 3
    3
      src/libutil/rrd.h
  66. 2
    2
      src/libutil/shingles.h
  67. 1
    1
      src/libutil/str_util.c
  68. 1
    1
      src/libutil/upstream.c
  69. 1
    1
      src/libutil/upstream.h
  70. 2
    2
      src/libutil/util.c
  71. 1
    1
      src/libutil/util.h
  72. 9
    9
      src/lua/lua_config.c
  73. 5
    5
      src/lua/lua_dns_resolver.c
  74. 3
    3
      src/lua/lua_mimepart.c
  75. 1
    1
      src/lua/lua_redis.c
  76. 1
    1
      src/lua/lua_regexp.c
  77. 9
    9
      src/lua/lua_task.c
  78. 1
    1
      src/lua/lua_tcp.c
  79. 2
    2
      src/lua/lua_text.c
  80. 1
    1
      src/lua/lua_thread_pool.h
  81. 3
    3
      src/lua/lua_url.c
  82. 5
    5
      src/lua/lua_util.c
  83. 4
    4
      src/lua/lua_xmlrpc.c
  84. 1
    1
      src/plugins/chartable.c
  85. 3
    3
      src/plugins/fuzzy_check.c
  86. 11
    11
      src/plugins/lua/bayes_expiry.lua
  87. 1
    1
      src/plugins/lua/bimi.lua
  88. 1
    1
      src/plugins/lua/clickhouse.lua
  89. 1
    1
      src/plugins/lua/maillist.lua
  90. 1
    1
      src/plugins/lua/multimap.lua
  91. 4
    4
      src/plugins/lua/rbl.lua
  92. 1
    1
      src/plugins/lua/whitelist.lua
  93. 6
    6
      src/rspamd.c
  94. 2
    2
      src/rspamd.h
  95. 2
    2
      test/coverage.md
  96. 5
    5
      test/rspamd_cryptobox_test.c
  97. 2
    2
      test/rspamd_mem_pool_test.c
  98. 1
    1
      test/rspamd_test_suite.c
  99. 1
    1
      test/tools/dump_coveralls.py
  100. 0
    0
      test/tools/gcov_coveralls.py

+ 1
- 1
CMakeLists.txt View File

OPTION(ENABLE_LUA_TRACE "Trace all Lua C API invocations [default: OFF]" OFF) OPTION(ENABLE_LUA_TRACE "Trace all Lua C API invocations [default: OFF]" OFF)
OPTION(ENABLE_LUA_REPL "Enables Lua repl (requires C++11 compiler) [default: ON]" ON) OPTION(ENABLE_LUA_REPL "Enables Lua repl (requires C++11 compiler) [default: ON]" ON)
OPTION(SYSTEM_ZSTD "Use system zstd instead of bundled one [default: OFF]" OFF) OPTION(SYSTEM_ZSTD "Use system zstd instead of bundled one [default: OFF]" OFF)
OPTION(SYSTEM_FMT "Use system fmt instead of bundled one [defalut: OFF]" OFF)
OPTION(SYSTEM_FMT "Use system fmt instead of bundled one [default: OFF]" OFF)
OPTION(SYSTEM_DOCTEST "Use system doctest instead of bundled one [default: OFF]" OFF) OPTION(SYSTEM_DOCTEST "Use system doctest instead of bundled one [default: OFF]" OFF)


############################# INCLUDE SECTION ############################################# ############################# INCLUDE SECTION #############################################

+ 1
- 1
conf/groups.conf View File

.include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/mime_types_group.conf" .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/mime_types_group.conf"
} }


# Used to limit maximium score
# Used to limit maximum score
group "excessqp" { group "excessqp" {
max_score = 2.4; max_score = 2.4;
.include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/excessqp_group.conf" .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/excessqp_group.conf"

+ 5
- 5
doc/doxydown/doxydown.pl View File

} }
} }


if ($m->{'stucts'}) {
if ( scalar(@{ $m->{'stucts'} }) > 0 ) {
print "\n## Stucts\n\nThe module `$mname` defines the following stucts.\n\n";
if ($m->{'structs'}) {
if ( scalar(@{ $m->{'structs'} }) > 0 ) {
print "\n## Structs\n\nThe module `$mname` defines the following structs.\n\n";


foreach ( @{ $m->{'stucts'} } ) {
print_stuct_markdown( "Stuct", $_->{'name'}, $_ );
foreach ( @{ $m->{'structs'} } ) {
print_struct_markdown( "Struct", $_->{'name'}, $_ );


print "\nBack to [module description](#$m->{'id'}).\n\n"; print "\nBack to [module description](#$m->{'id'}).\n\n";



+ 1
- 1
interface/README.md View File

count = 1; count = 1;
# Password for normal commands (use rspamadm pw) # Password for normal commands (use rspamadm pw)
password = "$2$anydoddx67ggcs74owybhcwqsq3z67q4$udympbo8pfcfqkeiiuj7gegabk5jpt8edmhseujhar9ooyuzig5b"; password = "$2$anydoddx67ggcs74owybhcwqsq3z67q4$udympbo8pfcfqkeiiuj7gegabk5jpt8edmhseujhar9ooyuzig5b";
# Password for privilleged commands (use rspamadm pw)
# Password for privileged commands (use rspamadm pw)
enable_password = "$2$nx6sqkxtewx9c5s3hxjmabaxdcr46pk9$45qajkbyqx77abapiqugpjpsojj38zcqn7xnp3ekqyu674koux4b"; enable_password = "$2$nx6sqkxtewx9c5s3hxjmabaxdcr46pk9$45qajkbyqx77abapiqugpjpsojj38zcqn7xnp3ekqyu674koux4b";
# Path to webiu static files # Path to webiu static files
static_dir = "${WWWDIR}"; static_dir = "${WWWDIR}";

+ 1
- 1
interface/index.html View File

<div class="row"> <div class="row">
<div class="col-lg-6"> <div class="col-lg-6">
<div class="card bg-light shadow card-body card p-2"> <div class="card bg-light shadow card-body card p-2">
<p>Learn Bayessian classifier:</p>
<p>Learn Bayesian classifier:</p>
<form class="form-inline"> <form class="form-inline">
<div class="form-group"> <div class="form-group">
<div class="btn-group"> <div class="btn-group">

+ 1
- 1
lua_style.md View File

end end
``` ```


- Never name a parameter `arg`, this will take precendence over the `arg` object that is given to every function scope in older versions of Lua.
- Never name a parameter `arg`, this will take precedence over the `arg` object that is given to every function scope in older versions of Lua.


```lua ```lua
-- bad -- bad

+ 1
- 1
lualib/lua_cfg_transform.lua View File



local actions_set = lua_util.list_to_hash(actions_defs) local actions_set = lua_util.list_to_hash(actions_defs)


-- Now check actions section for garbadge
-- Now check actions section for garbage
actions_set['unknown_weight'] = true actions_set['unknown_weight'] = true
actions_set['grow_factor'] = true actions_set['grow_factor'] = true
actions_set['subject'] = true actions_set['subject'] = true

+ 8
- 8
lualib/lua_content/pdf.lua View File

local extracted_js = maybe_extract_object_stream(js, pdf, task) local extracted_js = maybe_extract_object_stream(js, pdf, task)


if not extracted_js then if not extracted_js then
lua_util.debugm(N, task, 'invalid type for javascript from %s:%s: %s',
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
obj.major, obj.minor, js) obj.major, obj.minor, js)
else else
js = extracted_js js = extracted_js
lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s', lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s',
obj.major, obj.minor, obj.js.data) obj.major, obj.minor, obj.js.data)
else else
lua_util.debugm(N, task, 'invalid type for javascript from %s:%s: %s',
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
obj.major, obj.minor, js) obj.major, obj.minor, js)
end end
elseif obj.dict.F then elseif obj.dict.F then


if obj.dict.S and obj.dict.JS then if obj.dict.S and obj.dict.JS then
obj.type = 'Javascript' obj.type = 'Javascript'
lua_util.debugm(N, task, 'implicit type for Javascript object %s:%s',
lua_util.debugm(N, task, 'implicit type for JavaScript object %s:%s',
obj.major, obj.minor) obj.major, obj.minor)
else else
lua_util.debugm(N, task, 'no type for %s:%s', lua_util.debugm(N, task, 'no type for %s:%s',
local extracted_js = maybe_extract_object_stream(js, pdf, task) local extracted_js = maybe_extract_object_stream(js, pdf, task)


if not extracted_js then if not extracted_js then
lua_util.debugm(N, task, 'invalid type for javascript from %s:%s: %s',
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
obj.major, obj.minor, js) obj.major, obj.minor, js)
else else
js = extracted_js js = extracted_js
lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s', lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s',
obj.major, obj.minor, obj.js.data) obj.major, obj.minor, obj.js.data)
else else
lua_util.debugm(N, task, 'invalid type for javascript from %s:%s: %s',
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
obj.major, obj.minor, js) obj.major, obj.minor, js)
end end
end end
pdf_object.openaction.object.major, pdf_object.openaction.object.minor) pdf_object.openaction.object.major, pdf_object.openaction.object.minor)
table.insert(pdf_output.fuzzy_hashes, pdf_object.openaction.bin_hash) table.insert(pdf_output.fuzzy_hashes, pdf_object.openaction.bin_hash)
else else
lua_util.debugm(N, task, "pdf: skip fuzzy hash from Javascript: %s, too short: %s",
lua_util.debugm(N, task, "pdf: skip fuzzy hash from JavaScript: %s, too short: %s",
pdf_object.openaction.hash, #pdf_object.openaction.data) pdf_object.openaction.hash, #pdf_object.openaction.data)
end end
end end
-- All hashes -- All hashes
for h,sc in pairs(pdf_object.scripts) do for h,sc in pairs(pdf_object.scripts) do
if config.min_js_fuzzy and #sc.data >= config.min_js_fuzzy then if config.min_js_fuzzy and #sc.data >= config.min_js_fuzzy then
lua_util.debugm(N, task, "pdf: add fuzzy hash from Javascript: %s; size = %s; object: %s:%s",
lua_util.debugm(N, task, "pdf: add fuzzy hash from JavaScript: %s; size = %s; object: %s:%s",
sc.hash, sc.hash,
#sc.data, #sc.data,
sc.object.major, sc.object.minor) sc.object.major, sc.object.minor)
table.insert(pdf_output.fuzzy_hashes, h) table.insert(pdf_output.fuzzy_hashes, h)
else else
lua_util.debugm(N, task, "pdf: skip fuzzy hash from Javascript: %s, too short: %s",
lua_util.debugm(N, task, "pdf: skip fuzzy hash from JavaScript: %s, too short: %s",
sc.hash, #sc.data) sc.hash, #sc.data)
end end
end end

+ 1
- 1
lualib/lua_fuzzy.lua View File

local id = part:get_id() local id = part:get_id()
lua_util.debugm(N, task, 'check binary part %s: %s', id, ct) lua_util.debugm(N, task, 'check binary part %s: %s', id, ct)


-- For bad mime mime parts we implicitly enable fuzzy check
-- For bad mime parts we implicitly enable fuzzy check
local mime_trace = (task:get_symbol('MIME_TRACE') or {})[1] local mime_trace = (task:get_symbol('MIME_TRACE') or {})[1]
local opts = {} local opts = {}



+ 2
- 2
lualib/lua_meta.lua View File

0, -- ascii characters rate 0, -- ascii characters rate
0, -- non-ascii characters rate 0, -- non-ascii characters rate
0, -- capital characters rate 0, -- capital characters rate
0, -- numeric cahracters
0, -- numeric characters
} }
for _,p in ipairs(tp) do for _,p in ipairs(tp) do
local stats = p:get_stats() local stats = p:get_stats()
'ascii_characters_rate', 'ascii_characters_rate',
'non_ascii_characters_rate', 'non_ascii_characters_rate',
'capital_characters_rate', 'capital_characters_rate',
'numeric_cahracters'
'numeric_characters'
}, },
description = [[Functions for words data matching: description = [[Functions for words data matching:
- average length of the words - average length of the words

+ 2
- 2
lualib/lua_redis.lua View File

for _,m in ipairs(result) do for _,m in ipairs(result) do
local master = flatten_redis_table(m) local master = flatten_redis_table(m)


-- Wrap IPv6-adresses in brackets
-- Wrap IPv6-addresses in brackets
if (master.ip:match(":")) then if (master.ip:match(":")) then
master.ip = "["..master.ip.."]" master.ip = "["..master.ip.."]"
end end
lutil.debugm(N, rspamd_config, lutil.debugm(N, rspamd_config,
'found slave for master %s with ip %s and port %s', 'found slave for master %s with ip %s and port %s',
v.name, slave.ip, slave.port) v.name, slave.ip, slave.port)
-- Wrap IPv6-adresses in brackets
-- Wrap IPv6-addresses in brackets
if (slave.ip:match(":")) then if (slave.ip:match(":")) then
slave.ip = "["..slave.ip.."]" slave.ip = "["..slave.ip.."]"
end end

+ 1
- 1
lualib/lua_scanners/common.lua View File

lua_util.debugm(rule.name, task, '%s: extension matched: |%s|%s|', rule.log_prefix, ext, ext2) lua_util.debugm(rule.name, task, '%s: extension matched: |%s|%s|', rule.log_prefix, ext, ext2)
return true return true
elseif match_filter(task, rule, fname, rule.mime_parts_filter_regex, 'regex') then elseif match_filter(task, rule, fname, rule.mime_parts_filter_regex, 'regex') then
lua_util.debugm(rule.name, task, '%s: filname regex matched', rule.log_prefix)
lua_util.debugm(rule.name, task, '%s: filename regex matched', rule.log_prefix)
return true return true
end end
end end

+ 1
- 1
lualib/lua_scanners/fprot.lua View File

rule['symbol'], rule['type']) rule['symbol'], rule['type'])
end end
else else
-- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occured
-- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occurred
-- see http://www.f-prot.com/support/helpfiles/unix/appendix_c.html for more detail -- see http://www.f-prot.com/support/helpfiles/unix/appendix_c.html for more detail
local vname = string.match(data, '^[1-3] <[%w%s]-: (.-)>') local vname = string.match(data, '^[1-3] <[%w%s]-: (.-)>')
if not vname then if not vname then

+ 1
- 1
lualib/lua_scanners/sophos.lua View File

rspamd_logger.warnx(task, 'Message is oversized (REJ 4): %s', data) rspamd_logger.warnx(task, 'Message is oversized (REJ 4): %s', data)
common.yield_result(task, rule, 'SAVDI: Message oversized (REJ 4)', common.yield_result(task, rule, 'SAVDI: Message oversized (REJ 4)',
0.0, 'fail', maybe_part) 0.0, 'fail', maybe_part)
-- excplicitly set REJ1 message when SAVDIreports a protocol error
-- explicitly set REJ1 message when SAVDIreports a protocol error
elseif string.find(data, 'REJ 1') then elseif string.find(data, 'REJ 1') then
rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data) rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data)
common.yield_result(task, rule, 'SAVDI: Protocol error (REJ 1)', common.yield_result(task, rule, 'SAVDI: Protocol error (REJ 1)',

+ 1
- 0
lualib/lua_scanners/virustotal.lua View File



--[[[ --[[[
-- @module virustotal -- @module virustotal
-- This module contains Virustotal integration support
-- https://www.virustotal.com/ -- https://www.virustotal.com/
--]] --]]



+ 1
- 1
lualib/lua_selectors/init.lua View File

return rspamd_text.fromtable(selectors, delimiter) return rspamd_text.fromtable(selectors, delimiter)
end end
else else
-- We need to do a spill on each table selector and make a cortezian product
-- We need to do a spill on each table selector and make a cortesian product
-- e.g. s:tbl:s -> s:telt1:s + s:telt2:s ... -- e.g. s:tbl:s -> s:telt1:s + s:telt2:s ...
local tbl = {} local tbl = {}
local res = {} local res = {}

+ 1
- 0
lualib/lua_stat.lua View File

-- symbol_ham = YYY -- symbol_ham = YYY
-- db_spam = XXX.sqlite -- db_spam = XXX.sqlite
-- db_ham = YYY.sqlite -- db_ham = YYY.sqlite
-- learn_cache = ZZZ.sqlite
-- per_user = true/false -- per_user = true/false
-- label = str -- label = str
-- } -- }

+ 4
- 0
lualib/lua_util.lua View File

end end


--[[[ --[[[
-- @function lua_util.str_split(text, delimiter)
-- Splits text into a numeric table by delimiter
-- @param {string} text delimited text
-- @param {string} delimiter the delimiter
-- @return {table} numeric table containing string parts -- @return {table} numeric table containing string parts
--]] --]]



+ 1
- 1
lualib/lua_verdict.lua View File

}, },
uncertain = { uncertain = {
can_learn = false, can_learn = false,
description = 'not certainity in verdict'
description = 'not certainty in verdict'
} }
} }



+ 3
- 2
lualib/plugins/dmarc.lua View File

if munging_opts.munge_map_condition then if munging_opts.munge_map_condition then
local accepted,trace = munging_opts.munge_map_condition:process(task) local accepted,trace = munging_opts.munge_map_condition:process(task)
if not accepted then if not accepted then
lua_util.debugm(task, 'skip munging, maps condition not satisified: (%s)',
lua_util.debugm(task, 'skip munging, maps condition not satisfied: (%s)',
trace) trace)
-- Excepted -- Excepted
return return
-- Used to check dmarc record, check elements and produce dmarc policy processed -- Used to check dmarc record, check elements and produce dmarc policy processed
-- result. -- result.
-- Returns: -- Returns:
-- false,false - record is garbage
-- false,error_message - record is invalid -- false,error_message - record is invalid
-- true,policy_table - record is valid and parsed -- true,policy_table - record is valid and parsed
]] ]]
end end
result.raw_elts = elts result.raw_elts = elts
else else
return false,false -- Ignore garbadge
return false,false -- Ignore garbage
end end


return true, result return true, result

+ 3
- 3
lualib/plugins/neural.lua View File

if nspam <= train_opts.max_trains then if nspam <= train_opts.max_trains then
if train_opts.spam_skip_prob then if train_opts.spam_skip_prob then
if coin <= train_opts.spam_skip_prob then if coin <= train_opts.spam_skip_prob then
rspamd_logger.infox(task, 'skip %s sample probabilisticaly; probability %s (%s skip chance)', learn_type,
rspamd_logger.infox(task, 'skip %s sample probabilistically; probability %s (%s skip chance)', learn_type,
coin, train_opts.spam_skip_prob) coin, train_opts.spam_skip_prob)
return false return false
end end
if nham <= train_opts.max_trains then if nham <= train_opts.max_trains then
if train_opts.ham_skip_prob then if train_opts.ham_skip_prob then
if coin <= train_opts.ham_skip_prob then if coin <= train_opts.ham_skip_prob then
rspamd_logger.infox(task, 'skip %s sample probabilisticaly; probability %s (%s skip chance)', learn_type,
rspamd_logger.infox(task, 'skip %s sample probabilistically; probability %s (%s skip chance)', learn_type,
coin, train_opts.ham_skip_prob) coin, train_opts.ham_skip_prob)
return false return false
end end
else else
local inputs, outputs = {}, {} local inputs, outputs = {}, {}


-- Used to show sparsed vectors in a convenient format (for debugging only)
-- Used to show parsed vectors in a convenient format (for debugging only)
local function debug_vec(t) local function debug_vec(t)
local ret = {} local ret = {}
for i,v in ipairs(t) do for i,v in ipairs(t) do

+ 1
- 1
lualib/rspamadm/clickhouse.lua View File

end end


local function get_excluded_symbols(known_symbols, correlations, seen_total) local function get_excluded_symbols(known_symbols, correlations, seen_total)
-- Walk results once to collect all symbols & count ocurrences
-- Walk results once to collect all symbols & count occurrences


local remove = {} local remove = {}
local known_symbols_list = {} local known_symbols_list = {}

+ 5
- 5
lualib/rspamadm/configgraph.lua View File

return fname return fname
end end


local function output_dot(opts, nodes, adjastency)
local function output_dot(opts, nodes, adjacency)
rspamd_logger.messagex("digraph rspamd {") rspamd_logger.messagex("digraph rspamd {")
for k,node in pairs(nodes) do for k,node in pairs(nodes) do
local attrs = {"shape=box"} local attrs = {"shape=box"}
table.concat(attrs, ',')) table.concat(attrs, ','))
end end
end end
for _,adj in ipairs(adjastency) do
for _,adj in ipairs(adjacency) do
local attrs = {} local attrs = {}
local skip = false local skip = false




local function load_config_traced(opts) local function load_config_traced(opts)
local glob_traces = {} local glob_traces = {}
local adjastency = {}
local adjacency = {}
local nodes = {} local nodes = {}


local function maybe_match_glob(file) local function maybe_match_glob(file)
end end


local function add_dep(from, node, args) local function add_dep(from, node, args)
adjastency[#adjastency + 1] = {
adjacency[#adjacency + 1] = {
from = from, from = from,
to = node, to = node,
args = args args = args
os.exit(1) os.exit(1)
end end


output_dot(opts, nodes, adjastency)
output_dot(opts, nodes, adjacency)
end end





+ 1
- 1
lualib/rspamadm/configwizard.lua View File

end end


if not symbol_spam or not symbol_ham then if not symbol_spam or not symbol_ham then
printf("Calssifier has no symbols defined")
printf("Classifier has no symbols defined")
return return
end end



+ 8
- 8
lualib/rspamadm/mime.lua View File

end end


if opts.words then if opts.words then
local howw = opts['words_format'] or 'stem'
local how_words = opts['words_format'] or 'stem'
table.insert(out_elts[fname], 'meta_words: ' .. table.insert(out_elts[fname], 'meta_words: ' ..
print_words(task:get_meta_words(howw), howw == 'full'))
print_words(task:get_meta_words(how_words), how_words == 'full'))
end end


if opts.text or opts.html then if opts.text or opts.html then
end end


if opts.words then if opts.words then
local howw = opts['words_format'] or 'stem'
table.insert(out_elts[fname], print_words(part:get_words(howw),
howw == 'full'))
local how_words = opts['words_format'] or 'stem'
table.insert(out_elts[fname], print_words(part:get_words(how_words),
how_words == 'full'))
else else
table.insert(out_elts[fname], tostring(part:get_content(how))) table.insert(out_elts[fname], tostring(part:get_content(how)))
end end
end end


if opts.words then if opts.words then
local howw = opts['words_format'] or 'stem'
table.insert(out_elts[fname], print_words(part:get_words(howw),
howw == 'full'))
local how_words = opts['words_format'] or 'stem'
table.insert(out_elts[fname], print_words(part:get_words(how_words),
how_words == 'full'))
else else
if opts.structure then if opts.structure then
local hc = part:get_html() local hc = part:get_html()

+ 2
- 2
rules/bounce.lua View File

return true, 1.0, 'DSN' return true, 1.0, 'DSN'
end end


-- Apply heuristics for non-standard bounecs
-- Apply heuristics for non-standard bounces
local bounce_sender local bounce_sender
local mime_from = task:get_from('mime') local mime_from = task:get_from('mime')
if mime_from then if mime_from then


-- Look for a message/rfc822(-headers) part inside -- Look for a message/rfc822(-headers) part inside
local rfc822_part local rfc822_part
parts[10] = nil -- limit numbe of parts to check
parts[10] = nil -- limit number of parts to check
for _, p in ipairs(parts) do for _, p in ipairs(parts) do
local mime_type, mime_subtype = p:get_type() local mime_type, mime_subtype = p:get_type()
if (mime_subtype == 'rfc822' or mime_subtype == 'rfc822-headers') and if (mime_subtype == 'rfc822' or mime_subtype == 'rfc822-headers') and

+ 1
- 1
rules/headers_checks.lua View File

missing_mime = true missing_mime = true
end end


-- Check presense of MIME specific headers
-- Check presence of MIME specific headers
local has_ct_header = task:has_header('Content-Type') local has_ct_header = task:has_header('Content-Type')
local has_cte_header = task:has_header('Content-Transfer-Encoding') local has_cte_header = task:has_header('Content-Transfer-Encoding')



+ 4
- 3
rules/regexp/headers.lua View File

group = 'headers' group = 'headers'
} }


-- Mime-OLE is needed but absent (e.g. fake Outlook or fake Exchange)
local has_msmail_pri = 'header_exists(X-MSMail-Priority)' local has_msmail_pri = 'header_exists(X-MSMail-Priority)'
local has_mimeole = 'header_exists(X-MimeOLE)' local has_mimeole = 'header_exists(X-MimeOLE)'
local has_squirrelmail_in_mailer = 'X-Mailer=/SquirrelMail\\b/H' local has_squirrelmail_in_mailer = 'X-Mailer=/SquirrelMail\\b/H'


reconf['SUBJECT_ENDS_EXCLAIM'] = { reconf['SUBJECT_ENDS_EXCLAIM'] = {
re = 'Subject=/!\\s*$/H', re = 'Subject=/!\\s*$/H',
description = 'Subject ends with an exclaimation',
description = 'Subject ends with an exclamation',
score = 0.0, score = 0.0,
group = 'headers' group = 'headers'
} }


reconf['SUBJECT_HAS_EXCLAIM'] = { reconf['SUBJECT_HAS_EXCLAIM'] = {
re = string.format('%s & !%s', 'Subject=/!/H', 'Subject=/!\\s*$/H'), re = string.format('%s & !%s', 'Subject=/!/H', 'Subject=/!\\s*$/H'),
description = 'Subject contains an exclaimation',
description = 'Subject contains an exclamation',
score = 0.0, score = 0.0,
group = 'headers' group = 'headers'
} }
[[(?:Mozilla )?Thunderbird \d]], [[(?:Mozilla )?Thunderbird \d]],
-- Was used by Yahoo Groups in 2000s, no one expected to use this in 2020s -- Was used by Yahoo Groups in 2000s, no one expected to use this in 2020s
[[eGroups Message Poster]], [[eGroups Message Poster]],
-- Regexp for genuene iOS X-Mailer is below, anything which doesn't match it,
-- Regexp for genuine iOS X-Mailer is below, anything which doesn't match it,
-- but starts with 'iPhone Mail' or 'iPad Mail' is likely fake -- but starts with 'iPhone Mail' or 'iPad Mail' is likely fake
[[i(?:Phone|Pad) Mail]], [[i(?:Phone|Pad) Mail]],
} }

+ 3
- 3
src/controller.c View File

gboolean use_ssl; gboolean use_ssl;
/* Webui password */ /* Webui password */
gchar *password; gchar *password;
/* Privilleged password */
/* Privileged password */
gchar *enable_password; gchar *enable_password;
/* Cached versions of the passwords */ /* Cached versions of the passwords */
rspamd_ftok_t cached_password; rspamd_ftok_t cached_password;
g_strstrip (syms[j]); g_strstrip (syms[j]);


if (strlen (syms[j]) == 0) { if (strlen (syms[j]) == 0) {
/* Empty garbadge */
/* Empty garbage */
continue; continue;
} }


} }


if (!rspamd_controller_check_password (conn_ent, session, msg, if (!rspamd_controller_check_password (conn_ent, session, msg,
cmd->privilleged)) {
cmd->privileged)) {
return 0; return 0;
} }
if (cmd->require_message && (rspamd_http_message_get_body (msg, NULL) == NULL)) { if (cmd->require_message && (rspamd_http_message_get_body (msg, NULL) == NULL)) {

+ 1
- 1
src/fuzzy_storage.c View File

p = buf; p = buf;
/* /*
* Memory layout: n_ext of struct rspamd_fuzzy_cmd_extension * Memory layout: n_ext of struct rspamd_fuzzy_cmd_extension
* payload for each extension in a continious data segment
* payload for each extension in a continuous data segment
*/ */
storage = g_malloc (n_ext * sizeof (struct rspamd_fuzzy_cmd_extension) + storage = g_malloc (n_ext * sizeof (struct rspamd_fuzzy_cmd_extension) +
st_len); st_len);

+ 1
- 1
src/libcryptobox/catena/catena.c View File

*/ */
/* /*
* Sigma function that defines the diagonal connections of a DBG * Sigma function that defines the diagonal connections of a DBG
* diagonal front: flip the (g-i)th bit (Inverse Buttferly Graph)
* diagonal front: flip the (g-i)th bit (Inverse Butterfly Graph)
* diagonal back: flip the i-(g-1)th bit (Regular Butterfly Graph) * diagonal back: flip the i-(g-1)th bit (Regular Butterfly Graph)
*/ */
static uint64_t static uint64_t

+ 32
- 32
src/libmime/lang_detection.c View File

const gchar *name; /* e.g. "en" or "ru" */ const gchar *name; /* e.g. "en" or "ru" */
gint flags; /* enum rspamd_language_elt_flags */ gint flags; /* enum rspamd_language_elt_flags */
enum rspamd_language_category category; enum rspamd_language_category category;
guint trigramms_words;
guint trigrams_words;
guint stop_words; guint stop_words;
gdouble mean; gdouble mean;
gdouble std; gdouble std;
guint occurencies; /* total number of parts with this language */
guint occurrences; /* total number of parts with this language */
}; };


struct rspamd_ngramm_elt { struct rspamd_ngramm_elt {


struct rspamd_lang_detector { struct rspamd_lang_detector {
GPtrArray *languages; GPtrArray *languages;
khash_t(rspamd_trigram_hash) *trigramms[RSPAMD_LANGUAGE_MAX]; /* trigramms frequencies */
khash_t(rspamd_trigram_hash) *trigrams[RSPAMD_LANGUAGE_MAX]; /* trigrams frequencies */
struct rspamd_stop_word_elt stop_words[RSPAMD_LANGUAGE_MAX]; struct rspamd_stop_word_elt stop_words[RSPAMD_LANGUAGE_MAX];
khash_t(rspamd_stopwords_hash) *stop_words_norm; khash_t(rspamd_stopwords_hash) *stop_words_norm;
UConverter *uchar_converter; UConverter *uchar_converter;
gsize short_text_limit; gsize short_text_limit;
gsize total_occurencies; /* number of all languages found */
gsize total_occurrences; /* number of all languages found */
ref_entry_t ref; ref_entry_t ref;
}; };


return; return;
} }
else { else {
nelt->trigramms_words = ucl_object_toint (ucl_array_find_index (n_words,
nelt->trigrams_words = ucl_object_toint (ucl_array_find_index (n_words,
2)); 2));
} }


} }


nelt->category = cat; nelt->category = cat;
htb = d->trigramms[cat];
htb = d->trigrams[cat];


GPtrArray *ngramms; GPtrArray *ngramms;
guint nsym; guint nsym;
if (!(nelt->flags & RS_LANGUAGE_LATIN) && if (!(nelt->flags & RS_LANGUAGE_LATIN) &&
rspamd_language_detector_ucs_is_latin (ucs_elt->s, nsym)) { rspamd_language_detector_ucs_is_latin (ucs_elt->s, nsym)) {
ucs_elt->freq = 0; ucs_elt->freq = 0;
/* Skip latin ngramm for non-latin language to avoid garbadge */
/* Skip latin ngramm for non-latin language to avoid garbage */
skipped ++; skipped ++;
continue; continue;
} }
nelt->mean = mean; nelt->mean = mean;
nelt->std = std; nelt->std = std;


msg_debug_lang_det_cfg ("loaded %s language, %d trigramms, "
msg_debug_lang_det_cfg ("loaded %s language, %d trigrams, "
"%d ngramms loaded; " "%d ngramms loaded; "
"std=%.2f, mean=%.2f, skipped=%d, loaded=%d, stop_words=%d; " "std=%.2f, mean=%.2f, skipped=%d, loaded=%d, stop_words=%d; "
"(%s)", "(%s)",
nelt->name, nelt->name,
(gint)nelt->trigramms_words,
(gint)nelt->trigrams_words,
total, total,
std, mean, std, mean,
skipped, loaded, nelt->stop_words, skipped, loaded, nelt->stop_words,
{ {
if (d) { if (d) {
for (guint i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { for (guint i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) {
kh_destroy (rspamd_trigram_hash, d->trigramms[i]);
kh_destroy (rspamd_trigram_hash, d->trigrams[i]);
rspamd_multipattern_destroy (d->stop_words[i].mp); rspamd_multipattern_destroy (d->stop_words[i].mp);
g_array_free (d->stop_words[i].ranges, TRUE); g_array_free (d->stop_words[i].ranges, TRUE);
} }


/* Map from ngramm in ucs32 to GPtrArray of rspamd_language_elt */ /* Map from ngramm in ucs32 to GPtrArray of rspamd_language_elt */
for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) {
ret->trigramms[i] = kh_init (rspamd_trigram_hash);
ret->trigrams[i] = kh_init (rspamd_trigram_hash);
#ifdef WITH_HYPERSCAN #ifdef WITH_HYPERSCAN
ret->stop_words[i].mp = rspamd_multipattern_create ( ret->stop_words[i].mp = rspamd_multipattern_create (
RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8| RSPAMD_MULTIPATTERN_ICASE|RSPAMD_MULTIPATTERN_UTF8|
for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) { for (i = 0; i < RSPAMD_LANGUAGE_MAX; i ++) {
GError *err = NULL; GError *err = NULL;


kh_foreach_value (ret->trigramms[i], schain, {
kh_foreach_value (ret->trigrams[i], schain, {
chain = &schain; chain = &schain;
rspamd_language_detector_process_chain (cfg, chain); rspamd_language_detector_process_chain (cfg, chain);
}); });
g_error_free (err); g_error_free (err);
} }


total += kh_size (ret->trigramms[i]);
total += kh_size (ret->trigrams[i]);
} }


msg_info_config ("loaded %d languages, " msg_info_config ("loaded %d languages, "
"%d trigramms",
"%d trigrams",
(gint)ret->languages->len, (gint)ret->languages->len,
(gint)total); (gint)total);


struct rspamd_lang_detector *d, struct rspamd_lang_detector *d,
UChar32 *window, UChar32 *window,
khash_t(rspamd_candidates_hash) *candidates, khash_t(rspamd_candidates_hash) *candidates,
khash_t(rspamd_trigram_hash) *trigramms)
khash_t(rspamd_trigram_hash) *trigrams)
{ {
guint i; guint i;
gint ret; gint ret;
khiter_t k; khiter_t k;
gdouble prob; gdouble prob;


k = kh_get (rspamd_trigram_hash, trigramms, window);
if (k != kh_end (trigramms)) {
chain = &kh_value (trigramms, k);
k = kh_get (rspamd_trigram_hash, trigrams, window);
if (k != kh_end (trigrams)) {
chain = &kh_value (trigrams, k);
} }


if (chain) { if (chain) {
struct rspamd_lang_detector *d, struct rspamd_lang_detector *d,
rspamd_stat_token_t *tok, rspamd_stat_token_t *tok,
khash_t(rspamd_candidates_hash) *candidates, khash_t(rspamd_candidates_hash) *candidates,
khash_t(rspamd_trigram_hash) *trigramms)
khash_t(rspamd_trigram_hash) *trigrams)
{ {
const guint wlen = 3; const guint wlen = 3;
UChar32 window[3]; UChar32 window[3];
while ((cur = rspamd_language_detector_next_ngramm (tok, window, wlen, cur)) while ((cur = rspamd_language_detector_next_ngramm (tok, window, wlen, cur))
!= -1) { != -1) {
rspamd_language_detector_process_ngramm_full (task, rspamd_language_detector_process_ngramm_full (task,
d, window, candidates, trigramms);
d, window, candidates, trigrams);
} }
} }




if (tok->unicode.len >= 3) { if (tok->unicode.len >= 3) {
rspamd_language_detector_detect_word (task, d, tok, candidates, rspamd_language_detector_detect_word (task, d, tok, candidates,
d->trigramms[cat]);
d->trigrams[cat]);
} }
} }


gdouble adj; gdouble adj;
gdouble proba_adjusted, probb_adjusted, freqa, freqb; gdouble proba_adjusted, probb_adjusted, freqa, freqb;


if (cbd->d->total_occurencies == 0) {
if (cbd->d->total_occurrences == 0) {
return 0; return 0;
} }


freqa = ((gdouble)canda->elt->occurencies) /
(gdouble)cbd->d->total_occurencies;
freqb = ((gdouble)candb->elt->occurencies) /
(gdouble)cbd->d->total_occurencies;
freqa = ((gdouble)canda->elt->occurrences) /
(gdouble)cbd->d->total_occurrences;
freqb = ((gdouble)candb->elt->occurrences) /
(gdouble)cbd->d->total_occurrences;


proba_adjusted = canda->prob; proba_adjusted = canda->prob;
probb_adjusted = candb->prob; probb_adjusted = candb->prob;
if (!ret) { if (!ret) {
if (part->utf_words->len < default_short_text_limit) { if (part->utf_words->len < default_short_text_limit) {
r = rs_detect_none; r = rs_detect_none;
msg_debug_lang_det ("text is too short for trigramms detection: "
msg_debug_lang_det ("text is too short for trigrams detection: "
"%d words; at least %d words required", "%d words; at least %d words required",
(int)part->utf_words->len, (int)part->utf_words->len,
(int)default_short_text_limit); (int)default_short_text_limit);
candidates); candidates);


if (r == rs_detect_none) { if (r == rs_detect_none) {
msg_debug_lang_det ("no trigramms found, fallback to english");
msg_debug_lang_det ("no trigrams found, fallback to english");
rspamd_language_detector_set_language (task, part, "en", NULL); rspamd_language_detector_set_language (task, part, "en", NULL);
} else if (r == rs_detect_multiple) { } else if (r == rs_detect_multiple) {
/* Check our guess */ /* Check our guess */
std = 0.0; std = 0.0;
cand_len = 0; cand_len = 0;


/* Check distirbution */
/* Check distribution */
kh_foreach_value (candidates, cand, { kh_foreach_value (candidates, cand, {
if (!isnan (cand->prob)) { if (!isnan (cand->prob)) {
mean += cand->prob; mean += cand->prob;
std /= cand_len; std /= cand_len;
} }


msg_debug_lang_det ("trigramms checked, %d candidates, %.3f mean, %.4f stddev",
msg_debug_lang_det ("trigrams checked, %d candidates, %.3f mean, %.4f stddev",
cand_len, mean, std); cand_len, mean, std);


if (cand_len > 0 && std / fabs (mean) < 0.25) { if (cand_len > 0 && std / fabs (mean) < 0.25) {


if (result->len > 0 && !frequency_heuristic_applied) { if (result->len > 0 && !frequency_heuristic_applied) {
cand = g_ptr_array_index (result, 0); cand = g_ptr_array_index (result, 0);
cand->elt->occurencies++;
d->total_occurencies++;
cand->elt->occurrences++;
d->total_occurrences++;
} }


if (part->languages != NULL) { if (part->languages != NULL) {

+ 2
- 2
src/libmime/mime_parser.c View File

cd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*cd)); cd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*cd));
cd->type = RSPAMD_CT_INLINE; cd->type = RSPAMD_CT_INLINE;


/* We can also have content dispositon definitions in Content-Type */
/* We can also have content disposition definitions in Content-Type */
if (part->ct && part->ct->attrs) { if (part->ct && part->ct->attrs) {
RSPAMD_FTOK_ASSIGN (&srch, "name"); RSPAMD_FTOK_ASSIGN (&srch, "name");
found = g_hash_table_lookup (part->ct->attrs, &srch); found = g_hash_table_lookup (part->ct->attrs, &srch);
cd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*cd)); cd = rspamd_mempool_alloc0 (task->task_pool, sizeof (*cd));
cd->type = RSPAMD_CT_INLINE; cd->type = RSPAMD_CT_INLINE;


/* We can also have content dispositon definitions in Content-Type */
/* We can also have content disposition definitions in Content-Type */
if (part->ct->attrs) { if (part->ct->attrs) {
RSPAMD_FTOK_ASSIGN (&srch, "name"); RSPAMD_FTOK_ASSIGN (&srch, "name");
found = g_hash_table_lookup (part->ct->attrs, &srch); found = g_hash_table_lookup (part->ct->attrs, &srch);

+ 1
- 1
src/libmime/received.cxx View File



if (!seen_ip_in_data) { if (!seen_ip_in_data) {
if (rh.real_ip.size() != 0) { if (rh.real_ip.size() != 0) {
/* Get anounced hostname (usually helo) */
/* Get announced hostname (usually helo) */
received_process_rdns(pool, received_process_rdns(pool,
rpart.data.as_view(), rpart.data.as_view(),
rh.from_hostname); rh.from_hostname);

+ 1
- 1
src/libmime/scan_result.c View File

scan_result = task->result; scan_result = task->result;
} }


/* Find the speicific action config */
/* Find the specific action config */
struct rspamd_action_config *action_config = NULL; struct rspamd_action_config *action_config = NULL;


for (unsigned int i = 0; i < scan_result->nactions; i ++) { for (unsigned int i = 0; i < scan_result->nactions; i ++) {

+ 1
- 1
src/libserver/cfg_file.h View File

RSPAMD_SYMBOL_FLAG_NORMAL = 0, RSPAMD_SYMBOL_FLAG_NORMAL = 0,
RSPAMD_SYMBOL_FLAG_IGNORE_METRIC = (1 << 1), RSPAMD_SYMBOL_FLAG_IGNORE_METRIC = (1 << 1),
RSPAMD_SYMBOL_FLAG_ONEPARAM = (1 << 2), RSPAMD_SYMBOL_FLAG_ONEPARAM = (1 << 2),
RSPAMD_SYMBOL_FLAG_UNGROUPPED = (1 << 3),
RSPAMD_SYMBOL_FLAG_UNGROUPED = (1 << 3),
RSPAMD_SYMBOL_FLAG_DISABLED = (1 << 4), RSPAMD_SYMBOL_FLAG_DISABLED = (1 << 4),
RSPAMD_SYMBOL_FLAG_UNSCORED = (1 << 5), RSPAMD_SYMBOL_FLAG_UNSCORED = (1 << 5),
}; };

+ 11
- 11
src/libserver/cfg_utils.c View File

* $RUNDIR - local states directory * $RUNDIR - local states directory
* $DBDIR - databases dir * $DBDIR - databases dir
* $LOGDIR - logs dir * $LOGDIR - logs dir
* $PLUGINSDIR - pluggins dir
* $PLUGINSDIR - plugins dir
* $PREFIX - installation prefix * $PREFIX - installation prefix
* $VERSION - rspamd version * $VERSION - rspamd version
*/ */
/* We have only one statfile */ /* We have only one statfile */
return FALSE; return FALSE;
} }
/* We have not detected any statfile that has different class, so turn on euristic based on symbol's name */
/* We have not detected any statfile that has different class, so turn on heuristic based on symbol's name */
has_other = FALSE; has_other = FALSE;
cur = cf->statfiles; cur = cf->statfiles;
while (cur) { while (cur) {
/* Search for symbol group */ /* Search for symbol group */
if (group == NULL) { if (group == NULL) {
group = "ungrouped"; group = "ungrouped";
sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPPED;
sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPED;
} }
else { else {
if (strcmp (group, "ungrouped") == 0) { if (strcmp (group, "ungrouped") == 0) {
sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPPED;
sym_def->flags |= RSPAMD_SYMBOL_FLAG_UNGROUPED;
} }
} }


sym_def->gr = sym_group; sym_def->gr = sym_group;
g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def);


if (!(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPPED)) {
if (!(sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED)) {
g_ptr_array_add (sym_def->groups, sym_group); g_ptr_array_add (sym_def->groups, sym_group);
} }
} }
} }


if (!has_group) { if (!has_group) {
/* Non-empty group has a priority over non-groupped one */
/* Non-empty group has a priority over non-grouped one */
sym_group = g_hash_table_lookup (cfg->groups, group); sym_group = g_hash_table_lookup (cfg->groups, group);


if (sym_group == NULL) { if (sym_group == NULL) {
sym_group = rspamd_config_new_group (cfg, group); sym_group = rspamd_config_new_group (cfg, group);
} }


if ((!sym_def->gr) || (sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPPED)) {
if ((!sym_def->gr) || (sym_def->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED)) {
sym_def->gr = sym_group; sym_def->gr = sym_group;
sym_def->flags &= ~RSPAMD_SYMBOL_FLAG_UNGROUPPED;
sym_def->flags &= ~RSPAMD_SYMBOL_FLAG_UNGROUPED;
} }


g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def);
sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPPED);
sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPED);
g_ptr_array_add (sym_def->groups, sym_group); g_ptr_array_add (sym_def->groups, sym_group);
} }
} }
} }


if (!has_group) { if (!has_group) {
/* Non-empty group has a priority over non-groupped one */
/* Non-empty group has a priority over non-grouped one */
sym_group = g_hash_table_lookup (cfg->groups, group); sym_group = g_hash_table_lookup (cfg->groups, group);


if (sym_group == NULL) { if (sym_group == NULL) {
} }


g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def);
sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPPED);
sym_def->flags &= ~(RSPAMD_SYMBOL_FLAG_UNGROUPED);
g_ptr_array_add (sym_def->groups, sym_group); g_ptr_array_add (sym_def->groups, sym_group);


return TRUE; return TRUE;

+ 1
- 1
src/libserver/css/css_rule.hxx View File



/** /**
* Compile CSS declaration to the html block * Compile CSS declaration to the html block
* @param pool used to carry memory requred for html_block
* @param pool used to carry memory required for html_block
* @return html block structure * @return html block structure
*/ */
auto compile_to_block(rspamd_mempool_t *pool) const -> rspamd::html::html_block *; auto compile_to_block(rspamd_mempool_t *pool) const -> rspamd::html::html_block *;

+ 1
- 1
src/libserver/dkim.h View File

GError **err); GError **err);


/** /**
* Canonocalise header using relaxed algorithm
* Canonicalise header using relaxed algorithm
* @param hname * @param hname
* @param hvalue * @param hvalue
* @param out * @param out

+ 1
- 1
src/libserver/fuzzy_backend/fuzzy_backend.c View File

{ {
guint ret; guint ret;


/* Distirbuted uniformly already */
/* Distributed uniformly already */
memcpy (&ret, key, sizeof (ret)); memcpy (&ret, key, sizeof (ret));


return ret; return ret;

+ 5
- 5
src/libserver/html/html.cxx View File

ignore_bad_tag, ignore_bad_tag,
tag_end, tag_end,
slash_after_value, slash_after_value,
slash_in_unqouted_value,
slash_in_unquoted_value,
} state; } state;


state = static_cast<enum tag_parser_state>(parser_env.cur_state); state = static_cast<enum tag_parser_state>(parser_env.cur_state);
*/ */
if (*in == '>') { if (*in == '>') {
/* /*
* Attribtute name followed by end of tag
* Attribute name followed by end of tag
* Should be okay (empty attribute). The rest is handled outside * Should be okay (empty attribute). The rest is handled outside
* this automata. * this automata.
*/ */


case parse_value: case parse_value:
if (*in == '/') { if (*in == '/') {
state = slash_in_unqouted_value;
state = slash_in_unquoted_value;
} }
else if (g_ascii_isspace (*in) || *in == '>' || *in == '"') { else if (g_ascii_isspace (*in) || *in == '>' || *in == '"') {
store_component_value(); store_component_value();
state = parse_attr_name; state = parse_attr_name;
} }
break; break;
case slash_in_unqouted_value:
case slash_in_unquoted_value:
if (*in == '>') { if (*in == '>') {
/* That slash was in fact closing tag slash, wohoo */
/* That slash was in fact closing tag slash, woohoo */
tag->flags |= FL_CLOSED; tag->flags |= FL_CLOSED;
state = tag_end; state = tag_end;
store_component_value(); store_component_value();

+ 1
- 1
src/libserver/html/html_url.cxx View File

gsize dlen; gsize dlen;


if (visible_part.empty()) { if (visible_part.empty()) {
/* No dispalyed url, just some text within <a> tag */
/* No displayed url, just some text within <a> tag */
return; return;
} }



+ 1
- 1
src/libserver/http/http_connection.c View File

if (msg->body_buf.len > 0) { if (msg->body_buf.len > 0) {


if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) { if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
/* Avoid copying by just maping a shared segment */
/* Avoid copying by just mapping a shared segment */
new_msg->flags |= RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE; new_msg->flags |= RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE;


storage = &new_msg->body_buf.c; storage = &new_msg->body_buf.c;

+ 1
- 1
src/libserver/http/http_context.c View File

struct upstream_list *uls; struct upstream_list *uls;


if (!ctx->ups_ctx) { if (!ctx->ups_ctx) {
msg_err ("cannot parse http_proxy %s - upstreams context is udefined", name);
msg_err ("cannot parse http_proxy %s - upstreams context is undefined", name);
return; return;
} }



+ 4
- 4
src/libserver/logger/logger.c View File



if (!(rspamd_log->flags & RSPAMD_LOG_FLAG_RSPAMADM)) { if (!(rspamd_log->flags & RSPAMD_LOG_FLAG_RSPAMADM)) {
if ((nescaped = rspamd_log_line_need_escape (logbuf, end - logbuf)) != 0) { if ((nescaped = rspamd_log_line_need_escape (logbuf, end - logbuf)) != 0) {
gsize unsecaped_len = end - logbuf;
gchar *logbuf_escaped = g_alloca (unsecaped_len + nescaped * 4);
gsize unescaped_len = end - logbuf;
gchar *logbuf_escaped = g_alloca (unescaped_len + nescaped * 4);
log_line = logbuf_escaped; log_line = logbuf_escaped;


end = rspamd_log_line_hex_escape (logbuf, unsecaped_len,
logbuf_escaped, unsecaped_len + nescaped * 4);
end = rspamd_log_line_hex_escape (logbuf, unescaped_len,
logbuf_escaped, unescaped_len + nescaped * 4);
} }
} }



+ 1
- 1
src/libserver/maps/map.c View File



/* Now, we do some sanity checks for jittered seconds */ /* Now, we do some sanity checks for jittered seconds */
if (!(how & RSPAMD_MAP_SCHEDULE_INIT)) { if (!(how & RSPAMD_MAP_SCHEDULE_INIT)) {
/* Never allow too low interval between timer checks, it is epxensive */
/* Never allow too low interval between timer checks, it is expensive */
if (jittered_sec < min_timer_interval) { if (jittered_sec < min_timer_interval) {
jittered_sec = rspamd_time_jitter (min_timer_interval, 0); jittered_sec = rspamd_time_jitter (min_timer_interval, 0);
} }

+ 1
- 1
src/libserver/maps/map_helpers.c View File



#if !defined(__aarch64__) && !defined(__powerpc64__) #if !defined(__aarch64__) && !defined(__powerpc64__)
if (!(map->cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) { if (!(map->cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) {
msg_info_map ("disable hyperscan for map %s, ssse3 instructons are not supported by CPU",
msg_info_map ("disable hyperscan for map %s, ssse3 instructions are not supported by CPU",
map->name); map->name);
return; return;
} }

+ 2
- 2
src/libserver/re_cache.c View File

sizeof (fl)); sizeof (fl));
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl, rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
sizeof (fl)); sizeof (fl));
/* Numberic order */
/* Numeric order */
rspamd_cryptobox_hash_update (re_class->st, (const guchar *)&i, rspamd_cryptobox_hash_update (re_class->st, (const guchar *)&i,
sizeof (i)); sizeof (i));
rspamd_cryptobox_hash_update (&st_global, (const guchar *)&i, rspamd_cryptobox_hash_update (&st_global, (const guchar *)&i,
rspamd_re_cache_hs_pattern_from_pcre (rspamd_regexp_t *re) rspamd_re_cache_hs_pattern_from_pcre (rspamd_regexp_t *re)
{ {
/* /*
* Workaroung for bug in ragel 7.0.0.11
* Workaround for bug in ragel 7.0.0.11
* https://github.com/intel/hyperscan/issues/133 * https://github.com/intel/hyperscan/issues/133
*/ */
const gchar *pat = rspamd_regexp_get_pattern (re); const gchar *pat = rspamd_regexp_get_pattern (re);

+ 2
- 2
src/libserver/rspamd_symcache.c View File





if (o1 == o2) { if (o1 == o2) {
/* Heurstic */
/* Heuristic */
if (i1->priority == i2->priority) { if (i1->priority == i2->priority) {
avg_freq = ((gdouble) cache->total_hits / cache->used_items); avg_freq = ((gdouble) cache->total_hits / cache->used_items);
avg_weight = (cache->total_weight / cache->used_items); avg_weight = (cache->total_weight / cache->used_items);
* one more time * one more time
*/ */
msg_debug_cache_task ("postpone finalisation of %s(%d) as there are %d " msg_debug_cache_task ("postpone finalisation of %s(%d) as there are %d "
"async events pendning",
"async events pending",
item->symbol, item->id, dyn_item->async_events); item->symbol, item->id, dyn_item->async_events);


return; return;

+ 2
- 2
src/libserver/rspamd_symcache.h View File

gboolean exec_only); gboolean exec_only);


/** /**
* Returns symbcache item flags
* Returns symcache item flags
* @param item * @param item
* @return * @return
*/ */
const struct rspamd_symcache_item_stat * const struct rspamd_symcache_item_stat *
rspamd_symcache_item_stat (struct rspamd_symcache_item *item); rspamd_symcache_item_stat (struct rspamd_symcache_item *item);
/** /**
* Returns if an item is enabled (for virutal it also means that parent should be enabled)
* Returns if an item is enabled (for virtual it also means that parent should be enabled)
* @param item * @param item
* @return * @return
*/ */

+ 5
- 5
src/libserver/spf.c View File

struct spf_resolved_element { struct spf_resolved_element {
GPtrArray *elts; GPtrArray *elts;
gchar *cur_domain; gchar *cur_domain;
gboolean redirected; /* Ingnore level, it's redirected */
gboolean redirected; /* Ignore level, it's redirected */
}; };


struct spf_record { struct spf_record {
gboolean ret = FALSE; gboolean ret = FALSE;


/* /*
* We prefer spf version 1 as other records are mostly likely garbadge
* We prefer spf version 1 as other records are mostly likely garbage
* or incorrect records (e.g. spf2 records) * or incorrect records (e.g. spf2 records)
*/ */
LL_FOREACH (reply->entries, elt) { LL_FOREACH (reply->entries, elt) {
parse_ipv4_mask, parse_ipv4_mask,
parse_second_slash, parse_second_slash,
parse_ipv6_mask, parse_ipv6_mask,
skip_garbadge
skip_garbage
} state = 0; } state = 0;
const gchar *p = addr->spf_string, *host, *c; const gchar *p = addr->spf_string, *host, *c;
gchar *hostbuf; gchar *hostbuf;
state = parse_ipv4_mask; state = parse_ipv4_mask;
} }
else { else {
state = skip_garbadge;
state = skip_garbage;
} }
cur_mask = 0; cur_mask = 0;
break; break;
} }
p++; p++;
break; break;
case skip_garbadge:
case skip_garbage:
p++; p++;
break; break;
} }

+ 1
- 1
src/libserver/task.h View File

enum rspamd_command { enum rspamd_command {
CMD_SKIP = 0, CMD_SKIP = 0,
CMD_PING, CMD_PING,
CMD_CHECK_SPAMC, /* Legacy spamasassin format */
CMD_CHECK_SPAMC, /* Legacy spamassassin format */
CMD_CHECK_RSPAMC, /* Legacy rspamc format (like SA one) */ CMD_CHECK_RSPAMC, /* Legacy rspamc format (like SA one) */
CMD_CHECK, /* Legacy check - metric json reply */ CMD_CHECK, /* Legacy check - metric json reply */
CMD_CHECK_V2, /* Modern check - symbols in json reply */ CMD_CHECK_V2, /* Modern check - symbols in json reply */

+ 1
- 1
src/libserver/url.c View File

{"schema_encoded", RSPAMD_URL_FLAG_SCHEMAENCODED, -1}, {"schema_encoded", RSPAMD_URL_FLAG_SCHEMAENCODED, -1},
{"path_encoded", RSPAMD_URL_FLAG_PATHENCODED, -1}, {"path_encoded", RSPAMD_URL_FLAG_PATHENCODED, -1},
{"query_encoded", RSPAMD_URL_FLAG_QUERYENCODED, -1}, {"query_encoded", RSPAMD_URL_FLAG_QUERYENCODED, -1},
{"missing_slahes", RSPAMD_URL_FLAG_MISSINGSLASHES, -1},
{"missing_slashes", RSPAMD_URL_FLAG_MISSINGSLASHES, -1},
{"idn", RSPAMD_URL_FLAG_IDN, -1}, {"idn", RSPAMD_URL_FLAG_IDN, -1},
{"has_port", RSPAMD_URL_FLAG_HAS_PORT, -1}, {"has_port", RSPAMD_URL_FLAG_HAS_PORT, -1},
{"has_user", RSPAMD_URL_FLAG_HAS_USER, -1}, {"has_user", RSPAMD_URL_FLAG_HAS_USER, -1},

+ 1
- 1
src/libserver/url.h View File

* @param pool memory pool * @param pool memory pool
* @param task task object * @param task task object
* @param part current text part * @param part current text part
* @param is_html turn on html euristic
* @param is_html turn on html heuristic
*/ */
void rspamd_url_text_extract(rspamd_mempool_t *pool, void rspamd_url_text_extract(rspamd_mempool_t *pool,
struct rspamd_task *task, struct rspamd_task *task,

+ 1
- 1
src/libserver/worker_util.c View File

static void static void
rspamd_worker_drop_priv (struct rspamd_main *rspamd_main) rspamd_worker_drop_priv (struct rspamd_main *rspamd_main)
{ {
if (rspamd_main->is_privilleged) {
if (rspamd_main->is_privileged) {
if (setgid (rspamd_main->workers_gid) == -1) { if (setgid (rspamd_main->workers_gid) == -1) {
msg_err_main ("cannot setgid to %d (%s), aborting", msg_err_main ("cannot setgid to %d (%s), aborting",
(gint) rspamd_main->workers_gid, (gint) rspamd_main->workers_gid,

+ 1
- 1
src/libserver/worker_util.h View File

struct rspamd_custom_controller_command { struct rspamd_custom_controller_command {
const gchar *command; const gchar *command;
struct module_ctx *ctx; struct module_ctx *ctx;
gboolean privilleged;
gboolean privileged;
gboolean require_message; gboolean require_message;
rspamd_controller_func_t handler; rspamd_controller_func_t handler;
}; };

+ 3
- 3
src/libstat/learn_cache/redis_cache.c View File

gchar *h; gchar *h;


if (rspamd_session_blocked (task->s)) { if (rspamd_session_blocked (task->s)) {
return RSPAMD_LEARN_INGORE;
return RSPAMD_LEARN_IGNORE;
} }


h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); h = rspamd_mempool_get_variable (task->task_pool, "words_hash");


if (h == NULL) { if (h == NULL) {
return RSPAMD_LEARN_INGORE;
return RSPAMD_LEARN_IGNORE;
} }


if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_get, rt, if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_get, rt,
gint flag; gint flag;


if (rt == NULL || rt->ctx == NULL || rspamd_session_blocked (task->s)) { if (rt == NULL || rt->ctx == NULL || rspamd_session_blocked (task->s)) {
return RSPAMD_LEARN_INGORE;
return RSPAMD_LEARN_IGNORE;
} }


h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); h = rspamd_mempool_get_variable (task->task_pool, "words_hash");

+ 3
- 3
src/libstat/learn_cache/sqlite3_cache.c View File

gint64 flag; gint64 flag;


if (task->tokens == NULL || task->tokens->len == 0) { if (task->tokens == NULL || task->tokens->len == 0) {
return RSPAMD_LEARN_INGORE;
return RSPAMD_LEARN_IGNORE;
} }


if (ctx != NULL && ctx->db != NULL) { if (ctx != NULL && ctx->db != NULL) {
/* Already learned */ /* Already learned */
msg_warn_task ("already seen stat hash: %*bs", msg_warn_task ("already seen stat hash: %*bs",
rspamd_cryptobox_HASHBYTES, out); rspamd_cryptobox_HASHBYTES, out);
return RSPAMD_LEARN_INGORE;
return RSPAMD_LEARN_IGNORE;
} }
else { else {
/* Need to relearn */ /* Need to relearn */
h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); h = rspamd_mempool_get_variable (task->task_pool, "words_hash");


if (h == NULL) { if (h == NULL) {
return RSPAMD_LEARN_INGORE;
return RSPAMD_LEARN_IGNORE;
} }


flag = !!is_spam ? 1 : 0; flag = !!is_spam ? 1 : 0;

+ 1
- 1
src/libstat/stat_internal.h View File

typedef enum rspamd_learn_cache_result { typedef enum rspamd_learn_cache_result {
RSPAMD_LEARN_OK = 0, RSPAMD_LEARN_OK = 0,
RSPAMD_LEARN_UNLEARN, RSPAMD_LEARN_UNLEARN,
RSPAMD_LEARN_INGORE
RSPAMD_LEARN_IGNORE
} rspamd_learn_t; } rspamd_learn_t;


struct rspamd_stat_ctx *rspamd_stat_get_ctx (void); struct rspamd_stat_ctx *rspamd_stat_get_ctx (void);

+ 4
- 4
src/libstat/stat_process.c View File

#define RSPAMD_LEARN_OP 1 #define RSPAMD_LEARN_OP 1
#define RSPAMD_UNLEARN_OP 2 #define RSPAMD_UNLEARN_OP 2


static const gdouble similarity_treshold = 80.0;
static const gdouble similarity_threshold = 80.0;


static void static void
rspamd_stat_tokenize_parts_metadata (struct rspamd_stat_ctx *st_ctx, rspamd_stat_tokenize_parts_metadata (struct rspamd_stat_ctx *st_ctx,
} }




if (pdiff != NULL && (1.0 - *pdiff) * 100.0 > similarity_treshold) {
if (pdiff != NULL && (1.0 - *pdiff) * 100.0 > similarity_threshold) {
msg_debug_bayes ("message has two common parts (%.2f), so skip the last one", msg_debug_bayes ("message has two common parts (%.2f), so skip the last one",
*pdiff); *pdiff);
break; break;
learn_res = cl->cache->check (task, spam, rt); learn_res = cl->cache->check (task, spam, rt);
} }


if (learn_res == RSPAMD_LEARN_INGORE) {
if (learn_res == RSPAMD_LEARN_IGNORE) {
/* Do not learn twice */ /* Do not learn twice */
g_set_error (err, rspamd_stat_quark (), 404, "<%s> has been already " g_set_error (err, rspamd_stat_quark (), 404, "<%s> has been already "
"learned as %s, ignore it", MESSAGE_FIELD (task, message_id), "learned as %s, ignore it", MESSAGE_FIELD (task, message_id),
} }
} }
else if (ucl_object_type (obj) == UCL_STRING) { else if (ucl_object_type (obj) == UCL_STRING) {
/* Legacy sript */
/* Legacy script */
lua_script = ucl_object_tostring (obj); lua_script = ucl_object_tostring (obj);


if (luaL_dostring (L, lua_script) != 0) { if (luaL_dostring (L, lua_script) != 0) {

+ 1
- 1
src/libutil/addr.h View File

void rspamd_inet_library_destroy (void); void rspamd_inet_library_destroy (void);


/** /**
* Create new inet address structure based on the address familiy and opaque init pointer
* Create new inet address structure based on the address family and opaque init pointer
* @param af * @param af
* @param init * @param init
* @return new inet addr * @return new inet addr

+ 2
- 2
src/libutil/hash.h View File

/** /**
* Create new lru hash * Create new lru hash
* @param maxsize maximum elements in a hash * @param maxsize maximum elements in a hash
* @param maxage maximum age of elemnt
* @param maxage maximum age of element
* @param hash_func pointer to hash function * @param hash_func pointer to hash function
* @param key_equal_func pointer to function for comparing keys * @param key_equal_func pointer to function for comparing keys
* @return new rspamd_hash object * @return new rspamd_hash object
/** /**
* Create new lru hash * Create new lru hash
* @param maxsize maximum elements in a hash * @param maxsize maximum elements in a hash
* @param maxage maximum age of elemnt
* @param maxage maximum age of element
* @param hash_func pointer to hash function * @param hash_func pointer to hash function
* @param key_equal_func pointer to function for comparing keys * @param key_equal_func pointer to function for comparing keys
* @return new rspamd_hash object * @return new rspamd_hash object

+ 1
- 1
src/libutil/mem_pool.c View File

qsort (sz, G_N_ELEMENTS (sz), sizeof (gint), cmp_int); qsort (sz, G_N_ELEMENTS (sz), sizeof (gint), cmp_int);
jitter = rspamd_random_uint64_fast () % 10; jitter = rspamd_random_uint64_fast () % 10;
/* /*
* Take stochaistic quantiles
* Take stochastic quantiles
*/ */
sel_pos = sz[50 + jitter]; sel_pos = sz[50 + jitter];
sel_neg = sz[4 + jitter]; sel_neg = sz[4 + jitter];

+ 1
- 1
src/libutil/mem_pool.h View File

* Set memory pool variable * Set memory pool variable
* @param pool memory pool object * @param pool memory pool object
* @param name name of variable * @param name name of variable
* @param gpointer value value of variable
* @param gpointer value of variable
* @param destructor pointer to function-destructor * @param destructor pointer to function-destructor
*/ */
void rspamd_mempool_set_variable (rspamd_mempool_t *pool, void rspamd_mempool_set_variable (rspamd_mempool_t *pool,

+ 3
- 3
src/libutil/rrd.h View File

gdouble float_cookie; /* is it the correct double representation ? */ gdouble float_cookie; /* is it the correct double representation ? */


/* Data Base Structure Definition **** */ /* Data Base Structure Definition **** */
gulong ds_cnt; /* how many different ds provid input to the rrd */
gulong ds_cnt; /* how many different ds provide input to the rrd */
gulong rra_cnt; /* how many rras will be maintained in the rrd */ gulong rra_cnt; /* how many rras will be maintained in the rrd */
gulong pdp_step; /* pdp interval in seconds */ gulong pdp_step; /* pdp interval in seconds */


* prediction algorithm. */ * prediction algorithm. */
CDP_hw_last_intercept, CDP_hw_last_intercept,
/* Last iteration intercept coefficient for the Holt-Winters /* Last iteration intercept coefficient for the Holt-Winters
* prediction algorihtm. */
* prediction algorithm. */
CDP_hw_slope, CDP_hw_slope,
/* Current slope coefficient for the Holt-Winters /* Current slope coefficient for the Holt-Winters
* prediction algorithm. */ * prediction algorithm. */
CDP_hw_last_slope, CDP_hw_last_slope,
/* Last iteration slope coeffient. */
/* Last iteration slope coefficient. */
CDP_null_count, CDP_null_count,
/* Number of sequential Unknown (DNAN) values + 1 preceding /* Number of sequential Unknown (DNAN) values + 1 preceding
* the current prediction. * the current prediction.

+ 2
- 2
src/libutil/shingles.h View File

* if needed * if needed
* @param input array of `rspamd_fstring_t` * @param input array of `rspamd_fstring_t`
* @param key secret key used to generate shingles * @param key secret key used to generate shingles
* @param pool pool to allocate shigles array
* @param pool pool to allocate shingles array
* @param filter hashes filtering function * @param filter hashes filtering function
* @param filterd opaque data for filtering function * @param filterd opaque data for filtering function
* @return shingles array * @return shingles array
* Generate shingles from the DCT matrix of an image * Generate shingles from the DCT matrix of an image
* @param dct discrete cosine transfor matrix (must be 64x64) * @param dct discrete cosine transfor matrix (must be 64x64)
* @param key secret key used to generate shingles * @param key secret key used to generate shingles
* @param pool pool to allocate shigles array
* @param pool pool to allocate shingles array
* @param filter hashes filtering function * @param filter hashes filtering function
* @param filterd opaque data for filtering function * @param filterd opaque data for filtering function
* @return shingles array * @return shingles array

+ 1
- 1
src/libutil/str_util.c View File

continue; continue;
} }
else { else {
/* Hack, hack, hack, treat =<garbadge> as =<garbadge> */
/* Hack, hack, hack, treat =<garbage> as =<garbage> */
if (end - o > 1) { if (end - o > 1) {
*o++ = '='; *o++ = '=';
*o++ = *(p - 1); *o++ = *(p - 1);

+ 1
- 1
src/libutil/upstream.c View File

g_ptr_array_remove_index (ls->alive, upstream->active_idx); g_ptr_array_remove_index (ls->alive, upstream->active_idx);
upstream->active_idx = -1; upstream->active_idx = -1;


/* We need to update all indicies */
/* We need to update all indices */
for (i = 0; i < ls->alive->len; i ++) { for (i = 0; i < ls->alive->len; i ++) {
cur = g_ptr_array_index (ls->alive, i); cur = g_ptr_array_index (ls->alive, i);
cur->active_idx = i; cur->active_idx = i;

+ 1
- 1
src/libutil/upstream.h View File

const gchar *rspamd_upstream_name (struct upstream *up); const gchar *rspamd_upstream_name (struct upstream *up);


/** /**
* Returns the port of the current addres for the upstream
* Returns the port of the current address for the upstream
* @param up * @param up
* @return * @return
*/ */

+ 2
- 2
src/libutil/util.c View File

* Make a universal socket * Make a universal socket
* @param credits host, ip or path to unix socket * @param credits host, ip or path to unix socket
* @param port port (used for network sockets) * @param port port (used for network sockets)
* @param async make this socket asynced
* @param async make this socket async
* @param is_server make this socket as server socket * @param is_server make this socket as server socket
* @param try_resolve try name resolution for a socket (BLOCKING) * @param try_resolve try name resolution for a socket (BLOCKING)
*/ */


/* /*
* Open the PID file and obtain exclusive lock. * Open the PID file and obtain exclusive lock.
* We truncate PID file here only to remove old PID immediatelly,
* We truncate PID file here only to remove old PID immediately,
* PID file will be truncated again in pidfile_write(), so * PID file will be truncated again in pidfile_write(), so
* pidfile_write() can be called multiple times. * pidfile_write() can be called multiple times.
*/ */

+ 1
- 1
src/libutil/util.h View File

* @param credits host, ip or path to unix socket * @param credits host, ip or path to unix socket
* @param port port (used for network sockets) * @param port port (used for network sockets)
* @param type type of socket (SO_STREAM or SO_DGRAM) * @param type type of socket (SO_STREAM or SO_DGRAM)
* @param async make this socket asynced
* @param async make this socket async
* @param is_server make this socket as server socket * @param is_server make this socket as server socket
* @param try_resolve try name resolution for a socket (BLOCKING) * @param try_resolve try name resolution for a socket (BLOCKING)
*/ */

+ 9
- 9
src/lua/lua_config.c View File

/*** /***
* @method rspamd_config:register_dependency(id|name, depname) * @method rspamd_config:register_dependency(id|name, depname)
* Create a dependency on symbol identified by name for symbol identified by ID or name. * Create a dependency on symbol identified by name for symbol identified by ID or name.
* This affects order of checks only (a symbol is still checked if its dependencys are disabled).
* This affects order of checks only (a symbol is still checked if its dependencies are disabled).
* @param {number|string} id id or name of source (numeric id is returned by all register_*_symbol) * @param {number|string} id id or name of source (numeric id is returned by all register_*_symbol)
* @param {string} depname dependency name * @param {string} depname dependency name
* @example * @example


/** /**
* @method rspamd_config:get_action(name) * @method rspamd_config:get_action(name)
* Gets data for a specific action in config. This function returns number reperesenting action's score
* Gets data for a specific action in config. This function returns number representing action's score
* *
* @param {string} name name of action * @param {string} name name of action
* @return {number} action's score or nil in case of undefined score or action * @return {number} action's score or nil in case of undefined score or action


/*** /***
* @method rspamd_config:__newindex(name, callback) * @method rspamd_config:__newindex(name, callback)
* This metamethod is called if new indicies are added to the `rspamd_config` object.
* This metamethod is called if new indices are added to the `rspamd_config` object.
* Technically, it is the equivalent of @see rspamd_config:register_symbol where `weight` is 1.0. * Technically, it is the equivalent of @see rspamd_config:register_symbol where `weight` is 1.0.
* There is also table form invocation that allows to control more things: * There is also table form invocation that allows to control more things:
* *
* *
* - `default`: default option value * - `default`: default option value
* - `type`: type of an option (`string`, `number`, `object`, `array` etc) * - `type`: type of an option (`string`, `number`, `object`, `array` etc)
* - `reqired`: if an option is required
* - `required`: if an option is required
* *
* @param {string} path documentation path (e.g. module name) * @param {string} path documentation path (e.g. module name)
* @param {string} option name of the option * @param {string} option name of the option
} }
else else
{ {
/* Fill in missing fields from lua defintion if they are not set */
/* Fill in missing fields from lua definition if they are not set */
if (sym->description == NULL) { if (sym->description == NULL) {
lua_pushstring (L, "description"); lua_pushstring (L, "description");
lua_gettable (L, -2); lua_gettable (L, -2);
} }
lua_pop (L, 1); lua_pop (L, 1);
if (group) { if (group) {
if (sym->flags & RSPAMD_SYMBOL_FLAG_UNGROUPPED) {
if (sym->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED) {
/* Unset the "ungrouped" group */ /* Unset the "ungrouped" group */
sym->gr = NULL; sym->gr = NULL;
} }
/* Add the group. If the symbol was ungrouped, this will /* Add the group. If the symbol was ungrouped, this will
* clear RSPAMD_SYMBOL_FLAG_UNGROUPPED from the flags. */
* clear RSPAMD_SYMBOL_FLAG_UNGROUPED from the flags. */
rspamd_config_add_symbol_group (cfg, name, group); rspamd_config_add_symbol_group (cfg, name, group);
} }
} }
lua_pushboolean (L, true); lua_pushboolean (L, true);
lua_settable (L, -3); lua_settable (L, -3);
} }
if (s->flags & RSPAMD_SYMBOL_FLAG_UNGROUPPED) {
lua_pushstring (L, "ungroupped");
if (s->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED) {
lua_pushstring (L, "ungrouped");
lua_pushboolean (L, true); lua_pushboolean (L, true);
lua_settable (L, -3); lua_settable (L, -3);
} }

+ 5
- 5
src/lua/lua_dns_resolver.c View File

* * `mempool` - pool memory pool for storing intermediate data * * `mempool` - pool memory pool for storing intermediate data
* * `name` - host name to resolve * * `name` - host name to resolve
* * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
* * `forced` - true if needed to override notmal limit for DNS requests
* * `forced` - true if needed to override normal limit for DNS requests
* @return {boolean} `true` if DNS request has been scheduled * @return {boolean} `true` if DNS request has been scheduled
*/ */
static int static int
* * `mempool` - pool memory pool for storing intermediate data * * `mempool` - pool memory pool for storing intermediate data
* * `name` - host name to resolve * * `name` - host name to resolve
* * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
* * `forced` - true if needed to override notmal limit for DNS requests
* * `forced` - true if needed to override normal limit for DNS requests
* @return {boolean} `true` if DNS request has been scheduled * @return {boolean} `true` if DNS request has been scheduled
*/ */
static int static int
* * `mempool` - pool memory pool for storing intermediate data * * `mempool` - pool memory pool for storing intermediate data
* * `name` - host name to resolve * * `name` - host name to resolve
* * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
* * `forced` - true if needed to override notmal limit for DNS requests
* * `forced` - true if needed to override normal limit for DNS requests
* @return {boolean} `true` if DNS request has been scheduled * @return {boolean} `true` if DNS request has been scheduled
*/ */
static int static int
* * `mempool` - pool memory pool for storing intermediate data * * `mempool` - pool memory pool for storing intermediate data
* * `name` - host name to resolve * * `name` - host name to resolve
* * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
* * `forced` - true if needed to override notmal limit for DNS requests
* * `forced` - true if needed to override normal limit for DNS requests
* @return {boolean} `true` if DNS request has been scheduled * @return {boolean} `true` if DNS request has been scheduled
*/ */
static int static int
* * `mempool` - pool memory pool for storing intermediate data * * `mempool` - pool memory pool for storing intermediate data
* * `name` - host name to resolve * * `name` - host name to resolve
* * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)` * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
* * `forced` - true if needed to override notmal limit for DNS requests
* * `forced` - true if needed to override normal limit for DNS requests
* @return {boolean} `true` if DNS request has been scheduled * @return {boolean} `true` if DNS request has been scheduled
*/ */
static int static int

+ 3
- 3
src/lua/lua_mimepart.c View File

* * By default headers are searched in caseless matter. * * By default headers are searched in caseless matter.
* @param {string} name name of header to get * @param {string} name name of header to get
* @param {boolean} case_sensitive case sensitiveness flag to search for a header * @param {boolean} case_sensitive case sensitiveness flag to search for a header
* @return {number} number of header's occurrencies or 0 if not found
* @return {number} number of header's occurrences or 0 if not found
*/ */
LUA_FUNCTION_DEF (mimepart, get_header_count); LUA_FUNCTION_DEF (mimepart, get_header_count);


/*** /***
* @method mime_part:get_urls([need_emails|list_protos][, need_images]) * @method mime_part:get_urls([need_emails|list_protos][, need_images])
* Get all URLs found in a mime part. Telephone urls and emails are not included unless explicitly asked in `list_protos` * Get all URLs found in a mime part. Telephone urls and emails are not included unless explicitly asked in `list_protos`
* @param {boolean} need_emails if `true` then reutrn also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`)
* @param {boolean} need_emails if `true` then return also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`)
* @param {boolean} need_images return urls from images (<img src=...>) as well * @param {boolean} need_images return urls from images (<img src=...>) as well
* @return {table rspamd_url} list of all urls found * @return {table rspamd_url} list of all urls found
*/ */
lua_pushboolean (L, true); lua_pushboolean (L, true);
} }
else { else {
/* Image or an embeded object */
/* Image or an embedded object */
lua_pushboolean (L, false); lua_pushboolean (L, false);
} }
} }

+ 1
- 1
src/lua/lua_redis.c View File

struct redisAsyncContext *ac; struct redisAsyncContext *ac;


ud = &ctx->async; ud = &ctx->async;
msg_debug_lua_redis ("desctructing %p", ctx);
msg_debug_lua_redis ("destructing %p", ctx);


if (ud->ctx) { if (ud->ctx) {



+ 1
- 1
src/lua/lua_regexp.c View File



/*** /***
* @method re:destroy() * @method re:destroy()
* Destroy regexp from caches if needed (the pointer is removed by garbadge collector)
* Destroy regexp from caches if needed (the pointer is removed by garbage collector)
*/ */
static gint static gint
lua_regexp_destroy (lua_State *L) lua_regexp_destroy (lua_State *L)

+ 9
- 9
src/lua/lua_task.c View File



/*** /***
* @method task:remove_result(symbol[, shadow_result]) * @method task:remove_result(symbol[, shadow_result])
* Removes the symbol from a named or unamed/default result
* Removes the symbol from a named or unnamed/default result
* @param {string} symbol symbol to remove * @param {string} symbol symbol to remove
* @param {string} shadow_result name of shadow result * @param {string} shadow_result name of shadow result
* @return {boolean} true if a symbol has been removed * @return {boolean} true if a symbol has been removed
/*** /***
* @method task:get_urls([need_emails|list_protos][, need_images]) * @method task:get_urls([need_emails|list_protos][, need_images])
* Get all URLs found in a message. Telephone urls and emails are not included unless explicitly asked in `list_protos` * Get all URLs found in a message. Telephone urls and emails are not included unless explicitly asked in `list_protos`
* @param {boolean} need_emails if `true` then reutrn also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`)
* @param {boolean} need_emails if `true` then return also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`)
* @param {boolean} need_images return urls from images (<img src=...>) as well * @param {boolean} need_images return urls from images (<img src=...>) as well
* @return {table rspamd_url} list of all urls found * @return {table rspamd_url} list of all urls found
@example @example
* - If both parameters are nil then all urls are included * - If both parameters are nil then all urls are included
* @param {table} flags_include included flags * @param {table} flags_include included flags
* @param {table} flags_exclude excluded flags * @param {table} flags_exclude excluded flags
* @param {table} protocols_mask incude only specific protocols
* @param {table} protocols_mask include only specific protocols
* @return {table rspamd_url} list of urls matching conditions * @return {table rspamd_url} list of urls matching conditions
*/ */
LUA_FUNCTION_DEF (task, get_urls_filtered); LUA_FUNCTION_DEF (task, get_urls_filtered);
/*** /***
* @method task:has_urls([need_emails]) * @method task:has_urls([need_emails])
* Returns 'true' if a task has urls listed * Returns 'true' if a task has urls listed
* @param {boolean} need_emails if `true` then reutrn also email urls
* @param {boolean} need_emails if `true` then return also email urls
* @return {boolean} true if a task has urls (urls or emails if `need_emails` is true) * @return {boolean} true if a task has urls (urls or emails if `need_emails` is true)
*/ */
LUA_FUNCTION_DEF (task, has_urls); LUA_FUNCTION_DEF (task, has_urls);
* * By default headers are searched in caseless matter. * * By default headers are searched in caseless matter.
* @param {string} name name of header to get * @param {string} name name of header to get
* @param {boolean} case_sensitive case sensitiveness flag to search for a header * @param {boolean} case_sensitive case sensitiveness flag to search for a header
* @return {number} number of header's occurrencies or 0 if not found
* @return {number} number of header's occurrences or 0 if not found
*/ */
LUA_FUNCTION_DEF (task, get_header_count); LUA_FUNCTION_DEF (task, get_header_count);
/*** /***
* Order in remove starts from 1, where 0 means 'remove all', and negative value means * Order in remove starts from 1, where 0 means 'remove all', and negative value means
* remove from the end * remove from the end
* Order in addition means addition from the top: 0 means the most top header, 1 one after, etc * Order in addition means addition from the top: 0 means the most top header, 1 one after, etc
* negative order means addtion to the end, e.g. -1 is appending header.
* negative order means addition to the end, e.g. -1 is appending header.
* @return {bool} true if header could be modified (always true unless we don't have an unparsed message) * @return {bool} true if header could be modified (always true unless we don't have an unparsed message)
*/ */
LUA_FUNCTION_DEF (task, modify_header); LUA_FUNCTION_DEF (task, modify_header);
RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT, RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
"*action=S;message=S;module=S;score=D;priority=i;flags=S;result=S", "*action=S;message=S;module=S;score=D;priority=i;flags=S;result=S",
&act_str, &message, &module, &score, &priority, &fl_str, &res_name)) { &act_str, &message, &module, &score, &priority, &fl_str, &res_name)) {
gint ret = luaL_error (L, "invald arguments: %s", err->message);
gint ret = luaL_error (L, "invalid arguments: %s", err->message);
g_error_free (err); g_error_free (err);


return ret; return ret;
* UCL itself cannot do it directly. So the trick is to extract the * UCL itself cannot do it directly. So the trick is to extract the
* original object, pack it into an array and then insert it back. * original object, pack it into an array and then insert it back.
* *
* I wish there was a simplier way to do it...
* I wish there was a simpler way to do it...
*/ */
const ucl_object_t *add_hdrs = ucl_object_lookup (prev, "add_headers"); const ucl_object_t *add_hdrs = ucl_object_lookup (prev, "add_headers");
const ucl_object_t *nadd_hdrs = ucl_object_lookup (reply, "add_headers"); const ucl_object_t *nadd_hdrs = ucl_object_lookup (reply, "add_headers");
return 1; return 1;
} }


/* Arvhive methods */
/* Archive methods */
static gint static gint
lua_archive_get_type (lua_State *L) lua_archive_get_type (lua_State *L)
{ {

+ 1
- 1
src/lua/lua_tcp.c View File

* @function rspamd_tcp.connect_sync() * @function rspamd_tcp.connect_sync()
* *
* Creates pseudo-synchronous TCP connection. * Creates pseudo-synchronous TCP connection.
* Each method of the connection requiring IO, becames a yielding point,
* Each method of the connection requiring IO, becomes a yielding point,
* i.e. current thread Lua thread is get suspended and resumes as soon as IO is done * i.e. current thread Lua thread is get suspended and resumes as soon as IO is done
* *
* This class represents low-level API, using of "lua_tcp_sync" module is recommended. * This class represents low-level API, using of "lua_tcp_sync" module is recommended.

+ 2
- 2
src/lua/lua_text.c View File

LUA_FUNCTION_DEF (text, span); LUA_FUNCTION_DEF (text, span);
/*** /***
* @method rspamd_text:sub(start[, len]) * @method rspamd_text:sub(start[, len])
* Returns a substrin for lua_text similar to string.sub from Lua
* Returns a substring for lua_text similar to string.sub from Lua
* @return {rspamd_text} new rspamd_text with span (must be careful when using with owned texts...) * @return {rspamd_text} new rspamd_text with span (must be careful when using with owned texts...)
*/ */
LUA_FUNCTION_DEF (text, sub); LUA_FUNCTION_DEF (text, sub);
* @method rspamd_text:base64([line_length, [nline, [fold]]]) * @method rspamd_text:base64([line_length, [nline, [fold]]])
* Returns a text encoded in base64 (new rspamd_text is allocated) * Returns a text encoded in base64 (new rspamd_text is allocated)
* *
* @param {number} line_length return text splited with newlines up to this attribute
* @param {number} line_length return text split with newlines up to this attribute
* @param {string} nline newline type: `cr`, `lf`, `crlf` * @param {string} nline newline type: `cr`, `lf`, `crlf`
* @param {boolean} fold use folding when splitting into lines (false by default) * @param {boolean} fold use folding when splitting into lines (false by default)
* @return {rspamd_text} new text encoded in base64 * @return {rspamd_text} new text encoded in base64

+ 1
- 1
src/lua/lua_thread_pool.h View File



/** /**
* Extracts a thread from the list of available ones. * Extracts a thread from the list of available ones.
* It immediately becames running one and should be used to run a Lua script/function straight away.
* It immediately becomes the running one and should be used to run a Lua script/function straight away.
* as soon as the code is finished, it should be either returned into list of available threads by * as soon as the code is finished, it should be either returned into list of available threads by
* calling lua_thread_pool_return() or terminated by calling lua_thread_pool_terminate_entry() * calling lua_thread_pool_return() or terminated by calling lua_thread_pool_terminate_entry()
* if the code finished with error. * if the code finished with error.

+ 3
- 3
src/lua/lua_url.c View File



/*** /***
* @method url:get_count() * @method url:get_count()
* Return number of occurrencies for this particular URL
* @return {number} number of occurrencies
* Return number of occurrences for this particular URL
* @return {number} number of occurrences
*/ */
static gint static gint
lua_url_get_count (lua_State *L) lua_url_get_count (lua_State *L)
* - `host_encoded`: URL host part is encoded * - `host_encoded`: URL host part is encoded
* - `schema_encoded`: URL schema part is encoded * - `schema_encoded`: URL schema part is encoded
* - `query_encoded`: URL query part is encoded * - `query_encoded`: URL query part is encoded
* - `missing_slahes`: URL has some slashes missing
* - `missing_slashes`: URL has some slashes missing
* - `idn`: URL has international characters * - `idn`: URL has international characters
* - `has_port`: URL has port * - `has_port`: URL has port
* - `has_user`: URL has user part * - `has_user`: URL has user part

+ 5
- 5
src/lua/lua_util.c View File

LUA_FUNCTION_DEF (util, load_rspamd_config); LUA_FUNCTION_DEF (util, load_rspamd_config);
/*** /***
* @function util.config_from_ucl(any, string) * @function util.config_from_ucl(any, string)
* Load rspamd config from ucl reperesented by any lua table
* Load rspamd config from ucl represented by any lua table
* @return {confg} new configuration object suitable for access * @return {confg} new configuration object suitable for access
*/ */
LUA_FUNCTION_DEF (util, config_from_ucl); LUA_FUNCTION_DEF (util, config_from_ucl);
LUA_FUNCTION_DEF (util, encode_base64); LUA_FUNCTION_DEF (util, encode_base64);
/*** /***
* @function util.encode_qp(input[, str_len, [newlines_type]]) * @function util.encode_qp(input[, str_len, [newlines_type]])
* Encodes data in quouted printable breaking lines if needed
* Encodes data in quoted printable breaking lines if needed
* @param {text or string} input input data * @param {text or string} input input data
* @param {number} str_len optional size of lines or 0 if split is not needed * @param {number} str_len optional size of lines or 0 if split is not needed
* @return {rspamd_text} encoded data chunk * @return {rspamd_text} encoded data chunk


/*** /***
* @function util.decode_qp(input) * @function util.decode_qp(input)
* Decodes data from quouted printable
* Decodes data from quoted printable
* @param {text or string} input input data * @param {text or string} input input data
* @return {rspamd_text} decoded data chunk * @return {rspamd_text} decoded data chunk
*/ */
LUA_FUNCTION_DEF (util, process_message); LUA_FUNCTION_DEF (util, process_message);
/*** /***
* @function util.tanh(num) * @function util.tanh(num)
* Calculates hyperbolic tanhent of the specified floating point value
* Calculates hyperbolic tangent of the specified floating point value
* @param {number} num input number * @param {number} num input number
* @return {number} hyperbolic tanhent of the variable
* @return {number} hyperbolic tangent of the variable
*/ */
LUA_FUNCTION_DEF (util, tanh); LUA_FUNCTION_DEF (util, tanh);



+ 4
- 4
src/lua/lua_xmlrpc.c View File

INIT_LOG_MODULE(xmlrpc) INIT_LOG_MODULE(xmlrpc)


enum lua_xmlrpc_state { enum lua_xmlrpc_state {
read_method_responce = 0,
read_method_response = 0,
read_params = 1, read_params = 1,
read_param = 2, read_param = 2,
read_param_value = 3, read_param_value = 3,
msg_debug_xmlrpc ("got start element %s on state %d", name, last_state); msg_debug_xmlrpc ("got start element %s on state %d", name, last_state);


switch (ud->parser_state) { switch (ud->parser_state) {
case read_method_responce:
case read_method_response:
/* Expect tag methodResponse */ /* Expect tag methodResponse */
if (g_ascii_strcasecmp (name, "methodResponse") == 0) { if (g_ascii_strcasecmp (name, "methodResponse") == 0) {
ud->parser_state = read_params; ud->parser_state = read_params;
msg_debug_xmlrpc ("got end element %s on state %d", name, last_state); msg_debug_xmlrpc ("got end element %s on state %d", name, last_state);


switch (ud->parser_state) { switch (ud->parser_state) {
case read_method_responce:
case read_method_response:
ud->parser_state = error_state; ud->parser_state = error_state;
break; break;
case read_params: case read_params:


if (data != NULL) { if (data != NULL) {
ud.L = L; ud.L = L;
ud.parser_state = read_method_responce;
ud.parser_state = read_method_response;
ud.param_count = 0; ud.param_count = 0;
ud.st = g_queue_new (); ud.st = g_queue_new ();



+ 1
- 1
src/plugins/chartable.c View File

* Allowed options: * Allowed options:
* - symbol (string): symbol to insert (default: 'R_BAD_CHARSET') * - symbol (string): symbol to insert (default: 'R_BAD_CHARSET')
* - threshold (double): value that would be used as threshold in expression characters_changed / total_characters * - threshold (double): value that would be used as threshold in expression characters_changed / total_characters
* (e.g. if threshold is 0.1 than charset change should occure more often than in 10 symbols), default: 0.1
* (e.g. if threshold is 0.1 than charset change should occur more often than in 10 symbols), default: 0.1
*/ */





+ 3
- 3
src/plugins/fuzzy_check.c View File

struct rspamd_custom_controller_command *cmd; struct rspamd_custom_controller_command *cmd;


cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd)); cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd));
cmd->privilleged = TRUE;
cmd->privileged = TRUE;
cmd->require_message = TRUE; cmd->require_message = TRUE;
cmd->handler = fuzzy_add_handler; cmd->handler = fuzzy_add_handler;
cmd->ctx = ctx; cmd->ctx = ctx;
g_hash_table_insert (commands, "/fuzzyadd", cmd); g_hash_table_insert (commands, "/fuzzyadd", cmd);


cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd)); cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd));
cmd->privilleged = TRUE;
cmd->privileged = TRUE;
cmd->require_message = TRUE; cmd->require_message = TRUE;
cmd->handler = fuzzy_delete_handler; cmd->handler = fuzzy_delete_handler;
cmd->ctx = ctx; cmd->ctx = ctx;
g_hash_table_insert (commands, "/fuzzydel", cmd); g_hash_table_insert (commands, "/fuzzydel", cmd);


cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd)); cmd = rspamd_mempool_alloc (fctx->fuzzy_pool, sizeof (*cmd));
cmd->privilleged = TRUE;
cmd->privileged = TRUE;
cmd->require_message = FALSE; cmd->require_message = FALSE;
cmd->handler = fuzzy_deletehash_handler; cmd->handler = fuzzy_deletehash_handler;
cmd->ctx = ctx; cmd->ctx = ctx;

+ 11
- 11
src/plugins/lua/bayes_expiry.lua View File

local tokens = {} local tokens = {}


-- Tokens occurrences distribution counters -- Tokens occurrences distribution counters
local occurr = {
local occur = {
ham = {}, ham = {},
spam = {}, spam = {},
total = {} total = {}


for k,v in pairs({['ham']=ham, ['spam']=spam, ['total']=total}) do for k,v in pairs({['ham']=ham, ['spam']=spam, ['total']=total}) do
if tonumber(v) > 19 then v = 20 end if tonumber(v) > 19 then v = 20 end
occurr[k][v] = occurr[k][v] and occurr[k][v] + 1 or 1
occur[k][v] = occur[k][v] and occur[k][v] + 1 or 1
end end
end end
end end


local occ_distr = {} local occ_distr = {}
for _,cl in pairs({'ham', 'spam', 'total'}) do for _,cl in pairs({'ham', 'spam', 'total'}) do
local occurr_key = pattern_sha1 .. '_occurrence_' .. cl
local occur_key = pattern_sha1 .. '_occurrence_' .. cl


if cursor ~= 0 then if cursor ~= 0 then
local n local n
for i,v in ipairs(redis.call('HGETALL', occurr_key)) do
for i,v in ipairs(redis.call('HGETALL', occur_key)) do
if i % 2 == 1 then if i % 2 == 1 then
n = tonumber(v) n = tonumber(v)
else else
occurr[cl][n] = occurr[cl][n] and occurr[cl][n] + v or v
occur[cl][n] = occur[cl][n] and occur[cl][n] + v or v
end end
end end


local str = '' local str = ''
if occurr[cl][0] ~= nil then
str = '0:' .. occurr[cl][0] .. ','
if occur[cl][0] ~= nil then
str = '0:' .. occur[cl][0] .. ','
end end
for k,v in ipairs(occurr[cl]) do
for k,v in ipairs(occur[cl]) do
if k == 20 then k = '>19' end if k == 20 then k = '>19' end
str = str .. k .. ':' .. v .. ',' str = str .. k .. ':' .. v .. ','
end end
table.insert(occ_distr, str) table.insert(occ_distr, str)
else else
redis.call('DEL', occurr_key)
redis.call('DEL', occur_key)
end end


if next(occurr[cl]) ~= nil then
redis.call('HMSET', occurr_key, unpack_function(hash2list(occurr[cl])))
if next(occur[cl]) ~= nil then
redis.call('HMSET', occur_key, unpack_function(hash2list(occur[cl])))
end end
end end



+ 1
- 1
src/plugins/lua/bimi.lua View File

-- We can either check BIMI via DNS or check Redis cache -- We can either check BIMI via DNS or check Redis cache
-- BIMI check is an external check, so we might prefer Redis to be checked -- BIMI check is an external check, so we might prefer Redis to be checked
-- first. On the other hand, DNS request is cheaper and counting low BIMI -- first. On the other hand, DNS request is cheaper and counting low BIMI
-- adoptation we would need to have both Redis and DNS request to hit no
-- adaptation we would need to have both Redis and DNS request to hit no
-- result. So, it might be better to check DNS first at this stage... -- result. So, it might be better to check DNS first at this stage...
check_bimi_dns(task, dmarc_domain_maybe) check_bimi_dns(task, dmarc_domain_maybe)
end end

+ 1
- 1
src/plugins/lua/clickhouse.lua View File

max_memory = 50 * 1024 * 1024, -- How many memory should be occupied before sending collection max_memory = 50 * 1024 * 1024, -- How many memory should be occupied before sending collection
max_interval = 60, -- Maximum collection interval max_interval = 60, -- Maximum collection interval
}, },
collect_garbage = false, -- Peform GC collection after sending the data
collect_garbage = false, -- Perform GC collection after sending the data
check_timeout = 10.0, -- Periodic timeout check_timeout = 10.0, -- Periodic timeout
timeout = 5.0, timeout = 5.0,
bayes_spam_symbols = {'BAYES_SPAM'}, bayes_spam_symbols = {'BAYES_SPAM'},

+ 1
- 1
src/plugins/lua/maillist.lua View File

return false return false
end end


-- Mailmain 3 allows to disable all List-* headers in settings, but by default it adds them.
-- Mailman 3 allows to disable all List-* headers in settings, but by default it adds them.
-- In all other cases all Mailman message should have List-Id header -- In all other cases all Mailman message should have List-Id header
if not task:has_header('List-Id') then if not task:has_header('List-Id') then
return false return false

+ 1
- 1
src/plugins/lua/multimap.lua View File

end end
end, fun.filter(function(r) return not r['prefilter'] end, rules)) end, fun.filter(function(r) return not r['prefilter'] end, rules))


-- prefilter symbils
-- prefilter symbols
fun.each(function(rule) fun.each(function(rule)
rspamd_config:register_symbol({ rspamd_config:register_symbol({
type = 'prefilter', type = 'prefilter',

+ 4
- 4
src/plugins/lua/rbl.lua View File

end end
else else
local to_resolve local to_resolve
local orign = req
local origin = req


if not resolve_ip then if not resolve_ip then
orign = maybe_make_hash(req, rule)
origin = maybe_make_hash(req, rule)
to_resolve = string.format('%s.%s', to_resolve = string.format('%s.%s',
orign,
origin,
rule.rbl) rule.rbl)
else else
-- First, resolve origin stuff without hashing or anything -- First, resolve origin stuff without hashing or anything
to_resolve = orign
to_resolve = origin
end end


nreq = { nreq = {

+ 1
- 1
src/plugins/lua/whitelist.lua View File

local mult local mult
local how = 'wl' local how = 'wl'


-- Can be overriden
-- Can be overridden
if rule.blacklist then how = 'bl' end if rule.blacklist then how = 'bl' end


local function parse_val(val) local function parse_val(val)

+ 6
- 6
src/rspamd.c View File

return -1; return -1;
} }


if (main->is_privilleged) {
if (main->is_privileged) {
/* Force root user as owner of pid file */ /* Force root user as owner of pid file */
#ifdef HAVE_PIDFILE_FILENO #ifdef HAVE_PIDFILE_FILENO
if (fchown (pidfile_fileno (main->pfh), 0, 0) == -1) { if (fchown (pidfile_fileno (main->pfh), 0, 0) == -1) {
return 0; return 0;
} }


/* Detect privilleged mode */
/* Detect privileged mode */
static void static void
detect_priv (struct rspamd_main *rspamd_main) detect_priv (struct rspamd_main *rspamd_main)
{ {
if (euid == 0) { if (euid == 0) {
if (!rspamd_main->cfg->rspamd_user && !is_insecure) { if (!rspamd_main->cfg->rspamd_user && !is_insecure) {
msg_err_main ( msg_err_main (
"cannot run rspamd workers as root user, please add -u and -g options to select a proper unprivilleged user or specify --insecure flag");
"cannot run rspamd workers as root user, please add -u and -g options to select a proper unprivileged user or specify --insecure flag");
exit (EXIT_FAILURE); exit (EXIT_FAILURE);
} }
else if (is_insecure) { else if (is_insecure) {
rspamd_main->is_privilleged = TRUE;
rspamd_main->is_privileged = TRUE;
rspamd_main->workers_uid = 0; rspamd_main->workers_uid = 0;
rspamd_main->workers_gid = 0; rspamd_main->workers_gid = 0;
} }
else { else {
rspamd_main->is_privilleged = TRUE;
rspamd_main->is_privileged = TRUE;
pwd = getpwnam (rspamd_main->cfg->rspamd_user); pwd = getpwnam (rspamd_main->cfg->rspamd_user);
if (pwd == NULL) { if (pwd == NULL) {
msg_err_main ("user specified does not exists (%s), aborting", msg_err_main ("user specified does not exists (%s), aborting",
} }
} }
else { else {
rspamd_main->is_privilleged = FALSE;
rspamd_main->is_privileged = FALSE;
rspamd_main->workers_uid = (uid_t)-1; rspamd_main->workers_uid = (uid_t)-1;
rspamd_main->workers_gid = (gid_t)-1; rspamd_main->workers_gid = (gid_t)-1;
} }

+ 2
- 2
src/rspamd.h View File

rspamd_logger_t *logger; rspamd_logger_t *logger;
uid_t workers_uid; /**< worker's uid running to */ uid_t workers_uid; /**< worker's uid running to */
gid_t workers_gid; /**< worker's gid running to */ gid_t workers_gid; /**< worker's gid running to */
gboolean is_privilleged; /**< true if run in privilleged mode */
gboolean is_privileged; /**< true if run in privileged mode */
gboolean wanna_die; /**< no respawn of processes */ gboolean wanna_die; /**< no respawn of processes */
gboolean cores_throttling; /**< turn off cores when limits are exceeded */ gboolean cores_throttling; /**< turn off cores when limits are exceeded */
struct roll_history *history; /**< rolling history */ struct roll_history *history; /**< rolling history */
*/ */
void register_custom_controller_command (const gchar *name, void register_custom_controller_command (const gchar *name,
controller_func_t handler, controller_func_t handler,
gboolean privilleged,
gboolean privileged,
gboolean require_message); gboolean require_message);


#ifdef __cplusplus #ifdef __cplusplus

+ 2
- 2
test/coverage.md View File

- To make it possible, we explicitly run `umask 0000` in "build" and "functional" stages in .circleci/config.yml - To make it possible, we explicitly run `umask 0000` in "build" and "functional" stages in .circleci/config.yml
- After run, we persist coverage data in "coverage.${CIRCLE\_JOB}.dump" during this build flow, see `capture_coverage_data`, - After run, we persist coverage data in "coverage.${CIRCLE\_JOB}.dump" during this build flow, see `capture_coverage_data`,
to use it on the final stage. to use it on the final stage.
- we user `cpp-coverals` because it is able to save data for coveralls without actually sending it. We send on our own
- we use `cpp-coveralls` because it is able to save data for coveralls without actually sending it. We send on our own
along with Lua-coverage. along with Lua-coverage.


Lua coverage Lua coverage


1. Coverage collecting is initiated and dumped in `test/functional/lua/test_coverage.lua` (there are a lot of comments inside). 1. Coverage collecting is initiated and dumped in `test/functional/lua/test_coverage.lua` (there are a lot of comments inside).
This file should be included on the very early stage of test run. Usually it's included via config. This file should be included on the very early stage of test run. Usually it's included via config.
2. Coverage is dumped into ${TMPDIR}/%{woker_name}.luacov.stats.out
2. Coverage is dumped into ${TMPDIR}/%{worker_name}.luacov.stats.out
3. All worker coverage reports are merged into `lua_coverage_report.json` (see `collect_lua_coverage()`) 3. All worker coverage reports are merged into `lua_coverage_report.json` (see `collect_lua_coverage()`)
4. finally, `lua_coverage_report.json` is persisted in build flow (see `functional` stage) 4. finally, `lua_coverage_report.json` is persisted in build flow (see `functional` stage)



+ 5
- 5
test/rspamd_cryptobox_test.c View File

} }


static int static int
create_constrainted_split (struct rspamd_cryptobox_segment *seg, int mseg,
create_constrained_split (struct rspamd_cryptobox_segment *seg, int mseg,
int constraint, int constraint,
guchar *begin, guchar *end) guchar *begin, guchar *end)
{ {


msg_info ("realistic split of %d chunks encryption: %.0f", cnt, t2 - t1); msg_info ("realistic split of %d chunks encryption: %.0f", cnt, t2 - t1);


cnt = create_constrainted_split (seg, max_seg + 1, 32, begin, end);
cnt = create_constrained_split (seg, max_seg + 1, 32, begin, end);
t1 = rspamd_get_ticks (TRUE); t1 = rspamd_get_ticks (TRUE);
rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac, mode); rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac, mode);
t2 = rspamd_get_ticks (TRUE); t2 = rspamd_get_ticks (TRUE);


check_result (key, nonce, mac, begin, end); check_result (key, nonce, mac, begin, end);


msg_info ("constrainted split of %d chunks encryption: %.0f", cnt, t2 - t1);
msg_info ("constrained split of %d chunks encryption: %.0f", cnt, t2 - t1);


for (i = 0; i < random_fuzz_cnt; i ++) { for (i = 0; i < random_fuzz_cnt; i ++) {
ms = ottery_rand_range (i % max_seg * 2) + 1; ms = ottery_rand_range (i % max_seg * 2) + 1;
} }
for (i = 0; i < random_fuzz_cnt; i ++) { for (i = 0; i < random_fuzz_cnt; i ++) {
ms = ottery_rand_range (i % max_seg * 10) + 1; ms = ottery_rand_range (i % max_seg * 10) + 1;
cnt = create_constrainted_split (seg, ms, i, begin, end);
cnt = create_constrained_split (seg, ms, i, begin, end);
t1 = rspamd_get_ticks (TRUE); t1 = rspamd_get_ticks (TRUE);
rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac, mode); rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac, mode);
t2 = rspamd_get_ticks (TRUE); t2 = rspamd_get_ticks (TRUE);
check_result (key, nonce, mac, begin, end); check_result (key, nonce, mac, begin, end);


if (i % 1000 == 0) { if (i % 1000 == 0) {
msg_info ("constrainted fuzz iterations: %d", i);
msg_info ("constrained fuzz iterations: %d", i);
} }
} }



+ 2
- 2
test/rspamd_mem_pool_test.c View File

#include <sys/wait.h> #include <sys/wait.h>
#endif #endif


#define TEST_BUF "test bufffer"
#define TEST2_BUF "test bufffertest bufffer"
#define TEST_BUF "test buffer"
#define TEST2_BUF "test buffertest buffer"


void void
rspamd_mem_pool_test_func () rspamd_mem_pool_test_func ()

+ 1
- 1
test/rspamd_test_suite.c View File

rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0);
cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT);
cfg->libs_ctx = rspamd_init_libs (); cfg->libs_ctx = rspamd_init_libs ();
/* More agressive GC, workaround for 'not enough memory' test failures */
/* More aggressive GC, workaround for 'not enough memory' test failures */
cfg->lua_gc_step *= 2; cfg->lua_gc_step *= 2;
rspamd_main->cfg = cfg; rspamd_main->cfg = cfg;
cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0); cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL, 0);

+ 1
- 1
test/tools/dump_coveralls.py View File

warn("{}: no source_files, not a coveralls.io payload?".format(json_file)) warn("{}: no source_files, not a coveralls.io payload?".format(json_file))
return 1 return 1


print("{} ({} soource files)".format(json_file, len(data['source_files'])))
print("{} ({} source files)".format(json_file, len(data['source_files'])))


for src_file in sorted(data['source_files'], key=itemgetter('name')): for src_file in sorted(data['source_files'], key=itemgetter('name')):
covered_lines = not_skipped_lines = 0 covered_lines = not_skipped_lines = 0

+ 0
- 0
test/tools/gcov_coveralls.py View File


Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save