diff options
69 files changed, 1787 insertions, 672 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml index b1fb59779..19a2d3057 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,6 +55,10 @@ jobs: - run: sudo apt-get install -qq cmake libevent-dev libglib2.0-dev libicu-dev libluajit-5.1-dev libmagic-dev libsqlite3-dev libssl-dev ragel libunwind-dev libunwind8 - run: mkdir ../build ; mkdir ../install ; cd ../build + # this weird peice is needed to properly collect coverage + # rspamd works under "nobody" user and will not be able to dump + # the coverage if directories have restrictive permissions + - run: umask 0000 - run: cmake ../project -DDBDIR=/nana -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON -DCMAKE_INSTALL_PREFIX=../install - run: make install -j`nproc` @@ -111,7 +115,8 @@ jobs: - run: sudo luarocks install luacov - run: cd ../build - - run: set +e; RSPAMD_INSTALLROOT=../install sudo -E robot -x xunit.xml --exclude isbroken ../project/test/functional/cases; echo "export RETURN_CODE=$?" >> $BASH_ENV + # see coverage notice in "build" stage + - run: set +e; RSPAMD_INSTALLROOT=../install sudo -E bash -c "umask 0000; robot -x xunit.xml --exclude isbroken ../project/test/functional/cases"; echo "export RETURN_CODE=$?" >> $BASH_ENV - *capture_coverage_data diff --git a/.eslintrc.json b/.eslintrc.json index 6af989a1f..722f2b8ae 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -23,7 +23,7 @@ "singleLine": { "afterColon": false } }], "max-params": ["warn", 6], - "max-statements": ["warn", 28], + "max-statements": ["warn", 30], "max-statements-per-line": ["error", { "max": 2 }], "multiline-comment-style": "off", "multiline-ternary": ["error", "always-multiline"], diff --git a/.luacheckrc b/.luacheckrc index a56ac63fc..9b899c216 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -30,7 +30,9 @@ globals = { 'rspamd_maps', 'rspamd_plugins_state', 'rspamadm', - 'loadstring' + 'loadstring', + 'rspamadm_ev_base', + 'rspamadm_session', } ignore = { diff --git a/conf/composites.conf b/conf/composites.conf index 49a3f43d9..7197adc8f 100644 --- a/conf/composites.conf +++ b/conf/composites.conf @@ -46,7 +46,7 @@ composites { } RBL_SPAMHAUS_XBL_ANY { expression = "RBL_SPAMHAUS_XBL & RECEIVED_SPAMHAUS_XBL"; - description = "Message was relayed through at least one hop listed in Spamhaus XBL"; + description = "From and Received address are listed in Spamhaus XBL"; } AUTH_NA { expression = "R_DKIM_NA & R_SPF_NA & DMARC_NA"; diff --git a/conf/groups.conf b/conf/groups.conf index 04b777e6b..02e714174 100644 --- a/conf/groups.conf +++ b/conf/groups.conf @@ -59,6 +59,12 @@ group "policies" { .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/policies_group.conf" } +group "whitelist" { + .include "$CONFDIR/scores.d/whitelist_group.conf" + .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/whitelist_group.conf" + .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/whitelist_group.conf" +} + group "surbl" { .include "$CONFDIR/scores.d/surbl_group.conf" .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/surbl_group.conf" diff --git a/conf/modules.d/antivirus.conf b/conf/modules.d/antivirus.conf index 16f38fbda..803820dbb 100644 --- a/conf/modules.d/antivirus.conf +++ b/conf/modules.d/antivirus.conf @@ -18,8 +18,12 @@ antivirus { clamav { # If set force this action if any virus is found (default unset: no action is forced) # action = "reject"; - # if `true` only messages with non-image attachments will be checked (default true) - attachments_only = true; + # message = '${SCANNER}: virus found: "${VIRUS}"'; + # Scan mime_parts seperately - otherwise the complete mail will be transfered to AV Scanner + #scan_mime_parts = true; + # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity) + #scan_text_mime = false; + #scan_image_mime = false; # If `max_size` is set, messages > n bytes in size are not scanned #max_size = 20000000; # symbol to add (add it to metric if you want non-zero weight) diff --git a/conf/modules.d/dcc.conf b/conf/modules.d/dcc.conf index ea774bdca..d7622374a 100644 --- a/conf/modules.d/dcc.conf +++ b/conf/modules.d/dcc.conf @@ -14,10 +14,14 @@ # See https://rspamd.com/doc/tutorials/writing_rules.html for details dcc { - # host = "/var/dcc/dccifd"; - # Port is only required if `dccifd` listens on a TCP socket - # port = 1234 - timeout = 2s; + + enabled = false; + + # Define local socket or TCP servers in upstreams syntax + # When sockets and servers are definined - servers is used! + socket = "/var/dcc/dccifd"; # Unix socket + #servers = "127.0.0.1:10045" # OR TCP upstreams + timeout = 2s; # Timeout to wait for checks .include(try=true,priority=5) "${DBDIR}/dynamic/dcc.conf" .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/dcc.conf" diff --git a/conf/modules.d/whitelist.conf b/conf/modules.d/whitelist.conf index 0b19002c8..df4c30248 100644 --- a/conf/modules.d/whitelist.conf +++ b/conf/modules.d/whitelist.conf @@ -22,7 +22,7 @@ whitelist { "fallback+file://${CONFDIR}/spf_whitelist.inc" ]; score = -1.0 - description = "Mail comes from the whitelisted domain and has a valid SPF policy"; + inverse_symbol = "BLACKLIST_SPF"; } "WHITELIST_DKIM" = { @@ -31,8 +31,8 @@ whitelist { "${DBDIR}/dkim_whitelist.inc.local", "fallback+file://${CONFDIR}/dkim_whitelist.inc" ]; - description = "Mail comes from the whitelisted domain and has a valid DKIM signature"; - score = -1.0 + score = -1.0; + inverse_symbol = "BLACKLIST_DKIM"; } "WHITELIST_SPF_DKIM" = { valid_spf = true; @@ -43,17 +43,17 @@ whitelist { "fallback+file://${CONFDIR}/spf_dkim_whitelist.inc" ]; score = -3.0; - description = "Mail comes from the whitelisted domain and has valid SPF and DKIM policies"; + inverse_symbol = "BLACKLIST_SPF_DKIM"; } "WHITELIST_DMARC" = { valid_dmarc = true; domains = [ - "https://maps.rspamd.com/rspamd/dmarc_whitelist.inc.zst", + "https://maps.rspamd.com/rspamd/dmarc_whitelist_new.inc.zst", "${DBDIR}/dmarc_whitelist.inc.local", "fallback+file://${CONFDIR}/dmarc_whitelist.inc" ]; score = -7.0; - description = "Mail comes from the whitelisted domain and has valid DMARC and DKIM policies"; + inverse_symbol = "BLACKLIST_DMARC"; } } diff --git a/conf/scores.d/whitelist_group.conf b/conf/scores.d/whitelist_group.conf new file mode 100644 index 000000000..900aa3680 --- /dev/null +++ b/conf/scores.d/whitelist_group.conf @@ -0,0 +1,54 @@ +# Whitelist rules scores +# +# Please don't modify this file as your changes might be overwritten with +# the next update. +# +# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine +# parameters defined on the top level +# +# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add +# parameters defined on the top level +# +# For specific modules or configuration you can also modify +# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults +# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults +# +# See https://rspamd.com/doc/tutorials/writing_rules.html for details + + +max_score = 10.0; + +symbols = { + "WHITELIST_SPF" { + weight = -1.0; + description = "Mail comes from the whitelisted domain and has a valid SPF policy"; + } + "BLACKLIST_SPF" { + weight = 1.0; + description = "Mail comes from the whitelisted domain and has no valid SPF policy"; + } + "WHITELIST_DKIM" { + weight = -1.0; + description = "Mail comes from the whitelisted domain and has a valid DKIM signature"; + } + "BLACKLIST_DKIM" { + weight = 2.0; + description = "Mail comes from the whitelisted domain and has non-valid DKIM signature"; + } + "WHITELIST_SPF_DKIM" { + weight = -3.0; + description = "Mail comes from the whitelisted domain and has valid SPF and DKIM policies"; + } + "BLACKLIST_SPF_DKIM" { + weight = 3.0; + description = "Mail comes from the whitelisted domain and has no valid SPF policy or a bad DKIM signature"; + } + "WHITELIST_DMARC" { + weight = -7.0; + description = "Mail comes from the whitelisted domain and has valid DMARC and DKIM policies"; + } + "BLACKLIST_DMARC" { + weight = 6.0; + description = "Mail comes from the whitelisted domain and has valid failed DMARC and DKIM policies"; + } +} diff --git a/contrib/aho-corasick/acism_create.c b/contrib/aho-corasick/acism_create.c index bd8586c82..15f9e801e 100644 --- a/contrib/aho-corasick/acism_create.c +++ b/contrib/aho-corasick/acism_create.c @@ -50,7 +50,7 @@ static TNODE* find_child(TNODE*, SYMBOL); static inline void set_tran(ac_trie_t *psp, STATE s, SYMBOL sym, int match, int suffix, TRAN ns) { - psp->tranv[s + sym] = sym | (match ? IS_MATCH : 0) + psp->tranv[s + sym] = sym | (match ? IS_MATCH : 0) | (suffix ? IS_SUFFIX : 0) | (ns << SYM_BITS); } @@ -151,7 +151,7 @@ fill_symv(ac_trie_t *psp, ac_trie_pat_t const *strv, int nstrs) ++psp->nsyms; #if ACISM_SIZE < 8 psp->sym_bits = bitwid(psp->nsyms); - psp->sym_mask = ~(-1 << psp->sym_bits); + psp->sym_mask = ~((~(uint32_t)0u) << psp->sym_bits); #endif } diff --git a/contrib/librdns/packet.c b/contrib/librdns/packet.c index e3020d7e8..35f4a9601 100644 --- a/contrib/librdns/packet.c +++ b/contrib/librdns/packet.c @@ -235,18 +235,23 @@ rdns_format_dns_name (struct rdns_resolver *resolver, const char *in, return false; } +#define U16_TO_WIRE_ADVANCE(val, p8) \ + *p8++ = (uint8_t)(((uint16_t)(val)) >> 8); \ + *p8++ = (uint8_t)(((uint16_t)(val)) & 0xFF); + bool rdns_add_rr (struct rdns_request *req, const char *name, unsigned int len, enum dns_type type, struct rdns_compression_entry **comp) { - uint16_t *p; + uint8_t *p8; if (!rdns_write_name_compressed (req, name, len, comp)) { return false; } - p = (uint16_t *)(req->packet + req->pos); - *p++ = htons (type); - *p = htons (DNS_C_IN); + + p8 = (req->packet + req->pos); + U16_TO_WIRE_ADVANCE (type, p8); + U16_TO_WIRE_ADVANCE (DNS_C_IN, p8); req->pos += sizeof (uint16_t) * 2; return true; @@ -256,18 +261,13 @@ bool rdns_add_edns0 (struct rdns_request *req) { uint8_t *p8; - uint16_t *p16; - - p8 = (uint8_t *)(req->packet + req->pos); - *p8 = '\0'; /* Name is root */ - p16 = (uint16_t *)(req->packet + req->pos + 1); - *p16++ = htons (DNS_T_OPT); - /* UDP packet length */ - *p16++ = htons (UDP_PACKET_SIZE); - /* Extended rcode 00 00 */ - *p16++ = 0; - /* Z 10000000 00000000 to allow dnssec */ - p8 = (uint8_t *)p16; + + p8 = (req->packet + req->pos); + *p8++ = '\0'; /* Name is root */ + U16_TO_WIRE_ADVANCE (DNS_T_OPT, p8); + U16_TO_WIRE_ADVANCE (UDP_PACKET_SIZE, p8); + U16_TO_WIRE_ADVANCE (0, p8); + if (req->resolver->enable_dnssec) { *p8++ = 0x80; } @@ -275,9 +275,9 @@ rdns_add_edns0 (struct rdns_request *req) *p8++ = 0x00; } *p8++ = 0; - p16 = (uint16_t *)p8; /* Length */ - *p16 = 0; + U16_TO_WIRE_ADVANCE (0, p8); + req->pos += sizeof (uint8_t) + sizeof (uint16_t) * 5; return true; diff --git a/contrib/libucl/ucl_schema.c b/contrib/libucl/ucl_schema.c index 4d7c871ff..d11d66909 100644 --- a/contrib/libucl/ucl_schema.c +++ b/contrib/libucl/ucl_schema.c @@ -49,7 +49,16 @@ static bool ucl_schema_validate (const ucl_object_t *schema, /* * Create validation error */ -static void + +#ifdef __GNUC__ +static inline void +ucl_schema_create_error (struct ucl_schema_error *err, + enum ucl_schema_error_code code, const ucl_object_t *obj, + const char *fmt, ...) +__attribute__ (( format( printf, 4, 5) )); +#endif + +static inline void ucl_schema_create_error (struct ucl_schema_error *err, enum ucl_schema_error_code code, const ucl_object_t *obj, const char *fmt, ...) @@ -69,7 +78,7 @@ ucl_schema_create_error (struct ucl_schema_error *err, * Check whether we have a pattern specified */ static const ucl_object_t * -ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern) +ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern, bool recursive) { const ucl_object_t *res = NULL; #ifdef HAVE_REGEX_H @@ -78,11 +87,16 @@ ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern) ucl_object_iter_t iter = NULL; if (regcomp (®, pattern, REG_EXTENDED | REG_NOSUB) == 0) { - while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { - if (regexec (®, ucl_object_key (elt), 0, NULL, 0) == 0) { - res = elt; - break; + if (recursive) { + while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { + if (regexec (®, ucl_object_key (elt), 0, NULL, 0) == 0) { + res = elt; + break; + } } + } else { + if (regexec (®, ucl_object_key (obj), 0, NULL, 0) == 0) + res = obj; } regfree (®); } @@ -205,12 +219,17 @@ ucl_schema_validate_object (const ucl_object_t *schema, } } else if (strcmp (ucl_object_key (elt), "patternProperties") == 0) { + const ucl_object_t *vobj; + ucl_object_iter_t viter; piter = NULL; while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) { - found = ucl_schema_test_pattern (obj, ucl_object_key (prop)); - if (found) { - ret = ucl_schema_validate (prop, found, true, err, root, - ext_ref); + viter = NULL; + while (ret && (vobj = ucl_object_iterate (obj, &viter, true)) != NULL) { + found = ucl_schema_test_pattern (vobj, ucl_object_key (prop), false); + if (found) { + ret = ucl_schema_validate (prop, found, true, err, root, + ext_ref); + } } } } @@ -234,7 +253,7 @@ ucl_schema_validate_object (const ucl_object_t *schema, piter = NULL; pat = ucl_object_lookup (schema, "patternProperties"); while ((pelt = ucl_object_iterate (pat, &piter, true)) != NULL) { - found = ucl_schema_test_pattern (obj, ucl_object_key (pelt)); + found = ucl_schema_test_pattern (obj, ucl_object_key (pelt), true); if (found != NULL) { break; } @@ -301,7 +320,7 @@ ucl_schema_validate_number (const ucl_object_t *schema, if (fabs (remainder (val, constraint)) > alpha) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "number %.4f is not multiple of %.4f, remainder is %.7f", - val, constraint); + val, constraint, remainder (val, constraint)); ret = false; break; } @@ -361,7 +380,7 @@ ucl_schema_validate_string (const ucl_object_t *schema, constraint = ucl_object_toint (elt); if (obj->len > constraint) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, - "string is too big: %.3f, maximum is: %.3f", + "string is too big: %u, maximum is: %" PRId64, obj->len, constraint); ret = false; break; @@ -372,7 +391,7 @@ ucl_schema_validate_string (const ucl_object_t *schema, constraint = ucl_object_toint (elt); if (obj->len < constraint) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, - "string is too short: %.3f, minimum is: %.3f", + "string is too short: %u, minimum is: %" PRId64, obj->len, constraint); ret = false; break; diff --git a/interface/index.html b/interface/index.html index 6eebef1d4..8d40da510 100644 --- a/interface/index.html +++ b/interface/index.html @@ -291,7 +291,8 @@ <div class="form-group widget-title-form"> <label for="selSymOrder">Symbols order:</label> <select id="selSymOrder" class="form-control"> - <option value="score" selected>Score</option> + <option value="magnitude" selected>Score magnitude</option> + <option value="score">Score value</option> <option value="name">Name</option> </select> </div> @@ -356,11 +357,11 @@ </div> <!-- login modal --> -<div id="connectDialog" class="modal" tabindex="-1" role="dialog" aria-labelledby="RSPAMD Connect"> +<div id="connectDialog" class="modal" tabindex="-1" role="dialog" style="display:none"> <div class="modal-dialog"> <div class="modal-content"> <div class="modal-header"> - <h3>RSPAMD Connect</h3> + <h3>Login to Rspamd</h3> </div> <div class="modal-body" id="connectBody"> <form id="connectForm"> diff --git a/interface/js/app/history.js b/interface/js/app/history.js index f82925b26..ca6b21c47 100644 --- a/interface/js/app/history.js +++ b/interface/js/app/history.js @@ -42,6 +42,7 @@ define(["jquery", "footable", "humanize"], "=": "=" }; var htmlEscaper = /[&<>"'/`=]/g; + var symbols = []; var symbolDescriptions = {}; var escapeHTML = function (string) { @@ -112,22 +113,43 @@ define(["jquery", "footable", "humanize"], }; } - function process_history_v2(data) { - // Display no more than rcpt_lim recipients - var rcpt_lim = 3; - var items = []; + function get_compare_function() { + var compare_functions = { + magnitude: function (e1, e2) { + return Math.abs(e2.score) - Math.abs(e1.score); + }, + name: function (e1, e2) { + return e1.name.localeCompare(e2.name); + }, + score: function (e1, e2) { + return e2.score - e1.score; + } + }; function getSelector(id) { var e = document.getElementById(id); return e.options[e.selectedIndex].value; } - var compare = (getSelector("selSymOrder") === "score") - ? function (e1, e2) { - return Math.abs(e2.score) - Math.abs(e1.score); - } - : function (e1, e2) { - return e1.name.localeCompare(e2.name); - }; + + return compare_functions[getSelector("selSymOrder")]; + } + + function sort_symbols(o, compare_function) { + return Object.keys(o) + .map(function (key) { + return o[key]; + }) + .sort(compare_function) + .map(function (e) { return e.str; }) + .join("<br>\n"); + } + + function process_history_v2(data) { + // Display no more than rcpt_lim recipients + var rcpt_lim = 3; + var items = []; + var unsorted_symbols = []; + var compare_function = get_compare_function(); $("#selSymOrder, label[for='selSymOrder']").show(); @@ -172,15 +194,10 @@ define(["jquery", "footable", "humanize"], if (sym.options) { str += "[" + sym.options.join(",") + "]"; } - item.symbols[key].str = str; + sym.str = str; }); - item.symbols = Object.keys(item.symbols) - .map(function (key) { - return item.symbols[key]; - }) - .sort(compare) - .map(function (e) { return e.str; }) - .join("<br>\n"); + unsorted_symbols.push(item.symbols); + item.symbols = sort_symbols(item.symbols, compare_function); item.time = { value: unix_time_format(item.unix_time), options: { @@ -214,7 +231,7 @@ define(["jquery", "footable", "humanize"], items.push(item); }); - return items; + return {items:items, symbols:unsorted_symbols}; } function process_history_legacy(data) { @@ -252,7 +269,7 @@ define(["jquery", "footable", "humanize"], items.push(item); }); - return items; + return {items:items}; } function columns_v2() { @@ -650,7 +667,9 @@ define(["jquery", "footable", "humanize"], // Legacy version data = [].concat.apply([], neighbours_data); } - var items = process_history_data(data); + var o = process_history_data(data); + var items = o.items; + symbols = o.symbols; if (Object.prototype.hasOwnProperty.call(tables, "history") && version === prevVersion) { @@ -684,7 +703,12 @@ define(["jquery", "footable", "humanize"], ui.getHistory(rspamd, tables); }); $("#selSymOrder").unbind().change(function () { - ui.getHistory(rspamd, tables); + var compare_function = get_compare_function(); + $.each(tables.history.rows.all, function (i, row) { + var cell_val = sort_symbols(symbols[i], compare_function); + row.cells[8].val(cell_val, false, true); + }); + drawTooltips(); }); // @reset history log diff --git a/interface/js/app/rspamd.js b/interface/js/app/rspamd.js index fde541285..29bbfbcf1 100644 --- a/interface/js/app/rspamd.js +++ b/interface/js/app/rspamd.js @@ -134,13 +134,6 @@ function ($, D3pie, visibility, NProgress, tab_stat, tab_graph, tab_config, sessionStorage.setItem("Password", password); } - function isLogged() { - if (sessionStorage.getItem("Credentials") !== null) { - return true; - } - return false; - } - function displayUI() { // In many browsers local storage can only store string. // So when we store the boolean true or false, it actually stores the strings "true" or "false". @@ -300,53 +293,59 @@ function ($, D3pie, visibility, NProgress, tab_stat, tab_graph, tab_config, }; ui.connect = function () { - if (isLogged()) { - displayUI(); - return; - } - - var dialog = $("#connectDialog"); - var backdrop = $("#backDrop"); - $("#mainUI").hide(); - $(dialog).show(); - $(backdrop).show(); - $("#connectPassword").focus(); - $("#connectForm").off("submit"); - - $("#connectForm").on("submit", function (e) { - e.preventDefault(); - var password = $("#connectPassword").val(); - if (!(/^[\u0020-\u007e]*$/).test(password)) { - alertMessage("alert-modal alert-error", "Invalid characters in the password"); + // Query "/stat" to check if user is already logged in or client ip matches "secure_ip" + $.ajax({ + type: "GET", + url: "stat", + async: false, + success: function () { + displayUI(); + }, + error: function () { + var dialog = $("#connectDialog"); + var backdrop = $("#backDrop"); + $("#mainUI").hide(); + $(dialog).show(); + $(backdrop).show(); $("#connectPassword").focus(); - return; - } - - ui.query("auth", { - headers: { - Password: password - }, - success: function (json) { - var data = json[0].data; - $("#connectPassword").val(""); - if (data.auth === "ok") { - sessionStorage.setItem("read_only", data.read_only); - saveCredentials(password); - $(dialog).hide(); - $(backdrop).hide(); - displayUI(); + $("#connectForm").off("submit"); + + $("#connectForm").on("submit", function (e) { + e.preventDefault(); + var password = $("#connectPassword").val(); + if (!(/^[\u0020-\u007e]*$/).test(password)) { + alertMessage("alert-modal alert-error", "Invalid characters in the password"); + $("#connectPassword").focus(); + return; } - }, - error: function (jqXHR) { - ui.alertMessage("alert-modal alert-error", jqXHR.statusText); - $("#connectPassword").val(""); - $("#connectPassword").focus(); - }, - params: { - global: false, - }, - server: "local" - }); + + ui.query("auth", { + headers: { + Password: password + }, + success: function (json) { + var data = json[0].data; + $("#connectPassword").val(""); + if (data.auth === "ok") { + sessionStorage.setItem("read_only", data.read_only); + saveCredentials(password); + $(dialog).hide(); + $(backdrop).hide(); + displayUI(); + } + }, + error: function (jqXHR) { + ui.alertMessage("alert-modal alert-error", jqXHR.statusText); + $("#connectPassword").val(""); + $("#connectPassword").focus(); + }, + params: { + global: false, + }, + server: "local" + }); + }); + } }); }; diff --git a/lualib/lua_dkim_tools.lua b/lualib/lua_dkim_tools.lua index c9c969f4f..5469ac138 100644 --- a/lualib/lua_dkim_tools.lua +++ b/lualib/lua_dkim_tools.lua @@ -20,9 +20,24 @@ local exports = {} local E = {} local lua_util = require "lua_util" local rspamd_util = require "rspamd_util" +local logger = require "rspamd_logger" + +local function check_violation(N, task, domain, selector) + -- Check for DKIM_REJECT + local sym_check = 'R_DKIM_REJECT' + + if N == 'arc' then sym_check = 'ARC_REJECT' end + if task:has_symbol(sym_check) then + local sym = task:get_symbol(sym_check) + logger.infox(task, 'skip signing for %s:%s: violation %s found: %s', + domain, selector, sym_check, sym.options) + return false + end + + return true +end local function parse_dkim_http_headers(N, task, settings) - local logger = require "rspamd_logger" -- Configure headers local headers = { sign_header = settings.http_sign_header or "PerformDkimSign", @@ -46,15 +61,8 @@ local function parse_dkim_http_headers(N, task, settings) -- Now check if we need to check the existing auth local hdr = task:get_request_header(headers.sign_on_reject_header) if not hdr or tostring(hdr) == '0' or tostring(hdr) == 'false' then - -- Check for DKIM_REJECT - local sym_check = 'R_DKIM_REJECT' - - if N == 'arc' then sym_check = 'ARC_REJECT' end - if task:has_symbol(sym_check) then - local sym = task:get_symbol(sym_check) - logger.infox(task, 'skip signing for %s:%s: %s found: %s', - domain, selector, sym_check, sym.options) - return false,{} + if not check_violation(N, task, domain, selector) then + return false, {} end end @@ -269,6 +277,12 @@ local function prepare_dkim_signing(N, task, settings) lua_util.debugm(N, task, 'use default selector "%s"', p.selector) end + if settings.check_violation then + if not check_violation(N, task, p.domain, p.selector) then + return false,{} + end + end + p.domain = dkim_domain return true,p diff --git a/lualib/lua_meta.lua b/lualib/lua_meta.lua index d81709dab..f2b008dd4 100644 --- a/lualib/lua_meta.lua +++ b/lualib/lua_meta.lua @@ -17,6 +17,7 @@ limitations under the License. local exports = {} local N = "metatokens" +local ts = require("tableshape").types -- Metafunctions local function meta_size_function(task) @@ -272,9 +273,10 @@ local metafunctions = { { cb = meta_size_function, ninputs = 1, - desc = { + names = { "size" - } + }, + description = 'Describes size of the message', }, { cb = meta_images_function, @@ -284,65 +286,93 @@ local metafunctions = { -- 3 - number of jpeg images -- 4 - number of large images (> 128 x 128) -- 5 - number of small images (< 128 x 128) - desc = { + names = { 'nimages', 'npng_images', 'njpeg_images', 'nlarge_images', 'nsmall_images' - } + }, + description = [[Functions for images matching: + - number of images, + - number of png images, + - number of jpeg images + - number of large images (> 128 x 128) + - number of small images (< 128 x 128) +]] }, { cb = meta_nparts_function, ninputs = 2, -- 1 - number of text parts -- 2 - number of attachments - desc = { + names = { 'ntext_parts', 'nattachments' - } + }, + description = [[Functions for images matching: + - number of text parts + - number of attachments +]] }, { cb = meta_encoding_function, ninputs = 2, -- 1 - number of utf parts -- 2 - number of non-utf parts - desc = { + names = { 'nutf_parts', 'nascii_parts' - } + }, + description = [[Functions for encoding matching: + - number of utf parts + - number of non-utf parts +]] }, { cb = meta_recipients_function, ninputs = 2, -- 1 - number of mime rcpt -- 2 - number of smtp rcpt - desc = { + names = { 'nmime_rcpt', 'nsmtp_rcpt' - } + }, + description = [[Functions for recipients data matching: + - number of mime rcpt + - number of smtp rcpt +]] }, { cb = meta_received_function, ninputs = 4, - desc = { + names = { 'nreceived', 'nreceived_invalid', 'nreceived_bad_time', 'nreceived_secure' - } + }, + description = [[Functions for received headers data matching: + - number of received headers + - number of bad received headers + - number of skewed time received headers + - number of received via secured relays +]] }, { cb = meta_urls_function, ninputs = 1, - desc = { + names = { 'nurls' - } + }, + description = [[Functions for urls data matching: + - number of urls +]] }, { cb = meta_words_function, ninputs = 9, - desc = { + names = { 'avg_words_len', 'nshort_words', 'spaces_rate', @@ -352,32 +382,95 @@ local metafunctions = { 'non_ascii_characters_rate', 'capital_characters_rate', 'numeric_cahracters' - } + }, + description = [[Functions for words data matching: + - average length of the words + - number of short words + - rate of spaces in the text + - rate of multiple spaces + - rate of non space characters + - rate of ascii characters + - rate of non-ascii characters + - rate of capital letters + - rate of numbers +]] }, } -local function rspamd_gen_metatokens(task) +local meta_schema = ts.shape{ + cb = ts.func, + ninputs = ts.number, + names = ts.array_of(ts.string), + description = ts.string:is_optional() +} + +local metatokens_by_name = {} + +local function fill_metatokens_by_name() + metatokens_by_name = {} + + for _,mt in ipairs(metafunctions) do + for i=1,mt.ninputs do + local name = mt.names[i] + + metatokens_by_name[name] = function(task) + local results = mt.cb(task) + return results[i] + end + end + end +end + +local function calculate_digest() + local cr = require "rspamd_cryptobox_hash" + + local h = cr.create() + for _,mt in ipairs(metafunctions) do + for i=1,mt.ninputs do + local name = mt.names[i] + h:update(name) + end + end + + exports.digest = h:hex() +end + +local function rspamd_gen_metatokens(task, names) local lua_util = require "lua_util" local ipairs = ipairs local metatokens = {} - local cached = task:cache_get('metatokens') - if cached then - return cached - else - for _,mt in ipairs(metafunctions) do - local ct = mt.cb(task) - for i,tok in ipairs(ct) do - lua_util.debugm(N, task, "metatoken: %s = %s", mt.desc[i], tok) - table.insert(metatokens, tok) + if not names then + local cached = task:cache_get('metatokens') + + if cached then + return cached + else + for _,mt in ipairs(metafunctions) do + local ct = mt.cb(task) + for i,tok in ipairs(ct) do + lua_util.debugm(N, task, "metatoken: %s = %s", + mt.names[i], tok) + table.insert(metatokens, tok) + end end + + task:cache_set('metatokens', metatokens) end - task:cache_set('metatokens', metatokens) + else + local logger = require "rspamd_logger" + for _,n in ipairs(names) do + if metatokens_by_name[n] then + table.insert(metatokens, metatokens_by_name[n](task)) + else + logger.errx(task, 'unknown metatoken: %s', n) + end + end end return metatokens -end + end exports.rspamd_gen_metatokens = rspamd_gen_metatokens exports.gen_metatokens = rspamd_gen_metatokens @@ -388,7 +481,7 @@ local function rspamd_gen_metatokens_table(task) for _,mt in ipairs(metafunctions) do local ct = mt.cb(task) for i,tok in ipairs(ct) do - metatokens[mt.desc[i]] = tok + metatokens[mt.names[i]] = tok end end @@ -411,4 +504,20 @@ end exports.rspamd_count_metatokens = rspamd_count_metatokens exports.count_metatokens = rspamd_count_metatokens +exports.add_metafunction = function(tbl) + local ret, err = meta_schema(tbl) + + if not ret then + local logger = require "rspamd_logger" + logger.errx('cannot add metafunction: %s', err) + else + table.insert(metafunctions, tbl) + fill_metatokens_by_name() + calculate_digest() + end +end + +fill_metatokens_by_name() +calculate_digest() + return exports diff --git a/lualib/lua_redis.lua b/lualib/lua_redis.lua index 92ff4cc12..e95981c82 100644 --- a/lualib/lua_redis.lua +++ b/lualib/lua_redis.lua @@ -997,12 +997,6 @@ local function redis_connect_sync(redis_params, is_write, key, cfg, ev_base) if not redis_params then return false,nil end - if not cfg then - cfg = rspamd_config - end - if not ev_base then - ev_base = rspamadm_ev_base - end local rspamd_redis = require "rspamd_redis" local addr @@ -1028,14 +1022,30 @@ local function redis_connect_sync(redis_params, is_write, key, cfg, ev_base) local options = { host = addr:get_addr(), timeout = redis_params['timeout'], - config = cfg, - ev_base = ev_base + config = cfg or rspamd_config, + ev_base = ev_base or rspamadm_ev_base, + session = redis_params.session or rspamadm_session } for k,v in pairs(redis_params) do options[k] = v end + if not options.config then + logger.errx('config is not set') + return false,nil,addr + end + + if not options.ev_base then + logger.errx('ev_base is not set') + return false,nil,addr + end + + if not options.session then + logger.errx('session is not set') + return false,nil,addr + end + local ret,conn = rspamd_redis.connect_sync(options) if not ret then logger.errx('cannot execute redis request: %s', conn) diff --git a/lualib/lua_squeeze_rules.lua b/lualib/lua_squeeze_rules.lua index 9c751a33a..e85d7e012 100644 --- a/lualib/lua_squeeze_rules.lua +++ b/lualib/lua_squeeze_rules.lua @@ -271,15 +271,29 @@ exports.squeeze_init = function() if metric_sym then v.group = metric_sym.group + v.groups = metric_sym.groups v.score = metric_sym.score v.description = metric_sym.description - if not squeezed_groups[v.group] then - logger.debugm(SN, rspamd_config, 'added squeezed group: %s', v.group) - squeezed_groups[v.group] = {} + if v.group then + if not squeezed_groups[v.group] then + logger.debugm(SN, rspamd_config, 'added squeezed group: %s', v.group) + squeezed_groups[v.group] = {} + end + + table.insert(squeezed_groups[v.group], k) + end + if v.groups then + for _,gr in ipairs(v.groups) do + if not squeezed_groups[gr] then + logger.debugm(SN, rspamd_config, 'added squeezed group: %s', gr) + squeezed_groups[gr] = {} + end - table.insert(squeezed_groups[v.group], k) + table.insert(squeezed_groups[gr], k) + end + end else logger.debugm(SN, rspamd_config, 'no metric symbol found for %s, maybe bug', k) end @@ -334,6 +348,7 @@ exports.handle_settings = function(task, settings) if settings.symbols_disabled then found = true for _,s in ipairs(settings.symbols_disabled) do + logger.debugm(SN, task, 'try disable symbol %s as it is in `symbols_disabled`', s) if not symbols_enabled[s] then symbols_disabled[s] = true logger.debugm(SN, task, 'disable symbol %s as it is in `symbols_disabled`', s) @@ -344,6 +359,7 @@ exports.handle_settings = function(task, settings) if settings.groups_disabled then found = true for _,gr in ipairs(settings.groups_disabled) do + logger.debugm(SN, task, 'try disable group %s as it is in `groups_disabled`: %s', gr) if squeezed_groups[gr] then for _,sym in ipairs(squeezed_groups[gr]) do if not symbols_enabled[sym] then diff --git a/lualib/rspamadm/corpus_test.lua b/lualib/rspamadm/corpus_test.lua index 07051a196..eb144107a 100644 --- a/lualib/rspamadm/corpus_test.lua +++ b/lualib/rspamadm/corpus_test.lua @@ -130,9 +130,9 @@ end local function handler(args) opts = parser:parse(args) - local ham_directory = opts['ham_directory'] - local spam_directory = opts['spam_directory'] - local connections = opts["connections"] + local ham_directory = opts['ham'] + local spam_directory = opts['spam'] + local connections = opts["conns"] local output = opts["output"] local results = {} @@ -182,4 +182,4 @@ return { name = 'corpus_test', handler = handler, description = parser._description -}
\ No newline at end of file +} diff --git a/lualib/rspamadm/rescore.lua b/lualib/rspamadm/rescore.lua index cc331c6e8..dfa73f2d5 100644 --- a/lualib/rspamadm/rescore.lua +++ b/lualib/rspamadm/rescore.lua @@ -18,8 +18,6 @@ if not rspamd_config:has_torch() then return end -local torch = require "torch" -local nn = require "nn" local lua_util = require "lua_util" local ucl = require "ucl" local logger = require "rspamd_logger" @@ -28,6 +26,10 @@ local rspamd_util = require "rspamd_util" local argparse = require "argparse" local rescore_utility = require "rescore_utility" +-- Load these lazily +local torch +local nn + local opts local ignore_symbols = { ['DATE_IN_PAST'] =true, @@ -509,6 +511,8 @@ local function get_threshold() end local function handler(args) + torch = require "torch" + nn = require "nn" opts = parser:parse(args) if not opts['log'] then parser:error('no log specified') diff --git a/rules/headers_checks.lua b/rules/headers_checks.lua index d1f972a46..5dc78063a 100644 --- a/rules/headers_checks.lua +++ b/rules/headers_checks.lua @@ -26,6 +26,8 @@ local E = {} local rcvd_cb_id = rspamd_config:register_symbol{ name = 'CHECK_RECEIVED', type = 'callback', + score = 0.0, + group = 'headers', callback = function(task) local cnts = { [1] = 'ONE', @@ -113,6 +115,8 @@ rspamd_config:register_symbol{ local prio_cb_id = rspamd_config:register_symbol { name = 'HAS_X_PRIO', type = 'callback', + score = 0.0, + group = 'headers', callback = function (task) local cnts = { [1] = 'ONE', @@ -180,10 +184,16 @@ local function get_raw_header(task, name) return ((task:get_header_full(name) or {})[1] or {})['value'] end -local check_replyto_id = rspamd_config:register_callback_symbol('CHECK_REPLYTO', 1.0, - function (task) +local check_replyto_id = rspamd_config:register_symbol({ + type = 'callback', + name = 'CHECK_REPLYTO', + score = 0.0, + group = 'headers', + callback = function(task) local replyto = get_raw_header(task, 'Reply-To') - if not replyto then return false end + if not replyto then + return false + end local rt = util.parse_mail_address(replyto, task:get_mempool()) if not (rt and rt[1] and (string.len(rt[1].addr) > 0)) then task:insert_result('REPLYTO_UNPARSEABLE', 1.0) @@ -205,7 +215,9 @@ local check_replyto_id = rspamd_config:register_callback_symbol('CHECK_REPLYTO', -- See if Reply-To matches From in some way local from = task:get_from(2) local from_h = get_raw_header(task, 'From') - if not (from and from[1]) then return false end + if not (from and from[1]) then + return false + end if (from_h and from_h == replyto) then -- From and Reply-To are identical task:insert_result('REPLYTO_EQ_FROM', 1.0) @@ -221,17 +233,17 @@ local check_replyto_id = rspamd_config:register_callback_symbol('CHECK_REPLYTO', -- See if Reply-To matches the To address local to = task:get_recipients(2) if (to and to[1] and to[1].addr:lower() == rt[1].addr:lower()) then - -- Ignore this for mailing-lists and automatic submissions - if (not (task:get_header('List-Unsubscribe') or - task:get_header('X-To-Get-Off-This-List') or - task:get_header('X-List') or - task:get_header('Auto-Submitted'))) - then - task:insert_result('REPLYTO_EQ_TO_ADDR', 1.0) - end + -- Ignore this for mailing-lists and automatic submissions + if (not (task:get_header('List-Unsubscribe') or + task:get_header('X-To-Get-Off-This-List') or + task:get_header('X-List') or + task:get_header('Auto-Submitted'))) + then + task:insert_result('REPLYTO_EQ_TO_ADDR', 1.0) + end else task:insert_result('REPLYTO_DOM_NEQ_FROM_DOM', 1.0) - end + end end end -- See if the Display Names match @@ -242,7 +254,7 @@ local check_replyto_id = rspamd_config:register_callback_symbol('CHECK_REPLYTO', end end end -) +}) rspamd_config:register_symbol{ name = 'REPLYTO_UNPARSEABLE', @@ -322,6 +334,8 @@ rspamd_config:register_dependency('CHECK_REPLYTO', 'CHECK_FROM') local check_mime_id = rspamd_config:register_symbol{ name = 'CHECK_MIME', type = 'callback', + group = 'headers', + score = 0.0, callback = function(task) local parts = task:get_parts() if not parts then return false end @@ -420,6 +434,7 @@ rspamd_config.PREVIOUSLY_DELIVERED = { end end, description = 'Message either to a list or was forwarded', + group = 'headers', score = 0.0 } rspamd_config.BROKEN_HEADERS = { @@ -612,6 +627,8 @@ rspamd_config.FAKE_REPLY = { local check_from_id = rspamd_config:register_symbol{ name = 'CHECK_FROM', type = 'callback', + score = 0.0, + group = 'headers', callback = function(task) local envfrom = task:get_from(1) local from = task:get_from(2) @@ -735,6 +752,8 @@ rspamd_config:register_symbol{ local check_to_cc_id = rspamd_config:register_symbol{ name = 'CHECK_TO_CC', type = 'callback', + score = 0.0, + group = 'headers', callback = function(task) local rcpts = task:get_recipients(1) local to = task:get_recipients(2) diff --git a/rules/html.lua b/rules/html.lua index c324b9de1..da4ef1d13 100644 --- a/rules/html.lua +++ b/rules/html.lua @@ -171,6 +171,7 @@ rspamd_config.R_SUSPICIOUS_IMAGES = { local vis_check_id = rspamd_config:register_symbol{ name = 'HTML_VISIBLE_CHECKS', type = 'callback', + group = 'html', callback = function(task) --local logger = require "rspamd_logger" local tp = task:get_text_parts() -- get text parts in a message diff --git a/rules/mid.lua b/rules/mid.lua index d89f217da..60df4bd1f 100644 --- a/rules/mid.lua +++ b/rules/mid.lua @@ -67,8 +67,13 @@ local function mid_check_func(task) end -- MID checks from Steve Freegard -local check_mid_id = rspamd_config:register_callback_symbol('CHECK_MID', 1.0, - mid_check_func) +local check_mid_id = rspamd_config:register_symbol({ + name = 'CHECK_MID', + score = 0.0, + group = 'mid', + type = 'callback', + callback = mid_check_func +}) rspamd_config:register_virtual_symbol('MID_BARE_IP', 1.0, check_mid_id) rspamd_config:set_metric_symbol('MID_BARE_IP', 2.0, 'Message-ID RHS is a bare IP address', 'default', 'Message ID') rspamd_config:register_virtual_symbol('MID_RHS_NOT_FQDN', 1.0, check_mid_id) diff --git a/rules/misc.lua b/rules/misc.lua index fe5325d46..3494d26ca 100644 --- a/rules/misc.lua +++ b/rules/misc.lua @@ -190,6 +190,7 @@ rspamd_config.ENVFROM_VERP = { local check_rcvd = rspamd_config:register_symbol{ name = 'CHECK_RCVD', + group = 'headers', callback = function (task) local rcvds = task:get_received_headers() if not rcvds then return false end @@ -521,6 +522,7 @@ local check_from_display_name = rspamd_config:register_symbol{ end return false end, + group = 'headers', } rspamd_config:register_symbol{ diff --git a/rules/regexp/compromised_hosts.lua b/rules/regexp/compromised_hosts.lua index 46a192978..47af04eec 100644 --- a/rules/regexp/compromised_hosts.lua +++ b/rules/regexp/compromised_hosts.lua @@ -51,7 +51,8 @@ rspamd_config.HAS_X_AS = { end end, description = 'Has X-Authenticated-Sender header', - group = "compromised_hosts" + group = "compromised_hosts", + score = 0.0 } -- X-Get-Message-Sender-Via: accord.host-care.com: authenticated_id: sales@cortaflex.si @@ -69,7 +70,8 @@ rspamd_config.HAS_X_GMSV = { end end, description = 'Has X-Get-Message-Sender-Via: header', - group = "compromised_hosts" + group = "compromised_hosts", + score = 0.0, } -- X-AntiAbuse: This header was added to track abuse, please include it with any abuse report diff --git a/rules/regexp/headers.lua b/rules/regexp/headers.lua index 4889c0e7f..bd286b764 100644 --- a/rules/regexp/headers.lua +++ b/rules/regexp/headers.lua @@ -31,7 +31,7 @@ reconf['SUBJECT_NEEDS_ENCODING'] = { re = string.format('!(%s) & !(%s) & (%s)', subject_encoded_b64, subject_encoded_qp, subject_needs_mime), score = 1.0, description = 'Subject needs encoding', - group = 'header' + group = 'headers' } local from_encoded_b64 = 'From=/=\\?\\S+\\?B\\?/iX' @@ -41,7 +41,7 @@ reconf['FROM_NEEDS_ENCODING'] = { re = string.format('!(%s) & !(%s) & (%s)', from_encoded_b64, from_encoded_qp, raw_from_needs_mime), score = 1.0, description = 'From header needs encoding', - group = 'header' + group = 'headers' } local to_encoded_b64 = 'To=/=\\?\\S+\\?B\\?/iX' @@ -51,7 +51,7 @@ reconf['TO_NEEDS_ENCODING'] = { re = string.format('!(%s) & !(%s) & (%s)', to_encoded_b64, to_encoded_qp, raw_to_needs_mime), score = 1.0, description = 'To header needs encoding', - group = 'header' + group = 'headers' } -- Detects that there is no space in From header (e.g. Some Name<some@host>) @@ -59,14 +59,14 @@ reconf['R_NO_SPACE_IN_FROM'] = { re = 'From=/\\S<[-\\w\\.]+\\@[-\\w\\.]+>/X', score = 1.0, description = 'No space in from header', - group = 'header' + group = 'headers' } reconf['TO_WRAPPED_IN_SPACES'] = { re = [[To=/<\s[-.\w]+\@[-.\w]+\s>/X]], score = 2.0, description = 'To address is wrapped in spaces inside angle brackets (e.g. display-name < local-part@domain >)', - group = 'header' + group = 'headers' } -- Detects missing Subject header @@ -74,13 +74,13 @@ reconf['MISSING_SUBJECT'] = { re = '!raw_header_exists(Subject)', score = 2.0, description = 'Subject header is missing', - group = 'header' + group = 'headers' } rspamd_config.EMPTY_SUBJECT = { score = 1.0, description = 'Subject header is empty', - group = 'header', + group = 'headers', callback = function(task) local hdr = task:get_header('Subject') if hdr and #hdr == 0 then @@ -95,7 +95,7 @@ reconf['MISSING_TO'] = { re = '!raw_header_exists(To)', score = 2.0, description = 'To header is missing', - group = 'header' + group = 'headers' } -- Detects undisclosed recipients @@ -104,7 +104,7 @@ reconf['R_UNDISC_RCPT'] = { re = string.format('(%s)', undisc_rcpt), score = 3.0, description = 'Recipients are absent or undisclosed', - group = 'header' + group = 'headers' } -- Detects missing Message-Id @@ -113,7 +113,7 @@ reconf['MISSING_MID'] = { re = '!header_exists(Message-Id)', score = 2.5, description = 'Message id is missing', - group = 'header' + group = 'headers' } -- Received seems to be fake @@ -121,7 +121,7 @@ reconf['R_RCVD_SPAMBOTS'] = { re = 'Received=/^from \\[\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\] by [-.\\w+]{5,255}; [SMTWF][a-z][a-z], [\\s\\d]?\\d [JFMAJSOND][a-z][a-z] \\d{4} \\d{2}:\\d{2}:\\d{2} [-+]\\d{4}$/mH', score = 3.0, description = 'Spambots signatures in received headers', - group = 'header' + group = 'headers' } -- Charset is missing in message @@ -130,7 +130,7 @@ reconf['R_MISSING_CHARSET'] = { 'compare_transfer_encoding(7bit)'), score = 2.5, description = 'Charset is missing in a message', - group = 'header' + group = 'headers' } -- Subject seems to be spam @@ -138,7 +138,7 @@ reconf['R_SAJDING'] = { re = 'Subject=/\\bsajding(?:om|a)?\\b/iH', score = 8.0, description = 'Subject seems to be spam', - group = 'header' + group = 'headers' } -- Find forged Outlook MUA @@ -151,7 +151,7 @@ reconf['FORGED_OUTLOOK_HTML'] = { re = string.format('!%s & %s & %s', yahoo_bulk, outlook_mua, 'has_only_html_part()'), score = 5.0, description = 'Forged outlook HTML signature', - group = 'header' + group = 'headers' } -- Recipients seems to be likely with each other (only works when recipients count is more than 5 recipients) @@ -159,7 +159,7 @@ reconf['SUSPICIOUS_RECIPS'] = { re = 'compare_recipients_distance(0.65)', score = 1.5, description = 'Recipients seems to be autogenerated (works if recipients count is more than 5)', - group = 'header' + group = 'headers' } -- Recipients list seems to be sorted @@ -167,7 +167,7 @@ reconf['SORTED_RECIPS'] = { re = 'is_recipients_sorted()', score = 3.5, description = 'Recipients list seems to be sorted', - group = 'header' + group = 'headers' } -- Spam string at the end of message to make statistics faults @@ -175,7 +175,7 @@ reconf['TRACKER_ID'] = { re = '/^[a-z0-9]{6,24}[-_a-z0-9]{12,36}[a-z0-9]{6,24}\\s*\\z/isPr', score = 3.84, description = 'Spam string at the end of message to make statistics fault', - group = 'header' + group = 'headers' } -- From contains only 7bit characters (parsed headers are used) @@ -335,7 +335,7 @@ reconf['FORGED_OUTLOOK_TAGS'] = { tag_exists_meta, tag_exists_body), score = 2.1, description = "Message pretends to be send from Outlook but has 'strange' tags", - group = 'header' + group = 'headers' } -- Forged OE/MSO boundary @@ -504,7 +504,7 @@ reconf['INVALID_MSGID'] = { re = string.format('(%s) & !((%s) | (%s))', has_mid, sane_msgid, msgid_comment), score = 1.7, description = 'Message id is incorrect', - group = 'header' + group = 'headers' } @@ -518,7 +518,7 @@ reconf['MIME_HEADER_CTYPE_ONLY'] = { re = string.format('!(%s) & !(%s) & (%s) & !(%s) & !(%s)', cd, cte, ct, mime_version, ct_text_plain), score = 2.0, description = 'Only Content-Type header without other MIME headers', - group = 'header' + group = 'headers' } @@ -530,7 +530,7 @@ reconf['RATWARE_MS_HASH'] = { re = string.format('(%s) & !(%s) & !(%s)', msgid_dollars_ok, mimeole_ms, rcvd_with_exchange), score = 2.0, description = 'Forged Exchange messages', - group = 'header' + group = 'headers' } -- Reply-type in content-type @@ -538,7 +538,7 @@ reconf['STOX_REPLY_TYPE'] = { re = 'Content-Type=/text\\/plain; .* reply-type=original/H', score = 1.0, description = 'Reply-type in content-type', - group = 'header' + group = 'headers' } -- Fake Verizon headers @@ -548,7 +548,7 @@ reconf['FM_FAKE_HELO_VERIZON'] = { re = string.format('(%s) & !(%s)', fhelo_verizon, fhost_verizon), score = 2.0, description = 'Fake helo for verizon provider', - group = 'header' + group = 'headers' } -- Forged yahoo msgid @@ -558,7 +558,7 @@ reconf['FORGED_MSGID_YAHOO'] = { re = string.format('(%s) & !(%s)', at_yahoo_msgid, from_yahoo_com), score = 2.0, description = 'Forged yahoo msgid', - group = 'header' + group = 'headers' } -- Forged The Bat! MUA headers @@ -570,7 +570,7 @@ reconf['FORGED_MUA_THEBAT_BOUN'] = { re = string.format('(%s) & (%s) & !(%s) & !(%s)', thebat_mua_v1, ctype_has_boundary, bat_boundary, mailman_21), score = 2.0, description = 'Forged The Bat! MUA headers', - group = 'header' + group = 'headers' } -- Detect Mail.Ru web-mail @@ -580,7 +580,7 @@ reconf['MAIL_RU_MAILER'] = { re = string.format('(%s) & (%s)', xm_mail_ru_mailer_1_0, rcvd_e_mail_ru), score = 0.0, description = 'Sent with Mail.Ru web-mail', - group = 'header' + group = 'headers' } -- Detect yandex.ru web-mail @@ -590,7 +590,7 @@ reconf['YANDEX_RU_MAILER'] = { re = string.format('(%s) & (%s)', xm_yandex_ru_mailer_5_0, rcvd_web_yandex_ru), score = 0.0, description = 'Sent with yandex.ru web-mail', - group = 'header' + group = 'headers' } -- Detect 1C v8.2 and v8.3 mailers @@ -598,7 +598,7 @@ reconf['MAILER_1C_8'] = { re = 'X-Mailer=/^1C:Enterprise 8\\.[23]$/H', score = 0.0, description = 'Sent with 1C:Enterprise 8', - group = 'header' + group = 'headers' } -- Detect rogue 'strongmail' MTA with IPv4 and '(-)' in Received line @@ -606,7 +606,7 @@ reconf['STRONGMAIL'] = { re = [[Received=/^from\s+strongmail\s+\(\[\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\]\) by \S+ \(-\); /mH]], score = 6.0, description = 'Sent via rogue "strongmail" MTA', - group = 'header' + group = 'headers' } -- Two received headers with ip addresses @@ -616,7 +616,7 @@ reconf['RCVD_DOUBLE_IP_SPAM'] = { re = string.format('(%s) | (%s)', double_ip_spam_1, double_ip_spam_2), score = 2.0, description = 'Two received headers with ip addresses', - group = 'header' + group = 'headers' } -- Quoted reply-to from yahoo (seems to be forged) @@ -625,7 +625,7 @@ reconf['REPTO_QUOTE_YAHOO'] = { re = string.format('(%s) & ((%s) | (%s))', repto_quote, from_yahoo_com, at_yahoo_msgid), score = 2.0, description = 'Quoted reply-to from yahoo (seems to be forged)', - group = 'header' + group = 'headers' } -- MUA definitions @@ -665,7 +665,7 @@ reconf['MISSING_MIMEOLE'] = { has_office_version_in_mailer), score = 2.0, description = 'Mime-OLE is needed but absent (e.g. fake Outlook or fake Exchange)', - group = 'header' + group = 'headers' } -- Header delimiters @@ -685,31 +685,31 @@ reconf['HEADER_FROM_DELIMITER_TAB'] = { re = string.format('(%s) & !(%s)', 'check_header_delimiter_tab(From)', yandex), score = 1.0, description = 'Header From begins with tab', - group = 'header' + group = 'headers' } reconf['HEADER_TO_DELIMITER_TAB'] = { re = string.format('(%s) & !(%s)', 'check_header_delimiter_tab(To)', yandex), score = 1.0, description = 'Header To begins with tab', - group = 'header' + group = 'headers' } reconf['HEADER_CC_DELIMITER_TAB'] = { re = string.format('(%s) & !(%s)', 'check_header_delimiter_tab(Cc)', yandex), score = 1.0, description = 'Header To begins with tab', - group = 'header' + group = 'headers' } reconf['HEADER_REPLYTO_DELIMITER_TAB'] = { re = string.format('(%s) & !(%s)', 'check_header_delimiter_tab(Reply-To)', yandex), score = 1.0, description = 'Header Reply-To begins with tab', - group = 'header' + group = 'headers' } reconf['HEADER_DATE_DELIMITER_TAB'] = { re = string.format('(%s) & !(%s)', 'check_header_delimiter_tab(Date)', yandex), score = 1.0, description = 'Header Date begins with tab', - group = 'header' + group = 'headers' } -- Empty delimiters between header names and header values function check_header_delimiter_empty(task, header_name) @@ -722,31 +722,31 @@ reconf['HEADER_FROM_EMPTY_DELIMITER'] = { re = string.format('(%s)', 'check_header_delimiter_empty(From)'), score = 1.0, description = 'Header From has no delimiter between header name and header value', - group = 'header' + group = 'headers' } reconf['HEADER_TO_EMPTY_DELIMITER'] = { re = string.format('(%s)', 'check_header_delimiter_empty(To)'), score = 1.0, description = 'Header To has no delimiter between header name and header value', - group = 'header' + group = 'headers' } reconf['HEADER_CC_EMPTY_DELIMITER'] = { re = string.format('(%s)', 'check_header_delimiter_empty(Cc)'), score = 1.0, description = 'Header Cc has no delimiter between header name and header value', - group = 'header' + group = 'headers' } reconf['HEADER_REPLYTO_EMPTY_DELIMITER'] = { re = string.format('(%s)', 'check_header_delimiter_empty(Reply-To)'), score = 1.0, description = 'Header Reply-To has no delimiter between header name and header value', - group = 'header' + group = 'headers' } reconf['HEADER_DATE_EMPTY_DELIMITER'] = { re = string.format('(%s)', 'check_header_delimiter_empty(Date)'), score = 1.0, description = 'Header Date has no delimiter between header name and header value', - group = 'header' + group = 'headers' } -- Definitions of received headers regexp @@ -754,7 +754,7 @@ reconf['RCVD_ILLEGAL_CHARS'] = { re = 'Received=/[\\x80-\\xff]/X', score = 4.0, description = 'Header Received has raw illegal character', - group = 'header' + group = 'headers' } local MAIL_RU_Return_Path = 'Return-path=/^\\s*<.+\\@mail\\.ru>$/iX' @@ -766,7 +766,7 @@ reconf['FAKE_RECEIVED_mail_ru'] = { re = string.format('(%s) & !(((%s) | (%s)) & (%s))', MAIL_RU_Received, MAIL_RU_Return_Path, MAIL_RU_X_Envelope_From, MAIL_RU_From), score = 4.0, description = 'Fake helo mail.ru in header Received from non mail.ru sender address', - group = 'header' + group = 'headers' } local GMAIL_COM_Return_Path = 'Return-path=/^\\s*<.+\\@gmail\\.com>$/iX' @@ -791,70 +791,70 @@ reconf['FAKE_RECEIVED_smtp_yandex_ru'] = { re = string.format('(((%s) & ((%s) | (%s))) | ((%s) & ((%s) | (%s))) | ((%s) & ((%s) | (%s)))) & (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s)', MAIL_RU_From, MAIL_RU_Return_Path, MAIL_RU_X_Envelope_From, GMAIL_COM_From, GMAIL_COM_Return_Path, GMAIL_COM_X_Envelope_From, UKR_NET_From, UKR_NET_Return_Path, UKR_NET_X_Envelope_From, RECEIVED_smtp_yandex_ru_1, RECEIVED_smtp_yandex_ru_2, RECEIVED_smtp_yandex_ru_3, RECEIVED_smtp_yandex_ru_4, RECEIVED_smtp_yandex_ru_5, RECEIVED_smtp_yandex_ru_6, RECEIVED_smtp_yandex_ru_7, RECEIVED_smtp_yandex_ru_8, RECEIVED_smtp_yandex_ru_9), score = 4.0, description = 'Fake smtp.yandex.ru Received', - group = 'header' + group = 'headers' } reconf['FORGED_GENERIC_RECEIVED'] = { re = 'Received=/^\\s*(.+\\n)*from \\[\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\] by (([\\w\\d-]+\\.)+[a-zA-Z]{2,6}|\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}); \\w{3}, \\d+ \\w{3} 20\\d\\d \\d\\d\\:\\d\\d\\:\\d\\d [+-]\\d\\d\\d0/X', score = 3.6, description = 'Forged generic Received', - group = 'header' + group = 'headers' } reconf['FORGED_GENERIC_RECEIVED2'] = { re = 'Received=/^\\s*(.+\\n)*from \\[\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\] by ([\\w\\d-]+\\.)+[a-z]{2,6} id [\\w\\d]{12}; \\w{3}, \\d+ \\w{3} 20\\d\\d \\d\\d\\:\\d\\d\\:\\d\\d [+-]\\d\\d\\d0/X', score = 3.6, description = 'Forged generic Received', - group = 'header' + group = 'headers' } reconf['FORGED_GENERIC_RECEIVED3'] = { re = 'Received=/^\\s*(.+\\n)*by \\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3} with SMTP id [a-zA-Z]{14}\\.\\d{13};[\\r\\n\\s]*\\w{3}, \\d+ \\w{3} 20\\d\\d \\d\\d\\:\\d\\d\\:\\d\\d [+-]\\d\\d\\d0 \\(GMT\\)/X', score = 3.6, description = 'Forged generic Received', - group = 'header' + group = 'headers' } reconf['FORGED_GENERIC_RECEIVED4'] = { re = 'Received=/^\\s*(.+\\n)*from localhost by \\S+;\\s+\\w{3}, \\d+ \\w{3} 20\\d\\d \\d\\d\\:\\d\\d\\:\\d\\d [+-]\\d\\d\\d0[\\s\\r\\n]*$/X', score = 3.6, description = 'Forged generic Received', - group = 'header' + group = 'headers' } reconf['INVALID_POSTFIX_RECEIVED'] = { re = 'Received=/ \\(Postfix\\) with ESMTP id [A-Z\\d]+([\\s\\r\\n]+for <\\S+?>)?;[\\s\\r\\n]*[A-Z][a-z]{2}, \\d{1,2} [A-Z][a-z]{2} \\d\\d\\d\\d \\d\\d:\\d\\d:\\d\\d [\\+\\-]\\d\\d\\d\\d$/X', score = 3.0, description = 'Invalid Postfix Received', - group = 'header' + group = 'headers' } reconf['X_PHP_FORGED_0X'] = { re = "X-PHP-Originating-Script=/^0\\d/X", score = 4.0, description = "X-PHP-Originating-Script header appears forged", - group = 'header' + group = 'headers' } reconf['GOOGLE_FORWARDING_MID_MISSING'] = { re = "Message-ID=/SMTPIN_ADDED_MISSING\\@mx\\.google\\.com>$/X", score = 2.5, description = "Message was missing Message-ID pre-forwarding", - group = 'header' + group = 'headers' } reconf['GOOGLE_FORWARDING_MID_BROKEN'] = { re = "Message-ID=/SMTPIN_ADDED_BROKEN\\@mx\\.google\\.com>$/X", score = 1.7, description = "Message had invalid Message-ID pre-forwarding", - group = 'header' + group = 'headers' } reconf['CTE_CASE'] = { re = 'Content-Transfer-Encoding=/^[78]B/X', description = '[78]Bit .vs. [78]bit', score = 0.5, - group = 'header' + group = 'headers' } reconf['HAS_INTERSPIRE_SIG'] = { @@ -866,14 +866,14 @@ reconf['HAS_INTERSPIRE_SIG'] = { 'List-Unsubscribe=/\\/unsubscribe\\.php\\?M=[^&]+&C=[^&]+&L=[^&]+&N=[^>]+>$/Xi'), description = "Has Interspire fingerprint", score = 1.0, - group = 'header' + group = 'headers' } reconf['CT_EXTRA_SEMI'] = { re = 'Content-Type=/;$/X', description = 'Content-Type ends with a semi-colon', score = 1.0, - group = 'header' + group = 'headers' } reconf['SUBJECT_ENDS_EXCLAIM'] = { diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c index 5c8d96f66..f4811f18b 100644 --- a/src/libmime/lang_detection.c +++ b/src/libmime/lang_detection.c @@ -795,6 +795,10 @@ rspamd_language_detector_init (struct rspamd_config *cfg) (gint)ret->languages->len, (gint)total); + if (stop_words) { + ucl_object_unref (stop_words); + } + REF_INIT_RETAIN (ret, rspamd_language_detector_dtor); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)rspamd_language_detector_unref, diff --git a/src/libserver/events.c b/src/libserver/events.c index 9563753e0..325da4452 100644 --- a/src/libserver/events.c +++ b/src/libserver/events.c @@ -395,12 +395,25 @@ rspamd_session_watch_start (struct rspamd_async_session *session, if (session->cur_watcher == NULL) { session->cur_watcher = rspamd_mempool_alloc0 (session->pool, sizeof (*session->cur_watcher)); - } - st_elt = rspamd_mempool_alloc (session->pool, sizeof (*st_elt)); - st_elt->cb = cb; - st_elt->ud = ud; - LL_PREPEND (session->cur_watcher->st, st_elt); + st_elt = rspamd_mempool_alloc (session->pool, sizeof (*st_elt)); + st_elt->cb = cb; + st_elt->ud = ud; + LL_PREPEND (session->cur_watcher->st, st_elt); + } + else { + if (session->cur_watcher->st) { + /* Reuse the existing (empty) watcher */ + session->cur_watcher->st->cb = cb; + session->cur_watcher->st->ud = ud; + } + else { + st_elt = rspamd_mempool_alloc (session->pool, sizeof (*st_elt)); + st_elt->cb = cb; + st_elt->ud = ud; + LL_PREPEND (session->cur_watcher->st, st_elt); + } + } session->cur_watcher->id = id; session->flags |= RSPAMD_SESSION_FLAG_WATCHING; diff --git a/src/libutil/expression.c b/src/libutil/expression.c index 2469d0415..65465ceb6 100644 --- a/src/libutil/expression.c +++ b/src/libutil/expression.c @@ -1098,16 +1098,15 @@ rspamd_process_expression_track (struct rspamd_expression *expr, struct rspamd_e /* Ensure that stack is empty at this point */ g_assert (expr->expression_stack->len == 0); + expr->evals ++; ret = rspamd_ast_process_node (expr, expr->ast, process_data); /* Cleanup */ g_node_traverse (expr->ast, G_IN_ORDER, G_TRAVERSE_ALL, -1, rspamd_ast_cleanup_traverse, NULL); - expr->evals ++; - /* Check if we need to resort */ - if (expr->evals == expr->next_resort) { + if (expr->evals % expr->next_resort == 0) { expr->next_resort = ottery_rand_range (MAX_RESORT_EVALS) + MIN_RESORT_EVALS; /* Set priorities for branches */ diff --git a/src/libutil/fstring.c b/src/libutil/fstring.c index 6c51ad62e..fac3b364b 100644 --- a/src/libutil/fstring.c +++ b/src/libutil/fstring.c @@ -160,14 +160,19 @@ rspamd_fstring_grow (rspamd_fstring_t *str, gsize needed_len) rspamd_fstring_t * rspamd_fstring_append (rspamd_fstring_t *str, const char *in, gsize len) { - gsize avail = fstravail (str); - - if (avail < len) { - str = rspamd_fstring_grow (str, len); + if (str == NULL) { + str = rspamd_fstring_new_init (in, len); } + else { + gsize avail = fstravail (str); + + if (avail < len) { + str = rspamd_fstring_grow (str, len); + } - memcpy (str->str + str->len, in, len); - str->len += len; + memcpy (str->str + str->len, in, len); + str->len += len; + } return str; } @@ -176,14 +181,22 @@ rspamd_fstring_t * rspamd_fstring_append_chars (rspamd_fstring_t *str, char c, gsize len) { - gsize avail = fstravail (str); + if (str == NULL) { + str = rspamd_fstring_sized_new (len); - if (avail < len) { - str = rspamd_fstring_grow (str, len); + memset (str->str + str->len, c, len); + str->len += len; } + else { + gsize avail = fstravail (str); - memset (str->str + str->len, c, len); - str->len += len; + if (avail < len) { + str = rspamd_fstring_grow (str, len); + } + + memset (str->str + str->len, c, len); + str->len += len; + } return str; } diff --git a/src/libutil/map.c b/src/libutil/map.c index d0fb9ff9e..d96cdc2ad 100644 --- a/src/libutil/map.c +++ b/src/libutil/map.c @@ -124,9 +124,23 @@ write_http_request (struct http_callback_data *cbd) g_assert_not_reached (); } + msg->url = rspamd_fstring_append (msg->url, cbd->data->rest, + strlen (cbd->data->rest)); + + if (cbd->data->userinfo) { + rspamd_http_message_add_header (msg, "Authorization", + cbd->data->userinfo); + } + MAP_RETAIN (cbd, "http_callback_data"); - rspamd_http_connection_write_message (cbd->conn, msg, cbd->data->host, - NULL, cbd, cbd->fd, &cbd->tv, cbd->ev_base); + rspamd_http_connection_write_message (cbd->conn, + msg, + cbd->data->host, + NULL, + cbd, + cbd->fd, + &cbd->tv, + cbd->ev_base); } else { msg_err_map ("cannot connect to %s: %s", cbd->data->host, @@ -1285,7 +1299,8 @@ rspamd_map_dns_callback (struct rdns_reply *reply, void *arg) } else { /* We could not resolve host, so cowardly fail here */ - msg_err_map ("cannot resolve %s", cbd->data->host); + msg_err_map ("cannot resolve %s: %s", cbd->data->host, + rdns_strerror (reply->code)); cbd->periodic->errored = 1; rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); } @@ -2244,6 +2259,11 @@ rspamd_map_backend_dtor (struct rspamd_map_backend *bk) g_free (data->host); g_free (data->path); + g_free (data->rest); + + if (data->userinfo) { + g_free (data->userinfo); + } if (data->etag) { rspamd_fstring_free (data->etag); @@ -2338,7 +2358,7 @@ rspamd_map_parse_backend (struct rspamd_config *cfg, const gchar *map_line) goto err; } else { - if (!(up.field_set & 1 << UF_HOST)) { + if (!(up.field_set & 1u << UF_HOST)) { msg_err_config ("cannot parse HTTP url: %s: no host", bk->uri); goto err; } @@ -2347,7 +2367,7 @@ rspamd_map_parse_backend (struct rspamd_config *cfg, const gchar *map_line) tok.len = up.field_data[UF_HOST].len; hdata->host = rspamd_ftokdup (&tok); - if (up.field_set & 1 << UF_PORT) { + if (up.field_set & (1u << UF_PORT)) { hdata->port = up.port; } else { @@ -2359,11 +2379,32 @@ rspamd_map_parse_backend (struct rspamd_config *cfg, const gchar *map_line) } } - if (up.field_set & 1 << UF_PATH) { + if (up.field_set & (1u << UF_PATH)) { tok.begin = bk->uri + up.field_data[UF_PATH].off; - tok.len = strlen (tok.begin); + tok.len = up.field_data[UF_PATH].len; hdata->path = rspamd_ftokdup (&tok); + + /* We also need to check query + fragment */ + if (up.field_set & ((1u << UF_QUERY) | (1u << UF_FRAGMENT))) { + tok.begin = bk->uri + up.field_data[UF_PATH].off + + up.field_data[UF_PATH].len; + tok.len = strlen (tok.begin); + hdata->rest = rspamd_ftokdup (&tok); + } + else { + hdata->rest = g_strdup (""); + } + } + + if (up.field_set & (1u << UF_USERINFO)) { + /* Create authorisation header for basic auth */ + guint len = sizeof ("Basic ") + + up.field_data[UF_USERINFO].len * 8 / 5 + 4; + hdata->userinfo = g_malloc (len); + rspamd_snprintf (hdata->userinfo, len, "Basic %*Bs", + (int)up.field_data[UF_USERINFO].len, + bk->uri + up.field_data[UF_USERINFO].off); } } diff --git a/src/libutil/map_private.h b/src/libutil/map_private.h index 7e0390a6c..68415d0e0 100644 --- a/src/libutil/map_private.h +++ b/src/libutil/map_private.h @@ -84,8 +84,10 @@ struct http_map_data { struct rspamd_map_cachepoint *cache; /* Non-shared for cache owner, used to cleanup cache */ struct rspamd_http_map_cached_cbdata *cur_cache_cbd; + gchar *userinfo; gchar *path; gchar *host; + gchar *rest; gchar *last_signature; rspamd_fstring_t *etag; time_t last_modified; diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index 9bfaee0d2..f15c04e46 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -1739,14 +1739,21 @@ lua_config_register_symbol (lua_State * L) FALSE, no_squeeze); - if (!isnan (score)) { + if (!isnan (score) || group) { if (one_shot) { nshots = 1; } - rspamd_config_add_symbol (cfg, name, - score, description, group, flags, - (guint) priority, nshots); + if (!isnan (score)) { + rspamd_config_add_symbol (cfg, name, + score, description, group, flags, + (guint) priority, nshots); + } + else { + rspamd_config_add_symbol (cfg, name, + 0.0, description, group, flags, + (guint) priority, nshots); + } lua_pushstring (L, "groups"); lua_gettable (L, 2); @@ -2385,7 +2392,7 @@ lua_config_newindex (lua_State *L) } else if (lua_type (L, 3) == LUA_TTABLE) { gint type = SYMBOL_TYPE_NORMAL, priority = 0, idx; - gdouble weight = 1.0, score; + gdouble weight = 1.0, score = NAN; const char *type_str, *group = NULL, *description = NULL; guint flags = 0; @@ -2500,22 +2507,23 @@ lua_config_newindex (lua_State *L) */ if (g_hash_table_lookup (cfg->symbols, name) == NULL) { nshots = cfg->default_max_shots; + lua_pushstring (L, "score"); lua_gettable (L, -2); - if (lua_type (L, -1) == LUA_TNUMBER) { score = lua_tonumber (L, -1); - lua_pop (L, 1); - /* If score defined, then we can check other metric fields */ - lua_pushstring (L, "group"); - lua_gettable (L, -2); + } + lua_pop (L, 1); - if (lua_type (L, -1) == LUA_TSTRING) { - group = lua_tostring (L, -1); - } - lua_pop (L, 1); + lua_pushstring (L, "group"); + lua_gettable (L, -2); + if (lua_type (L, -1) == LUA_TSTRING) { + group = lua_tostring (L, -1); + } + lua_pop (L, 1); + if (!isnan (score) || group != NULL) { lua_pushstring (L, "description"); lua_gettable (L, -2); @@ -2548,8 +2556,15 @@ lua_config_newindex (lua_State *L) * Do not override the existing symbols (using zero priority), * since we are defining default values here */ - rspamd_config_add_symbol (cfg, name, score, - description, group, flags, 0, nshots); + if (!isnan (score)) { + rspamd_config_add_symbol (cfg, name, score, + description, group, flags, 0, nshots); + } + else if (group) { + /* Add with zero score */ + rspamd_config_add_symbol (cfg, name, 0.0, + description, group, flags, 0, nshots); + } lua_pushstring (L, "groups"); lua_gettable (L, -2); @@ -2568,9 +2583,6 @@ lua_config_newindex (lua_State *L) lua_pop (L, 1); } - else { - lua_pop (L, 1); - } } /* Remove table from stack */ diff --git a/src/lua/lua_dns.c b/src/lua/lua_dns.c index 77a7ef855..fe4b7ff2e 100644 --- a/src/lua/lua_dns.c +++ b/src/lua/lua_dns.c @@ -39,6 +39,7 @@ lua_dns_request (lua_State *L) { GError *err = NULL; struct rspamd_async_session *session = NULL; + struct rspamd_config *cfg = NULL; struct lua_rspamd_dns_cbdata *cbdata = NULL; const gchar *to_resolve = NULL; const gchar *type_str = NULL; @@ -49,8 +50,8 @@ lua_dns_request (lua_State *L) /* Check arguments */ if (!rspamd_lua_parse_table_arguments (L, 1, &err, - "*name=S;*task=U{task};*type=S;forced=B", - &to_resolve, &task, &type_str, &forced)) { + "*name=S;task=U{task};*type=S;forced=B;session=U{session};config=U{config}", + &to_resolve, &task, &type_str, &forced, &session, &cfg)) { if (err) { ret = luaL_error (L, "invalid arguments: %s", err->message); @@ -65,6 +66,10 @@ lua_dns_request (lua_State *L) if (task) { session = task->s; pool = task->task_pool; + cfg = task->cfg; + } + else if (session && cfg) { + pool = cfg->cfg_pool; } else { return luaL_error (L, "invalid arguments: either task or session/config should be set"); @@ -97,23 +102,35 @@ lua_dns_request (lua_State *L) free (ptr_str); } - if (forced) { - ret = make_dns_request_task_forced (task, - lua_dns_callback, - cbdata, - type, - to_resolve); + + if (task == NULL) { + ret = make_dns_request (cfg->dns_resolver, + session, + pool, + lua_dns_callback, + cbdata, + type, + to_resolve); } else { - ret = make_dns_request_task (task, - lua_dns_callback, - cbdata, - type, - to_resolve); + if (forced) { + ret = make_dns_request_task_forced (task, + lua_dns_callback, + cbdata, + type, + to_resolve); + } + else { + ret = make_dns_request_task (task, + lua_dns_callback, + cbdata, + type, + to_resolve); + } } if (ret) { - cbdata->thread = lua_thread_pool_get_running_entry (task->cfg->lua_thread_pool); + cbdata->thread = lua_thread_pool_get_running_entry (cfg->lua_thread_pool); cbdata->s = session; cbdata->w = rspamd_session_get_watcher (session); rspamd_session_watcher_push (session); diff --git a/src/lua/lua_dns_resolver.c b/src/lua/lua_dns_resolver.c index 7c224243e..e7b6eeaab 100644 --- a/src/lua/lua_dns_resolver.c +++ b/src/lua/lua_dns_resolver.c @@ -185,8 +185,12 @@ lua_dns_resolver_callback (struct rdns_reply *reply, gpointer arg) lua_pushboolean (L, reply->authenticated); if (lua_pcall (L, 6, 0, err_idx) != 0) { - msg_info_pool_check ("call to dns callback failed: %s", tb->str); - g_string_free (tb, TRUE); + tb = lua_touserdata (L, -1); + + if (tb) { + msg_err_pool_check ("call to dns callback failed: %s", tb->str); + g_string_free (tb, TRUE); + } } lua_settop (L, err_idx - 1); diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c index 96872108c..3f3ee72e0 100644 --- a/src/lua/lua_http.c +++ b/src/lua/lua_http.c @@ -506,16 +506,27 @@ lua_http_push_headers (lua_State *L, struct rspamd_http_message *msg) * Required params are: * * - `url` - * - `callback` * - `task` + * + * In taskless mode, instead of `task` required are: + * + * - `ev_base` + * - `config` + * * @param {string} url specifies URL for a request in the standard URI form (e.g. 'http://example.com/path') - * @param {function} callback specifies callback function in format `function (err_message, code, body, headers)` that is called on HTTP request completion + * @param {function} callback specifies callback function in format `function (err_message, code, body, headers)` that is called on HTTP request completion. if this parameter is missing, the function performs "pseudo-synchronous" call (see [Synchronous and Asynchronous API overview](/doc/lua/sync_async.html#API-example-http-module) * @param {task} task if called from symbol handler it is generally a good idea to use the common task objects: event base, DNS resolver and events session * @param {table} headers optional headers in form `[name='value', name='value']` * @param {string} mime_type MIME type of the HTTP content (for example, `text/html`) * @param {string/text} body full body content, can be opaque `rspamd{text}` to avoid data copying * @param {number} timeout floating point request timeout value in seconds (default is 5.0 seconds) - * @return {boolean} `true` if a request has been successfully scheduled. If this value is `false` then some error occurred, the callback thus will not be called + * @param {resolver} resolver to perform DNS-requests. Usually got from either `task` or `config` + * @param {boolean} gzip if true, body of the requests will be compressed + * @param {boolean} no_ssl_verify disable SSL peer checks + * @param {string} user for HTTP authentication + * @param {string} password for HTTP authentication, only if "user" present + * @return {boolean} `true`, in **async** mode, if a request has been successfully scheduled. If this value is `false` then some error occurred, the callback thus will not be called. + * @return In **sync** mode `string|nil, nil|table` In sync mode error message if any and response as table: `int` _code_, `string` _content_ and `table` _headers_ (header -> value) */ static gint lua_http_request (lua_State *L) @@ -854,6 +865,9 @@ lua_http_request (lua_State *L) if (task == NULL && cfg == NULL) { return luaL_error (L, "Bad params to rspamd_http:request(): either task or config should be set"); } + if (ev_base == NULL) { + return luaL_error (L, "Bad params to rspamd_http:request(): ev_base isn't passed"); + } cbd = g_malloc0 (sizeof (*cbd)); cbd->cbref = cbref; diff --git a/src/lua/lua_tcp.c b/src/lua/lua_tcp.c index 91cf3726d..f9c1a477d 100644 --- a/src/lua/lua_tcp.c +++ b/src/lua/lua_tcp.c @@ -1194,7 +1194,8 @@ lua_tcp_register_event (struct lua_tcp_cbdata *cbd) if (cbd->session) { event_finalizer_t fin = IS_SYNC (cbd) ? lua_tcp_void_finalyser : lua_tcp_fin; - cbd->async_ev = rspamd_session_add_event (cbd->session, NULL, fin, cbd, g_quark_from_static_string ("lua tcp")); + cbd->async_ev = rspamd_session_add_event (cbd->session, NULL, fin, cbd, + g_quark_from_static_string ("lua tcp")); if (!cbd->async_ev) { return FALSE; @@ -1240,7 +1241,14 @@ lua_tcp_make_connection (struct lua_tcp_cbdata *cbd) return FALSE; } - cbd->flags |= LUA_TCP_FLAG_RESOLVED; + if (!(cbd->flags & LUA_TCP_FLAG_RESOLVED)) { + /* We come here without resolving, so we need to add a watcher */ + lua_tcp_register_watcher (cbd); + } + else { + cbd->flags |= LUA_TCP_FLAG_RESOLVED; + } + lua_tcp_register_event (cbd); cbd->fd = fd; @@ -1262,6 +1270,12 @@ lua_tcp_dns_handler (struct rdns_reply *reply, gpointer ud) TCP_RELEASE (cbd); } else { + /* + * We set this flag as it means that we have already registered the watcher + * when started DNS query + */ + cbd->flags |= LUA_TCP_FLAG_RESOLVED; + if (reply->entries->type == RDNS_REQUEST_A) { cbd->addr = rspamd_inet_address_new (AF_INET, &reply->entries->content.a.addr); @@ -1649,8 +1663,6 @@ lua_tcp_request (lua_State *L) return 1; } - - lua_tcp_register_watcher (cbd); } if (rspamd_parse_inet_address (&cbd->addr, host, 0)) { @@ -1675,6 +1687,9 @@ lua_tcp_request (lua_State *L) return 1; } + else { + lua_tcp_register_watcher (cbd); + } } else { if (!make_dns_request_task (task, lua_tcp_dns_handler, cbd, @@ -1686,6 +1701,9 @@ lua_tcp_request (lua_State *L) return 1; } + else { + lua_tcp_register_watcher (cbd); + } } } @@ -1812,8 +1830,6 @@ lua_tcp_connect_sync (lua_State *L) return 2; } - - lua_tcp_register_watcher (cbd); } if (rspamd_parse_inet_address (&cbd->addr, host, 0)) { @@ -1839,6 +1855,9 @@ lua_tcp_connect_sync (lua_State *L) return 2; } + else { + lua_tcp_register_watcher (cbd); + } } else { if (!make_dns_request_task (task, lua_tcp_dns_handler, cbd, @@ -1849,6 +1868,9 @@ lua_tcp_connect_sync (lua_State *L) return 2; } + else { + lua_tcp_register_watcher (cbd); + } } } @@ -2202,7 +2224,6 @@ lua_tcp_sync_write (lua_State *L) lua_tcp_plan_handler_event (cbd, TRUE, TRUE); TCP_RETAIN (cbd); - // lua_tcp_register_event (cbd); return lua_thread_yield (thread, 0); } diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index 67aa8eb8e..1784612f0 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -320,15 +320,26 @@ dkim_module_config (struct rspamd_config *cfg) lua_pop (cfg->lua_state, 1); /* Remove global function */ dkim_module_ctx->whitelist_ip = NULL; - if ((value = - rspamd_config_get_module_opt (cfg, "dkim", "check_local")) != NULL) { + value = rspamd_config_get_module_opt (cfg, "dkim", "check_local"); + + if (value == NULL) { + rspamd_config_get_module_opt (cfg, "options", "check_local"); + } + + if (value != NULL) { dkim_module_ctx->check_local = ucl_object_toboolean (value); } else { dkim_module_ctx->check_local = FALSE; } - if ((value = - rspamd_config_get_module_opt (cfg, "dkim", "check_authed")) != NULL) { + + value = rspamd_config_get_module_opt (cfg, "dkim", "check_authed"); + + if (value == NULL) { + rspamd_config_get_module_opt (cfg, "options", "check_authed"); + } + + if (value != NULL) { dkim_module_ctx->check_authed = ucl_object_toboolean (value); } else { @@ -488,36 +499,43 @@ dkim_module_config (struct rspamd_config *cfg) } cb_id = rspamd_symbols_cache_add_symbol (cfg->cache, - dkim_module_ctx->symbol_reject, - 0, - dkim_symbol_callback, - NULL, - SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_FINE, - -1); + "DKIM_CHECK", + 0, + dkim_symbol_callback, + NULL, + SYMBOL_TYPE_CALLBACK, + -1); rspamd_symbols_cache_add_symbol (cfg->cache, - dkim_module_ctx->symbol_na, - 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, - cb_id); + dkim_module_ctx->symbol_reject, + 0, + NULL, + NULL, + SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + cb_id); rspamd_symbols_cache_add_symbol (cfg->cache, - dkim_module_ctx->symbol_permfail, - 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, - cb_id); + dkim_module_ctx->symbol_na, + 0, + NULL, NULL, + SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + cb_id); rspamd_symbols_cache_add_symbol (cfg->cache, - dkim_module_ctx->symbol_tempfail, - 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, - cb_id); + dkim_module_ctx->symbol_permfail, + 0, + NULL, NULL, + SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + cb_id); rspamd_symbols_cache_add_symbol (cfg->cache, - dkim_module_ctx->symbol_allow, - 0, - NULL, NULL, - SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, - cb_id); + dkim_module_ctx->symbol_tempfail, + 0, + NULL, NULL, + SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + cb_id); + rspamd_symbols_cache_add_symbol (cfg->cache, + dkim_module_ctx->symbol_allow, + 0, + NULL, NULL, + SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, + cb_id); rspamd_symbols_cache_add_symbol (cfg->cache, "DKIM_TRACE", diff --git a/src/plugins/lua/antivirus.lua b/src/plugins/lua/antivirus.lua index 46ea8c40d..91a6c0a9b 100644 --- a/src/plugins/lua/antivirus.lua +++ b/src/plugins/lua/antivirus.lua @@ -35,8 +35,11 @@ antivirus { # action = "reject"; # If set, then rejection message is set to this value (mention single quotes) # message = '${SCANNER}: virus found: "${VIRUS}"'; - # if `true` only messages with non-image attachments will be checked (default true) - attachments_only = true; + # Scan mime_parts seperately - otherwise the complete mail will be transfered to AV Scanner + #scan_mime_parts = true; + # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity) + #scan_text_mime = false; + #scan_image_mime = false; # If `max_size` is set, messages > n bytes in size are not scanned max_size = 20000000; # symbol to add (add it to metric if you want non-zero weight) @@ -126,7 +129,9 @@ end local function clamav_config(opts) local clamav_conf = { - attachments_only = true, + scan_mime_parts = true; + scan_text_mime = false; + scan_image_mime = false; default_port = 3310, log_clean = false, timeout = 15.0, @@ -164,7 +169,9 @@ end local function fprot_config(opts) local fprot_conf = { - attachments_only = true, + scan_mime_parts = true; + scan_text_mime = false; + scan_image_mime = false; default_port = 10200, timeout = 15.0, log_clean = false, @@ -202,13 +209,17 @@ end local function sophos_config(opts) local sophos_conf = { - attachments_only = true, + scan_mime_parts = true; + scan_text_mime = false; + scan_image_mime = false; default_port = 4010, timeout = 15.0, log_clean = false, retransmits = 2, cache_expire = 3600, -- expire redis in one hour message = default_message, + savdi_report_encrypted = false, + savdi_report_oversize = false, } for k,v in pairs(opts) do @@ -240,7 +251,9 @@ end local function savapi_config(opts) local savapi_conf = { - attachments_only = true, + scan_mime_parts = true; + scan_text_mime = false; + scan_image_mime = false; default_port = 4444, -- note: You must set ListenAddress in savapi.conf product_id = 0, log_clean = false, @@ -378,30 +391,32 @@ local function fprot_check(task, content, digest, rule) local function fprot_callback(err, data) if err then - if err == 'IO timeout' then - if retransmits > 0 then - retransmits = retransmits - 1 - -- Select a different upstream! - upstream = rule.upstreams:get_upstream_round_robin() - addr = upstream:get_addr() - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = fprot_callback, - data = { header, content, footer }, - stop_pattern = '\n' - }) - else - rspamd_logger.errx(task, 'failed to scan, maximum retransmits exceed') - task:insert_result(rule['symbol_fail'], 0.0, 'retransmits exceed') - upstream:fail() - end + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = fprot_callback, + data = { header, content, footer }, + stop_pattern = '\n' + }) else - rspamd_logger.errx(task, 'failed to scan: %s', err) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan') - upstream:fail() + rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') end else upstream:ok() @@ -411,7 +426,7 @@ local function fprot_check(task, content, digest, rule) if clean then cached = 'OK' if rule['log_clean'] then - rspamd_logger.infox(task, '%s [%s]: message is clean', rule['symbol'], rule['type']) + rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) end else -- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occured @@ -461,32 +476,35 @@ local function clamav_check(task, content, digest, rule) local function clamav_callback(err, data) if err then - if err == 'IO timeout' then - if retransmits > 0 then - retransmits = retransmits - 1 - -- Select a different upstream! - upstream = rule.upstreams:get_upstream_round_robin() - addr = upstream:get_addr() - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = clamav_callback, - data = { header, content, footer }, - stop_pattern = '\0' - }) - else - rspamd_logger.errx(task, 'failed to scan, maximum retransmits exceed') - task:insert_result(rule['symbol_fail'], 0.0, 'retransmits exceed') - upstream:fail() - end + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = clamav_callback, + data = { header, content, footer }, + stop_pattern = '\0' + }) else - rspamd_logger.errx(task, 'failed to scan: %s', err) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan') - upstream:fail() + rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') end + else upstream:ok() data = tostring(data) @@ -495,9 +513,9 @@ local function clamav_check(task, content, digest, rule) if data == 'stream: OK' then cached = 'OK' if rule['log_clean'] then - rspamd_logger.infox(task, '%s [%s]: message is clean', rule['symbol'], rule['type']) + rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) else - lua_util.debugm(N, task, '%s [%s]: message is clean', rule['symbol'], rule['type']) + lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) end else local vname = string.match(data, 'stream: (.+) FOUND') @@ -547,12 +565,21 @@ local function sophos_check(task, content, digest, rule) local function sophos_callback(err, data, conn) if err then - if err == 'IO timeout' then + + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds if retransmits > 0 then + retransmits = retransmits - 1 + -- Select a different upstream! upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() + + lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + tcp.request({ task = task, host = addr:to_string(), @@ -562,18 +589,13 @@ local function sophos_check(task, content, digest, rule) data = { protocol, streamsize, content, bye } }) else - rspamd_logger.errx(task, 'failed to scan, maximum retransmits exceed') - task:insert_result(rule['symbol_fail'], 0.0, 'retransmits exceed') - upstream:fail() + rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') end - else - rspamd_logger.errx(task, 'failed to scan: %s', err) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan') - upstream:fail() - end else upstream:ok() data = tostring(data) + lua_util.debugm(N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data) local vname = string.match(data, 'VIRUS (%S+) ') if vname then yield_result(task, rule, vname) @@ -581,29 +603,40 @@ local function sophos_check(task, content, digest, rule) else if string.find(data, 'DONE OK') then if rule['log_clean'] then - rspamd_logger.infox(task, '%s [%s]: message is clean', rule['symbol'], rule['type']) + rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) + else + lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type']) end save_av_cache(task, digest, rule, 'OK') + -- not finished - continue + elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then + conn:add_read(sophos_callback) + -- set pseudo virus if configured, else do nothing since it's no fatal elseif string.find(data, 'FAIL 0212') then + rspamd_logger.infox(task, 'Message is ENCRYPTED (0212 SOPHOS_SAVI_ERROR_FILE_ENCRYPTED): %s', data) if rule['savdi_report_encrypted'] then - rspamd_logger.infox(task, 'Message is ENCRYPTED (0212 SOPHOS_SAVI_ERROR_FILE_ENCRYPTED): %s', data) yield_result(task, rule, "SAVDI_FILE_ENCRYPTED") save_av_cache(task, digest, rule, "SAVDI_FILE_ENCRYPTED") end + -- set pseudo virus if configured, else set fail since part was not scanned elseif string.find(data, 'REJ 4') then if rule['savdi_report_oversize'] then - rspamd_logger.infox(task, 'Message is OVERSIZED (SSSP reject code 4): %s', data) - yield_result(task, digest, rule, "SAVDI_FILE_OVERSIZED") + rspamd_logger.infox(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data) + yield_result(task, rule, "SAVDI_FILE_OVERSIZED") save_av_cache(task, digest, rule, "SAVDI_FILE_OVERSIZED") + else + rspamd_logger.errx(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data) + task:insert_result(rule['symbol_fail'], 0.0, 'Message is OVERSIZED (SSSP reject code 4):' .. data) end + -- excplicitly set REJ1 message when SAVDIreports a protocol error elseif string.find(data, 'REJ 1') then rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data) - elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then - conn:add_read(sophos_callback) + task:insert_result(rule['symbol_fail'], 0.0, 'SAVDI (Protocol error (REJ 1)):' .. data) else rspamd_logger.errx(task, 'unhandled response: %s', data) task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response') end + end end end @@ -666,7 +699,7 @@ local function savapi_check(task, content, digest, rule) -- Terminal response - clean if string.find(result, '200') or string.find(result, '210') then if rule['log_clean'] then - rspamd_logger.infox(task, '%s: message is clean', rule['type']) + rspamd_logger.infox(task, '%s: message or mime_part is clean', rule['type']) end save_av_cache(task, digest, rule, 'OK') conn:add_write(savapi_fin_cb, 'QUIT\n') @@ -715,29 +748,32 @@ local function savapi_check(task, content, digest, rule) local function savapi_callback_init(err, data, conn) if err then - if err == 'IO timeout' then - if retransmits > 0 then - retransmits = retransmits - 1 - -- Select a different upstream! - upstream = rule.upstreams:get_upstream_round_robin() - addr = upstream:get_addr() - tcp.request({ - task = task, - host = addr:to_string(), - port = addr:get_port(), - timeout = rule['timeout'], - callback = savapi_callback_init, - stop_pattern = {'\n'}, - }) - else - rspamd_logger.errx(task, 'failed to scan, maximum retransmits exceed') - upstream:fail() - task:insert_result(rule['symbol_fail'], 0.0, 'retransmits exceed') - end + + -- set current upstream to fail because an error occurred + upstream:fail() + + -- retry with another upstream until retransmits exceeds + if retransmits > 0 then + + retransmits = retransmits - 1 + + -- Select a different upstream! + upstream = rule.upstreams:get_upstream_round_robin() + addr = upstream:get_addr() + + lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + + tcp.request({ + task = task, + host = addr:to_string(), + port = addr:get_port(), + timeout = rule['timeout'], + callback = savapi_callback_init, + stop_pattern = {'\n'}, + }) else - rspamd_logger.errx(task, 'failed to scan: %s', err) - task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan') - upstream:fail() + rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type']) + task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed') end else upstream:ok() @@ -801,6 +837,14 @@ local function add_antivirus_rule(sym, opts) opts['symbol_fail'] = string.upper(opts['type']) .. '_FAIL' end + -- WORKAROUND for deprecated attachments_only + if opts['attachments_only'] ~= nil then + opts['scan_mime_parts'] = opts['attachments_only'] + rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. '.. + 'Please use scan_mime_parts = %s instead', opts['symbol'], opts['type'], opts['attachments_only']) + end + -- WORKAROUND for deprecated attachments_only + if not cfg then rspamd_logger.errx(rspamd_config, 'unknown antivirus type: %s', opts['type']) @@ -843,11 +887,17 @@ local function add_antivirus_rule(sym, opts) end return function(task) - if rule.attachments_only then + if rule.scan_mime_parts then local parts = task:get_parts() or {} for _,p in ipairs(parts) do - if not p:is_image() and not p:is_text() and not p:is_multipart() then + if ( + (p:is_image() and rule.scan_image_mime) + or (p:is_text() and rule.scan_text_mime) + or (p:is_multipart() and rule.scan_text_mime) + or (not p:is_image() and not p:is_text() and not p:is_multipart()) + ) then + local content = p:get_content() if content and #content > 0 then @@ -876,12 +926,15 @@ if opts and type(opts) == 'table' then type = 'normal', name = m['symbol'], callback = cb, + score = 0.0, + group = 'antivirus' }) rspamd_config:register_symbol({ type = 'virtual', name = m['symbol_fail'], parent = id, score = 0.0, + group = 'antivirus' }) has_valid = true if type(m['patterns']) == 'table' then @@ -927,7 +980,7 @@ if opts and type(opts) == 'table' then name = m['symbol'], score = m['score'], description = description, - group = group + group = group or 'antivirus' }) end end diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua index bf6010c3f..eeae65289 100644 --- a/src/plugins/lua/arc.lua +++ b/src/plugins/lua/arc.lua @@ -343,6 +343,8 @@ end local id = rspamd_config:register_symbol({ name = 'ARC_CALLBACK', type = 'callback', + group = 'policies', + groups = {'arc'}, callback = arc_callback }) @@ -353,6 +355,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = -1.0, group = 'policies', + groups = {'arc'}, }) rspamd_config:register_symbol({ name = arc_symbols['reject'], @@ -360,6 +363,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 2.0, group = 'policies', + groups = {'arc'}, }) rspamd_config:register_symbol({ name = arc_symbols['invalid'], @@ -367,6 +371,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 1.0, group = 'policies', + groups = {'arc'}, }) rspamd_config:register_symbol({ name = arc_symbols['dnsfail'], @@ -374,6 +379,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 0.0, group = 'policies', + groups = {'arc'}, }) rspamd_config:register_symbol({ name = arc_symbols['na'], @@ -381,6 +387,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 0.0, group = 'policies', + groups = {'arc'}, }) rspamd_config:register_dependency('ARC_CALLBACK', symbols['spf_allow_symbol']) diff --git a/src/plugins/lua/dcc.lua b/src/plugins/lua/dcc.lua index 131c83d36..311dc608e 100644 --- a/src/plugins/lua/dcc.lua +++ b/src/plugins/lua/dcc.lua @@ -20,18 +20,19 @@ limitations under the License. local N = 'dcc' local symbol_bulk = "DCC_BULK" local opts = rspamd_config:get_all_opt(N) -local logger = require "rspamd_logger" +local rspamd_logger = require "rspamd_logger" +local lua_util = require "lua_util" local tcp = require "rspamd_tcp" +local upstream_list = require "rspamd_upstream_list" local fun = require "fun" -local lua_util = require "lua_util" if confighelp then rspamd_config:add_example(nil, 'dcc', "Check messages for 'bulkiness' using DCC", [[ dcc { - host = "/var/dcc/dccifd"; # Unix socket or hostname - port = 1234 # Port to use (needed for TCP socket) + socket = "/var/dcc/dccifd"; # Unix socket + servers = "127.0.0.1:10045" # OR TCP upstreams timeout = 2s; # Timeout to wait for checks } ]]) @@ -42,6 +43,22 @@ local function check_dcc (task) -- Connection local client = '0.0.0.0' local client_ip = task:get_from_ip() + local dcc_upstream + local upstream + local addr + local port + local retransmits = 2 + + if opts['servers'] then + dcc_upstream = upstream_list.create(rspamd_config, opts['servers']) + upstream = dcc_upstream:get_upstream_round_robin() + addr = upstream:get_addr() + port = addr:get_port() + else + lua_util.debugm(N, task, 'using socket %s', opts['socket']) + addr = opts['socket'] + end + if client_ip and client_ip:is_valid() then client = client_ip:to_string() end @@ -74,27 +91,64 @@ local function check_dcc (task) -- Callback function to receive async result from DCC local function cb(err, data) - if (err) then - logger.warnx(task, 'DCC error: %1', err) - return - end - -- Parse the response - local _,_,result,disposition,header = tostring(data):find("(.-)\n(.-)\n(.-)\n") - logger.debugm(N, task, 'DCC result=%1 disposition=%2 header="%3"', - result, disposition, header) - - if header then - local _,_,info = header:find("; (.-)$") - if (result == 'R') then - -- Reject - task:insert_result(symbol_bulk, 1.0, info) - elseif (result == 'T') then - -- Temporary failure - logger.warnx(task, 'DCC returned a temporary failure result') + + if err then + if retransmits > 0 then + retransmits = retransmits - 1 + -- Select a different upstream or the socket again + if opts['servers'] then + upstream = dcc_upstream:get_upstream_round_robin() + addr = upstream:get_addr() + port = addr:get_port() + else + addr = opts['socket'] + end + + lua_util.debugm(N, task, "sending query to %s:%s",tostring(addr), port) + + data = { + "header\n", + client .. "\n", + helo .. "\n", + envfrom .. "\n", + envrcpt .. "\n", + "\n", + task:get_content() + } + + tcp.request({ + task = task, + host = tostring(addr), + port = port or 1, + timeout = opts['timeout'] or 2.0, + shutdown = true, + data = data, + callback = cb + }) + else - if result ~= 'A' and result ~= 'G' and result ~= 'S' then - -- Unknown result - logger.warnx(task, 'DCC result error: %1', result); + rspamd_logger.errx(task, 'failed to scan, maximum retransmits exceed') + upstream:fail() + end + else + -- Parse the response + local _,_,result,disposition,header = tostring(data):find("(.-)\n(.-)\n(.-)\n") + lua_util.debugm(N, task, 'DCC result=%1 disposition=%2 header="%3"', + result, disposition, header) + + if header then + local _,_,info = header:find("; (.-)$") + if (result == 'R') then + -- Reject + task:insert_result(symbol_bulk, 1.0, info) + elseif (result == 'T') then + -- Temporary failure + rspamd_logger.warnx(task, 'DCC returned a temporary failure result') + else + if result ~= 'A' and result ~= 'G' and result ~= 'S' then + -- Unknown result + rspamd_logger.warnx(task, 'DCC result error: %1', result); + end end end end @@ -112,13 +166,12 @@ local function check_dcc (task) task:get_content() } - logger.debugm(N, task, 'sending to dcc: client=%1 helo="%2" envfrom="%3" envrcpt="%4"', - client, helo, envfrom, envrcpt) + rspamd_logger.warnx(task, "sending to %s:%s",tostring(addr), port) tcp.request({ task = task, - host = opts['host'], - port = opts['port'] or 1, + host = tostring(addr), + port = port or 1, timeout = opts['timeout'] or 2.0, shutdown = true, data = data, @@ -127,7 +180,21 @@ local function check_dcc (task) end -- Configuration -if opts and opts['host'] then + +-- WORKAROUND for deprecated host and port settings +if opts['host'] ~= nil and opts['port'] ~= nil then + opts['servers'] = opts['host'] .. ':' .. opts['port'] + rspamd_logger.warnx(rspamd_config, 'Using host and port parameters is deprecated. '.. + 'Please use servers = "%s:%s"; instead', opts['host'], opts['port']) +end +if opts['host'] ~= nil and not opts['port'] then + opts['socket'] = opts['host'] + rspamd_logger.warnx(rspamd_config, 'Using host parameters is deprecated. '.. + 'Please use socket = "%s"; instead', opts['host']) +end +-- WORKAROUND for deprecated host and port settings + +if opts and ( opts['servers'] or opts['socket'] ) then rspamd_config:register_symbol({ name = symbol_bulk, callback = check_dcc @@ -141,5 +208,5 @@ if opts and opts['host'] then }) else lua_util.disable_module(N, "config") - logger.infox('DCC module not configured'); + rspamd_logger.infox('DCC module not configured'); end diff --git a/src/plugins/lua/dkim_signing.lua b/src/plugins/lua/dkim_signing.lua index f15816afd..8d621bbb2 100644 --- a/src/plugins/lua/dkim_signing.lua +++ b/src/plugins/lua/dkim_signing.lua @@ -221,3 +221,6 @@ rspamd_config:register_symbol({ groups = {"policies", "dkim"}, score = 0.0, }) + +-- Add dependency on DKIM checks +rspamd_config:register_dependency(settings['symbol'], 'DKIM_CHECK')
\ No newline at end of file diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua index c08b1dfa9..69e210e47 100644 --- a/src/plugins/lua/dmarc.lua +++ b/src/plugins/lua/dmarc.lua @@ -561,17 +561,26 @@ local function dmarc_callback(task) forced = true}) end -local opts = rspamd_config:get_all_opt('options') -if type(opts) == 'table' then - if type(opts['check_local']) == 'boolean' then - check_local = opts['check_local'] - end - if type(opts['check_authed']) == 'boolean' then - check_authed = opts['check_authed'] +local function try_opts(where) + local ret = false + local opts = rspamd_config:get_all_opt(where) + if type(opts) == 'table' then + if type(opts['check_local']) == 'boolean' then + check_local = opts['check_local'] + ret = true + end + if type(opts['check_authed']) == 'boolean' then + check_authed = opts['check_authed'] + ret = true + end end + + return ret end -opts = rspamd_config:get_all_opt('dmarc') +if not try_opts(N) then try_opts('options') end + +local opts = rspamd_config:get_all_opt('dmarc') if not opts or type(opts) ~= 'table' then return end @@ -1266,37 +1275,51 @@ end local id = rspamd_config:register_symbol({ name = 'DMARC_CALLBACK', type = 'callback', + group = 'policies', + groups = {'dmarc'}, callback = dmarc_callback }) rspamd_config:register_symbol({ name = dmarc_symbols['allow'], flags = 'nice', parent = id, + group = 'policies', + groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ name = dmarc_symbols['reject'], parent = id, + group = 'policies', + groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ name = dmarc_symbols['quarantine'], parent = id, + group = 'policies', + groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ name = dmarc_symbols['softfail'], parent = id, + group = 'policies', + groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ name = dmarc_symbols['dnsfail'], parent = id, + group = 'policies', + groups = {'dmarc'}, type = 'virtual' }) rspamd_config:register_symbol({ name = dmarc_symbols['na'], parent = id, + group = 'policies', + groups = {'dmarc'}, type = 'virtual' }) diff --git a/src/plugins/lua/forged_recipients.lua b/src/plugins/lua/forged_recipients.lua index 25d6c8f8c..887b1bf82 100644 --- a/src/plugins/lua/forged_recipients.lua +++ b/src/plugins/lua/forged_recipients.lua @@ -97,6 +97,8 @@ if opts then name = 'FORGED_CALLBACK', callback = check_forged_headers, type = 'callback', + group = 'headers', + score = 0.0, }) if opts['symbol_rcpt'] then symbol_rcpt = opts['symbol_rcpt'] diff --git a/src/plugins/lua/hfilter.lua b/src/plugins/lua/hfilter.lua index 70992b8f3..334378ba4 100644 --- a/src/plugins/lua/hfilter.lua +++ b/src/plugins/lua/hfilter.lua @@ -569,17 +569,26 @@ local symbols_from = { "HFILTER_FROM_BOUNCE" } -local opts = rspamd_config:get_all_opt('options') -if type(opts) == 'table' then - if type(opts['check_local']) == 'boolean' then - check_local = opts['check_local'] - end - if type(opts['check_authed']) == 'boolean' then - check_authed = opts['check_authed'] +local function try_opts(where) + local ret = false + local opts = rspamd_config:get_all_opt(where) + if type(opts) == 'table' then + if type(opts['check_local']) == 'boolean' then + check_local = opts['check_local'] + ret = true + end + if type(opts['check_authed']) == 'boolean' then + check_authed = opts['check_authed'] + ret = true + end end + + return ret end -opts = rspamd_config:get_all_opt('hfilter') +if not try_opts(N) then try_opts('options') end + +local opts = rspamd_config:get_all_opt('hfilter') if opts then for k,v in pairs(opts) do config[k] = v @@ -618,6 +627,19 @@ end --dumper(symbols_enabled) if #symbols_enabled > 0 then rspamd_config:register_symbols(hfilter, 1.0, "HFILTER", symbols_enabled); + rspamd_config:set_metric_symbol({ + name = 'HFILTER', + score = 0.0, + group = 'hfilter' + }) + + for _,s in ipairs(symbols_enabled) do + rspamd_config:set_metric_symbol({ + name = s, + score = 0.0, + group = 'hfilter' + }) + end else lua_util.disable_module(N, "config") end diff --git a/src/plugins/lua/ip_score.lua b/src/plugins/lua/ip_score.lua index 7ada08413..d3aec2264 100644 --- a/src/plugins/lua/ip_score.lua +++ b/src/plugins/lua/ip_score.lua @@ -314,8 +314,12 @@ local ip_score_check = function(task) return cmd, args end - if task:get_user() or (ip and ip:is_local()) then - rspamd_logger.infox(task, "skip IP Score for local networks and authorized users") + if task:get_user() and not check_authed then + rspamd_logger.infox(task, "skip IP Score for authorized users") + return + end + if ip and ip:is_local() and not check_local then + rspamd_logger.infox(task, "skip IP Score for local networks") return end if ip:is_valid() then @@ -352,19 +356,29 @@ local ip_score_check = function(task) end end - --- Configuration options -local configure_ip_score_module = function() - local opts = rspamd_config:get_all_opt('options') +local function try_opts(where) + local ret = false + local opts = rspamd_config:get_all_opt(where) if type(opts) == 'table' then - if type(opts['check_authed']) == 'boolean' then - check_authed = opts['check_authed'] - end if type(opts['check_local']) == 'boolean' then check_local = opts['check_local'] + ret = true + end + if type(opts['check_authed']) == 'boolean' then + check_authed = opts['check_authed'] + ret = true end end - opts = rspamd_config:get_all_opt('ip_score') + + return ret +end + +if not try_opts(N) then try_opts('options') end + +-- Configuration options +local configure_ip_score_module = function() + local opts = rspamd_config:get_all_opt(N) + if not opts then return end for k,v in pairs(opts) do options[k] = v diff --git a/src/plugins/lua/mid.lua b/src/plugins/lua/mid.lua index 4baa8867c..5410e7f9f 100644 --- a/src/plugins/lua/mid.lua +++ b/src/plugins/lua/mid.lua @@ -79,16 +79,19 @@ if opts then local id = rspamd_config:register_symbol({ name = 'KNOWN_MID_CALLBACK', type = 'callback', + group = 'mid', callback = known_mid_cb }) rspamd_config:register_symbol({ name = settings['symbol_known_mid'], parent = id, + group = 'mid', type = 'virtual' }) rspamd_config:register_symbol({ name = settings['symbol_known_no_mid'], parent = id, + group = 'mid', type = 'virtual' }) rspamd_config:add_composite(settings['csymbol_invalid_msgid_allowed'], diff --git a/src/plugins/lua/mime_types.lua b/src/plugins/lua/mime_types.lua index f97f22d2a..e8ce709da 100644 --- a/src/plugins/lua/mime_types.lua +++ b/src/plugins/lua/mime_types.lua @@ -376,7 +376,7 @@ local full_extensions_map = { {"hxx", "text/plain"}, {"i", "text/plain"}, {"ico", "image/x-icon"}, - {"ics", "application/octet-stream"}, + {"ics", {"text/calendar", "application/octet-stream"}}, {"idl", "text/plain"}, {"ief", "image/ief"}, {"iii", "application/x-iphone"}, @@ -1018,49 +1018,58 @@ if opts then local id = rspamd_config:register_symbol({ name = 'MIME_TYPES_CALLBACK', callback = check_mime_type, - type = 'callback,nostat' + type = 'callback,nostat', + group = 'mime_types', }) rspamd_config:register_symbol({ type = 'virtual', name = settings['symbol_unknown'], - parent = id + parent = id, + group = 'mime_types', }) rspamd_config:register_symbol({ type = 'virtual', name = settings['symbol_bad'], - parent = id + parent = id, + group = 'mime_types', }) rspamd_config:register_symbol({ type = 'virtual', name = settings['symbol_good'], flags = 'nice', - parent = id + parent = id, + group = 'mime_types', }) rspamd_config:register_symbol({ type = 'virtual', name = settings['symbol_attachment'], - parent = id + parent = id, + group = 'mime_types', }) rspamd_config:register_symbol({ type = 'virtual', name = settings['symbol_encrypted_archive'], - parent = id + parent = id, + group = 'mime_types', }) rspamd_config:register_symbol({ type = 'virtual', name = settings['symbol_archive_in_archive'], - parent = id + parent = id, + group = 'mime_types', }) rspamd_config:register_symbol({ type = 'virtual', name = settings['symbol_double_extension'], - parent = id + parent = id, + group = 'mime_types', }) rspamd_config:register_symbol({ type = 'virtual', name = settings['symbol_bad_extension'], - parent = id + parent = id, + group = 'mime_types', }) else lua_util.disable_module(N, "config") diff --git a/src/plugins/lua/neural.lua b/src/plugins/lua/neural.lua index 9d0bbb446..022160083 100644 --- a/src/plugins/lua/neural.lua +++ b/src/plugins/lua/neural.lua @@ -33,8 +33,6 @@ local N = "neural" if rspamd_config:has_torch() then use_torch = true - torch = require "torch" - nn = require "nn" end -- Module vars @@ -949,6 +947,9 @@ else if opts.disable_torch then use_torch = false + else + torch = require "torch" + nn = require "nn" end local id = rspamd_config:register_symbol({ @@ -1017,9 +1018,9 @@ else if worker:is_primary_controller() then -- We also want to train neural nets when they have enough data rspamd_config:add_periodic(ev_base, 0.0, - function(_, _) - return maybe_train_anns(rule, cfg, ev_base, worker) - end) + function(_, _) + return maybe_train_anns(rule, cfg, ev_base, worker) + end) end end) end diff --git a/src/plugins/lua/once_received.lua b/src/plugins/lua/once_received.lua index b09a87dbe..c8c47ddb1 100644 --- a/src/plugins/lua/once_received.lua +++ b/src/plugins/lua/once_received.lua @@ -32,6 +32,7 @@ local whitelist = nil local rspamd_logger = require "rspamd_logger" local fun = require "fun" +local N = 'once_received' local check_local = false local check_authed = false @@ -152,17 +153,27 @@ local function check_quantity_received (task) end end -local opts = rspamd_config:get_all_opt('options') -if type(opts) == 'table' then - if type(opts['check_local']) == 'boolean' then - check_local = opts['check_local'] - end - if type(opts['check_authed']) == 'boolean' then - check_authed = opts['check_authed'] +local function try_opts(where) + local ret = false + local opts = rspamd_config:get_all_opt(where) + if type(opts) == 'table' then + if type(opts['check_local']) == 'boolean' then + check_local = opts['check_local'] + ret = true + end + if type(opts['check_authed']) == 'boolean' then + check_authed = opts['check_authed'] + ret = true + end end + + return ret end + +if not try_opts(N) then try_opts('options') end + -- Configuration -opts = rspamd_config:get_all_opt('once_received') +local opts = rspamd_config:get_all_opt(N) if opts then if opts['symbol'] then symbol = opts['symbol'] diff --git a/src/plugins/lua/spamtrap.lua b/src/plugins/lua/spamtrap.lua index cf0b89957..85af46dd2 100644 --- a/src/plugins/lua/spamtrap.lua +++ b/src/plugins/lua/spamtrap.lua @@ -31,10 +31,11 @@ local settings = { fuzzy_flag = 1, fuzzy_weight = 10.0, key_prefix = 'sptr_', - check_authed = true, - check_local = true, } +local check_authed = true +local check_local = true + local function spamtrap_cb(task) local rcpts = task:get_recipients('smtp') local authed_user = task:get_user() @@ -42,8 +43,8 @@ local function spamtrap_cb(task) local called_for_domain = false local target - if ((not settings['check_authed'] and authed_user) or - (not settings['check_local'] and ip_addr and ip_addr:is_local())) then + if ((not check_authed and authed_user) or + (not check_local and ip_addr and ip_addr:is_local())) then rspamd_logger.infox(task, "skip spamtrap checks for local networks or authenticated user"); return end @@ -136,11 +137,31 @@ local function spamtrap_cb(task) end -- Module setup +local function try_opts(where) + local ret = false + local opts = rspamd_config:get_all_opt(where) + if type(opts) == 'table' then + if type(opts['check_local']) == 'boolean' then + check_local = opts['check_local'] + ret = true + end + if type(opts['check_authed']) == 'boolean' then + check_authed = opts['check_authed'] + ret = true + end + end + + return ret +end + local opts = rspamd_config:get_all_opt('spamtrap') if not (opts and type(opts) == 'table') then rspamd_logger.infox(rspamd_config, 'module is unconfigured') return end + +if not try_opts(M) then try_opts('options') end + if opts then for k,v in pairs(opts) do settings[k] = v diff --git a/src/plugins/lua/whitelist.lua b/src/plugins/lua/whitelist.lua index 25add8c98..7e6052d24 100644 --- a/src/plugins/lua/whitelist.lua +++ b/src/plugins/lua/whitelist.lua @@ -39,18 +39,51 @@ local function whitelist_cb(symbol, rule, task) local domains = {} - local function find_domain(dom) - local mult = 1.0 + local function find_domain(dom, check) + local mult + local how = 'wl' + + -- Can be overriden + if rule.blacklist then how = 'bl' end + + local function parse_val(val) + local how_override + -- Strict is 'special' + if rule.strict then how_override = 'both' end + if val then + lua_util.debugm(N, task, "found whitelist key: %s=%s", dom, val) + if val == '' then + return (how_override or how),1.0 + elseif val:match('^bl:') then + return (how_override or 'bl'),(tonumber(val:sub(4)) or 1.0) + elseif val:match('^wl:') then + return (how_override or 'wl'),(tonumber(val:sub(4)) or 1.0) + elseif val:match('^both:') then + return (how_override or 'both'),(tonumber(val:sub(6)) or 1.0) + else + return (how_override or how),(tonumber(val) or 1.0) + end + end + + return (how_override or how),1.0 + end if rule['map'] then local val = rule['map']:get_key(dom) if val then - if #val > 0 then - mult = tonumber(val) + how,mult = parse_val(val) + + if not domains[check] then + domains[check] = {} end - table.insert(domains, dom) - return true,mult + domains[check] = { + [dom] = {how, mult} + } + + lua_util.debugm(N, task, "final result: %s: %s->%s", + dom, how, mult) + return true,mult,how end elseif rule['maps'] then for _,v in pairs(rule['maps']) do @@ -58,43 +91,48 @@ local function whitelist_cb(symbol, rule, task) if map then local val = map:get_key(dom) if val then - if #val > 0 then - mult = tonumber(val) - else - mult = v.mult or 1.0 + how,mult = parse_val(val) + + if not domains[check] then + domains[check] = {} end - table.insert(domains, dom) - return true,mult + + domains[check] = { + [dom] = {how, mult} + } + + lua_util.debugm(N, task, "final result: %s: %s->%s", + dom, how, mult) + return true,mult,how end end end else mult = rule['domains'][dom] if mult then - table.insert(domains, dom) - return true, mult + if not domains[check] then + domains[check] = {} + end + + domains[check] = { + [dom] = {how, mult} + } + + return true, mult,how end end - return false,0.0 + return false,0.0,how end - local found = false - local mult = 1.0 local spf_violated = false - local dkim_violated = false local dmarc_violated = false if rule['valid_spf'] then if not task:has_symbol(options['spf_allow_symbol']) then -- Not whitelisted - if not rule['blacklist'] and not rule['strict'] then - return - end - spf_violated = true end - -- Now we can check from domain or helo local from = task:get_from(1) @@ -102,7 +140,7 @@ local function whitelist_cb(symbol, rule, task) local tld = rspamd_util.get_tld(from[1]['domain']) if tld then - found,mult = find_domain(tld) + find_domain(tld, 'spf') end else local helo = task:get_helo() @@ -111,46 +149,33 @@ local function whitelist_cb(symbol, rule, task) local tld = rspamd_util.get_tld(helo) if tld then - found, mult = find_domain(tld) + find_domain(tld) end end end end if rule['valid_dkim'] then - local sym = task:get_symbol(options['dkim_allow_symbol']) - if not sym then - if not rule['blacklist'] and not rule['strict'] then - return - end - - dkim_violated = true - else - found = false + if task:has_symbol('DKIM_TRACE') then + local sym = task:get_symbol('DKIM_TRACE') local dkim_opts = sym[1]['options'] if dkim_opts then fun.each(function(val) - if not found then - local tld = rspamd_util.get_tld(val) - - if tld then - found, mult = find_domain(tld) - if not found then - found, mult = find_domain(val) - end + if val[2] == '+' then + find_domain(val[1], 'dkim_success') + elseif val[2] == '-' then + find_domain(val[1], 'dkim_fail') end - end - end, dkim_opts) + end, + fun.map(function(s) + return lua_util.rspamd_str_split(s, ':') + end, dkim_opts)) end end end if rule['valid_dmarc'] then if not task:has_symbol(options['dmarc_allow_symbol']) then - if not rule['blacklist'] and not rule['strict'] then - return - end - dmarc_violated = true end @@ -160,37 +185,107 @@ local function whitelist_cb(symbol, rule, task) local tld = rspamd_util.get_tld(from[1]['domain']) if tld then - found, mult = find_domain(tld) + local found = find_domain(tld, 'dmarc') if not found then - found, mult = find_domain(from[1]['domain']) + find_domain(from[1]['domain'], 'dmarc') end end end end - if found then - if not rule['blacklist'] and not rule['strict'] then - task:insert_result(symbol, mult, domains) - else - -- Additional constraints for blacklist - if rule['valid_spf'] or rule['valid_dkim'] or rule['valid_dmarc'] then - if dmarc_violated or dkim_violated or spf_violated then - if rule['strict'] then - -- Inverse multiplier to convert whitelist to blacklist - mult = -mult - end + local final_mult = 1.0 + local found_wl, found_bl = false, false + local opts = {} + + if rule.valid_dkim then + for dom,val in pairs(domains.dkim_success or E) do + if val[1] == 'wl' or val[1] == 'both' then + -- We have valid and whitelisted signature + table.insert(opts, dom .. ':d:+') + found_wl = true - task:insert_result(symbol, mult, domains) - elseif rule['strict'] then - -- Add whitelist score (negative) - task:insert_result(symbol, mult, domains) + if not found_bl then + final_mult = val[2] + lua_util.debugm(N, task, "hui4 final mult: %s", final_mult) end - else - -- Unconstrained input - task:insert_result(symbol, mult, domains) end end + + -- Blacklist counterpart + for dom,val in pairs(domains.dkim_fail or E) do + if val[1] == 'bl' or val[1] == 'both' then + -- We have valid and whitelisted signature + table.insert(opts, dom .. ':d:-') + found_bl = true + final_mult = val[2] + lua_util.debugm(N, task, "hui2 final mult: %s", final_mult) + end + end + end + + local function check_domain_violation(what, dom, val, violated) + if violated then + if val[1] == 'both' or val[1] == 'bl' then + found_bl = true + final_mult = val[2] + lua_util.debugm(N, task, "hui3 final mult: %s", final_mult) + table.insert(opts, string.format("%s:%s:-", dom, what)) + end + else + if val[1] == 'both' or val[1] == 'wl' then + found_wl = true + table.insert(opts, string.format("%s:%s:+", dom, what)) + if not found_bl then + final_mult = val[2] + lua_util.debugm(N, task, "hui1 final mult: %s", final_mult) + end + end + end + end + + if rule.valid_dmarc then + found_wl = false + + for dom,val in pairs(domains.dmarc or E) do + check_domain_violation('D', dom, val, dmarc_violated) + end + end + + if rule.valid_spf then + found_wl = false + + for dom,val in pairs(domains.spf or E) do + check_domain_violation('s', dom, val, spf_violated) + end + end + + lua_util.debugm(N, task, "final mult: %s", final_mult) + + local function add_symbol(violated, mult) + local sym = symbol + + if violated then + if rule.inverse_symbol then + sym = rule.inverse_symbol + elseif not rule.blacklist then + mult = -mult + end + + if rule.inverse_multiplier then + mult = mult * rule.inverse_multiplier + end + + task:insert_result(sym, mult, opts) + else + task:insert_result(sym, mult, opts) + end + end + + if found_bl then + add_symbol(true, final_mult) + elseif found_wl then + add_symbol(false, final_mult) end end @@ -261,12 +356,20 @@ local configure_whitelist_module = function() flags = 'empty' end - rspamd_config:register_symbol({ + local id = rspamd_config:register_symbol({ name = symbol, flags = flags, callback = gen_whitelist_cb(symbol, rule) }) + if rule.inverse_symbol then + rspamd_config:register_symbol({ + name = rule.inverse_symbol, + type = 'virtual', + parent = id + }) + end + local spf_dep = false local dkim_dep = false if rule['valid_spf'] then @@ -293,6 +396,13 @@ local configure_whitelist_module = function() end rule['name'] = symbol rspamd_config:set_metric_symbol(rule) + + if rule.inverse_symbol then + local inv_rule = lua_util.shallowcopy(rule) + inv_rule.name = rule.inverse_symbol + inv_rule.score = -rule.score + rspamd_config:set_metric_symbol(inv_rule) + end end end end, options['rules']) diff --git a/src/plugins/spf.c b/src/plugins/spf.c index 5ec5bfcfc..46160878f 100644 --- a/src/plugins/spf.c +++ b/src/plugins/spf.c @@ -206,15 +206,26 @@ spf_module_config (struct rspamd_config *cfg) spf_module_ctx->whitelist_ip = NULL; - if ((value = - rspamd_config_get_module_opt (cfg, "options", "check_local")) != NULL) { + value = rspamd_config_get_module_opt (cfg, "spf", "check_local"); + + if (value == NULL) { + rspamd_config_get_module_opt (cfg, "options", "check_local"); + } + + if (value != NULL) { spf_module_ctx->check_local = ucl_obj_toboolean (value); } else { spf_module_ctx->check_local = FALSE; } - if ((value = - rspamd_config_get_module_opt (cfg, "options", "check_authed")) != NULL) { + + value = rspamd_config_get_module_opt (cfg, "spf", "check_authed"); + + if (value == NULL) { + rspamd_config_get_module_opt (cfg, "options", "check_authed"); + } + + if (value != NULL) { spf_module_ctx->check_authed = ucl_obj_toboolean (value); } else { diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index 052dcd1d6..49242ccaf 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -423,6 +423,11 @@ main (gint argc, gchar **argv, gchar **env) rspamd_set_logger (cfg, process_quark, &rspamd_main->logger, rspamd_main->server_pool); (void) rspamd_log_open (rspamd_main->logger); + + (void) dns_resolver_init (rspamd_main->logger, + rspamd_main->ev_base, + cfg); + g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger); g_set_printerr_handler (rspamd_glib_printerr_function); rspamd_config_post_load (cfg, diff --git a/test/functional/cases/123_whitelist.robot b/test/functional/cases/123_whitelist.robot index 9ca57b9ec..570c43bf8 100644 --- a/test/functional/cases/123_whitelist.robot +++ b/test/functional/cases/123_whitelist.robot @@ -9,6 +9,10 @@ Variables ${TESTDIR}/lib/vars.py ${CONFIG} ${TESTDIR}/configs/plugins.conf ${M_DMARC_OK} ${TESTDIR}/messages/dmarc/pass_none.eml ${M_DMARC_BAD} ${TESTDIR}/messages/dmarc/fail_none.eml + +${M_DKIM_RSPAMD_OK} ${TESTDIR}/messages/dmarc/good_dkim_rspamd.eml +${M_DKIM_RSPAMD_BAD} ${TESTDIR}/messages/dmarc/bad_dkim_rspamd.eml + ${UTF_MESSAGE} ${TESTDIR}/messages/utf.eml ${RSPAMD_SCOPE} Suite ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat @@ -16,12 +20,12 @@ ${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat *** Test Cases *** WHITELISTS ${result} = Scan Message With Rspamc ${M_DMARC_OK} -i 8.8.4.4 -F foo@spf.cacophony.za.org - Check Rspamc ${result} WHITELIST_DKIM (-1 - Should Contain ${result.stdout} STRICT_DMARC (-3 - Should Contain ${result.stdout} WHITELIST_DDS (-3 - Should Contain ${result.stdout} WHITELIST_DMARC (-2 - Should Contain ${result.stdout} WHITELIST_DMARC_DKIM (-2 - Should Contain ${result.stdout} WHITELIST_SPF (-1 + Check Rspamc ${result} WHITELIST_DKIM (- + Should Contain ${result.stdout} STRICT_DMARC (- + Should Contain ${result.stdout} WHITELIST_DDS (- + Should Contain ${result.stdout} WHITELIST_DMARC (- + Should Contain ${result.stdout} WHITELIST_DMARC_DKIM (- + Should Contain ${result.stdout} WHITELIST_SPF (- Should Not Contain ${result.stdout} BLACKLIST_SPF ( Should Not Contain ${result.stdout} BLACKLIST_DKIM ( Should Not Contain ${result.stdout} BLACKLIST_DMARC ( @@ -38,13 +42,24 @@ BLACKLISTS Should Contain ${result.stdout} BLACKLIST_SPF (3 Should Contain ${result.stdout} STRICT_DMARC (3 Should Contain ${result.stdout} BLACKLIST_DDS (3 - Should Contain ${result.stdout} BLACKLIST_DMARC (3 + Should Contain ${result.stdout} BLACKLIST_DMARC (2 Should Not Contain ${result.stdout} WHITELIST_DDS ( Should Not Contain ${result.stdout} WHITELIST_SPF ( - Should Not Contain ${result.stdout} WHITEIST_DKIM ( + Should Not Contain ${result.stdout} WHITELIST_DKIM ( Should Not Contain ${result.stdout} WHITELIST_DMARC ( Should Not Contain ${result.stdout} WHITELIST_DMARC_DKIM ( +WHITELIST_WL_ONLY + ${result} = Scan Message With Rspamc ${M_DKIM_RSPAMD_OK} + Check Rspamc ${result} WHITELIST_DKIM (-2 + Should Not Contain ${result.stdout} BLACKLIST_DKIM ( + +BLACKLISTS_WL_ONLY + ${result} = Scan Message With Rspamc ${M_DKIM_RSPAMD_BAD} + Check Rspamc ${result} DKIM_REJECT ( + Should Not Contain ${result.stdout} WHITELIST_DKIM ( + Should Not Contain ${result.stdout} BLACKLIST_DKIM ( + *** Keywords *** Whitelist Setup ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/whitelist.conf diff --git a/test/functional/cases/151_rspamadm_async.robot b/test/functional/cases/151_rspamadm_async.robot index 31fa84245..a496bf12a 100644 --- a/test/functional/cases/151_rspamadm_async.robot +++ b/test/functional/cases/151_rspamadm_async.robot @@ -9,7 +9,9 @@ Suite Teardown Terminate All Processes kill=True *** Variables *** ${REDIS_SCOPE} Test - +${CONFIG} ${TESTDIR}/configs/plugins.conf +${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat +${PLUGIN_CONFIG} *** Test Cases *** Tcp client @@ -24,6 +26,16 @@ Redis client Should Be Equal As Integers ${result.rc} 0 Should Be Equal ${result.stdout} true\thello from lua on redis +DNS client + ${tmpdir} = Prepare temp directory ${CONFIG} + Set test variable ${tmpdir} + ${result} = Run Process ${RSPAMADM} --var\=CONFDIR\=${tmpdir} lua -b ${TESTDIR}/lua/rspamadm/test_dns_client.lua + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Be Equal ${result.stdout} true\tk=ed25519; p=yi50DjK5O9pqbFpNHklsv9lqaS0ArSYu02qp1S0DW1Y= + Cleanup Temporary Directory ${tmpdir} + *** Keywords *** Rspamadm test Setup @@ -40,3 +52,13 @@ Run Dummy Http [Arguments] ${result} = Start Process ${TESTDIR}/util/dummy_http.py Wait Until Created /tmp/dummy_http.pid + +Prepare temp directory + [Arguments] ${CONFIG} + ${template} = Get File ${CONFIG} + ${tmpdir} = Make Temporary Directory + ${config} = Replace Variables ${template} + ${config} = Replace Variables ${config} + Log ${config} + Create File ${tmpdir}/rspamd.conf ${config} + [Return] ${tmpdir}
\ No newline at end of file diff --git a/test/functional/cases/160_antivirus.robot b/test/functional/cases/160_antivirus.robot index 86c2e51b0..bdbd7456f 100644 --- a/test/functional/cases/160_antivirus.robot +++ b/test/functional/cases/160_antivirus.robot @@ -24,13 +24,13 @@ CLAMAV MISS CLAMAV HIT Run Dummy Clam ${PORT_CLAM} 1 ${result} = Scan Message With Rspamc ${MESSAGE2} - Check Rspamc ${result} CLAM_VIRUS (1.00)[Eicar-Test-Signature] + Check Rspamc ${result} CLAM_VIRUS Should Not Contain ${result.stdout} CLAMAV_FAIL Shutdown clamav CLAMAV CACHE HIT ${result} = Scan Message With Rspamc ${MESSAGE2} - Check Rspamc ${result} CLAM_VIRUS (1.00)[Eicar-Test-Signature] + Check Rspamc ${result} CLAM_VIRUS Should Not Contain ${result.stdout} CLAMAV_FAIL CLAMAV CACHE MISS @@ -49,7 +49,7 @@ FPROT HIT - PATTERN Run Dummy Fprot ${PORT_FPROT} 1 Run Dummy Fprot ${PORT_FPROT_DUPLICATE} 1 /tmp/dummy_fprot_dupe.pid ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} FPROT_EICAR (1.00)[EICAR_Test_File] + Check Rspamc ${result} FPROT_EICAR Should Not Contain ${result.stdout} CLAMAV_VIRUS # Also check ordered pattern match Should Contain ${result.stdout} FPROT_VIRUS_DUPLICATE_PATTERN @@ -60,7 +60,7 @@ FPROT HIT - PATTERN FPROT CACHE HIT ${result} = Scan Message With Rspamc ${MESSAGE} - Check Rspamc ${result} FPROT_EICAR (1.00)[EICAR_Test_File] + Check Rspamc ${result} FPROT_EICAR Should Not Contain ${result.stdout} CLAMAV_VIRUS # Also check ordered pattern match Should Contain ${result.stdout} FPROT_VIRUS_DUPLICATE_PATTERN diff --git a/test/functional/configs/maps/domains.list b/test/functional/configs/maps/domains.list index f0b635852..04b0a9b12 100644 --- a/test/functional/configs/maps/domains.list +++ b/test/functional/configs/maps/domains.list @@ -1,3 +1,5 @@ example.com -cacophony.za.org -#other.com +cacophony.za.org both:1.0 +highsecure.ru bl:1.0 +rspamd.com wl:2.0 +#other.com
\ No newline at end of file diff --git a/test/functional/configs/maps/domains.list.2 b/test/functional/configs/maps/domains.list.2 index 850c64045..45cd0b2ec 100644 --- a/test/functional/configs/maps/domains.list.2 +++ b/test/functional/configs/maps/domains.list.2 @@ -1,2 +1,2 @@ -rspamd.com +rspamd-test.com #other.com diff --git a/test/functional/configs/plugins.conf b/test/functional/configs/plugins.conf index 8097893ac..6d4fff803 100644 --- a/test/functional/configs/plugins.conf +++ b/test/functional/configs/plugins.conf @@ -23,6 +23,21 @@ options = { replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDEEXmNGQq7PUrr9Mg4UakTFHgXBCy2DOztkrZm+0OrVWtiRzGluxBkbOWTBwuU3/Yw97yTphBMQxzWFN603/f/KPAQcF/Lc1l+6kmIBBxNXjjGuOK/3PYKZVntUdKmqcQBYfnHdzH2Tohbuyx1a7xqnv6VSChqQrZU4CwkeT3+eQIDAQAB"]; }, { + name = "dkim._domainkey.rspamd.com", + type = "txt"; + replies = ["v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCd/XhZBEGGAss48lEuMmwZv9lOFf6FTllBiQ3sPhdTpDdIPaW9TInW7iYnYD/bXHeVxYAyD/sKhYk6+qGBRu10rEi+iyPvLCIED+Boq0tEQosuKuV6Fjoomb+QhZY9KdjyZTjsrFPZ+wCkUY/30uTmpX2SwSqyxxlK0pUIsRgMAQIDAQAB"]; + }, + { + name = "_dmarc.rspamd.com", + type = "txt"; + rcode = 'norec'; + }, + { + name = "dkim._domainkey.highsecure.ru", + type = "txt"; + replies = ["p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDK4ZQYky30GH0Ak9OQ1fv3IdFNbpOtpa4S/PR20ZLgPXfd/FCA//ztUmu7kHlELI+/+4f8W+xX0oZlOc/cFxhopRjXZMlSsQqmWOZ40/GxWFBtcqafKu78FCqO7URqZUmMCM5Jlp4zt/yzH3dbYNG3i5PVlB5QtQnZvY+dvBL3dwIDAQAB"]; + }, + { name = "_dmarc.cacophony.za.org", type = "txt"; replies = ["v=DMARC1; p=none; sp=reject"]; diff --git a/test/functional/configs/whitelist.conf b/test/functional/configs/whitelist.conf index 9b8c3af63..7769f5cd4 100644 --- a/test/functional/configs/whitelist.conf +++ b/test/functional/configs/whitelist.conf @@ -3,16 +3,6 @@ dmarc {} whitelist { rules { - "BLACKLIST_DDS" = { - blacklist = true; - valid_dkim = true; - valid_dmarc = true; - valid_spf = true; - domains = [ - "${TESTDIR}/configs/maps/domains.list", - ]; - score = 3.0 - } "WHITELIST_DDS" = { valid_dkim = true; @@ -21,7 +11,8 @@ whitelist { domains = [ "${TESTDIR}/configs/maps/domains.list", ]; - score = -3.0 + score = -3.0; + inverse_symbol = "BLACKLIST_DDS"; } "WHITELIST_DKIM" = { @@ -30,16 +21,8 @@ whitelist { "${TESTDIR}/configs/maps/domains.list", ]; description = "Mail comes from the whitelisted domain and has a valid DKIM signature"; - score = -1.0 - } - - "BLACKLIST_DKIM" = { - blacklist = true; - valid_dkim = true; - domains = [ - "${TESTDIR}/configs/maps/domains.list", - ]; - score = 3.0 + score = -1.0; + inverse_symbol = "BLACKLIST_DKIM"; } "WHITELIST_SPF" = { @@ -48,16 +31,9 @@ whitelist { "${TESTDIR}/configs/maps/domains.list", ]; score = -1.0 + inverse_multiplier = 3.0 description = "Mail comes from the whitelisted domain and has a valid SPF policy"; - } - - "BLACKLIST_SPF" = { - blacklist = true; - valid_spf = true; - domains = [ - "${TESTDIR}/configs/maps/domains.list", - ]; - score = 3.0 + inverse_symbol = "BLACKLIST_SPF"; } "WHITELIST_DMARC_DKIM" = { @@ -68,6 +44,7 @@ whitelist { ]; score = -2.0; description = "Mail comes from the whitelisted domain and has valid DMARC and DKIM policies"; + inverse_symbol = "BLACKLIST_DMARC_DKIM"; } "WHITELIST_DMARC" = { @@ -77,15 +54,7 @@ whitelist { ]; score = -2.0; description = "Mail comes from the whitelisted domain and has valid DMARC policy"; - } - - "BLACKLIST_DMARC" = { - blacklist = true; - valid_dmarc = true; - domains = [ - "${TESTDIR}/configs/maps/domains.list", - ]; - score = 3.0; + inverse_symbol = "BLACKLIST_DMARC"; } "STRICT_DMARC" = { diff --git a/test/functional/lua/mapreload.lua b/test/functional/lua/mapreload.lua index aa1de9392..572da10de 100644 --- a/test/functional/lua/mapreload.lua +++ b/test/functional/lua/mapreload.lua @@ -7,9 +7,9 @@ rspamd_config:register_symbol({ name = 'MAP_SET_HIT_AND_MISS', score = 1.0, callback = function() - if (test_map:get_key('example.com') and not test_map:get_key('rspamd.com')) then + if (test_map:get_key('example.com') and not test_map:get_key('rspamd-test.com')) then return true, 'example.com' - elseif (test_map:get_key('rspamd.com') and not test_map:get_key('example.com')) then + elseif (test_map:get_key('rspamd-test.com') and not test_map:get_key('example.com')) then return true, 'rspamd.com' end end diff --git a/test/functional/lua/rspamadm/test_dns_client.lua b/test/functional/lua/rspamadm/test_dns_client.lua new file mode 100644 index 000000000..c54d59499 --- /dev/null +++ b/test/functional/lua/rspamadm/test_dns_client.lua @@ -0,0 +1,30 @@ +local rspamd_dns = require "rspamd_dns" +local logger = require "rspamd_logger" + +local config_path = rspamd_paths['CONFDIR'] .. '/rspamd.conf' +local _r,err = rspamd_config:load_ucl(config_path) + +if not _r then + logger.errx('cannot parse %s: %s (r=%s)', config_path, err, _r) + os.exit(1) +end + +_r,err = rspamd_config:parse_rcl({'logging', 'worker'}) +if not _r then + logger.errx('cannot process %s: %s (r=%s)', config_path, err, _r) + os.exit(1) +end + +rspamd_config:init_subsystem('dns', rspamadm_ev_base) + + +local is_ok, results = rspamd_dns.request({ + config = rspamd_config, + session = rspamadm_session, + + type = 'txt', + name = 'test._domainkey.example.com', + -- name = '_dmarc.google.com', + }) + +print(is_ok, results[1]) diff --git a/test/functional/lua/rspamadm/test_redis_client.lua b/test/functional/lua/rspamadm/test_redis_client.lua index 7de82cd96..a7428a807 100644 --- a/test/functional/lua/rspamadm/test_redis_client.lua +++ b/test/functional/lua/rspamadm/test_redis_client.lua @@ -8,9 +8,9 @@ local upstreams_read = upstream_list.create('127.0.0.1', 56379) local is_ok, connection = redis.redis_connect_sync({ write_servers = upstreams_write, read_servers = upstreams_read, - config = rspamd_config, - ev_base = rspamadm_ev_base, - session = rspamadm_session, +-- config = rspamd_config, +-- ev_base = rspamadm_ev_base, +-- session = rspamadm_session, timeout = 2 }) diff --git a/test/functional/messages/dmarc/bad_dkim_highsecure.eml b/test/functional/messages/dmarc/bad_dkim_highsecure.eml new file mode 100644 index 000000000..4a895985f --- /dev/null +++ b/test/functional/messages/dmarc/bad_dkim_highsecure.eml @@ -0,0 +1,61 @@ +Delivered-To: vstakhov@gmail.com
+Received: by 2002:a6b:e610:0:0:0:0:0 with SMTP id g16-v6csp2167100ioh;
+ Mon, 1 Oct 2018 07:01:38 -0700 (PDT)
+X-Google-Smtp-Source: ACcGV600DIuvoFwRUFUQRw3uefS2vbyPNMVmB4EVMRmpQN5W/q6VFysTqCZxRezyjaLn+UH6TEhI
+X-Received: by 2002:a1c:4887:: with SMTP id v129-v6mr8698698wma.139.1538402498770;
+ Mon, 01 Oct 2018 07:01:38 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1538402498; cv=none;
+ d=google.com; s=arc-20160816;
+ b=uKHnpS+DExFKAe2EsUhcF9Nx9aYDFuqxV8cUYpabBGOXa3cO33wzlwEINeoG0GqESe
+ nacEwvHeHYplUkAM2SQz+AdSTjzk5TSUzqdzyNox6KU3OripFgApCaFGXVn7rQqU0YYh
+ OcB/0aBpU61ix2buEETdAV+umKd+qKzos5V4WxJmulVWES9pWi44SdXJK9yb+/savl49
+ Gipc0kSdiAkB7r1qMt13tm5oZ+66sCQxCK9IfvOj0QIf5sALgwHsTCSq5yfyCZdtFwjn
+ CZPduL1EwfFIUx+6axy1MEyImDsLhJv4pDKpFfko4X59hoVFGi63YU89YmsO6guTWOZ0
+ jwmw==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+ h=content-transfer-encoding:content-language:mime-version:user-agent
+ :date:message-id:autocrypt:openpgp:subject:from:to:dkim-signature;
+ bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=oGaANMvTT16cEY+U7T08L9TiRD7bE5KwtJa2Fl3p+2813bomvchlWQdCh65CMcxm3+
+ felXzope3YyDKJmf8waAEG/jMK6VnTz+2hTkUm/p08LLoOv6u6OoRWsI+h8cGgnS3ylG
+ xidRereLZxqaC6PC/U5OSSJ63UgOu6S//hL+o1WYdKhGoQyj2jOD3DxnCY7z/qq048Al
+ SNqycpynWXdoZjwrhB6HtydlO3VQSdPL5qaHNFLKCCTvaQ89Dlk5R/01rWeRKxnSM8bI
+ w4LBKGDUrpGYE/A3AvS+ByZu7ASGeg8cEL2GPfNlvnmy9sAQj0pU1Q/xmHyP1NPxUnra
+ xCpg==
+ARC-Authentication-Results: i=1; mx.google.com;
+ dkim=pass header.i=@highsecure.ru header.s=dkim header.b="oVxOH/NU";
+ spf=pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@highsecure.ru
+Return-Path: <vsevolod@highsecure.ru>
+Received: from mail.highsecure.ru (mail.highsecure.ru. [88.99.142.95])
+ by mx.google.com with ESMTPS id z129-v6si8080952wmg.157.2018.10.01.07.01.38
+ for <vstakhov@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+ Mon, 01 Oct 2018 07:01:38 -0700 (PDT)
+Received-SPF: pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) client-ip=88.99.142.95;
+Authentication-Results: mx.google.com;
+ dkim=pass header.i=@highsecure.ru header.s=dkim header.b="oVxOH/NU";
+ spf=pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@highsecure.ru
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=highsecure.ru;
+ s=dkim; t=1538402490; h=from:from:sender:reply-to:subject:subject:date:date:
+ message-id:message-id:to:to:cc:mime-version:mime-version:
+ content-type:content-type:
+ content-transfer-encoding:content-transfer-encoding:in-reply-to:
+ references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=oVxOH/NUK6jcI0tcG73U5qZr2kj4WWZqzOCi3n08KHHW3w/nDFs/V1GfJ5VJy8O0Si+Qtl
+ yx7E+loc2sAXpzlis3myUL8AdACQlyzuxujwY3kj4/7ZmdMaN7/x9mXfnxHfFy0MkbdINq
+ e6mVhumtfbbWA+rJBcMvxQBB+IDgDks=
+To: vstakhov@gmail.com
+From: Vsevolod Stakhov <vsevolod@highsecure.ru>
+Subject: test
+Message-ID: <5ab16dfe-f4e5-3b01-6ee2-9fccb68fc1fc@highsecure.ru>
+Date: Mon, 1 Oct 2018 15:01:29 +0100
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:52.0)
+ Gecko/20100101 Thunderbird/52.9.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Language: en-US
+Content-Transfer-Encoding: 7bit
+Authentication-Results: mail.highsecure.ru;
+ auth=pass smtp.auth=vsevolod@highsecure.ru smtp.mailfrom=vsevolod@highsecure.ru
+
+Malicious content.
\ No newline at end of file diff --git a/test/functional/messages/dmarc/bad_dkim_rspamd.eml b/test/functional/messages/dmarc/bad_dkim_rspamd.eml new file mode 100644 index 000000000..228112867 --- /dev/null +++ b/test/functional/messages/dmarc/bad_dkim_rspamd.eml @@ -0,0 +1,61 @@ +Delivered-To: vstakhov@gmail.com
+Received: by 2002:a6b:e610:0:0:0:0:0 with SMTP id g16-v6csp2201135ioh;
+ Mon, 1 Oct 2018 07:27:55 -0700 (PDT)
+X-Google-Smtp-Source: ACcGV6270qkcKzPIvBSxvaIpBZbNAdj6Qp7qqenTBQi6YaUXdBFD2+2ZYmZaw2WM/SxZP2kYTIz2
+X-Received: by 2002:adf:93e6:: with SMTP id 93-v6mr1901854wrp.81.1538404075350;
+ Mon, 01 Oct 2018 07:27:55 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1538404075; cv=none;
+ d=google.com; s=arc-20160816;
+ b=RaCtnaDmStierMmg+OOhEkzaXxQVAcFO/Rc/ey+6INIQJx+lKVO+dWT0qNA7cZcwUm
+ my6bQE0AZNf45s3bVmQeECtvfe2yS7zVSRx1HFTJJ+iiNR9iSvC8j5PUz1VShRez9Csm
+ 4tqy1ic5t0t9NoOL24f82ju5gTbpl0cc7aH9sMn8gr4DwBxnvuJu4+EdP1QcDKE9qTVa
+ QpjOOOpnkmA46PypufkX+ENaq+bfNDpgbAppKfz2rmutF49jouF8XkrB9Z2ZRWPHE4YA
+ gHJ78GT/4NPlFNo95Ik/nDdnUI6gHkTmiSS6aDJh1W5MiXbkuLT8DSa4Htc43nIr2/m6
+ uQ7g==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+ h=content-transfer-encoding:content-language:mime-version:user-agent
+ :date:message-id:autocrypt:openpgp:to:subject:from:dkim-signature;
+ bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=utFz5J7XQOwR6hdXC0uF+PX17r57baD9GXm5v+3ztobSCFAI0ex+psgbX2gBo6izq3
+ Vo/QjJ9SeJEYhTsLR7jZ3o5meWXJZJqRH073eZlisUGOnDJkJQ5aN/4DY0L5btqLYhwI
+ mJ7c3g4Uh9zFNK8eSIDLdLAIPXNXWRT3SvoS4Ck9ok7fivfZzNfKIPUXbQFIql4+vIAj
+ t1v47QwIrTU+ojwBfaaDjtQEnOB2t8c7RNXys+LQFawG6QZGmG8PCrkVZTU+1v23qbUb
+ M7kDhvSISDchgSrHFwSIniXnnqZe6MRm24xlfW5yebFgmjzMCZQLiyA+WuMIUVxDJpKO
+ V/6g==
+ARC-Authentication-Results: i=1; mx.google.com;
+ dkim=pass header.i=@rspamd.com header.s=dkim header.b=iuniqXuF;
+ spf=pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@rspamd.com
+Return-Path: <vsevolod@rspamd.com>
+Received: from mail.highsecure.ru (mail.highsecure.ru. [88.99.142.95])
+ by mx.google.com with ESMTPS id u13-v6si8362844wmd.167.2018.10.01.07.27.54
+ for <vstakhov@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+ Mon, 01 Oct 2018 07:27:55 -0700 (PDT)
+Received-SPF: pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) client-ip=88.99.142.95;
+Authentication-Results: mx.google.com;
+ dkim=pass header.i=@rspamd.com header.s=dkim header.b=iuniqXuF;
+ spf=pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@rspamd.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=rspamd.com; s=dkim;
+ t=1538404074; h=from:from:sender:reply-to:subject:subject:date:date:
+ message-id:message-id:to:to:cc:mime-version:mime-version:
+ content-type:content-type:
+ content-transfer-encoding:content-transfer-encoding:in-reply-to:
+ references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=iuniqXuFCfL63FjU4vXZQEjBRMXkRMwF3kAv+5heeVbgsAvpd+t4I/6w6CU+GVX8xLmsuW
+ jz+Ycmobj0O7eaw+93gfJ3rYnWNA4/WV2J3vAPSpqgBloYpoktEUOQcNm0VsWiCt6WNAq2
+ Ad2CNaQOOmvDMC5lBfp+YVJITiMcYoU=
+From: Vsevolod Stakhov <vsevolod@rspamd.com>
+Subject: test
+To: vstakhov@gmail.com
+Message-ID: <6f4415bf-ff61-f0f5-b60c-ba71a56b9e48@rspamd.com>
+Date: Mon, 1 Oct 2018 15:27:53 +0100
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:52.0)
+ Gecko/20100101 Thunderbird/52.9.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Language: en-US
+Content-Transfer-Encoding: 7bit
+Authentication-Results: mail.highsecure.ru;
+ auth=pass smtp.auth=vsevolod@highsecure.ru smtp.mailfrom=vsevolod@rspamd.com
+
+Evil content
\ No newline at end of file diff --git a/test/functional/messages/dmarc/good_dkim_highsecure.eml b/test/functional/messages/dmarc/good_dkim_highsecure.eml new file mode 100644 index 000000000..450e7a3a8 --- /dev/null +++ b/test/functional/messages/dmarc/good_dkim_highsecure.eml @@ -0,0 +1,60 @@ +Delivered-To: vstakhov@gmail.com
+Received: by 2002:a6b:e610:0:0:0:0:0 with SMTP id g16-v6csp2167100ioh;
+ Mon, 1 Oct 2018 07:01:38 -0700 (PDT)
+X-Google-Smtp-Source: ACcGV600DIuvoFwRUFUQRw3uefS2vbyPNMVmB4EVMRmpQN5W/q6VFysTqCZxRezyjaLn+UH6TEhI
+X-Received: by 2002:a1c:4887:: with SMTP id v129-v6mr8698698wma.139.1538402498770;
+ Mon, 01 Oct 2018 07:01:38 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1538402498; cv=none;
+ d=google.com; s=arc-20160816;
+ b=uKHnpS+DExFKAe2EsUhcF9Nx9aYDFuqxV8cUYpabBGOXa3cO33wzlwEINeoG0GqESe
+ nacEwvHeHYplUkAM2SQz+AdSTjzk5TSUzqdzyNox6KU3OripFgApCaFGXVn7rQqU0YYh
+ OcB/0aBpU61ix2buEETdAV+umKd+qKzos5V4WxJmulVWES9pWi44SdXJK9yb+/savl49
+ Gipc0kSdiAkB7r1qMt13tm5oZ+66sCQxCK9IfvOj0QIf5sALgwHsTCSq5yfyCZdtFwjn
+ CZPduL1EwfFIUx+6axy1MEyImDsLhJv4pDKpFfko4X59hoVFGi63YU89YmsO6guTWOZ0
+ jwmw==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+ h=content-transfer-encoding:content-language:mime-version:user-agent
+ :date:message-id:autocrypt:openpgp:subject:from:to:dkim-signature;
+ bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=oGaANMvTT16cEY+U7T08L9TiRD7bE5KwtJa2Fl3p+2813bomvchlWQdCh65CMcxm3+
+ felXzope3YyDKJmf8waAEG/jMK6VnTz+2hTkUm/p08LLoOv6u6OoRWsI+h8cGgnS3ylG
+ xidRereLZxqaC6PC/U5OSSJ63UgOu6S//hL+o1WYdKhGoQyj2jOD3DxnCY7z/qq048Al
+ SNqycpynWXdoZjwrhB6HtydlO3VQSdPL5qaHNFLKCCTvaQ89Dlk5R/01rWeRKxnSM8bI
+ w4LBKGDUrpGYE/A3AvS+ByZu7ASGeg8cEL2GPfNlvnmy9sAQj0pU1Q/xmHyP1NPxUnra
+ xCpg==
+ARC-Authentication-Results: i=1; mx.google.com;
+ dkim=pass header.i=@highsecure.ru header.s=dkim header.b="oVxOH/NU";
+ spf=pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@highsecure.ru
+Return-Path: <vsevolod@highsecure.ru>
+Received: from mail.highsecure.ru (mail.highsecure.ru. [88.99.142.95])
+ by mx.google.com with ESMTPS id z129-v6si8080952wmg.157.2018.10.01.07.01.38
+ for <vstakhov@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+ Mon, 01 Oct 2018 07:01:38 -0700 (PDT)
+Received-SPF: pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) client-ip=88.99.142.95;
+Authentication-Results: mx.google.com;
+ dkim=pass header.i=@highsecure.ru header.s=dkim header.b="oVxOH/NU";
+ spf=pass (google.com: domain of vsevolod@highsecure.ru designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@highsecure.ru
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=highsecure.ru;
+ s=dkim; t=1538402490; h=from:from:sender:reply-to:subject:subject:date:date:
+ message-id:message-id:to:to:cc:mime-version:mime-version:
+ content-type:content-type:
+ content-transfer-encoding:content-transfer-encoding:in-reply-to:
+ references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=oVxOH/NUK6jcI0tcG73U5qZr2kj4WWZqzOCi3n08KHHW3w/nDFs/V1GfJ5VJy8O0Si+Qtl
+ yx7E+loc2sAXpzlis3myUL8AdACQlyzuxujwY3kj4/7ZmdMaN7/x9mXfnxHfFy0MkbdINq
+ e6mVhumtfbbWA+rJBcMvxQBB+IDgDks=
+To: vstakhov@gmail.com
+From: Vsevolod Stakhov <vsevolod@highsecure.ru>
+Subject: test
+Message-ID: <5ab16dfe-f4e5-3b01-6ee2-9fccb68fc1fc@highsecure.ru>
+Date: Mon, 1 Oct 2018 15:01:29 +0100
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:52.0)
+ Gecko/20100101 Thunderbird/52.9.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Language: en-US
+Content-Transfer-Encoding: 7bit
+Authentication-Results: mail.highsecure.ru;
+ auth=pass smtp.auth=vsevolod@highsecure.ru smtp.mailfrom=vsevolod@highsecure.ru
+
diff --git a/test/functional/messages/dmarc/good_dkim_rspamd.eml b/test/functional/messages/dmarc/good_dkim_rspamd.eml new file mode 100644 index 000000000..13fe5b0d3 --- /dev/null +++ b/test/functional/messages/dmarc/good_dkim_rspamd.eml @@ -0,0 +1,60 @@ +Delivered-To: vstakhov@gmail.com
+Received: by 2002:a6b:e610:0:0:0:0:0 with SMTP id g16-v6csp2201135ioh;
+ Mon, 1 Oct 2018 07:27:55 -0700 (PDT)
+X-Google-Smtp-Source: ACcGV6270qkcKzPIvBSxvaIpBZbNAdj6Qp7qqenTBQi6YaUXdBFD2+2ZYmZaw2WM/SxZP2kYTIz2
+X-Received: by 2002:adf:93e6:: with SMTP id 93-v6mr1901854wrp.81.1538404075350;
+ Mon, 01 Oct 2018 07:27:55 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1538404075; cv=none;
+ d=google.com; s=arc-20160816;
+ b=RaCtnaDmStierMmg+OOhEkzaXxQVAcFO/Rc/ey+6INIQJx+lKVO+dWT0qNA7cZcwUm
+ my6bQE0AZNf45s3bVmQeECtvfe2yS7zVSRx1HFTJJ+iiNR9iSvC8j5PUz1VShRez9Csm
+ 4tqy1ic5t0t9NoOL24f82ju5gTbpl0cc7aH9sMn8gr4DwBxnvuJu4+EdP1QcDKE9qTVa
+ QpjOOOpnkmA46PypufkX+ENaq+bfNDpgbAppKfz2rmutF49jouF8XkrB9Z2ZRWPHE4YA
+ gHJ78GT/4NPlFNo95Ik/nDdnUI6gHkTmiSS6aDJh1W5MiXbkuLT8DSa4Htc43nIr2/m6
+ uQ7g==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+ h=content-transfer-encoding:content-language:mime-version:user-agent
+ :date:message-id:autocrypt:openpgp:to:subject:from:dkim-signature;
+ bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=utFz5J7XQOwR6hdXC0uF+PX17r57baD9GXm5v+3ztobSCFAI0ex+psgbX2gBo6izq3
+ Vo/QjJ9SeJEYhTsLR7jZ3o5meWXJZJqRH073eZlisUGOnDJkJQ5aN/4DY0L5btqLYhwI
+ mJ7c3g4Uh9zFNK8eSIDLdLAIPXNXWRT3SvoS4Ck9ok7fivfZzNfKIPUXbQFIql4+vIAj
+ t1v47QwIrTU+ojwBfaaDjtQEnOB2t8c7RNXys+LQFawG6QZGmG8PCrkVZTU+1v23qbUb
+ M7kDhvSISDchgSrHFwSIniXnnqZe6MRm24xlfW5yebFgmjzMCZQLiyA+WuMIUVxDJpKO
+ V/6g==
+ARC-Authentication-Results: i=1; mx.google.com;
+ dkim=pass header.i=@rspamd.com header.s=dkim header.b=iuniqXuF;
+ spf=pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@rspamd.com
+Return-Path: <vsevolod@rspamd.com>
+Received: from mail.highsecure.ru (mail.highsecure.ru. [88.99.142.95])
+ by mx.google.com with ESMTPS id u13-v6si8362844wmd.167.2018.10.01.07.27.54
+ for <vstakhov@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+ Mon, 01 Oct 2018 07:27:55 -0700 (PDT)
+Received-SPF: pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) client-ip=88.99.142.95;
+Authentication-Results: mx.google.com;
+ dkim=pass header.i=@rspamd.com header.s=dkim header.b=iuniqXuF;
+ spf=pass (google.com: domain of vsevolod@rspamd.com designates 88.99.142.95 as permitted sender) smtp.mailfrom=vsevolod@rspamd.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=rspamd.com; s=dkim;
+ t=1538404074; h=from:from:sender:reply-to:subject:subject:date:date:
+ message-id:message-id:to:to:cc:mime-version:mime-version:
+ content-type:content-type:
+ content-transfer-encoding:content-transfer-encoding:in-reply-to:
+ references; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;
+ b=iuniqXuFCfL63FjU4vXZQEjBRMXkRMwF3kAv+5heeVbgsAvpd+t4I/6w6CU+GVX8xLmsuW
+ jz+Ycmobj0O7eaw+93gfJ3rYnWNA4/WV2J3vAPSpqgBloYpoktEUOQcNm0VsWiCt6WNAq2
+ Ad2CNaQOOmvDMC5lBfp+YVJITiMcYoU=
+From: Vsevolod Stakhov <vsevolod@rspamd.com>
+Subject: test
+To: vstakhov@gmail.com
+Message-ID: <6f4415bf-ff61-f0f5-b60c-ba71a56b9e48@rspamd.com>
+Date: Mon, 1 Oct 2018 15:27:53 +0100
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:52.0)
+ Gecko/20100101 Thunderbird/52.9.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Language: en-US
+Content-Transfer-Encoding: 7bit
+Authentication-Results: mail.highsecure.ru;
+ auth=pass smtp.auth=vsevolod@highsecure.ru smtp.mailfrom=vsevolod@rspamd.com
+
|