diff options
| author | Vsevolod Stakhov <vsevolod@rspamd.com> | 2023-08-07 11:41:28 +0100 |
|---|---|---|
| committer | Vsevolod Stakhov <vsevolod@rspamd.com> | 2023-08-07 11:41:28 +0100 |
| commit | 662145d0554de5e769b92dab2d41173a98adcee5 (patch) | |
| tree | ec28311a0bce6181f248ba7b50304293ad764e44 /src | |
| parent | bbd88232db43d18f5e0de5a6502848d4074621c5 (diff) | |
| download | rspamd-662145d0554de5e769b92dab2d41173a98adcee5.tar.gz rspamd-662145d0554de5e769b92dab2d41173a98adcee5.zip | |
[Minor] Reformat all Lua code, no functional changes
Diffstat (limited to 'src')
46 files changed, 2371 insertions, 2012 deletions
diff --git a/src/plugins/lua/antivirus.lua b/src/plugins/lua/antivirus.lua index 1623d33e8..26b8509b8 100644 --- a/src/plugins/lua/antivirus.lua +++ b/src/plugins/lua/antivirus.lua @@ -27,49 +27,49 @@ local N = "antivirus" if confighelp then rspamd_config:add_example(nil, 'antivirus', - "Check messages for viruses", - [[ -antivirus { - # multiple scanners could be checked, for each we create a configuration block with an arbitrary name - clamav { - # If set force this action if any virus is found (default unset: no action is forced) - # action = "reject"; - # If set, then rejection message is set to this value (mention single quotes) - # message = '${SCANNER}: virus found: "${VIRUS}"'; - # Scan mime_parts separately - otherwise the complete mail will be transferred to AV Scanner - #scan_mime_parts = true; - # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity) - #scan_text_mime = false; - #scan_image_mime = false; - # If `max_size` is set, messages > n bytes in size are not scanned - max_size = 20000000; - # symbol to add (add it to metric if you want non-zero weight) - symbol = "CLAM_VIRUS"; - # type of scanner: "clamav", "fprot", "sophos" or "savapi" - type = "clamav"; - # For "savapi" you must also specify the following variable - product_id = 12345; - # You can enable logging for clean messages - log_clean = true; - # servers to query (if port is unspecified, scanner-specific default is used) - # can be specified multiple times to pool servers - # can be set to a path to a unix socket - # Enable this in local.d/antivirus.conf - servers = "127.0.0.1:3310"; - # if `patterns` is specified virus name will be matched against provided regexes and the related - # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. - patterns { - # symbol_name = "pattern"; - JUST_EICAR = "^Eicar-Test-Signature$"; + "Check messages for viruses", + [[ + antivirus { + # multiple scanners could be checked, for each we create a configuration block with an arbitrary name + clamav { + # If set force this action if any virus is found (default unset: no action is forced) + # action = "reject"; + # If set, then rejection message is set to this value (mention single quotes) + # message = '${SCANNER}: virus found: "${VIRUS}"'; + # Scan mime_parts separately - otherwise the complete mail will be transferred to AV Scanner + #scan_mime_parts = true; + # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity) + #scan_text_mime = false; + #scan_image_mime = false; + # If `max_size` is set, messages > n bytes in size are not scanned + max_size = 20000000; + # symbol to add (add it to metric if you want non-zero weight) + symbol = "CLAM_VIRUS"; + # type of scanner: "clamav", "fprot", "sophos" or "savapi" + type = "clamav"; + # For "savapi" you must also specify the following variable + product_id = 12345; + # You can enable logging for clean messages + log_clean = true; + # servers to query (if port is unspecified, scanner-specific default is used) + # can be specified multiple times to pool servers + # can be set to a path to a unix socket + # Enable this in local.d/antivirus.conf + servers = "127.0.0.1:3310"; + # if `patterns` is specified virus name will be matched against provided regexes and the related + # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. + patterns { + # symbol_name = "pattern"; + JUST_EICAR = "^Eicar-Test-Signature$"; + } + # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. + whitelist = "/etc/rspamd/antivirus.wl"; + # Replace content that exactly matches the following string to the EICAR pattern + # Useful for E2E testing when another party removes/blocks EICAR attachments + #eicar_fake_pattern = 'testpatterneicar'; } - # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. - whitelist = "/etc/rspamd/antivirus.wl"; - # Replace content that exactly matches the following string to the EICAR pattern - # Useful for E2E testing when another party removes/blocks EICAR attachments - #eicar_fake_pattern = 'testpatterneicar'; } -} -]]) + ]]) return end @@ -84,7 +84,9 @@ local function add_antivirus_rule(sym, opts) return nil end - if not opts.symbol then opts.symbol = sym:upper() end + if not opts.symbol then + opts.symbol = sym:upper() + end local cfg = lua_antivirus[opts.type] if not cfg then @@ -106,13 +108,15 @@ local function add_antivirus_rule(sym, opts) -- WORKAROUND for deprecated attachments_only if opts.attachments_only ~= nil then opts.scan_mime_parts = opts.attachments_only - rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. '.. + rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. ' .. 'Please use scan_mime_parts = %s instead', opts.symbol, opts.type, opts.attachments_only) end -- WORKAROUND for deprecated attachments_only local rule = cfg.configure(opts) - if not rule then return nil end + if not rule then + return nil + end rule.type = opts.type rule.symbol_fail = opts.symbol_fail @@ -169,7 +173,7 @@ local function add_antivirus_rule(sym, opts) if clen == #opts.eicar_fake_pattern and content == opts.eicar_fake_pattern then rspamd_logger.infox(task, 'found eicar fake replacement part in the part (filename="%s")', - p:get_filename()) + p:get_filename()) content = eicar_pattern end end @@ -190,8 +194,12 @@ if opts and type(opts) == 'table' then local has_valid = false for k, m in pairs(opts) do if type(m) == 'table' then - if not m.type then m.type = k end - if not m.name then m.name = k end + if not m.type then + m.type = k + end + if not m.name then + m.name = k + end local cb = add_antivirus_rule(k, m) if not cb then diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua index 697c38b48..ff19aef4c 100644 --- a/src/plugins/lua/arc.lua +++ b/src/plugins/lua/arc.lua @@ -86,13 +86,15 @@ local ar_settings = lua_auth_results.default_settings local function parse_arc_header(hdr, target, is_aar) -- Split elements by ';' and trim spaces local arr = fun.totable(fun.map( - function(val) - return fun.totable(fun.map(lua_util.rspamd_str_trim, - fun.filter(function(v) return v and #v > 0 end, - lua_util.rspamd_str_split(val.decoded, ';') - ) - )) - end, hdr + function(val) + return fun.totable(fun.map(lua_util.rspamd_str_trim, + fun.filter(function(v) + return v and #v > 0 + end, + lua_util.rspamd_str_split(val.decoded, ';') + ) + )) + end, hdr )) -- v[1] is the key and v[2] is the value @@ -106,8 +108,10 @@ local function parse_arc_header(hdr, target, is_aar) -- Now we have two tables in format: -- [arc_header] -> [{arc_header1_elts}, {arc_header2_elts}...] - for i,elts in ipairs(arr) do - if not target[i] then target[i] = {} end + for i, elts in ipairs(arr) do + if not target[i] then + target[i] = {} + end if not is_aar then -- For normal ARC headers we split by kv pair, like k=v fun.each(function(v) @@ -120,7 +124,7 @@ local function parse_arc_header(hdr, target, is_aar) else -- For AAR we check special case of i=%d and pass everything else to -- AAR specific parser - for _,elt in ipairs(elts) do + for _, elt in ipairs(elts) do if string.match(elt, "%s*i%s*=%s*%d+%s*") then local pair = lua_util.rspamd_str_split(elt, '=') fill_arc_header_table(pair, target[i]) @@ -149,26 +153,26 @@ end local function arc_validate_seals(task, seals, sigs, seal_headers, sig_headers) local fail_reason - for i = 1,#seals do + for i = 1, #seals do if (sigs[i].i or 0) ~= i then fail_reason = string.format('bad i for signature: %d, expected %d; d=%s', sigs[i].i, i, sigs[i].d) rspamd_logger.infox(task, fail_reason) task:insert_result(arc_symbols['invalid'], 1.0, fail_reason) - return false,fail_reason + return false, fail_reason end if (seals[i].i or 0) ~= i then fail_reason = string.format('bad i for seal: %d, expected %d; d=%s', seals[i].i, i, seals[i].d) rspamd_logger.infox(task, fail_reason) - task:insert_result(arc_symbols['invalid'], 1.0,fail_reason) - return false,fail_reason + task:insert_result(arc_symbols['invalid'], 1.0, fail_reason) + return false, fail_reason end if not seals[i].cv then fail_reason = string.format('no cv on i=%d', i) task:insert_result(arc_symbols['invalid'], 1.0, fail_reason) - return false,fail_reason + return false, fail_reason end if i == 1 then @@ -176,18 +180,18 @@ local function arc_validate_seals(task, seals, sigs, seal_headers, sig_headers) if seals[i].cv ~= 'none' then fail_reason = 'cv is not "none" for i=1' task:insert_result(arc_symbols['invalid'], 1.0, fail_reason) - return false,fail_reason + return false, fail_reason end else if seals[i].cv ~= 'pass' then - fail_reason = string.format('cv is %s on i=%d', seals[i].cv, i) + fail_reason = string.format('cv is %s on i=%d', seals[i].cv, i) task:insert_result(arc_symbols['reject'], 1.0, fail_reason) - return true,fail_reason + return true, fail_reason end end end - return true,nil + return true, nil end local function arc_callback(task) @@ -244,7 +248,7 @@ local function arc_callback(task) lua_util.debugm(N, task, 'got %s arc sections', #cbdata.seals) -- Now check sanity of what we have - local valid,validation_error = arc_validate_seals(task, cbdata.seals, cbdata.sigs, + local valid, validation_error = arc_validate_seals(task, cbdata.seals, cbdata.sigs, arc_seal_headers, arc_sig_headers) if not valid then task:cache_set('arc-failure', validation_error) @@ -261,7 +265,7 @@ local function arc_callback(task) end local function gen_arc_seal_cb(index, sig) - return function (_, res, err, domain) + return function(_, res, err, domain) lua_util.debugm(N, task, 'checked arc seal: %s(%s), %s processed', res, err, index) @@ -283,7 +287,7 @@ local function arc_callback(task) else task:cache_set(AR_TRUSTED_CACHE_KEY, cur_aar) local seen_dmarc - for _,ar in ipairs(cur_aar.ar) do + for _, ar in ipairs(cur_aar.ar) do if ar.dmarc then local dmarc_fwd = ar.dmarc seen_dmarc = true @@ -326,7 +330,7 @@ local function arc_callback(task) local function arc_signature_cb(_, res, err, domain) lua_util.debugm(N, task, 'checked arc signature %s: %s(%s)', - domain, res, err) + domain, res, err) if not res then cbdata.res = 'fail' @@ -336,7 +340,7 @@ local function arc_callback(task) end if cbdata.res == 'success' then -- Verify seals - for i,sig in ipairs(cbdata.seals) do + for i, sig in ipairs(cbdata.seals) do local ret, lerr = dkim_verify(task, sig.header, gen_arc_seal_cb(i, sig), 'arc-seal') if not ret then cbdata.res = 'fail' @@ -348,8 +352,8 @@ local function arc_callback(task) end else task:insert_result(arc_symbols['reject'], 1.0, - rspamd_logger.slog('signature check failed: %s, %s', cbdata.res, - cbdata.errors)) + rspamd_logger.slog('signature check failed: %s, %s', cbdata.res, + cbdata.errors)) end end @@ -397,7 +401,7 @@ local function arc_callback(task) local processed = 0 local sig = cbdata.sigs[#cbdata.sigs] -- last AMS - local ret,err = dkim_verify(task, sig.header, arc_signature_cb, 'arc-sign') + local ret, err = dkim_verify(task, sig.header, arc_signature_cb, 'arc-sign') if not ret then cbdata.res = 'fail' @@ -405,13 +409,13 @@ local function arc_callback(task) else processed = processed + 1 lua_util.debugm(N, task, 'processed arc signature %s[%s]: %s(%s), %s total', - sig.d, sig.i, ret, err, #cbdata.seals) + sig.d, sig.i, ret, err, #cbdata.seals) end if processed == 0 then task:insert_result(arc_symbols['reject'], 1.0, - rspamd_logger.slog('cannot verify %s of %s signatures: %s', - #arc_sig_headers - processed, #arc_sig_headers, cbdata.errors)) + rspamd_logger.slog('cannot verify %s of %s signatures: %s', + #arc_sig_headers - processed, #arc_sig_headers, cbdata.errors)) end end @@ -421,21 +425,20 @@ if not opts or type(opts) ~= 'table' then end if opts['symbols'] then - for k,_ in pairs(arc_symbols) do + for k, _ in pairs(arc_symbols) do if opts['symbols'][k] then arc_symbols[k] = opts['symbols'][k] end end end - local id = rspamd_config:register_symbol({ name = 'ARC_CHECK', type = 'callback', group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, callback = arc_callback, - augmentations = {lua_util.dns_timeout_augmentation(rspamd_config)}, + augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) }, }) rspamd_config:register_symbol({ name = 'ARC_CALLBACK', -- compatibility symbol @@ -449,7 +452,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = -1.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) rspamd_config:register_symbol({ name = arc_symbols['reject'], @@ -457,7 +460,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 2.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) rspamd_config:register_symbol({ name = arc_symbols['invalid'], @@ -465,7 +468,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 1.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) rspamd_config:register_symbol({ name = arc_symbols['dnsfail'], @@ -473,7 +476,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 0.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) rspamd_config:register_symbol({ name = arc_symbols['na'], @@ -481,7 +484,7 @@ rspamd_config:register_symbol({ type = 'virtual', score = 0.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) rspamd_config:register_dependency('ARC_CHECK', 'SPF_CHECK') @@ -535,13 +538,13 @@ local function arc_sign_seal(task, params, header) for i = 1, #arc_seals, 1 do if arc_auth_results[i] then local s = dkim_canonicalize('ARC-Authentication-Results', - arc_auth_results[i].raw_header) + arc_auth_results[i].raw_header) sha_ctx:update(s) lua_util.debugm(N, task, 'update signature with header: %s', s) end if arc_sigs[i] then local s = dkim_canonicalize('ARC-Message-Signature', - arc_sigs[i].raw_header) + arc_sigs[i].raw_header) sha_ctx:update(s) lua_util.debugm(N, task, 'update signature with header: %s', s) end @@ -554,8 +557,8 @@ local function arc_sign_seal(task, params, header) end header = lua_util.fold_header(task, - 'ARC-Message-Signature', - header) + 'ARC-Message-Signature', + header) cur_auth_results = string.format('i=%d; %s', cur_idx, cur_auth_results) cur_auth_results = lua_util.fold_header(task, @@ -563,7 +566,7 @@ local function arc_sign_seal(task, params, header) cur_auth_results, ';') local s = dkim_canonicalize('ARC-Authentication-Results', - cur_auth_results) + cur_auth_results) sha_ctx:update(s) lua_util.debugm(N, task, 'update signature with header: %s', s) s = dkim_canonicalize('ARC-Message-Signature', header) @@ -588,17 +591,17 @@ local function arc_sign_seal(task, params, header) local sig = rspamd_rsa.sign_memory(privkey, sha_ctx:bin()) cur_arc_seal = string.format('%s%s', cur_arc_seal, - sig:base64(70, nl_type)) + sig:base64(70, nl_type)) lua_mime.modify_headers(task, { add = { - ['ARC-Authentication-Results'] = {order = 1, value = cur_auth_results}, - ['ARC-Message-Signature'] = {order = 1, value = header}, - ['ARC-Seal'] = {order = 1, value = lua_util.fold_header(task, - 'ARC-Seal', cur_arc_seal) } + ['ARC-Authentication-Results'] = { order = 1, value = cur_auth_results }, + ['ARC-Message-Signature'] = { order = 1, value = header }, + ['ARC-Seal'] = { order = 1, value = lua_util.fold_header(task, + 'ARC-Seal', cur_arc_seal) } }, -- RFC requires a strict order for these headers to be inserted - order = {'ARC-Authentication-Results', 'ARC-Message-Signature', 'ARC-Seal'}, + order = { 'ARC-Authentication-Results', 'ARC-Message-Signature', 'ARC-Seal' }, }) task:insert_result(settings.sign_symbol, 1.0, string.format('%s:s=%s:i=%d', params.domain, params.selector, cur_idx)) @@ -733,7 +736,7 @@ local function arc_signing_cb(task) selector = cur_selector.selector }) - local exists,err = rspamd_util.file_exists(cur_selector.key) + local exists, err = rspamd_util.file_exists(cur_selector.key) if not exists then if err and err == 'No such file or directory' then lua_util.debugm(N, task, 'cannot read key from %s: %s', cur_selector.key, err) @@ -775,7 +778,7 @@ if settings.use_redis then redis_params = rspamd_parse_redis_server('arc') if not redis_params then - rspamd_logger.errx(rspamd_config, 'no servers are specified, '.. + rspamd_logger.errx(rspamd_config, 'no servers are specified, ' .. 'but module is configured to load keys from redis, disable arc signing') return end @@ -786,7 +789,7 @@ end local sym_reg_tbl = { name = settings['sign_symbol'], callback = arc_signing_cb, - groups = {"policies", "arc"}, + groups = { "policies", "arc" }, flags = 'ignore_passthrough', score = 0.0, } @@ -805,7 +808,7 @@ if settings.whitelisted_signers_map then type = 'virtual', score = -2.0, group = 'policies', - groups = {'arc'}, + groups = { 'arc' }, }) end @@ -827,7 +830,7 @@ if settings.adjust_dmarc and settings.whitelisted_signers_map then sym_to_adjust = ar_settings.dmarc_symbols.quarantine end if sym_to_adjust and trusted_arc_ar and trusted_arc_ar.ar then - for _,ar in ipairs(trusted_arc_ar.ar) do + for _, ar in ipairs(trusted_arc_ar.ar) do if ar.dmarc then local dmarc_fwd = ar.dmarc if dmarc_fwd == 'pass' then diff --git a/src/plugins/lua/asn.lua b/src/plugins/lua/asn.lua index e870fdb3b..24da19eb4 100644 --- a/src/plugins/lua/asn.lua +++ b/src/plugins/lua/asn.lua @@ -81,7 +81,7 @@ local function asn_check(task) end lua_util.debugm(N, task, 'got reply from %s when requesting %s: %s', - serv, req_name, results[1]) + serv, req_name, results[1]) local parts = rspamd_re:split(results[1]) -- "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8 @@ -106,9 +106,9 @@ end -- Configuration options local configure_asn_module = function() - local opts = rspamd_config:get_all_opt('asn') + local opts = rspamd_config:get_all_opt('asn') if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do options[k] = v end end @@ -145,7 +145,7 @@ if configure_asn_module() then callback = asn_check, priority = lua_util.symbols_priorities.high, flags = 'empty,nostat', - augmentations = {lua_util.dns_timeout_augmentation(rspamd_config)}, + augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) }, }) if options['symbol'] then rspamd_config:register_symbol({ @@ -156,7 +156,7 @@ if configure_asn_module() then score = 0, }) end - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = options['symbol_fail'], parent = id, type = 'virtual', diff --git a/src/plugins/lua/aws_s3.lua b/src/plugins/lua/aws_s3.lua index 60981df9e..30e88d2cd 100644 --- a/src/plugins/lua/aws_s3.lua +++ b/src/plugins/lua/aws_s3.lua @@ -35,7 +35,7 @@ local settings = { inline_content_limit = nil, } -local settings_schema = ts.shape{ +local settings_schema = ts.shape { s3_bucket = ts.string, s3_region = ts.string, s3_host = ts.string, @@ -103,11 +103,11 @@ local function structured_data(task, nonce, queue_id) local message_split = lua_mime.message_to_ucl(task) if settings.inline_content_limit and settings.inline_content_limit > 0 then - for i,part in ipairs(message_split.parts or {}) do + for i, part in ipairs(message_split.parts or {}) do if part.content and #part.content >= settings.inline_content_limit then local ref = convert_to_ref(task, nonce, queue_id, part, external_refs) lua_util.debugm(N, task, "convert part number %s to a reference %s", - i, ref) + i, ref) end end end @@ -137,7 +137,7 @@ local function s3_aws_callback(task) local aws_host = string.format('%s.%s', settings.s3_bucket, settings.s3_host) local function gen_s3_http_callback(path, what) - return function (http_err, code, body, headers) + return function(http_err, code, body, headers) if http_err then if settings.fail_action then @@ -201,7 +201,7 @@ local function s3_aws_callback(task) timeout = settings.s3_timeout, }) - for ref,part_content in pairs(external_refs) do + for ref, part_content in pairs(external_refs) do local part_hdrs = lua_aws.aws_request_enrich({ region = settings.s3_region, headers = { @@ -235,7 +235,7 @@ if not opts then end settings = lua_util.override_defaults(settings, opts) -local res,err = settings_schema:transform(settings) +local res, err = settings_schema:transform(settings) if not res then rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err) @@ -263,7 +263,7 @@ rspamd_config:register_symbol({ name = 'EXPORT_AWS_S3', type = is_postfilter and 'postfilter' or 'idempotent', callback = s3_aws_callback, - augmentations = {string.format("timeout=%f", settings.s3_timeout)}, + augmentations = { string.format("timeout=%f", settings.s3_timeout) }, priority = is_postfilter and lua_util.symbols_priorities.high or nil, flags = 'empty,explicit_disable,ignore_passthrough,nostat', })
\ No newline at end of file diff --git a/src/plugins/lua/bayes_expiry.lua b/src/plugins/lua/bayes_expiry.lua index 97d463f81..44ff9dafa 100644 --- a/src/plugins/lua/bayes_expiry.lua +++ b/src/plugins/lua/bayes_expiry.lua @@ -72,9 +72,9 @@ local function check_redis_classifier(cls, cfg) local statfiles = cls.statfile if statfiles[1] then - for _,stf in ipairs(statfiles) do + for _, stf in ipairs(statfiles) do if not stf.symbol then - for k,v in pairs(stf) do + for k, v in pairs(stf) do check_statfile_table(v, k) end else @@ -82,7 +82,7 @@ local function check_redis_classifier(cls, cfg) end end else - for stn,stf in pairs(statfiles) do + for stn, stf in pairs(statfiles) do check_statfile_table(stf, stn) end end @@ -132,8 +132,10 @@ local classifier = obj.classifier if classifier then if classifier[1] then - for _,cls in ipairs(classifier) do - if cls.bayes then cls = cls.bayes end + for _, cls in ipairs(classifier) do + if cls.bayes then + cls = cls.bayes + end if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, obj) end @@ -143,7 +145,7 @@ if classifier then classifier = classifier.bayes if classifier[1] then - for _,cls in ipairs(classifier) do + for _, cls in ipairs(classifier) do if cls.backend and cls.backend == 'redis' then check_redis_classifier(cls, obj) end @@ -157,11 +159,10 @@ if classifier then end end - local opts = rspamd_config:get_all_opt(N) if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end end @@ -171,11 +172,13 @@ end if settings.cluster_nodes == 0 then local neighbours = obj.neighbours or {} local n_neighbours = 0 - for _,_ in pairs(neighbours) do n_neighbours = n_neighbours + 1 end + for _, _ in pairs(neighbours) do + n_neighbours = n_neighbours + 1 + end settings.cluster_nodes = n_neighbours end - -- Fill template +-- Fill template template.count = settings.count template.threshold = settings.threshold template.common_ttl = settings.common_ttl @@ -184,7 +187,7 @@ template.significant_factor = settings.significant_factor template.expire_step = settings.interval template.hostname = rspamd_util.get_hostname() -for k,v in pairs(template) do +for k, v in pairs(template) do template[k] = tostring(v) end @@ -438,12 +441,12 @@ local function expire_step(cls, ev_base, worker) data[5] } logger.infox(rspamd_config, - 'finished expiry %s: %s items checked, %s significant (%s %s), ' .. - '%s insignificant (%s %s), %s common (%s discriminated), ' .. - '%s infrequent (%s %s), %s mean, %s std', - lutil.unpack(d)) + 'finished expiry %s: %s items checked, %s significant (%s %s), ' .. + '%s insignificant (%s %s), %s common (%s discriminated), ' .. + '%s infrequent (%s %s), %s mean, %s std', + lutil.unpack(d)) if cycle then - for i,cl in ipairs({'in ham', 'in spam', 'total'}) do + for i, cl in ipairs({ 'in ham', 'in spam', 'total' }) do logger.infox(rspamd_config, 'tokens occurrences, %s: {%s}', cl, occ_distr[i]) end end @@ -457,29 +460,31 @@ local function expire_step(cls, ev_base, worker) end end lredis.exec_redis_script(cls.script, - {ev_base = ev_base, is_write = true}, + { ev_base = ev_base, is_write = true }, redis_step_cb, - {'RS*_*', cls.expiry} + { 'RS*_*', cls.expiry } ) end -rspamd_config:add_on_load(function (_, ev_base, worker) +rspamd_config:add_on_load(function(_, ev_base, worker) -- Exit unless we're the first 'controller' worker - if not worker:is_primary_controller() then return end + if not worker:is_primary_controller() then + return + end local unique_redis_params = {} -- Push redis script to all unique redis servers - for _,cls in ipairs(settings.classifiers) do + for _, cls in ipairs(settings.classifiers) do if not unique_redis_params[cls.redis_params.hash] then unique_redis_params[cls.redis_params.hash] = cls.redis_params end end - for h,rp in pairs(unique_redis_params) do + for h, rp in pairs(unique_redis_params) do local script_id = lredis.add_redis_script(lutil.template(expiry_script, template), rp) - for _,cls in ipairs(settings.classifiers) do + for _, cls in ipairs(settings.classifiers) do if cls.redis_params.hash == h then cls.script = script_id end @@ -487,10 +492,10 @@ rspamd_config:add_on_load(function (_, ev_base, worker) end -- Expire tokens at regular intervals - for _,cls in ipairs(settings.classifiers) do + for _, cls in ipairs(settings.classifiers) do rspamd_config:add_periodic(ev_base, settings['interval'], - function () + function() expire_step(cls, ev_base, worker) return true end, true) diff --git a/src/plugins/lua/bimi.lua b/src/plugins/lua/bimi.lua index e90a4e3b9..e3f555e48 100644 --- a/src/plugins/lua/bimi.lua +++ b/src/plugins/lua/bimi.lua @@ -73,12 +73,12 @@ end local function gen_bimi_grammar() local lpeg = require "lpeg" lpeg.locale(lpeg) - local space = lpeg.space^0 - local name = lpeg.C(lpeg.alpha^1) * space - local sep = (lpeg.S("\\;") * space) + (lpeg.space^1) - local value = lpeg.C(lpeg.P(lpeg.graph - sep)^1) - local pair = lpeg.Cg(name * "=" * space * value) * sep^-1 - local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset) + local space = lpeg.space ^ 0 + local name = lpeg.C(lpeg.alpha ^ 1) * space + local sep = (lpeg.S("\\;") * space) + (lpeg.space ^ 1) + local value = lpeg.C(lpeg.P(lpeg.graph - sep) ^ 1) + local pair = lpeg.Cg(name * "=" * space * value) * sep ^ -1 + local list = lpeg.Cf(lpeg.Ct("") * pair ^ 0, rawset) local version = lpeg.P("v") * space * lpeg.P("=") * space * lpeg.P("BIMI1") local record = version * sep * list @@ -114,7 +114,7 @@ local function insert_bimi_headers(task, domain, bimi_content) local content = rspamd_util.encode_base64(rspamd_util.decode_base64(bimi_content), 73, task:get_newlines_type()) lua_mime.modify_headers(task, { - remove = {[hdr_name] = 0}, + remove = { [hdr_name] = 0 }, add = { [hdr_name] = { order = 0, @@ -123,12 +123,12 @@ local function insert_bimi_headers(task, domain, bimi_content) } } }) - task:insert_result('BIMI_VALID', 1.0, {domain}) + task:insert_result('BIMI_VALID', 1.0, { domain }) end local function process_bimi_json(task, domain, redis_data) local parser = ucl.parser() - local _,err = parser:parse_string(redis_data) + local _, err = parser:parse_string(redis_data) if err then rspamd_logger.errx(task, "cannot parse BIMI result from Redis for %s: %s", @@ -163,7 +163,7 @@ local function make_helper_request(task, domain, record, redis_server) end if is_sync then local parser = ucl.parser() - local _,err = parser:parse_string(body) + local _, err = parser:parse_string(body) if err then rspamd_logger.errx(task, "cannot parse BIMI result from helper for %s: %s", @@ -185,18 +185,18 @@ local function make_helper_request(task, domain, record, redis_server) upstream:fail() else lua_util.debugm(N, task, 'stored bimi image in Redis for domain %s; key=%s', - domain, redis_key) + domain, redis_key) end end - ret,_,upstream = lua_redis.redis_make_request(task, + ret, _, upstream = lua_redis.redis_make_request(task, redis_params, -- connect params redis_key, -- hash key true, -- is write redis_set_cb, --callback 'PSETEX', -- command - {redis_key, tostring(settings.redis_min_expiry * 1000.0), - ucl.to_format(d, "json-compact")}) + { redis_key, tostring(settings.redis_min_expiry * 1000.0), + ucl.to_format(d, "json-compact") }) if not ret then rspamd_logger.warnx(task, 'cannot make request to Redis when storing image; domain %s', @@ -227,7 +227,7 @@ local function make_helper_request(task, domain, record, redis_server) local serialised = ucl.to_format(request_data, 'json-compact') lua_util.debugm(N, task, "send request to BIMI helper: %s", - serialised) + serialised) rspamd_http.request({ task = task, mime_type = 'application/json', @@ -241,7 +241,7 @@ end local function check_bimi_vmc(task, domain, record) local redis_key = string.format('%s%s', settings.redis_prefix, - domain) + domain) local ret, _, upstream local function redis_cached_cb(err, data) @@ -276,13 +276,13 @@ local function check_bimi_vmc(task, domain, record) end -- We first check Redis and then try to use helper - ret,_,upstream = lua_redis.redis_make_request(task, + ret, _, upstream = lua_redis.redis_make_request(task, redis_params, -- connect params redis_key, -- hash key false, -- is write redis_cached_cb, --callback 'GET', -- command - {redis_key}) + { redis_key }) if not ret then rspamd_logger.warnx(task, 'cannot make request to Redis; domain %s', domain) @@ -291,12 +291,12 @@ end local function check_bimi_dns(task, domain) local resolve_name = string.format('default._bimi.%s', domain) - local dns_cb = function (_, _, results, err) + local dns_cb = function(_, _, results, err) if err then lua_util.debugm(N, task, "cannot resolve bimi for %s: %s", domain, err) else - for _,rec in ipairs(results) do + for _, rec in ipairs(results) do local res = check_bimi_record(task, rec) if res then @@ -319,7 +319,7 @@ local function check_bimi_dns(task, domain) end end task:get_resolver():resolve_txt({ - task=task, + task = task, name = resolve_name, callback = dns_cb, forced = true @@ -329,7 +329,9 @@ end local function bimi_callback(task) local dmarc_domain_maybe = check_dmarc_policy(task) - if not dmarc_domain_maybe then return end + if not dmarc_domain_maybe then + return + end -- We can either check BIMI via DNS or check Redis cache @@ -347,7 +349,7 @@ if not opts then end settings = lua_util.override_defaults(settings, opts) -local res,err = settings_schema:transform(settings) +local res, err = settings_schema:transform(settings) if not res then rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err) @@ -365,10 +367,10 @@ if redis_params then name = 'BIMI_CHECK', type = 'normal', callback = bimi_callback, - augmentations = {string.format("timeout=%f", settings.helper_timeout or - redis_params.timeout or 0.0)} + augmentations = { string.format("timeout=%f", settings.helper_timeout or + redis_params.timeout or 0.0) } }) - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = 'BIMI_VALID', type = 'virtual', parent = id, diff --git a/src/plugins/lua/clickhouse.lua b/src/plugins/lua/clickhouse.lua index 6deaa3112..25eabc760 100644 --- a/src/plugins/lua/clickhouse.lua +++ b/src/plugins/lua/clickhouse.lua @@ -44,26 +44,26 @@ local settings = { collect_garbage = false, -- Perform GC collection after sending the data check_timeout = 10.0, -- Periodic timeout timeout = 5.0, - bayes_spam_symbols = {'BAYES_SPAM'}, - bayes_ham_symbols = {'BAYES_HAM'}, - ann_symbols_spam = {'NEURAL_SPAM'}, - ann_symbols_ham = {'NEURAL_HAM'}, - fuzzy_symbols = {'FUZZY_DENIED'}, - whitelist_symbols = {'WHITELIST_DKIM', 'WHITELIST_SPF_DKIM', 'WHITELIST_DMARC'}, - dkim_allow_symbols = {'R_DKIM_ALLOW'}, - dkim_reject_symbols = {'R_DKIM_REJECT'}, - dkim_dnsfail_symbols = {'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL'}, - dkim_na_symbols = {'R_DKIM_NA'}, - dmarc_allow_symbols = {'DMARC_POLICY_ALLOW'}, - dmarc_reject_symbols = {'DMARC_POLICY_REJECT'}, - dmarc_quarantine_symbols = {'DMARC_POLICY_QUARANTINE'}, - dmarc_softfail_symbols = {'DMARC_POLICY_SOFTFAIL'}, - dmarc_na_symbols = {'DMARC_NA'}, - spf_allow_symbols = {'R_SPF_ALLOW'}, - spf_reject_symbols = {'R_SPF_FAIL'}, - spf_dnsfail_symbols = {'R_SPF_DNSFAIL', 'R_SPF_PERMFAIL'}, - spf_neutral_symbols = {'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL'}, - spf_na_symbols = {'R_SPF_NA'}, + bayes_spam_symbols = { 'BAYES_SPAM' }, + bayes_ham_symbols = { 'BAYES_HAM' }, + ann_symbols_spam = { 'NEURAL_SPAM' }, + ann_symbols_ham = { 'NEURAL_HAM' }, + fuzzy_symbols = { 'FUZZY_DENIED' }, + whitelist_symbols = { 'WHITELIST_DKIM', 'WHITELIST_SPF_DKIM', 'WHITELIST_DMARC' }, + dkim_allow_symbols = { 'R_DKIM_ALLOW' }, + dkim_reject_symbols = { 'R_DKIM_REJECT' }, + dkim_dnsfail_symbols = { 'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL' }, + dkim_na_symbols = { 'R_DKIM_NA' }, + dmarc_allow_symbols = { 'DMARC_POLICY_ALLOW' }, + dmarc_reject_symbols = { 'DMARC_POLICY_REJECT' }, + dmarc_quarantine_symbols = { 'DMARC_POLICY_QUARANTINE' }, + dmarc_softfail_symbols = { 'DMARC_POLICY_SOFTFAIL' }, + dmarc_na_symbols = { 'DMARC_NA' }, + spf_allow_symbols = { 'R_SPF_ALLOW' }, + spf_reject_symbols = { 'R_SPF_FAIL' }, + spf_dnsfail_symbols = { 'R_SPF_DNSFAIL', 'R_SPF_PERMFAIL' }, + spf_neutral_symbols = { 'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL' }, + spf_na_symbols = { 'R_SPF_NA' }, stop_symbols = {}, ipmask = 19, ipmask6 = 48, @@ -96,7 +96,7 @@ local settings = { } --- @language SQL -local clickhouse_schema = {[[ +local clickhouse_schema = { [[ CREATE TABLE IF NOT EXISTS rspamd ( Date Date COMMENT 'Date (used for partitioning)', @@ -156,8 +156,8 @@ CREATE TABLE IF NOT EXISTS rspamd PARTITION BY toMonday(Date) ORDER BY TS ]], -[[CREATE TABLE IF NOT EXISTS rspamd_version ( Version UInt32) ENGINE = TinyLog]], -{[[INSERT INTO rspamd_version (Version) Values (${SCHEMA_VERSION})]], true}, + [[CREATE TABLE IF NOT EXISTS rspamd_version ( Version UInt32) ENGINE = TinyLog]], + { [[INSERT INTO rspamd_version (Version) Values (${SCHEMA_VERSION})]], true }, } -- This describes SQL queries to migrate between versions @@ -305,7 +305,9 @@ local function clickhouse_main_row(res) 'SettingsId', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_attachments_row(res) @@ -316,7 +318,9 @@ local function clickhouse_attachments_row(res) 'Attachments.Digest', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_urls_row(res) @@ -325,14 +329,18 @@ local function clickhouse_urls_row(res) 'Urls.Url', 'Urls.Flags', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_emails_row(res) local fields = { 'Emails', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_symbols_row(res) @@ -341,7 +349,9 @@ local function clickhouse_symbols_row(res) 'Symbols.Scores', 'Symbols.Options', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_groups_row(res) @@ -349,7 +359,9 @@ local function clickhouse_groups_row(res) 'Groups.Names', 'Groups.Scores', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_asn_row(res) @@ -358,11 +370,15 @@ local function clickhouse_asn_row(res) 'Country', 'IPNet', } - for _,v in ipairs(fields) do table.insert(res, v) end + for _, v in ipairs(fields) do + table.insert(res, v) + end end local function clickhouse_extra_columns(res) - for _,v in ipairs(settings.extra_columns) do table.insert(res, v.name) end + for _, v in ipairs(settings.extra_columns) do + table.insert(res, v.name) + end end local function today(ts) @@ -371,7 +387,7 @@ end local function clickhouse_check_symbol(task, settings_field_name, fields_table, field_name, value, value_negative) - for _,s in ipairs(settings[settings_field_name] or {}) do + for _, s in ipairs(settings[settings_field_name] or {}) do if task:has_symbol(s) then if value_negative then local sym = task:get_symbol(s)[1] @@ -399,7 +415,7 @@ local function clickhouse_send_data(task, ev_base, why, gen_rows, cust_rows) #gen_rows + #cust_rows, ip_addr, why) local function gen_success_cb(what, how_many) - return function (_, _) + return function(_, _) rspamd_logger.messagex(log_object, "sent %s rows of %s to clickhouse server %s; started as %s", how_many, what, ip_addr, why) upstream:ok() @@ -407,7 +423,7 @@ local function clickhouse_send_data(task, ev_base, why, gen_rows, cust_rows) end local function gen_fail_cb(what, how_many) - return function (_, err) + return function(_, err) rspamd_logger.errx(log_object, "cannot send %s rows of %s data to clickhouse server %s: %s; started as %s", how_many, what, ip_addr, err, why) upstream:fail() @@ -453,9 +469,9 @@ local function clickhouse_send_data(task, ev_base, why, gen_rows, cust_rows) string.format('INSERT INTO rspamd (%s)', table.concat(fields, ','))) - for k,crows in pairs(cust_rows) do + for k, crows in pairs(cust_rows) do if #crows > 1 then - send_data('custom data ('..k..')', crows, + send_data('custom data (' .. k .. ')', crows, settings.custom_rules[k].first_row()) end end @@ -470,7 +486,7 @@ local function clickhouse_collect(task) return end - for _,sym in ipairs(settings.stop_symbols) do + for _, sym in ipairs(settings.stop_symbols) do if task:has_symbol(sym) then rspamd_logger.infox(task, 'skip Clickhouse storage for message: symbol %s has fired', sym) return @@ -478,7 +494,7 @@ local function clickhouse_collect(task) end if settings.exceptions then - local excepted,trace = settings.exceptions:process(task) + local excepted, trace = settings.exceptions:process(task) if excepted then rspamd_logger.infox(task, 'skipped Clickhouse storage for message: excepted (%s)', trace) @@ -490,7 +506,7 @@ local function clickhouse_collect(task) local from_domain = '' local from_user = '' if task:has_from('smtp') then - local from = task:get_from({'smtp','orig'})[1] + local from = task:get_from({ 'smtp', 'orig' })[1] if from then from_domain = from['domain']:lower() @@ -501,7 +517,7 @@ local function clickhouse_collect(task) local mime_domain = '' local mime_user = '' if task:has_from('mime') then - local from = task:get_from({'mime','orig'})[1] + local from = task:get_from({ 'mime', 'orig' })[1] if from then mime_domain = from['domain']:lower() mime_user = from['user'] @@ -510,7 +526,7 @@ local function clickhouse_collect(task) local mime_recipients = {} if task:has_recipients('mime') then - local recipients = task:get_recipients({'mime','orig'}) + local recipients = task:get_recipients({ 'mime', 'orig' }) for _, rcpt in ipairs(recipients) do table.insert(mime_recipients, rcpt['user'] .. '@' .. rcpt['domain']:lower()) end @@ -561,79 +577,76 @@ local function clickhouse_collect(task) local ret - ret = clickhouse_check_symbol(task,'bayes_spam_symbols', fields, + ret = clickhouse_check_symbol(task, 'bayes_spam_symbols', fields, 'bayes', 'spam') if not ret then - clickhouse_check_symbol(task,'bayes_ham_symbols', fields, + clickhouse_check_symbol(task, 'bayes_ham_symbols', fields, 'bayes', 'ham') end - clickhouse_check_symbol(task,'ann_symbols_spam', fields, + clickhouse_check_symbol(task, 'ann_symbols_spam', fields, 'ann', 'spam') if not ret then - clickhouse_check_symbol(task,'ann_symbols_ham', fields, + clickhouse_check_symbol(task, 'ann_symbols_ham', fields, 'ann', 'ham') end - clickhouse_check_symbol(task,'whitelist_symbols', fields, + clickhouse_check_symbol(task, 'whitelist_symbols', fields, 'whitelist', 'blacklist', 'whitelist') - clickhouse_check_symbol(task,'fuzzy_symbols', fields, + clickhouse_check_symbol(task, 'fuzzy_symbols', fields, 'fuzzy', 'deny') - - ret = clickhouse_check_symbol(task,'dkim_allow_symbols', fields, + ret = clickhouse_check_symbol(task, 'dkim_allow_symbols', fields, 'dkim', 'allow') if not ret then - ret = clickhouse_check_symbol(task,'dkim_reject_symbols', fields, + ret = clickhouse_check_symbol(task, 'dkim_reject_symbols', fields, 'dkim', 'reject') end if not ret then - ret = clickhouse_check_symbol(task,'dkim_dnsfail_symbols', fields, + ret = clickhouse_check_symbol(task, 'dkim_dnsfail_symbols', fields, 'dkim', 'dnsfail') end if not ret then - clickhouse_check_symbol(task,'dkim_na_symbols', fields, + clickhouse_check_symbol(task, 'dkim_na_symbols', fields, 'dkim', 'na') end - - ret = clickhouse_check_symbol(task,'dmarc_allow_symbols', fields, + ret = clickhouse_check_symbol(task, 'dmarc_allow_symbols', fields, 'dmarc', 'allow') if not ret then - ret = clickhouse_check_symbol(task,'dmarc_reject_symbols', fields, + ret = clickhouse_check_symbol(task, 'dmarc_reject_symbols', fields, 'dmarc', 'reject') end if not ret then - ret = clickhouse_check_symbol(task,'dmarc_quarantine_symbols', fields, + ret = clickhouse_check_symbol(task, 'dmarc_quarantine_symbols', fields, 'dmarc', 'quarantine') end if not ret then - ret = clickhouse_check_symbol(task,'dmarc_softfail_symbols', fields, + ret = clickhouse_check_symbol(task, 'dmarc_softfail_symbols', fields, 'dmarc', 'softfail') end if not ret then - clickhouse_check_symbol(task,'dmarc_na_symbols', fields, + clickhouse_check_symbol(task, 'dmarc_na_symbols', fields, 'dmarc', 'na') end - - ret = clickhouse_check_symbol(task,'spf_allow_symbols', fields, + ret = clickhouse_check_symbol(task, 'spf_allow_symbols', fields, 'spf', 'allow') if not ret then - ret = clickhouse_check_symbol(task,'spf_reject_symbols', fields, + ret = clickhouse_check_symbol(task, 'spf_reject_symbols', fields, 'spf', 'reject') end if not ret then - ret = clickhouse_check_symbol(task,'spf_neutral_symbols', fields, + ret = clickhouse_check_symbol(task, 'spf_neutral_symbols', fields, 'spf', 'neutral') end if not ret then - ret = clickhouse_check_symbol(task,'spf_dnsfail_symbols', fields, + ret = clickhouse_check_symbol(task, 'spf_dnsfail_symbols', fields, 'spf', 'dnsfail') end if not ret then - clickhouse_check_symbol(task,'spf_na_symbols', fields, + clickhouse_check_symbol(task, 'spf_na_symbols', fields, 'spf', 'na') end @@ -644,10 +657,10 @@ local function clickhouse_collect(task) local nurls = 0 local task_urls = task:get_urls({ - content = true, - images = true, - emails = false, - sort = true, + content = true, + images = true, + emails = false, + sort = true, }) or {} nurls = #task_urls @@ -753,9 +766,9 @@ local function clickhouse_collect(task) if #attachments_fnames > 0 then table.insert(row, attachments_fnames) - table.insert(row, attachments_ctypes) - table.insert(row, attachments_lengths) - table.insert(row, attachments_digests) + table.insert(row, attachments_ctypes) + table.insert(row, attachments_lengths) + table.insert(row, attachments_digests) else table.insert(row, {}) table.insert(row, {}) @@ -769,7 +782,7 @@ local function clickhouse_collect(task) local urls_flags = {} if settings.full_urls then - for i,u in ipairs(task_urls) do + for i, u in ipairs(task_urls) do urls_urls[i] = u:get_text() urls_tlds[i] = u:get_tld() or u:get_host() urls_flags[i] = u:get_flags_num() @@ -781,7 +794,7 @@ local function clickhouse_collect(task) idx_tbl = {}, -- indexed by host + flags, reference to an index in ord_tbl __newindex = function(t, k, v) local idx = getmetatable(t).idx_tbl - local ord = getmetatable(t).ord_tbl + local ord = getmetatable(t).ord_tbl local key = k:get_host() .. tostring(k:get_flags_num()) if idx[key] then ord[idx[key]] = v -- replace @@ -806,7 +819,7 @@ local function clickhouse_collect(task) -- Extra index needed for making this unique local urls_idx = {} setmetatable(urls_idx, mt) - for _,u in ipairs(task_urls) do + for _, u in ipairs(task_urls) do if not urls_idx[u] then urls_idx[u] = u urls_urls[#urls_urls + 1] = u:get_host() @@ -828,7 +841,7 @@ local function clickhouse_collect(task) if task:has_urls(true) then local emails = task:get_emails() or {} local emails_formatted = {} - for i,u in ipairs(emails) do + for i, u in ipairs(emails) do emails_formatted[i] = string.format('%s@%s', u:get_user(), u:get_host()) end table.insert(row, emails_formatted) @@ -862,7 +875,7 @@ local function clickhouse_collect(task) local scores_tab = {} local options_tab = {} - for _,s in ipairs(symbols) do + for _, s in ipairs(symbols) do table.insert(syms_tab, s.name or '') table.insert(scores_tab, s.score) @@ -880,7 +893,7 @@ local function clickhouse_collect(task) local groups = task:get_groups() local groups_tab = {} local gr_scores_tab = {} - for gr,sc in pairs(groups) do + for gr, sc in pairs(groups) do table.insert(groups_tab, gr) table.insert(gr_scores_tab, sc) end @@ -890,7 +903,7 @@ local function clickhouse_collect(task) -- Extra columns if #settings.extra_columns > 0 then - for _,col in ipairs(settings.extra_columns) do + for _, col in ipairs(settings.extra_columns) do local elts = col.real_selector(task) if elts then @@ -902,8 +915,10 @@ local function clickhouse_collect(task) end -- Custom data - for k,rule in pairs(settings.custom_rules) do - if not custom_rows[k] then custom_rows[k] = {} end + for k, rule in pairs(settings.custom_rules) do + if not custom_rows[k] then + custom_rows[k] = {} + end table.insert(custom_rows[k], lua_clickhouse.row_to_tsv(rule.get_row(task))) end @@ -923,9 +938,9 @@ local function do_remove_partition(ev_base, cfg, table_name, partition) local remove_partition_sql = "ALTER TABLE ${table_name} ${remove_method} PARTITION '${partition}'" local remove_method = (settings.retention.method == 'drop') and 'DROP' or 'DETACH' local sql_params = { - ['table_name'] = table_name, - ['remove_method'] = remove_method, - ['partition'] = partition + ['table_name'] = table_name, + ['remove_method'] = remove_method, + ['partition'] = partition } local sql = lua_util.template(remove_partition_sql, sql_params) @@ -939,9 +954,9 @@ local function do_remove_partition(ev_base, cfg, table_name, partition) local err, _ = lua_clickhouse.generic_sync(upstream, settings, ch_params, sql) if err then rspamd_logger.errx(rspamd_config, - "cannot detach partition %s:%s from server %s: %s", - table_name, partition, - settings['server'], err) + "cannot detach partition %s:%s from server %s: %s", + table_name, partition, + settings['server'], err) return end @@ -1080,15 +1095,14 @@ local function clickhouse_remove_old_partitions(cfg, ev_base) "GROUP BY partition, table " .. "HAVING max(max_date) < toDate(now() - interval ${month} month)" - local table_names = {'rspamd'} + local table_names = { 'rspamd' } local tables = table.concat(table_names, "', '") local sql_params = { tables = tables, - month = settings.retention.period_months, + month = settings.retention.period_months, } local sql = lua_util.template(partition_to_remove_sql, sql_params) - local ch_params = { ev_base = ev_base, config = cfg, @@ -1096,8 +1110,8 @@ local function clickhouse_remove_old_partitions(cfg, ev_base) local err, rows = lua_clickhouse.select_sync(upstream, settings, ch_params, sql) if err then rspamd_logger.errx(rspamd_config, - "cannot send data to clickhouse server %s: %s", - settings['server'], err) + "cannot send data to clickhouse server %s: %s", + settings['server'], err) else fun.each(function(row) do_remove_partition(ev_base, cfg, row.table, row.partition) @@ -1139,11 +1153,11 @@ local function upload_clickhouse_schema(upstream, ev_base, cfg, initial) -- Process element and return nil if statement should be skipped local function preprocess_schema_elt(v) if type(v) == 'string' then - return lua_util.template(v, {SCHEMA_VERSION = tostring(schema_version)}) + return lua_util.template(v, { SCHEMA_VERSION = tostring(schema_version) }) elseif type(v) == 'table' then -- Pair of statement + boolean if initial == v[2] then - return lua_util.template(v[1], {SCHEMA_VERSION = tostring(schema_version)}) + return lua_util.template(v[1], { SCHEMA_VERSION = tostring(schema_version) }) else rspamd_logger.debugm(N, rspamd_config, 'skip clickhouse schema element %s: schema already exists', v) @@ -1155,12 +1169,14 @@ local function upload_clickhouse_schema(upstream, ev_base, cfg, initial) -- Apply schema elements sequentially, users additions are concatenated to the tail fun.each(upload_schema_elt, - -- Also template schema version - fun.filter(function(v) return v ~= nil end, - fun.map(preprocess_schema_elt, - fun.chain(clickhouse_schema, settings.schema_additions) + -- Also template schema version + fun.filter(function(v) + return v ~= nil + end, + fun.map(preprocess_schema_elt, + fun.chain(clickhouse_schema, settings.schema_additions) + ) ) - ) ) end @@ -1171,7 +1187,7 @@ local function maybe_apply_migrations(upstream, ev_base, cfg, version) } -- Apply migrations sequentially local function migration_recursor(i) - if i < schema_version then + if i < schema_version then if migrations[i] then -- We also need to apply statements sequentially local function sql_recursor(j) @@ -1189,7 +1205,7 @@ local function maybe_apply_migrations(upstream, ev_base, cfg, version) -- Apply the next statement sql_recursor(j + 1) end - end , + end, function(_, err) rspamd_logger.errx(rspamd_config, "cannot apply migration %s: '%s' on clickhouse server %s: %s", @@ -1221,7 +1237,7 @@ local function add_extra_columns(upstream, ev_base, cfg) } -- Apply migrations sequentially local function columns_recursor(i) - if i <= #settings.extra_columns then + if i <= #settings.extra_columns then local col = settings.extra_columns[i] local prev_column if i == 1 then @@ -1242,7 +1258,7 @@ local function add_extra_columns(upstream, ev_base, cfg) col.name, col.type, prev_column) -- Apply the next statement columns_recursor(i + 1) - end , + end, function(_, err) rspamd_logger.errx(rspamd_config, "cannot apply add column alter %s: '%s' on clickhouse server %s: %s", @@ -1268,7 +1284,7 @@ local function check_rspamd_table(upstream, ev_base, cfg) local err, rows = lua_clickhouse.select_sync(upstream, settings, ch_params, sql) if err then rspamd_logger.errx(rspamd_config, "cannot check rspamd table in clickhouse server %s: %s", - upstream:get_addr():to_string(true), err) + upstream:get_addr():to_string(true), err) return end @@ -1290,20 +1306,19 @@ local function check_rspamd_table(upstream, ev_base, cfg) end end - local function check_clickhouse_upstream(upstream, ev_base, cfg) local ch_params = { ev_base = ev_base, config = cfg, } -- If we have some custom rules, we just send its schema to the upstream - for k,rule in pairs(settings.custom_rules) do + for k, rule in pairs(settings.custom_rules) do if rule.schema then local sql = lua_util.template(rule.schema, settings) local err, _ = lua_clickhouse.generic_sync(upstream, settings, ch_params, sql) if err then rspamd_logger.errx(rspamd_config, 'cannot send custom schema %s to clickhouse server %s: ' .. - 'cannot make request (%s)', + 'cannot make request (%s)', k, upstream:get_addr():to_string(true), err) end end @@ -1320,7 +1335,7 @@ local function check_clickhouse_upstream(upstream, ev_base, cfg) else rspamd_logger.errx(rspamd_config, "cannot get version on clickhouse server %s: %s", - upstream:get_addr():to_string(true), err) + upstream:get_addr():to_string(true), err) end else upload_clickhouse_schema(upstream, ev_base, cfg, false) @@ -1339,13 +1354,13 @@ if opts then if opts.limit and not opts.limits then settings.limits.max_rows = opts.limit end - for k,v in pairs(opts) do + for k, v in pairs(opts) do if k == 'custom_rules' then if not v[1] then - v = {v} + v = { v } end - for i,rule in ipairs(v) do + for i, rule in ipairs(v) do if rule.schema and rule.first_row and rule.get_row then local first_row, get_row local loadstring = loadstring or load @@ -1418,9 +1433,11 @@ if opts then -- Select traverse function depending on what we have local iter_func = settings.extra_columns[1] and ipairs or pairs - for col_name,col_data in iter_func(settings.extra_columns) do + for col_name, col_data in iter_func(settings.extra_columns) do -- Array based extra columns - if col_data.name then col_name = col_data.name end + if col_data.name then + col_name = col_data.name + end if not col_data.selector or not col_data.type then rspamd_logger.errx(rspamd_config, 'cannot add clickhouse extra row %s: no type or no selector', col_name) @@ -1459,7 +1476,9 @@ if opts then -- preserve strict order when doing altering if need_sort then rspamd_logger.infox(rspamd_config, 'sort extra columns as they are not configured as an array') - table.sort(columns_transformed, function(c1, c2) return c1.name < c2.name end) + table.sort(columns_transformed, function(c1, c2) + return c1.name < c2.name + end) end settings.extra_columns = columns_transformed end @@ -1469,7 +1488,7 @@ if opts then type = 'idempotent', callback = clickhouse_collect, flags = 'empty,explicit_disable,ignore_passthrough', - augmentations = {string.format("timeout=%f", settings.timeout)}, + augmentations = { string.format("timeout=%f", settings.timeout) }, }) rspamd_config:register_finish_script(function(task) if nrows > 0 then @@ -1499,7 +1518,7 @@ if opts then if worker:is_primary_controller() then local upstreams = settings.upstream:all_upstreams() - for _,up in ipairs(upstreams) do + for _, up in ipairs(upstreams) do check_clickhouse_upstream(up, ev_base, cfg) end diff --git a/src/plugins/lua/clustering.lua b/src/plugins/lua/clustering.lua index 3a8a361b1..d97bdb97e 100644 --- a/src/plugins/lua/clustering.lua +++ b/src/plugins/lua/clustering.lua @@ -44,7 +44,7 @@ local default_rule = { score_mult = 0.1, } -local rule_schema = ts.shape{ +local rule_schema = ts.shape { max_elts = ts.number + ts.string / tonumber, expire = ts.number + ts.string / lua_util.parse_time_interval, expire_overflow = ts.number + ts.string / lua_util.parse_time_interval, @@ -157,9 +157,9 @@ local function clusterting_filter_cb(task, rule) 'processed rule %s, selectors: source="%s", cluster="%s"; data: %s elts, %s score, %s elt score', rule.name, source_selector, cluster_selector, cur_elts, total_score, element_score) if final_score > 0.1 then - task:insert_result(rule.symbol, final_score, {source_selector, - tostring(size_score), - tostring(cluster_score)}) + task:insert_result(rule.symbol, final_score, { source_selector, + tostring(size_score), + tostring(cluster_score) }) end end @@ -182,14 +182,18 @@ local function clusterting_filter_cb(task, rule) end lua_redis.exec_redis_script(query_cluster_id, - {task = task, is_write = false, key = source_selector}, + { task = task, is_write = false, key = source_selector }, redis_get_cb, - {source_selector, cluster_selector}) + { source_selector, cluster_selector }) end local function clusterting_idempotent_cb(task, rule) - if task:has_flag('skip') then return end - if not rule.allow_local and lua_util.is_rspamc_or_controller(task) then return end + if task:has_flag('skip') then + return + end + if not rule.allow_local and lua_util.is_rspamc_or_controller(task) then + return + end local verdict = lua_verdict.get_specific_verdict(N, task) local score @@ -230,7 +234,7 @@ local function clusterting_idempotent_cb(task, rule) end lua_redis.exec_redis_script(update_cluster_id, - {task = task, is_write = true, key = source_selector}, + { task = task, is_write = true, key = source_selector }, redis_set_cb, { source_selector, @@ -258,22 +262,26 @@ if not redis_params then end if opts['rules'] then - for k,v in pairs(opts['rules']) do + for k, v in pairs(opts['rules']) do local raw_rule = lua_util.override_defaults(default_rule, v) - local rule,err = rule_schema:transform(raw_rule) + local rule, err = rule_schema:transform(raw_rule) if not rule then rspamd_logger.errx(rspamd_config, 'invalid clustering rule %s: %s', k, err) else - if not rule.symbol then rule.symbol = k end - if not rule.prefix then rule.prefix = k .. "_" end + if not rule.symbol then + rule.symbol = k + end + if not rule.prefix then + rule.prefix = k .. "_" + end rule.source_selector = lua_selectors.create_selector_closure(rspamd_config, rule.source_selector, '') - rule.cluster_selector = lua_selectors.create_selector_closure(rspamd_config, + rule.cluster_selector = lua_selectors.create_selector_closure(rspamd_config, rule.cluster_selector, '') if rule.source_selector and rule.cluster_selector then rule.name = k @@ -287,21 +295,23 @@ if opts['rules'] then query_cluster_id = lua_redis.add_redis_script(query_cluster_script, redis_params) update_cluster_id = lua_redis.add_redis_script(update_cluster_script, redis_params) local function callback_gen(f, rule) - return function(task) return f(task, rule) end + return function(task) + return f(task, rule) + end end - for _,rule in ipairs(rules) do - rspamd_config:register_symbol{ + for _, rule in ipairs(rules) do + rspamd_config:register_symbol { name = rule.symbol, type = 'normal', callback = callback_gen(clusterting_filter_cb, rule), } - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = rule.symbol .. '_STORE', type = 'idempotent', flags = 'empty,explicit_disable,ignore_passthrough', callback = callback_gen(clusterting_idempotent_cb, rule), - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)} + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) } } end else diff --git a/src/plugins/lua/dcc.lua b/src/plugins/lua/dcc.lua index 4cbe301e0..850832057 100644 --- a/src/plugins/lua/dcc.lua +++ b/src/plugins/lua/dcc.lua @@ -25,20 +25,19 @@ local lua_util = require "lua_util" local rspamd_logger = require "rspamd_logger" local dcc = require("lua_scanners").filter('dcc').dcc - if confighelp then rspamd_config:add_example(nil, 'dcc', - "Check messages for 'bulkiness' using DCC", - [[ -dcc { - socket = "/var/dcc/dccifd"; # Unix socket - servers = "127.0.0.1:10045" # OR TCP upstreams - timeout = 2s; # Timeout to wait for checks - body_max = 999999; # Bulkness threshold for body - fuz1_max = 999999; # Bulkness threshold for fuz1 - fuz2_max = 999999; # Bulkness threshold for fuz2 -} -]]) + "Check messages for 'bulkiness' using DCC", + [[ + dcc { + socket = "/var/dcc/dccifd"; # Unix socket + servers = "127.0.0.1:10045" # OR TCP upstreams + timeout = 2s; # Timeout to wait for checks + body_max = 999999; # Bulkness threshold for body + fuz1_max = 999999; # Bulkness threshold for fuz1 + fuz2_max = 999999; # Bulkness threshold for fuz2 + } + ]]) return end @@ -53,18 +52,22 @@ end -- WORKAROUND for deprecated host and port settings if opts['host'] ~= nil and opts['port'] ~= nil then opts['servers'] = opts['host'] .. ':' .. opts['port'] - rspamd_logger.warnx(rspamd_config, 'Using host and port parameters is deprecated. '.. - 'Please use servers = "%s:%s"; instead', opts['host'], opts['port']) + rspamd_logger.warnx(rspamd_config, 'Using host and port parameters is deprecated. ' .. + 'Please use servers = "%s:%s"; instead', opts['host'], opts['port']) end if opts['host'] ~= nil and not opts['port'] then opts['socket'] = opts['host'] - rspamd_logger.warnx(rspamd_config, 'Using host parameters is deprecated. '.. - 'Please use socket = "%s"; instead', opts['host']) + rspamd_logger.warnx(rspamd_config, 'Using host parameters is deprecated. ' .. + 'Please use socket = "%s"; instead', opts['host']) end -- WORKAROUND for deprecated host and port settings -if not opts.symbol_bulk then opts.symbol_bulk = symbol_bulk end -if not opts.symbol then opts.symbol = symbol end +if not opts.symbol_bulk then + opts.symbol_bulk = symbol_bulk +end +if not opts.symbol then + opts.symbol = symbol +end rule = dcc.configure(opts) @@ -74,17 +77,17 @@ if rule then callback = check_dcc, type = 'callback', }) - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', parent = id, name = opts.symbol } - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', parent = id, name = opts.symbol_bulk } - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', parent = id, name = 'DCC_FAIL' diff --git a/src/plugins/lua/dkim_signing.lua b/src/plugins/lua/dkim_signing.lua index 355ce89b8..6c05520ce 100644 --- a/src/plugins/lua/dkim_signing.lua +++ b/src/plugins/lua/dkim_signing.lua @@ -58,7 +58,7 @@ local function insert_sign_results(task, ret, hdr, dkim_params) if settings.use_milter_headers then lua_mime.modify_headers(task, { add = { - ['DKIM-Signature'] = {order = 1, value = hdr}, + ['DKIM-Signature'] = { order = 1, value = hdr }, } }) end @@ -106,7 +106,7 @@ local function sign_error(task, msg) end local function dkim_signing_cb(task) - local ret,selectors = dkim_sign_tools.prepare_dkim_signing(N, task, settings) + local ret, selectors = dkim_sign_tools.prepare_dkim_signing(N, task, settings) if not ret then return @@ -140,8 +140,10 @@ local function dkim_signing_cb(task) end end -local opts = rspamd_config:get_all_opt('dkim_signing') -if not opts then return end +local opts = rspamd_config:get_all_opt('dkim_signing') +if not opts then + return +end dkim_sign_tools.process_signing_settings(N, settings, opts) @@ -167,7 +169,7 @@ end local sym_reg_tbl = { name = settings['symbol'], callback = dkim_signing_cb, - groups = {"policies", "dkim"}, + groups = { "policies", "dkim" }, flags = 'ignore_passthrough', score = 0.0, } diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua index 17ef5c274..bf1e338da 100644 --- a/src/plugins/lua/dmarc.lua +++ b/src/plugins/lua/dmarc.lua @@ -115,7 +115,6 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) table.insert(reason, "No valid SPF") end - local opts = ((task:get_symbol('DKIM_TRACE') or E)[1] or E).options local dkim_results = { pass = {}, @@ -124,12 +123,11 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) fail = {}, } - if opts then dkim_results.pass = {} local dkim_violated - for _,opt in ipairs(opts) do + for _, opt in ipairs(opts) do local check_res = string.sub(opt, -1) local domain = string.sub(opt, 1, -3):lower() @@ -247,7 +245,7 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) therefore cannot apply the advertised DMARC policy. ]]-- if spf_tmpfail or dkim_tmpfail then - task:insert_result(settings.symbols['dnsfail'], 1.0, policy.domain.. + task:insert_result(settings.symbols['dnsfail'], 1.0, policy.domain .. ' : ' .. 'SPF/DKIM temp error', policy.dmarc_policy) else -- We can now check the failed policy and maybe send report data elt @@ -305,34 +303,33 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld) -- Prepare and send redis report element local period = os.date('%Y%m%d', - task:get_date({format = 'connect', gmt = false})) + task:get_date({ format = 'connect', gmt = false })) -- Dmarc domain key must include dmarc domain, rua and period local dmarc_domain_key = table.concat( - {settings.reporting.redis_keys.report_prefix, dmarc_esld, policy.rua, period}, + { settings.reporting.redis_keys.report_prefix, dmarc_esld, policy.rua, period }, settings.reporting.redis_keys.join_char) local report_data = dmarc_common.dmarc_report(task, settings, { - spf_ok = spf_ok and 'pass' or 'fail', - dkim_ok = dkim_ok and 'pass' or 'fail', - disposition = (disposition == "softfail") and "none" or disposition, - sampled_out = sampled_out, - domain = hdrfromdom, - spf_domain = spf_domain, - dkim_results = dkim_results, - spf_result = spf_result + spf_ok = spf_ok and 'pass' or 'fail', + dkim_ok = dkim_ok and 'pass' or 'fail', + disposition = (disposition == "softfail") and "none" or disposition, + sampled_out = sampled_out, + domain = hdrfromdom, + spf_domain = spf_domain, + dkim_results = dkim_results, + spf_result = spf_result }) - - local idx_key = table.concat({settings.reporting.redis_keys.index_prefix, period}, + local idx_key = table.concat({ settings.reporting.redis_keys.index_prefix, period }, settings.reporting.redis_keys.join_char) if report_data then lua_redis.exec_redis_script(take_report_id, - {task = task, is_write = true}, + { task = task, is_write = true }, dmarc_report_cb, - {idx_key, dmarc_domain_key, - tostring(settings.reporting.max_entries), tostring(settings.reporting.keys_expire)}, - {hdrfromdom, report_data}) + { idx_key, dmarc_domain_key, + tostring(settings.reporting.max_entries), tostring(settings.reporting.keys_expire) }, + { hdrfromdom, report_data }) end end end @@ -365,13 +362,12 @@ local function dmarc_callback(task) return maybe_force_action(task, 'na') elseif (from or E)[1] then task:insert_result(settings.symbols['na'], 1.0, 'No domain in From header') - return maybe_force_action(task,'na') + return maybe_force_action(task, 'na') else task:insert_result(settings.symbols['na'], 1.0, 'No From header') - return maybe_force_action(task,'na') + return maybe_force_action(task, 'na') end - local dns_checks_inflight = 0 local dmarc_domain_policy = {} local dmarc_tld_policy = {} @@ -403,7 +399,7 @@ local function dmarc_callback(task) policy_target = dmarc_tld_policy end - return function (_, _, results, err) + return function(_, _, results, err) dns_checks_inflight = dns_checks_inflight - 1 if not seen_invalid then @@ -421,8 +417,8 @@ local function dmarc_callback(task) else local has_valid_policy = false - for _,rec in ipairs(results) do - local ret,results_or_err = dmarc_common.dmarc_check_record(task, rec, is_tld) + for _, rec in ipairs(results) do + local ret, results_or_err = dmarc_common.dmarc_check_record(task, rec, is_tld) if not ret then if results_or_err then @@ -442,7 +438,7 @@ local function dmarc_callback(task) end has_valid_policy = true - for k,v in pairs(results_or_err) do + for k, v in pairs(results_or_err) do policy_target[k] = v end end @@ -471,7 +467,7 @@ local function dmarc_callback(task) local resolve_name = '_dmarc.' .. hfromdom task:get_resolver():resolve_txt({ - task=task, + task = task, name = resolve_name, callback = gen_dmarc_cb(hfromdom, false), forced = true @@ -482,7 +478,7 @@ local function dmarc_callback(task) resolve_name = '_dmarc.' .. dmarc_domain task:get_resolver():resolve_txt({ - task=task, + task = task, name = resolve_name, callback = gen_dmarc_cb(dmarc_domain, true), forced = true @@ -492,7 +488,6 @@ local function dmarc_callback(task) end end - local opts = rspamd_config:get_all_opt('dmarc') settings = lua_util.override_defaults(settings, opts) @@ -523,7 +518,6 @@ if type(settings.reporting) == 'table' then }) end - if settings.reporting == true then rspamd_logger.errx(rspamd_config, 'old style dmarc reporting is NO LONGER supported, please read the documentation') elseif settings.reporting.enabled then @@ -575,49 +569,49 @@ rspamd_config:register_symbol({ name = settings.symbols['allow'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['reject'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['quarantine'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['softfail'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['dnsfail'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['badpolicy'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) rspamd_config:register_symbol({ name = settings.symbols['na'], parent = id, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, type = 'virtual' }) @@ -639,7 +633,7 @@ if settings.munging then local munging_opts = lua_util.override_defaults(munging_defaults, settings.munging) - if not munging_opts.list_map then + if not munging_opts.list_map then rspamd_logger.errx(rspamd_config, 'cannot enable DMARC munging with no list_map parameter') return @@ -648,7 +642,7 @@ if settings.munging then munging_opts.list_map = lua_maps.map_add_from_ucl(munging_opts.list_map, 'set', 'DMARC munging map of the recipients addresses to munge') - if not munging_opts.list_map then + if not munging_opts.list_map then rspamd_logger.errx(rspamd_config, 'cannot enable DMARC munging with invalid list_map (invalid map)') return @@ -656,19 +650,18 @@ if settings.munging then if munging_opts.munge_map_condition then munging_opts.munge_map_condition = lua_maps_expressions.create(rspamd_config, - munging_opts.munge_map_condition, N) + munging_opts.munge_map_condition, N) end - rspamd_config:register_symbol({ name = 'DMARC_MUNGED', type = 'normal', flags = 'nostat', score = 0, group = 'policies', - groups = {'dmarc'}, + groups = { 'dmarc' }, callback = dmarc_common.gen_munging_callback(munging_opts, settings), - augmentations = {lua_util.dns_timeout_augmentation(rspamd_config)}, + augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) }, }) rspamd_config:register_dependency('DMARC_MUNGED', 'DMARC_CHECK') diff --git a/src/plugins/lua/dynamic_conf.lua b/src/plugins/lua/dynamic_conf.lua index 8498b9b7c..5af26a933 100644 --- a/src/plugins/lua/dynamic_conf.lua +++ b/src/plugins/lua/dynamic_conf.lua @@ -52,7 +52,7 @@ end local function apply_dynamic_actions(_, acts) fun.each(function(k, v) - if type(v) == 'table' then + if type(v) == 'table' then v['name'] = k if not v['priority'] then v['priority'] = settings.priority @@ -145,7 +145,7 @@ local function update_dynamic_conf(cfg, ev_base, recv) settings.redis_key, true, redis_version_set_cb, - 'HINCRBY', {settings.redis_key, 'v', '1'}) + 'HINCRBY', { settings.redis_key, 'v', '1' }) end end @@ -158,12 +158,12 @@ local function update_dynamic_conf(cfg, ev_base, recv) fun.each(function(k, v) cur_settings.data.scores[k] = v end, - fun.filter(function(k) - if cur_settings.updates.symbols[k] then - return false - end - return true - end, recv['scores'])) + fun.filter(function(k) + if cur_settings.updates.symbols[k] then + return false + end + return true + end, recv['scores'])) end if recv['actions'] then if not cur_settings.data.actions then @@ -172,18 +172,18 @@ local function update_dynamic_conf(cfg, ev_base, recv) fun.each(function(k, v) cur_settings.data.actions[k] = v end, - fun.filter(function(k) - if cur_settings.updates.actions[k] then - return false - end - return true - end, recv['actions'])) + fun.filter(function(k) + if cur_settings.updates.actions[k] then + return false + end + return true + end, recv['actions'])) end end local newdata = ucl.to_format(cur_settings.data, 'json-compact') rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params, settings.redis_key, true, - redis_data_set_cb, 'HSET', {settings.redis_key, 'd', newdata}) + redis_data_set_cb, 'HSET', { settings.redis_key, 'd', newdata }) end local function check_dynamic_conf(cfg, ev_base) @@ -192,7 +192,7 @@ local function check_dynamic_conf(cfg, ev_base) rspamd_logger.errx(cfg, "cannot read dynamic conf from redis: %s", redis_err) elseif data and type(data) == 'string' then local parser = ucl.parser() - local _,err = parser:parse_string(data) + local _, err = parser:parse_string(data) if err then rspamd_logger.errx(cfg, "cannot load dynamic conf from redis: %s", err) @@ -214,11 +214,11 @@ local function check_dynamic_conf(cfg, ev_base) if not cur_settings.version or (rver and rver > cur_settings.version) then rspamd_logger.infox(cfg, "need to load fresh dynamic settings with version %s, local version is %s", - rver, cur_settings.version) + rver, cur_settings.version) cur_settings.version = rver rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params, settings.redis_key, false, - redis_load_cb, 'HGET', {settings.redis_key, 'd'}) + redis_load_cb, 'HGET', { settings.redis_key, 'd' }) elseif cur_settings.updates.has_updates then -- Need to send our updates to Redis update_dynamic_conf(cfg, ev_base) @@ -231,7 +231,7 @@ local function check_dynamic_conf(cfg, ev_base) rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params, settings.redis_key, false, - redis_check_cb, 'HGET', {settings.redis_key, 'v'}) + redis_check_cb, 'HGET', { settings.redis_key, 'v' }) end local section = rspamd_config:get_all_opt("dynamic_conf") @@ -242,7 +242,7 @@ if section then return end - for k,v in pairs(section) do + for k, v in pairs(section) do settings[k] = v end diff --git a/src/plugins/lua/elastic.lua b/src/plugins/lua/elastic.lua index ce2d01b80..ccbb7c198 100644 --- a/src/plugins/lua/elastic.lua +++ b/src/plugins/lua/elastic.lua @@ -43,7 +43,7 @@ local settings = { limit = 500, index_pattern = 'rspamd-%Y.%m.%d', template_file = rspamd_paths['SHAREDIR'] .. '/elastic/rspamd_template.json', - kibana_file = rspamd_paths['SHAREDIR'] ..'/elastic/kibana.json', + kibana_file = rspamd_paths['SHAREDIR'] .. '/elastic/kibana.json', key_prefix = 'elastic-', expire = 3600, timeout = 5.0, @@ -61,23 +61,25 @@ local settings = { } local function read_file(path) - local file = io.open(path, "rb") - if not file then return nil end - local content = file:read "*a" - file:close() - return content + local file = io.open(path, "rb") + if not file then + return nil + end + local content = file:read "*a" + file:close() + return content end local function elastic_send_data(task) local es_index = os.date(settings['index_pattern']) local tbl = {} - for _,value in pairs(rows) do + for _, value in pairs(rows) do if settings.elasticsearch_version >= 7 then - table.insert(tbl, '{ "index" : { "_index" : "'..es_index.. - '","pipeline": "rspamd-geoip"} }') + table.insert(tbl, '{ "index" : { "_index" : "' .. es_index .. + '","pipeline": "rspamd-geoip"} }') else - table.insert(tbl, '{ "index" : { "_index" : "'..es_index.. - '", "_type" : "_doc" ,"pipeline": "rspamd-geoip"} }') + table.insert(tbl, '{ "index" : { "_index" : "' .. es_index .. + '", "_type" : "_doc" ,"pipeline": "rspamd-geoip"} }') end table.insert(tbl, ucl.to_format(value, 'json-compact')) end @@ -87,7 +89,7 @@ local function elastic_send_data(task) local upstream = settings.upstream:get_upstream_round_robin() local ip_addr = upstream:get_addr():to_string(true) - local push_url = connect_prefix .. ip_addr .. '/'..es_index..'/_bulk' + local push_url = connect_prefix .. ip_addr .. '/' .. es_index .. '/_bulk' local bulk_json = table.concat(tbl, "\n") local function http_callback(err, code, _, _) @@ -127,7 +129,7 @@ local function get_general_metadata(task) local r = {} local ip_addr = task:get_ip() - if ip_addr and ip_addr:is_valid() then + if ip_addr and ip_addr:is_valid() then r.is_local = ip_addr:is_local() r.ip = tostring(ip_addr) else @@ -153,10 +155,10 @@ local function get_general_metadata(task) r.action = task:get_metric_action() r.rspamd_server = HOSTNAME if r.user ~= 'unknown' then - r.direction = "Outbound" + r.direction = "Outbound" end local s = task:get_metric_score()[1] - r.score = s + r.score = s local rcpt = task:get_recipients('smtp') if rcpt then @@ -169,14 +171,14 @@ local function get_general_metadata(task) r.rcpt = 'unknown' end - local from = task:get_from{'smtp', 'orig'} + local from = task:get_from { 'smtp', 'orig' } if ((from or E)[1] or E).addr then r.from = from[1].addr else r.from = 'unknown' end - local mime_from = task:get_from{'mime', 'orig'} + local mime_from = task:get_from { 'mime', 'orig' } if ((mime_from or E)[1] or E).addr then r.mime_from = mime_from[1].addr else @@ -188,7 +190,7 @@ local function get_general_metadata(task) r.asn = {} local pool = task:get_mempool() r.asn.country = pool:get_variable("country") or 'unknown' - r.asn.asn = pool:get_variable("asn") or 0 + r.asn.asn = pool:get_variable("asn") or 0 r.asn.ipnet = pool:get_variable("ipnet") or 'unknown' local function process_header(name) @@ -244,12 +246,18 @@ local function get_general_metadata(task) end local function elastic_collect(task) - if not enabled then return end - if task:has_flag('skip') then return end - if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then return end + if not enabled then + return + end + if task:has_flag('skip') then + return + end + if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then + return + end - local row = {['rspamd_meta'] = get_general_metadata(task), - ['@timestamp'] = tostring(util.get_time() * 1000)} + local row = { ['rspamd_meta'] = get_general_metadata(task), + ['@timestamp'] = tostring(util.get_time() * 1000) } table.insert(rows, row) nrows = nrows + 1 if nrows > settings['limit'] then @@ -272,7 +280,6 @@ local function elastic_collect(task) end end - local opts = rspamd_config:get_all_opt('elastic') local function check_elastic_server(cfg, ev_base, _) @@ -282,7 +289,7 @@ local function check_elastic_server(cfg, ev_base, _) local function http_callback(err, code, body, _) if code == 200 then local parser = ucl.parser() - local res,ucl_err = parser:parse_string(body) + local res, ucl_err = parser:parse_string(body) if not res then rspamd_logger.infox(rspamd_config, 'failed to parse reply from %s: %s', plugins_url, ucl_err) @@ -290,9 +297,9 @@ local function check_elastic_server(cfg, ev_base, _) return end local obj = parser:get_object() - for node,value in pairs(obj['nodes']) do + for node, value in pairs(obj['nodes']) do local plugin_found = false - for _,plugin in pairs(value['plugins']) do + for _, plugin in pairs(value['plugins']) do if plugin['name'] == 'ingest-geoip' then plugin_found = true lua_util.debugm(N, "ingest-geoip plugin has been found") @@ -326,7 +333,9 @@ end -- import ingest pipeline and kibana dashboard/visualization local function initial_setup(cfg, ev_base, worker) - if not worker:is_primary_controller() then return end + if not worker:is_primary_controller() then + return + end local upstream = settings.upstream:get_upstream_round_robin() local ip_addr = upstream:get_addr():to_string(true) @@ -337,7 +346,7 @@ local function initial_setup(cfg, ev_base, worker) local kibana_mappings = read_file(settings['kibana_file']) if kibana_mappings then local parser = ucl.parser() - local res,parser_err = parser:parse_string(kibana_mappings) + local res, parser_err = parser:parse_string(kibana_mappings) if not res then rspamd_logger.infox(rspamd_config, 'kibana template cannot be parsed: %s', parser_err) @@ -347,14 +356,14 @@ local function initial_setup(cfg, ev_base, worker) end local obj = parser:get_object() local tbl = {} - for _,item in ipairs(obj) do - table.insert(tbl, '{ "index" : { "_index" : ".kibana", "_type" : "doc" ,"_id": "'.. - item['_type'] .. ':' .. item["_id"]..'"} }') + for _, item in ipairs(obj) do + table.insert(tbl, '{ "index" : { "_index" : ".kibana", "_type" : "doc" ,"_id": "' .. + item['_type'] .. ':' .. item["_id"] .. '"} }') table.insert(tbl, ucl.to_format(item['_source'], 'json-compact')) end table.insert(tbl, '') -- For last \n - local kibana_url = connect_prefix .. ip_addr ..'/.kibana/_bulk' + local kibana_url = connect_prefix .. ip_addr .. '/.kibana/_bulk' local function kibana_template_callback(err, code, body, _) if code ~= 200 then rspamd_logger.errx('cannot put template to %s: %s(%s) (%s)', kibana_url, @@ -389,7 +398,7 @@ local function initial_setup(cfg, ev_base, worker) if enabled then -- create ingest pipeline - local geoip_url = connect_prefix .. ip_addr ..'/_ingest/pipeline/rspamd-geoip' + local geoip_url = connect_prefix .. ip_addr .. '/_ingest/pipeline/rspamd-geoip' local function geoip_cb(err, code, body, _) if code ~= 200 then rspamd_logger.errx('cannot get data from %s: %s(%s) (%s)', @@ -425,7 +434,7 @@ local function initial_setup(cfg, ev_base, worker) timeout = settings.timeout, }) -- create template mappings if not exist - local template_url = connect_prefix .. ip_addr ..'/_template/rspamd' + local template_url = connect_prefix .. ip_addr .. '/_template/rspamd' local function http_template_put_callback(err, code, body, _) if code ~= 200 then rspamd_logger.errx('cannot put template to %s: %s(%s) (%s)', @@ -477,7 +486,7 @@ end redis_params = rspamd_redis.parse_redis_server('elastic') if redis_params and opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end @@ -494,11 +503,11 @@ if redis_params and opts then end settings.upstream = upstream_list.create(rspamd_config, - settings['server'] or settings['servers'], 9200) + settings['server'] or settings['servers'], 9200) if not settings.upstream then rspamd_logger.errx('cannot parse elastic address: %s', - settings['server'] or settings['servers']) + settings['server'] or settings['servers']) lua_util.disable_module(N, "config") return end @@ -511,7 +520,7 @@ if redis_params and opts then elastic_template = read_file(settings['template_file']); if not elastic_template then rspamd_logger.infox(rspamd_config, 'elastic unable to read %s, disabling module', - settings['template_file']) + settings['template_file']) lua_util.disable_module(N, "config") return end @@ -521,10 +530,10 @@ if redis_params and opts then type = 'idempotent', callback = elastic_collect, flags = 'empty,explicit_disable,ignore_passthrough', - augmentations = {string.format("timeout=%f", settings.timeout)}, + augmentations = { string.format("timeout=%f", settings.timeout) }, }) - rspamd_config:add_on_load(function(cfg, ev_base,worker) + rspamd_config:add_on_load(function(cfg, ev_base, worker) if worker:is_scanner() then check_elastic_server(cfg, ev_base, worker) -- check for elasticsearch requirements initial_setup(cfg, ev_base, worker) -- import mappings pipeline and visualizations diff --git a/src/plugins/lua/external_relay.lua b/src/plugins/lua/external_relay.lua index 9c0f06ff8..3660f921e 100644 --- a/src/plugins/lua/external_relay.lua +++ b/src/plugins/lua/external_relay.lua @@ -34,40 +34,40 @@ local settings = { rules = {}, } -local config_schema = ts.shape{ +local config_schema = ts.shape { enabled = ts.boolean:is_optional(), rules = ts.map_of( - ts.string, ts.one_of{ - ts.shape{ - priority = ts.number:is_optional(), - strategy = 'authenticated', - symbol = ts.string:is_optional(), - user_map = lua_maps.map_schema:is_optional(), - }, - ts.shape{ - count = ts.number, - priority = ts.number:is_optional(), - strategy = 'count', - symbol = ts.string:is_optional(), - }, - ts.shape{ - priority = ts.number:is_optional(), - strategy = 'local', - symbol = ts.string:is_optional(), - }, - ts.shape{ - hostname_map = lua_maps.map_schema, - priority = ts.number:is_optional(), - strategy = 'hostname_map', - symbol = ts.string:is_optional(), - }, - ts.shape{ - ip_map = lua_maps.map_schema, - priority = ts.number:is_optional(), - strategy = 'ip_map', - symbol = ts.string:is_optional(), - }, - } + ts.string, ts.one_of { + ts.shape { + priority = ts.number:is_optional(), + strategy = 'authenticated', + symbol = ts.string:is_optional(), + user_map = lua_maps.map_schema:is_optional(), + }, + ts.shape { + count = ts.number, + priority = ts.number:is_optional(), + strategy = 'count', + symbol = ts.string:is_optional(), + }, + ts.shape { + priority = ts.number:is_optional(), + strategy = 'local', + symbol = ts.string:is_optional(), + }, + ts.shape { + hostname_map = lua_maps.map_schema, + priority = ts.number:is_optional(), + strategy = 'hostname_map', + symbol = ts.string:is_optional(), + }, + ts.shape { + ip_map = lua_maps.map_schema, + priority = ts.number:is_optional(), + strategy = 'ip_map', + symbol = ts.string:is_optional(), + }, + } ), } diff --git a/src/plugins/lua/external_services.lua b/src/plugins/lua/external_services.lua index d6fedeece..a68517c08 100644 --- a/src/plugins/lua/external_services.lua +++ b/src/plugins/lua/external_services.lua @@ -27,91 +27,90 @@ local N = "external_services" if confighelp then rspamd_config:add_example(nil, 'external_services', - "Check messages using external services (e.g. OEM AS engines, DCC, Pyzor etc)", - [[ -external_services { - # multiple scanners could be checked, for each we create a configuration block with an arbitrary name - - oletools { - # If set force this action if any virus is found (default unset: no action is forced) - # action = "reject"; - # If set, then rejection message is set to this value (mention single quotes) - # If `max_size` is set, messages > n bytes in size are not scanned - # max_size = 20000000; - # log_clean = true; - # servers = "127.0.0.1:10050"; - # cache_expire = 86400; - # scan_mime_parts = true; - # extended = false; - # if `patterns` is specified virus name will be matched against provided regexes and the related - # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. - patterns { - # symbol_name = "pattern"; - JUST_EICAR = "^Eicar-Test-Signature$"; + "Check messages using external services (e.g. OEM AS engines, DCC, Pyzor etc)", + [[ + external_services { + # multiple scanners could be checked, for each we create a configuration block with an arbitrary name + + oletools { + # If set force this action if any virus is found (default unset: no action is forced) + # action = "reject"; + # If set, then rejection message is set to this value (mention single quotes) + # If `max_size` is set, messages > n bytes in size are not scanned + # max_size = 20000000; + # log_clean = true; + # servers = "127.0.0.1:10050"; + # cache_expire = 86400; + # scan_mime_parts = true; + # extended = false; + # if `patterns` is specified virus name will be matched against provided regexes and the related + # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. + patterns { + # symbol_name = "pattern"; + JUST_EICAR = "^Eicar-Test-Signature$"; + } + # mime-part regex matching in content-type or filename + mime_parts_filter_regex { + #GEN1 = "application\/octet-stream"; + DOC2 = "application\/msword"; + DOC3 = "application\/vnd\.ms-word.*"; + XLS = "application\/vnd\.ms-excel.*"; + PPT = "application\/vnd\.ms-powerpoint.*"; + GEN2 = "application\/vnd\.openxmlformats-officedocument.*"; + } + # Mime-Part filename extension matching (no regex) + mime_parts_filter_ext { + doc = "doc"; + dot = "dot"; + docx = "docx"; + dotx = "dotx"; + docm = "docm"; + dotm = "dotm"; + xls = "xls"; + xlt = "xlt"; + xla = "xla"; + xlsx = "xlsx"; + xltx = "xltx"; + xlsm = "xlsm"; + xltm = "xltm"; + xlam = "xlam"; + xlsb = "xlsb"; + ppt = "ppt"; + pot = "pot"; + pps = "pps"; + ppa = "ppa"; + pptx = "pptx"; + potx = "potx"; + ppsx = "ppsx"; + ppam = "ppam"; + pptm = "pptm"; + potm = "potm"; + ppsm = "ppsm"; + } + # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. + whitelist = "/etc/rspamd/antivirus.wl"; } - # mime-part regex matching in content-type or filename - mime_parts_filter_regex { - #GEN1 = "application\/octet-stream"; - DOC2 = "application\/msword"; - DOC3 = "application\/vnd\.ms-word.*"; - XLS = "application\/vnd\.ms-excel.*"; - PPT = "application\/vnd\.ms-powerpoint.*"; - GEN2 = "application\/vnd\.openxmlformats-officedocument.*"; + dcc { + # If set force this action if any virus is found (default unset: no action is forced) + # action = "reject"; + # If set, then rejection message is set to this value (mention single quotes) + # If `max_size` is set, messages > n bytes in size are not scanned + max_size = 20000000; + #servers = "127.0.0.1:10045; + # if `patterns` is specified virus name will be matched against provided regexes and the related + # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. + patterns { + # symbol_name = "pattern"; + JUST_EICAR = "^Eicar-Test-Signature$"; + } + # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. + whitelist = "/etc/rspamd/antivirus.wl"; } - # Mime-Part filename extension matching (no regex) - mime_parts_filter_ext { - doc = "doc"; - dot = "dot"; - docx = "docx"; - dotx = "dotx"; - docm = "docm"; - dotm = "dotm"; - xls = "xls"; - xlt = "xlt"; - xla = "xla"; - xlsx = "xlsx"; - xltx = "xltx"; - xlsm = "xlsm"; - xltm = "xltm"; - xlam = "xlam"; - xlsb = "xlsb"; - ppt = "ppt"; - pot = "pot"; - pps = "pps"; - ppa = "ppa"; - pptx = "pptx"; - potx = "potx"; - ppsx = "ppsx"; - ppam = "ppam"; - pptm = "pptm"; - potm = "potm"; - ppsm = "ppsm"; - } - # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. - whitelist = "/etc/rspamd/antivirus.wl"; - } - dcc { - # If set force this action if any virus is found (default unset: no action is forced) - # action = "reject"; - # If set, then rejection message is set to this value (mention single quotes) - # If `max_size` is set, messages > n bytes in size are not scanned - max_size = 20000000; - #servers = "127.0.0.1:10045; - # if `patterns` is specified virus name will be matched against provided regexes and the related - # symbol will be yielded if a match is found. If no match is found, default symbol is yielded. - patterns { - # symbol_name = "pattern"; - JUST_EICAR = "^Eicar-Test-Signature$"; - } - # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned. - whitelist = "/etc/rspamd/antivirus.wl"; } -} -]]) + ]]) return end - local function add_scanner_rule(sym, opts) if not opts.type then rspamd_logger.errx(rspamd_config, 'unknown type for external scanner rule %s', sym) @@ -130,7 +129,7 @@ local function add_scanner_rule(sym, opts) if not rule then rspamd_logger.errx(rspamd_config, 'cannot configure %s for %s', - opts.type, rule.symbol or sym:upper()) + opts.type, rule.symbol or sym:upper()) return nil end @@ -204,8 +203,12 @@ if opts and type(opts) == 'table' then local has_valid = false for k, m in pairs(opts) do if type(m) == 'table' and m.servers then - if not m.type then m.type = k end - if not m.name then m.name = k end + if not m.type then + m.type = k + end + if not m.name then + m.name = k + end local cb, nrule = add_scanner_rule(k, m) if not cb then @@ -334,7 +337,7 @@ if opts and type(opts) == 'table' then end if m.symbols then local function reg_symbols(tbl) - for _,sym in pairs(tbl) do + for _, sym in pairs(tbl) do if type(sym) == 'string' then rspamd_config:register_symbol({ type = 'virtual', @@ -389,7 +392,7 @@ if opts and type(opts) == 'table' then -- Add preloads if a module requires that if type(m.preloads) == 'table' then - for _,preload in ipairs(m.preloads) do + for _, preload in ipairs(m.preloads) do rspamd_config:add_on_load(function(cfg, ev_base, worker) preload(m, cfg, ev_base, worker) end) diff --git a/src/plugins/lua/force_actions.lua b/src/plugins/lua/force_actions.lua index 5f4620e6d..7f203b066 100644 --- a/src/plugins/lua/force_actions.lua +++ b/src/plugins/lua/force_actions.lua @@ -59,7 +59,7 @@ local function gen_cb(params) return 0 end - local e, err = rspamd_expression.create(params.expr, {parse_atom, process_atom}, params.pool) + local e, err = rspamd_expression.create(params.expr, { parse_atom, process_atom }, params.pool) if err then rspamd_logger.errx(rspamd_config, 'Couldnt create expression [%1]: %2', params.expr, err) return @@ -112,9 +112,9 @@ local function gen_cb(params) if type(params.message) == 'string' then -- process selector expressions in the message local message = string.gsub(params.message, '(${(.-)})', process_message_selectors) - task:set_pre_result{action = params.act, message = message, module = N, flags = flags} + task:set_pre_result { action = params.act, message = message, module = N, flags = flags } else - task:set_pre_result{action = params.act, module = N, flags = flags} + task:set_pre_result { action = params.act, module = N, flags = flags } end return true, params.act end @@ -143,11 +143,11 @@ local function configure_module() end if type(expr) == 'string' then -- expr, act, pool, message, subject, raction, honor, limit, flags - local cb, atoms = gen_cb{expr = expr, - act = action, - pool = rspamd_config:get_mempool(), - message = message, - subject = subject} + local cb, atoms = gen_cb { expr = expr, + act = action, + pool = rspamd_config:get_mempool(), + message = message, + subject = subject } if cb and atoms then local h = rspamd_cryptobox_hash.create() h:update(expr) @@ -175,19 +175,23 @@ local function configure_module() if action and expr then local flags = {} - if sett.least then table.insert(flags, "least") end - if sett.process_all then table.insert(flags, "process_all") end + if sett.least then + table.insert(flags, "least") + end + if sett.process_all then + table.insert(flags, "process_all") + end local raction = lua_util.list_to_hash(sett.require_action) local honor = lua_util.list_to_hash(sett.honor_action) - local cb, atoms = gen_cb{expr = expr, - act = action, - pool = rspamd_config:get_mempool(), - message = sett.message, - subject = sett.subject, - raction = raction, - honor = honor, - limit = sett.limit, - flags = table.concat(flags, ',')} + local cb, atoms = gen_cb { expr = expr, + act = action, + pool = rspamd_config:get_mempool(), + message = sett.message, + subject = sett.subject, + raction = raction, + honor = honor, + limit = sett.limit, + flags = table.concat(flags, ',') } if cb and atoms then local t = {} if (raction or honor) then @@ -196,7 +200,7 @@ local function configure_module() else t.type = 'normal' if not sett.least then - t.augmentations = {'passthrough', 'important'} + t.augmentations = { 'passthrough', 'important' } end end t.name = 'FORCE_ACTION_' .. name diff --git a/src/plugins/lua/forged_recipients.lua b/src/plugins/lua/forged_recipients.lua index 4b8e720c3..0d51db392 100644 --- a/src/plugins/lua/forged_recipients.lua +++ b/src/plugins/lua/forged_recipients.lua @@ -40,10 +40,14 @@ local function check_forged_headers(task) local smtp_rcpts = task:get_recipients(1) local smtp_from = task:get_from(1) - if not smtp_rcpts then return end - if #smtp_rcpts == 0 then return end + if not smtp_rcpts then + return + end + if #smtp_rcpts == 0 then + return + end - local mime_rcpts = task:get_recipients({ 'mime', 'orig'}) + local mime_rcpts = task:get_recipients({ 'mime', 'orig' }) if not mime_rcpts then return @@ -88,10 +92,10 @@ local function check_forged_headers(task) end end - for _,mime_rcpt in ipairs(mime_rcpts) do + for _, mime_rcpt in ipairs(mime_rcpts) do if mime_rcpt.addr and mime_rcpt.addr ~= '' then local addr = string.lower(mime_rcpt.addr) - local dom = string.lower(mime_rcpt.domain) + local dom = string.lower(mime_rcpt.domain) local matched_smtp_addr = smtp_rcpt_map[addr] if matched_smtp_addr then -- Direct match, go forward @@ -119,13 +123,13 @@ local function check_forged_headers(task) local opts = {} local seen_mime_unmatched = false local seen_smtp_unmatched = false - for _,mime_rcpt in ipairs(mime_rcpts) do + for _, mime_rcpt in ipairs(mime_rcpts) do if not mime_rcpt.matched then seen_mime_unmatched = true table.insert(opts, 'm:' .. mime_rcpt.addr) end end - for _,smtp_rcpt in ipairs(smtp_rcpts) do + for _, smtp_rcpt in ipairs(smtp_rcpts) do if not smtp_rcpt.matched then if not smtp_rcpt_domain_map[smtp_rcpt.domain:lower()]._seen_mime_domain then seen_smtp_unmatched = true @@ -149,7 +153,7 @@ local function check_forged_headers(task) end -- Configuration -local opts = rspamd_config:get_all_opt('forged_recipients') +local opts = rspamd_config:get_all_opt('forged_recipients') if opts then if opts['symbol_rcpt'] or opts['symbol_sender'] then local id = rspamd_config:register_symbol({ @@ -169,7 +173,7 @@ if opts then end if opts['symbol_sender'] then symbol_sender = opts['symbol_sender'] - rspamd_config:register_symbol({ + rspamd_config:register_symbol({ name = symbol_sender, type = 'virtual', parent = id, diff --git a/src/plugins/lua/fuzzy_collect.lua b/src/plugins/lua/fuzzy_collect.lua index d64830c6b..132ace90c 100644 --- a/src/plugins/lua/fuzzy_collect.lua +++ b/src/plugins/lua/fuzzy_collect.lua @@ -39,7 +39,7 @@ local function send_data_mirror(m, cfg, ev_base, body) rspamd_logger.infox(cfg, 'saved data on %s(%s)', m.server, m.name) end end - rspamd_http.request{ + rspamd_http.request { url = string.format('http://%s//update_v1/%s', m.server, m.name), resolver = cfg:get_resolver(), config = cfg, @@ -58,8 +58,10 @@ local function collect_fuzzy_hashes(cfg, ev_base) rspamd_logger.errx(cfg, 'cannot load data: %s', err) else -- Here, we actually copy body once for each mirror - fun.each(function(_, v) send_data_mirror(v, cfg, ev_base, body) end, - settings.mirrors) + fun.each(function(_, v) + send_data_mirror(v, cfg, ev_base, body) + end, + settings.mirrors) end end @@ -70,13 +72,13 @@ local function collect_fuzzy_hashes(cfg, ev_base) if settings.saved_cookie ~= tostring(body) then settings.saved_cookie = tostring(body) rspamd_logger.infox(cfg, 'received collection cookie %s', - tostring(rspamd_util.encode_base32(settings.saved_cookie:sub(1, 6)))) + tostring(rspamd_util.encode_base32(settings.saved_cookie:sub(1, 6)))) local sig = rspamd_cryptolib.sign_memory(settings.sign_keypair, - settings.saved_cookie) + settings.saved_cookie) if not sig then rspamd_logger.info(cfg, 'cannot sign cookie') else - rspamd_http.request{ + rspamd_http.request { url = string.format('http://%s/data', settings.collect_server), resolver = cfg:get_resolver(), config = cfg, @@ -96,8 +98,8 @@ local function collect_fuzzy_hashes(cfg, ev_base) end end rspamd_logger.infox(cfg, 'start fuzzy collection, next sync in %s seconds', - settings.sync_time) - rspamd_http.request{ + settings.sync_time) + rspamd_http.request { url = string.format('http://%s/cookie', settings.collect_server), resolver = cfg:get_resolver(), config = cfg, @@ -141,7 +143,7 @@ end local opts = rspamd_config:get_all_opt('fuzzy_collect') if opts and type(opts) == 'table' then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end local sane_config = true @@ -180,9 +182,9 @@ if opts and type(opts) == 'table' then rspamd_config:add_on_load(function(_, ev_base, worker) if worker:is_primary_controller() then rspamd_config:add_periodic(ev_base, 0.0, - function(cfg, _) - return collect_fuzzy_hashes(cfg, ev_base) - end) + function(cfg, _) + return collect_fuzzy_hashes(cfg, ev_base) + end) end end) else diff --git a/src/plugins/lua/greylist.lua b/src/plugins/lua/greylist.lua index db1afeed7..16f498417 100644 --- a/src/plugins/lua/greylist.lua +++ b/src/plugins/lua/greylist.lua @@ -111,7 +111,9 @@ local function data_key(task) local body = task:get_rawbody() - if not body then return nil end + if not body then + return nil + end local len = body:len() if len > settings['max_data_len'] then @@ -175,16 +177,16 @@ local function check_time(task, tm, type, now) if not t then rspamd_logger.errx(task, 'not a valid number: %s', tm) - return false,false + return false, false end if now - t < settings['timeout'] then - return true,true + return true, true else -- We just set variable to pass when in post-filter stage task:get_mempool():set_variable("grey_whitelisted", type) - return true,false + return true, false end end @@ -197,7 +199,7 @@ local function greylist_message(task, end_time, why) if settings.message_func then task:set_pre_result(settings['action'], - settings.message_func(task, end_time), N) + settings.message_func(task, end_time), N) else local message = settings['message'] if settings.report_time then @@ -237,12 +239,12 @@ local function greylist_check(task) local greylisted_meta = false if data then - local end_time_body,end_time_meta + local end_time_body, end_time_meta local now = rspamd_util.get_time() if data[1] and type(data[1]) ~= 'userdata' then local tm = tonumber(data[1]) or now - ret_body,greylisted_body = check_time(task, data[1], 'body', now) + ret_body, greylisted_body = check_time(task, data[1], 'body', now) if greylisted_body then end_time_body = tm + settings['timeout'] task:get_mempool():set_variable("grey_greylisted_body", @@ -253,7 +255,7 @@ local function greylist_check(task) if data[2] and type(data[2]) ~= 'userdata' then if not ret_body or greylisted_body then local tm = tonumber(data[2]) or now - ret_meta,greylisted_meta = check_time(task, data[2], 'meta', now) + ret_meta, greylisted_meta = check_time(task, data[2], 'meta', now) if greylisted_meta then end_time_meta = tm + settings['timeout'] @@ -298,7 +300,7 @@ local function greylist_check(task) false, -- is write redis_get_cb, --callback 'MGET', -- command - {body_key, meta_key} -- arguments + { body_key, meta_key } -- arguments ) if not ret then rspamd_logger.errx(task, 'cannot make redis request to check results') @@ -310,10 +312,12 @@ local function greylist_set(task) local ip = task:get_ip() -- Don't do anything if pre-result has been already set - if task:has_pre_result() then return end + if task:has_pre_result() then + return + end -- Check whitelist_symbols - for _,sym in ipairs(settings.whitelist_symbols) do + for _, sym in ipairs(settings.whitelist_symbols) do if task:has_symbol(sym) then rspamd_logger.infox(task, 'skip greylisting as we have found symbol %s', sym) if action == 'greylist' then @@ -387,7 +391,7 @@ local function greylist_set(task) local function redis_set_cb(err) if err then rspamd_logger.errx(task, 'got error %s when setting greylisting record on server %s', - err, upstream:get_addr()) + err, upstream:get_addr()) end end @@ -402,18 +406,20 @@ local function greylist_set(task) task:insert_result(settings['symbol'], 0.0, 'pass', is_whitelisted) rspamd_logger.infox(task, 'greylisting pass (%s) until %s', - is_whitelisted, - rspamd_util.time_to_string(rspamd_util.get_time() + settings['expire'])) + is_whitelisted, + rspamd_util.time_to_string(rspamd_util.get_time() + settings['expire'])) - if not settings.check_local and is_rspamc then return end + if not settings.check_local and is_rspamc then + return + end - ret,conn,upstream = lua_redis.redis_make_request(task, - redis_params, -- connect params - hash_key, -- hash key - true, -- is write - redis_set_cb, --callback - 'EXPIRE', -- command - {body_key, tostring(toint(settings['expire']))} -- arguments + ret, conn, upstream = lua_redis.redis_make_request(task, + redis_params, -- connect params + hash_key, -- hash key + true, -- is write + redis_set_cb, --callback + 'EXPIRE', -- command + { body_key, tostring(toint(settings['expire'])) } -- arguments ) -- Update greylisting record expire if ret then @@ -424,19 +430,21 @@ local function greylist_set(task) rspamd_logger.errx(task, 'got error while connecting to redis') end elseif do_greylisting or do_greylisting_required then - if not settings.check_local and is_rspamc then return end + if not settings.check_local and is_rspamc then + return + end local t = tostring(toint(rspamd_util.get_time())) local end_time = rspamd_util.time_to_string(t + settings['timeout']) rspamd_logger.infox(task, 'greylisted until "%s", new record', end_time) greylist_message(task, end_time, 'new record') -- Create new record - ret,conn,upstream = lua_redis.redis_make_request(task, - redis_params, -- connect params - hash_key, -- hash key - true, -- is write - redis_set_cb, --callback - 'SETEX', -- command - {body_key, tostring(toint(settings['expire'])), t} -- arguments + ret, conn, upstream = lua_redis.redis_make_request(task, + redis_params, -- connect params + hash_key, -- hash key + true, -- is write + redis_set_cb, --callback + 'SETEX', -- command + { body_key, tostring(toint(settings['expire'])), t } -- arguments ) if ret then @@ -472,7 +480,7 @@ if opts then settings.message_func = assert(load(opts['message_func']))() end - for k,v in pairs(opts) do + for k, v in pairs(opts) do if k ~= 'message_func' then settings[k] = v end @@ -493,9 +501,9 @@ if opts then end whitelisted_ip = lua_map.rspamd_map_add(N, 'whitelisted_ip', 'radix', - 'Greylist whitelist ip map') + 'Greylist whitelist ip map') whitelist_domains_map = lua_map.rspamd_map_add(N, 'whitelist_domains_url', - 'map', 'Greylist whitelist domains map') + 'map', 'Greylist whitelist domains map') redis_params = lua_redis.parse_redis_server(N) if not redis_params then @@ -515,14 +523,14 @@ if opts then type = 'postfilter', callback = greylist_set, priority = lua_util.symbols_priorities.medium, - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}, + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }, }) local id = rspamd_config:register_symbol({ name = 'GREYLIST_CHECK', type = 'prefilter', callback = greylist_check, priority = lua_util.symbols_priorities.medium, - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)} + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) } }) rspamd_config:register_symbol({ name = settings.symbol, diff --git a/src/plugins/lua/hfilter.lua b/src/plugins/lua/hfilter.lua index f68d5b579..601786f33 100644 --- a/src/plugins/lua/hfilter.lua +++ b/src/plugins/lua/hfilter.lua @@ -200,7 +200,7 @@ local function check_regexp(str, regexp_text) end local function add_static_map(data) - return rspamd_config:add_map{ + return rspamd_config:add_map { type = 'regexp_multi', url = { upstreams = 'static', @@ -227,13 +227,13 @@ local function check_host(task, host, symbol_suffix, eq_ip, eq_host) local function check_host_cb_mx(_, to_resolve, results, err) if err and (err ~= 'requested record is not found' and err ~= 'no records with this name') then - lua_util.debugm(N, task, 'error looking up %s: %s', to_resolve, err) + lua_util.debugm(N, task, 'error looking up %s: %s', to_resolve, err) end if not results then task:insert_result('HFILTER_' .. symbol_suffix .. '_NORES_A_OR_MX', 1.0, - to_resolve) + to_resolve) else - for _,mx in pairs(results) do + for _, mx in pairs(results) do if mx['name'] then local failed_mx_address = 0 -- Capture failed_mx_address @@ -244,12 +244,12 @@ local function check_host(task, host, symbol_suffix, eq_ip, eq_host) if failed_mx_address >= 2 then task:insert_result('HFILTER_' .. symbol_suffix .. '_NORESOLVE_MX', - 1.0, mx['name']) + 1.0, mx['name']) end end task:get_resolver():resolve('a', { - task=task, + task = task, name = mx['name'], callback = check_host_cb_mx_a }) @@ -266,7 +266,7 @@ local function check_host(task, host, symbol_suffix, eq_ip, eq_host) if not results then failed_address = failed_address + 1 else - for _,result in pairs(results) do + for _, result in pairs(results) do table.insert(resolved_address, result:to_string()) end end @@ -274,7 +274,7 @@ local function check_host(task, host, symbol_suffix, eq_ip, eq_host) if failed_address >= 2 then -- No A or AAAA records if eq_ip and eq_ip ~= '' then - for _,result in pairs(resolved_address) do + for _, result in pairs(resolved_address) do if result == eq_ip then return true end @@ -303,7 +303,7 @@ local function check_host(task, host, symbol_suffix, eq_ip, eq_host) if check_fqdn(host) then if eq_host == '' or eq_host ~= host then task:get_resolver():resolve('a', { - task=task, + task = task, name = host, callback = check_host_cb_a }) @@ -329,7 +329,7 @@ local function hfilter_callback(task) if parts then local plain_text_part, html_text_part - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do if p:is_html() then html_text_part = p else @@ -345,7 +345,9 @@ local function hfilter_callback(task) local rel = url_len / plen if rel > 0.8 then local sc = (rel - 0.8) * 5.0 - if sc > 1.0 then sc = 1.0 end + if sc > 1.0 then + sc = 1.0 + end task:insert_result('HFILTER_URL_ONLY', sc, tostring(sc)) local lines = part:get_lines_count() if lines > 0 and lines < 2 then @@ -405,7 +407,7 @@ local function hfilter_callback(task) if not find_badip and not find_bareip then -- Regexp check HELO (checks_hello) local weights = checks_hello_map:get_key(helo) - for _,weight in ipairs(weights or {}) do + for _, weight in ipairs(weights or {}) do weight = tonumber(weight) or 0 if weight > weight_helo then weight_helo = weight @@ -413,7 +415,7 @@ local function hfilter_callback(task) end -- Regexp check HELO (checks_hellohost) weights = checks_hellohost_map:get_key(helo) - for _,weight in ipairs(weights or {}) do + for _, weight in ipairs(weights or {}) do weight = tonumber(weight) or 0 if weight > weight_helo then weight_helo = weight @@ -436,7 +438,7 @@ local function hfilter_callback(task) if hostname then -- Check regexp HOSTNAME local weights = checks_hellohost_map:get_key(hostname) - for _,weight in ipairs(weights or {}) do + for _, weight in ipairs(weights or {}) do weight = tonumber(weight) or 0 if weight > weight_hostname then weight_hostname = weight @@ -460,7 +462,7 @@ local function hfilter_callback(task) local from = task:get_from(1) if from then --FROM host check - for _,fr in ipairs(from) do + for _, fr in ipairs(from) do local fr_split = rspamd_str_split(fr['addr'], '@') if #fr_split == 2 then check_host(task, fr_split[2], 'FROMHOST', '', '') @@ -485,7 +487,7 @@ local function hfilter_callback(task) if frombounce then if count_rcpt > 1 then task:insert_result('HFILTER_RCPT_BOUNCEMOREONE', 1.00, - tostring(count_rcpt)) + tostring(count_rcpt)) end end end @@ -555,13 +557,15 @@ local timeout = 0.0 local opts = rspamd_config:get_all_opt('hfilter') if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do config[k] = v end end local function append_t(t, a) - for _,v in ipairs(a) do table.insert(t, v) end + for _, v in ipairs(a) do + table.insert(t, v) + end end if config['helo_enabled'] then checks_hello_bareip_map = add_static_map(checks_hello_bareip) @@ -594,14 +598,14 @@ end --dumper(symbols_enabled) if #symbols_enabled > 0 then - local id = rspamd_config:register_symbol{ + local id = rspamd_config:register_symbol { name = 'HFILTER_CHECK', callback = hfilter_callback, type = 'callback', - augmentations = {string.format("timeout=%f", timeout)}, + augmentations = { string.format("timeout=%f", timeout) }, } - for _,sym in ipairs(symbols_enabled) do - rspamd_config:register_symbol{ + for _, sym in ipairs(symbols_enabled) do + rspamd_config:register_symbol { type = 'virtual', score = 1.0, parent = id, diff --git a/src/plugins/lua/history_redis.lua b/src/plugins/lua/history_redis.lua index cfedfeac3..9cbe7caf7 100644 --- a/src/plugins/lua/history_redis.lua +++ b/src/plugins/lua/history_redis.lua @@ -94,8 +94,13 @@ local function normalise_results(tbl, task) return type(v) == 'table' and v.score end, metric) - fun.each(function(k, v) v.name = nil; tbl.symbols[k] = v; end, symbols) - fun.each(function(k, v) tbl[k] = v end, others) + fun.each(function(k, v) + v.name = nil; + tbl.symbols[k] = v; + end, symbols) + fun.each(function(k, v) + tbl[k] = v + end, others) -- Reset the original metric tbl.default = nil @@ -141,7 +146,7 @@ local function history_save(task) return end - local data = task:get_protocol_reply{'metrics', 'basic'} + local data = task:get_protocol_reply { 'metrics', 'basic' } local prefix = settings.key_prefix .. hostname if data then @@ -160,19 +165,19 @@ local function history_save(task) end local ret, conn, _ = lua_redis.rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - true, -- is write - redis_llen_cb, --callback - 'LPUSH', -- command - {prefix, json} -- arguments + redis_params, -- connect params + nil, -- hash key + true, -- is write + redis_llen_cb, --callback + 'LPUSH', -- command + { prefix, json } -- arguments ) if ret then - conn:add_cmd('LTRIM', {prefix, '0', string.format('%d', settings.nrows-1)}) + conn:add_cmd('LTRIM', { prefix, '0', string.format('%d', settings.nrows - 1) }) if settings.expire and settings.expire > 0 then - conn:add_cmd('EXPIRE', {prefix, string.format('%d', settings.expire)}) + conn:add_cmd('EXPIRE', { prefix, string.format('%d', settings.expire) }) end end end @@ -188,19 +193,19 @@ local function handle_history_request(task, conn, from, to, reset) local function redis_ltrim_cb(err, _) if err then rspamd_logger.errx(task, 'got error %s when resetting history: %s', - err) + err) conn:send_error(504, '{"error": "' .. err .. '"}') else conn:send_string('{"success":true}') end end lua_redis.rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - true, -- is write - redis_ltrim_cb, --callback - 'LTRIM', -- command - {prefix, '0', '0'} -- arguments + redis_params, -- connect params + nil, -- hash key + true, -- is write + redis_ltrim_cb, --callback + 'LTRIM', -- command + { prefix, '0', '0' } -- arguments ) else local function redis_lrange_cb(err, data) @@ -211,14 +216,16 @@ local function handle_history_request(task, conn, from, to, reset) if settings.compress then local t1 = rspamd_util:get_ticks() - data = fun.totable(fun.filter(function(e) return e ~= nil end, - fun.map(function(e) - local _,dec = rspamd_util.zstd_decompress(e) - if dec then - return dec - end - return nil - end, data))) + data = fun.totable(fun.filter(function(e) + return e ~= nil + end, + fun.map(function(e) + local _, dec = rspamd_util.zstd_decompress(e) + if dec then + return dec + end + return nil + end, data))) lua_util.debugm(N, task, 'decompress took %s ms', (rspamd_util:get_ticks() - t1) * 1000.0) collectgarbage() @@ -226,23 +233,25 @@ local function handle_history_request(task, conn, from, to, reset) -- Parse elements using ucl local t1 = rspamd_util:get_ticks() data = fun.totable( - fun.map(function (_, obj) return obj end, - fun.filter(function(res, obj) - if res then - return true - end - return false + fun.map(function(_, obj) + return obj end, - fun.map(function(elt) - local parser = ucl.parser() - local res,_ = parser:parse_text(elt) - - if res then - return true, parser:get_object() - else - return false, nil - end - end, data)))) + fun.filter(function(res, obj) + if res then + return true + end + return false + end, + fun.map(function(elt) + local parser = ucl.parser() + local res, _ = parser:parse_text(elt) + + if res then + return true, parser:get_object() + else + return false, nil + end + end, data)))) lua_util.debugm(N, task, 'parse took %s ms', (rspamd_util:get_ticks() - t1) * 1000.0) collectgarbage() @@ -254,26 +263,26 @@ local function handle_history_request(task, conn, from, to, reset) collectgarbage() else rspamd_logger.errx(task, 'got error %s when getting history: %s', - err) + err) conn:send_error(504, '{"error": "' .. err .. '"}') end end lua_redis.rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - false, -- is write - redis_lrange_cb, --callback - 'LRANGE', -- command - {prefix, string.format('%d', from), string.format('%d', to)}, -- arguments - {opaque_data = true} + redis_params, -- connect params + nil, -- hash key + false, -- is write + redis_lrange_cb, --callback + 'LRANGE', -- command + { prefix, string.format('%d', from), string.format('%d', to) }, -- arguments + { opaque_data = true } ) end end -local opts = rspamd_config:get_all_opt('history_redis') +local opts = rspamd_config:get_all_opt('history_redis') if opts then settings = lua_util.override_defaults(settings, opts) - local res,err = settings_schema:transform(settings) + local res, err = settings_schema:transform(settings) if not res then rspamd_logger.warnx(rspamd_config, '%s: plugin is misconfigured: %s', N, err) @@ -292,7 +301,7 @@ if opts then type = 'idempotent', callback = history_save, flags = 'empty,explicit_disable,ignore_passthrough', - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)} + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) } }) lua_redis.register_prefix(settings.key_prefix .. hostname, N, "Redis history", { diff --git a/src/plugins/lua/http_headers.lua b/src/plugins/lua/http_headers.lua index c437392e4..1c6494a1e 100644 --- a/src/plugins/lua/http_headers.lua +++ b/src/plugins/lua/http_headers.lua @@ -55,7 +55,7 @@ local dmarc_symbols = { local opts = rspamd_config:get_all_opt('dmarc') if opts and opts['symbols'] then - for k,_ in pairs(dmarc_symbols) do + for k, _ in pairs(dmarc_symbols) do if opts['symbols'][k] then dmarc_symbols[k] = opts['symbols'][k] end @@ -64,7 +64,7 @@ end opts = rspamd_config:get_all_opt('dkim') if opts then - for k,_ in pairs(dkim_symbols) do + for k, _ in pairs(dkim_symbols) do if opts[k] then dkim_symbols[k] = opts[k] end @@ -73,7 +73,7 @@ end opts = rspamd_config:get_all_opt('spf') if opts then - for k,_ in pairs(spf_symbols) do + for k, _ in pairs(spf_symbols) do if opts[k] then spf_symbols[k] = opts[k] end @@ -95,28 +95,28 @@ rspamd_config:add_condition("DKIM_CHECK", function(task) local p_obj = parser:get_object() local results = p_obj['results'] if not results and p_obj['result'] then - results = {{result = p_obj['result'], domain = 'unknown'}} + results = { { result = p_obj['result'], domain = 'unknown' } } end if results then for _, obj in ipairs(results) do - local dkim_domain = obj['domain'] or 'unknown' + local dkim_domain = obj['domain'] or 'unknown' if obj['result'] == 'pass' or obj['result'] == 'allow' then task:insert_result(dkim_symbols['symbol_allow'], 1.0, 'http header') task:insert_result(dkim_symbols['symbol_trace'], 1.0, - string.format('%s:%s', dkim_domain, dkim_trace.pass)) + string.format('%s:%s', dkim_domain, dkim_trace.pass)) elseif obj['result'] == 'fail' or obj['result'] == 'reject' then task:insert_result(dkim_symbols['symbol_deny'], 1.0, 'http header') task:insert_result(dkim_symbols['symbol_trace'], 1.0, - string.format('%s:%s', dkim_domain, dkim_trace.fail)) + string.format('%s:%s', dkim_domain, dkim_trace.fail)) elseif obj['result'] == 'tempfail' or obj['result'] == 'softfail' then task:insert_result(dkim_symbols['symbol_tempfail'], 1.0, 'http header') task:insert_result(dkim_symbols['symbol_trace'], 1.0, - string.format('%s:%s', dkim_domain, dkim_trace.temperror)) + string.format('%s:%s', dkim_domain, dkim_trace.temperror)) elseif obj['result'] == 'permfail' then task:insert_result(dkim_symbols['symbol_permfail'], 1.0, 'http header') task:insert_result(dkim_symbols['symbol_trace'], 1.0, - string.format('%s:%s', dkim_domain, dkim_trace.permerror)) + string.format('%s:%s', dkim_domain, dkim_trace.permerror)) elseif obj['result'] == 'na' then task:insert_result(dkim_symbols['symbol_na'], 1.0, 'http header') end diff --git a/src/plugins/lua/maillist.lua b/src/plugins/lua/maillist.lua index 023867a8d..be1401c32 100644 --- a/src/plugins/lua/maillist.lua +++ b/src/plugins/lua/maillist.lua @@ -104,7 +104,8 @@ local function check_ml_mailman(task) -- not much elase we can check, Subjects can be changed in settings return true end - else -- Mailman 3 + else + -- Mailman 3 -- XXX not Mailman3 admin messages have this headers, but one -- which don't usually have List-* headers examined below if task:has_header('List-Administrivia') then @@ -113,7 +114,7 @@ local function check_ml_mailman(task) end -- List-Archive and List-Post are optional, check other headers - for _, h in ipairs({'List-Help', 'List-Subscribe', 'List-Unsubscribe'}) do + for _, h in ipairs({ 'List-Help', 'List-Subscribe', 'List-Unsubscribe' }) do header = task:get_header(h) if not (header and header:find('<mailto:', 1, true)) then return false @@ -173,7 +174,7 @@ local function check_generic_list_headers(task) end, } - for hname,hscore in pairs(common_list_headers) do + for hname, hscore in pairs(common_list_headers) do if task:has_header(hname) then if type(hscore) == 'number' then score = score + hscore @@ -210,7 +211,9 @@ local function check_maillist(task) elseif check_ml_cgp(task) then task:insert_result(symbol, 1, 'cgp') else - if score > 2 then score = 2 end + if score > 2 then + score = 2 + end task:insert_result(symbol, 0.5 * score, 'generic') end end @@ -219,7 +222,7 @@ end -- Configuration -local opts = rspamd_config:get_all_opt('maillist') +local opts = rspamd_config:get_all_opt('maillist') if opts then if opts['symbol'] then symbol = opts['symbol'] diff --git a/src/plugins/lua/maps_stats.lua b/src/plugins/lua/maps_stats.lua index bf03f7801..d4188103a 100644 --- a/src/plugins/lua/maps_stats.lua +++ b/src/plugins/lua/maps_stats.lua @@ -66,7 +66,7 @@ local function process_map(map, ev_base, _) true, -- is write redis_zrange_cb, --callback 'ZREMRANGEBYRANK', -- command - {key, '0', tostring(-(settings.count) - 1)} -- arguments + { key, '0', tostring(-(settings.count) - 1) } -- arguments ) end end @@ -78,14 +78,14 @@ local function process_map(map, ev_base, _) true, -- is write redis_card_cb, --callback 'ZCARD', -- command - {key} -- arguments + { key } -- arguments ) if ret and conn then local stats = map:get_stats(true) - for k,s in pairs(stats) do + for k, s in pairs(stats) do if s > 0 then - conn:add_cmd('ZINCRBY', {key, tostring(s), k}) + conn:add_cmd('ZINCRBY', { key, tostring(s), k }) end end end @@ -99,7 +99,7 @@ end local opts = rspamd_config:get_all_opt(N) if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end end @@ -107,7 +107,7 @@ end redis_params = lua_redis.parse_redis_server(N, opts) -- XXX, this is a poor approach as not all maps are defined here... local tmaps = rspamd_config:get_maps() -for _,m in ipairs(tmaps) do +for _, m in ipairs(tmaps) do if m:get_uri() ~= 'static' then lua_redis.register_prefix(settings.prefix .. m:get_uri(), N, 'Maps stats data', { @@ -118,13 +118,13 @@ for _,m in ipairs(tmaps) do end if redis_params then - rspamd_config:add_on_load(function (_, ev_base, worker) + rspamd_config:add_on_load(function(_, ev_base, worker) local maps = rspamd_config:get_maps() - for _,m in ipairs(maps) do + for _, m in ipairs(maps) do rspamd_config:add_periodic(ev_base, settings['interval'], - function () + function() process_map(m, ev_base, worker) return true end, true) diff --git a/src/plugins/lua/metadata_exporter.lua b/src/plugins/lua/metadata_exporter.lua index 6c4cf0019..9d0826735 100644 --- a/src/plugins/lua/metadata_exporter.lua +++ b/src/plugins/lua/metadata_exporter.lua @@ -84,7 +84,7 @@ local function get_general_metadata(task, flatten, no_content) local fuzzy = task:get_mempool():get_variable("fuzzy_hashes", "fstrings") if fuzzy and #fuzzy > 0 then local fz = {} - for _,h in ipairs(fuzzy) do + for _, h in ipairs(fuzzy) do table.insert(fz, h) end if not flatten then @@ -224,7 +224,7 @@ local formatters = { meta.mail_to = table.concat(display_emails, ', ') meta.our_message_id = rspamd_util.random_hex(12) .. '@rspamd' meta.date = rspamd_util.time_to_string(rspamd_util.get_time()) - return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets} + return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets } end, json = function(task) return ucl.to_format(get_general_metadata(task), 'json-compact') @@ -276,7 +276,7 @@ end local pushers = { redis_pubsub = function(task, formatted, rule) - local _,ret,upstream + local _, ret, upstream local function redis_pub_cb(err) if err then rspamd_logger.errx(task, 'got error %s when publishing on server %s', @@ -285,13 +285,13 @@ local pushers = { end return true end - ret,_,upstream = rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - true, -- is write - redis_pub_cb, --callback - 'PUBLISH', -- command - {rule.channel, formatted} -- arguments + ret, _, upstream = rspamd_redis_make_request(task, + redis_params, -- connect params + nil, -- hash key + true, -- is write + redis_pub_cb, --callback + 'PUBLISH', -- command + { rule.channel, formatted } -- arguments ) if not ret then rspamd_logger.errx(task, 'error connecting to redis') @@ -300,7 +300,7 @@ local pushers = { end, http = function(task, formatted, rule) local function http_callback(err, code) - local valid_status = {200, 201, 202, 204} + local valid_status = { 200, 201, 202, 204 } if err then rspamd_logger.errx(task, 'got error %s in http callback', err) @@ -327,14 +327,14 @@ local pushers = { end end rspamd_http.request({ - task=task, - url=rule.url, - user=rule.user, - password=rule.password, - body=formatted, - callback=http_callback, - mime_type=rule.mime_type or settings.mime_type, - headers=hdrs, + task = task, + url = rule.url, + user = rule.user, + password = rule.password, + body = formatted, + callback = http_callback, + mime_type = rule.mime_type or settings.mime_type, + headers = hdrs, }) end, send_mail = function(task, formatted, rule, extra) @@ -359,7 +359,9 @@ local pushers = { } local opts = rspamd_config:get_all_opt(N) -if not opts then return end +if not opts then + return +end local process_settings = { select = function(val) selectors.custom = assert(load(val))() @@ -632,8 +634,10 @@ for k, v in pairs(settings.rules) do end local function gen_exporter(rule) - return function (task) - if task:has_flag('skip') then return end + return function(task) + if task:has_flag('skip') then + return + end local selector = rule.selector or 'default' local selected = selectors[selector](task) if selected then @@ -661,6 +665,6 @@ for k, r in pairs(settings.rules) do type = 'idempotent', callback = gen_exporter(r), flags = 'empty,explicit_disable,ignore_passthrough', - augmentations = {string.format("timeout=%f", r.timeout or 0.0)} + augmentations = { string.format("timeout=%f", r.timeout or 0.0) } }) end diff --git a/src/plugins/lua/metric_exporter.lua b/src/plugins/lua/metric_exporter.lua index 6e48ae533..75885516c 100644 --- a/src/plugins/lua/metric_exporter.lua +++ b/src/plugins/lua/metric_exporter.lua @@ -132,7 +132,7 @@ local function graphite_push(kwargs) data = { metrics_str, '\n', }, - callback = (function (err) + callback = (function(err) if err then logger.errx('Push failed: %1', err) return @@ -171,12 +171,14 @@ if not configure_metric_exporter() then return end -rspamd_config:add_on_load(function (_, ev_base, worker) +rspamd_config:add_on_load(function(_, ev_base, worker) -- Exit unless we're the first 'controller' worker - if not worker:is_primary_controller() then return end + if not worker:is_primary_controller() then + return + end -- Persist mempool variable to statefile on shutdown pool = mempool.create() - rspamd_config:register_finish_script(function () + rspamd_config:register_finish_script(function() local stamp = pool:get_variable(VAR_NAME, 'double') if not stamp then logger.warn('No last metric exporter push to persist to disk') @@ -207,14 +209,14 @@ rspamd_config:add_on_load(function (_, ev_base, worker) end -- Push metrics at regular intervals local function schedule_regular_push() - rspamd_config:add_periodic(ev_base, settings['interval'], function () + rspamd_config:add_periodic(ev_base, settings['interval'], function() push_metrics() return true end) end -- Push metrics to backend and reschedule check local function schedule_intermediate_push(when) - rspamd_config:add_periodic(ev_base, when, function () + rspamd_config:add_periodic(ev_base, when, function() push_metrics() schedule_regular_push() return false diff --git a/src/plugins/lua/mid.lua b/src/plugins/lua/mid.lua index d7bf02d1d..b8650c835 100644 --- a/src/plugins/lua/mid.lua +++ b/src/plugins/lua/mid.lua @@ -48,7 +48,7 @@ local function known_mid_cb(task) local header = task:get_header('Message-Id') local das = task:get_symbol(settings['symbol_dkim_allow']) if ((das or E)[1] or E).options then - for _,dkim_domain in ipairs(das[1]['options']) do + for _, dkim_domain in ipairs(das[1]['options']) do if dkim_domain then local v = map:get_key(dkim_domain:match "[^:]+") if v then @@ -70,9 +70,9 @@ local function known_mid_cb(task) end end -local opts = rspamd_config:get_all_opt('mid') +local opts = rspamd_config:get_all_opt('mid') if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end @@ -82,7 +82,7 @@ if opts then return end - map = rspamd_config:add_map{ + map = rspamd_config:add_map { url = opts.source, description = "Message-IDs map", type = 'map' diff --git a/src/plugins/lua/milter_headers.lua b/src/plugins/lua/milter_headers.lua index 02f50be27..b53a45457 100644 --- a/src/plugins/lua/milter_headers.lua +++ b/src/plugins/lua/milter_headers.lua @@ -137,11 +137,15 @@ local function milter_headers(task) local settings_override = false local function skip_wanted(hdr) - if settings_override then return true end + if settings_override then + return true + end -- Normal checks local function match_extended_headers_rcpt() local rcpts = task:get_recipients('smtp') - if not rcpts then return false end + if not rcpts then + return false + end local found for _, r in ipairs(rcpts) do found = false @@ -165,7 +169,9 @@ local function milter_headers(task) found = true end end - if found then break end + if found then + break + end end return found end @@ -176,11 +182,15 @@ local function milter_headers(task) if settings.skip_local and not settings.local_headers[hdr] then local ip = task:get_ip() - if (ip and ip:is_local()) then return true end + if (ip and ip:is_local()) then + return true + end end if settings.skip_authenticated and not settings.authenticated_headers[hdr] then - if task:get_user() ~= nil then return true end + if task:get_user() ~= nil then + return true + end end if settings.skip_all then @@ -211,7 +221,9 @@ local function milter_headers(task) routines['x-spamd-result'] = function() local local_mod = settings.routines['x-spamd-result'] - if skip_wanted('x-spamd-result') then return end + if skip_wanted('x-spamd-result') then + return + end if not common.symbols then common.symbols = task:get_symbols_all() end @@ -227,7 +239,7 @@ local function milter_headers(task) local buf = {} local verdict = string.format('default: %s [%.2f / %.2f]', - --TODO: (common.metric_action == 'no action') and 'False' or 'True', + --TODO: (common.metric_action == 'no action') and 'False' or 'True', (common.metric_action == 'reject') and 'True' or 'False', common.metric_score[1], common.metric_score[2]) table.insert(buf, verdict) @@ -247,12 +259,12 @@ local function milter_headers(task) for _, s in ipairs(common.symbols) do local sym_str = string.format('%s(%.2f)[%s]', - s.name, s.score, table.concat(s.options or {}, ',')) + s.name, s.score, table.concat(s.options or {}, ',')) table.insert(buf, sym_str) end add_header('x-spamd-result', table.concat(buf, '; '), ';') - local has_pr,action,message,module = task:has_pre_result() + local has_pr, action, message, module = task:has_pre_result() if has_pr then local pr_header = {} @@ -270,7 +282,9 @@ local function milter_headers(task) end routines['x-rspamd-queue-id'] = function() - if skip_wanted('x-rspamd-queue-id') then return end + if skip_wanted('x-rspamd-queue-id') then + return + end if common.queue_id ~= false then common.queue_id = task:get_queue_id() if not common.queue_id then @@ -286,21 +300,27 @@ local function milter_headers(task) end routines['remove-header'] = function() - if skip_wanted('remove-header') then return end + if skip_wanted('remove-header') then + return + end if settings.routines['remove-header'].header and settings.routines['remove-header'].remove then remove[settings.routines['remove-header'].header] = settings.routines['remove-header'].remove end end routines['remove-headers'] = function() - if skip_wanted('remove-headers') then return end + if skip_wanted('remove-headers') then + return + end for h, r in pairs(settings.routines['remove-headers'].headers) do remove[h] = r end end routines['add-headers'] = function() - if skip_wanted('add-headers') then return end + if skip_wanted('add-headers') then + return + end for h, r in pairs(settings.routines['add-headers'].headers) do add[h] = r remove[h] = settings.routines['add-headers'].remove @@ -309,7 +329,9 @@ local function milter_headers(task) routines['x-rspamd-server'] = function() local local_mod = settings.routines['x-rspamd-server'] - if skip_wanted('x-rspamd-server') then return end + if skip_wanted('x-rspamd-server') then + return + end if local_mod.remove then remove[local_mod.header] = local_mod.remove end @@ -319,7 +341,9 @@ local function milter_headers(task) routines['x-spamd-bar'] = function() local local_mod = settings.routines['x-spamd-bar'] - if skip_wanted('x-rspamd-bar') then return end + if skip_wanted('x-rspamd-bar') then + return + end if not common['metric_score'] then common['metric_score'] = task:get_metric_score() end @@ -342,7 +366,9 @@ local function milter_headers(task) routines['x-spam-level'] = function() local local_mod = settings.routines['x-spam-level'] - if skip_wanted('x-spam-level') then return end + if skip_wanted('x-spam-level') then + return + end if not common['metric_score'] then common['metric_score'] = task:get_metric_score() end @@ -358,7 +384,9 @@ local function milter_headers(task) routines['x-rspamd-action'] = function() local local_mod = settings.routines['x-rspamd-action'] - if skip_wanted('x-rspamd-action') then return end + if skip_wanted('x-rspamd-action') then + return + end if not common['metric_action'] then common['metric_action'] = task:get_metric_action() end @@ -370,7 +398,9 @@ local function milter_headers(task) end local function spam_header (class, name, value, remove_v) - if skip_wanted(class) then return end + if skip_wanted(class) then + return + end if not common['metric_action'] then common['metric_action'] = task:get_metric_action() end @@ -396,7 +426,9 @@ local function milter_headers(task) routines['x-virus'] = function() local local_mod = settings.routines['x-virus'] - if skip_wanted('x-virus') then return end + if skip_wanted('x-virus') then + return + end if not common.symbols_hash then if not common.symbols then common.symbols = task:get_symbols_all() @@ -453,17 +485,20 @@ local function milter_headers(task) end routines['x-os-fingerprint'] = function() - if skip_wanted('x-os-fingerprint') then return end + if skip_wanted('x-os-fingerprint') then + return + end local local_mod = settings.routines['x-os-fingerprint'] - local os_string, link_type, uptime_min, distance = - task:get_mempool():get_variable('os_fingerprint', + local os_string, link_type, uptime_min, distance = task:get_mempool():get_variable('os_fingerprint', 'string, string, double, double'); - if not os_string then return end + if not os_string then + return + end local value = string.format('%s, (up: %i min), (distance %i, link: %s)', - os_string, uptime_min, distance, link_type) + os_string, uptime_min, distance, link_type) if local_mod.remove then remove[local_mod.header] = local_mod.remove @@ -473,7 +508,9 @@ local function milter_headers(task) end routines['x-spam-status'] = function() - if skip_wanted('x-spam-status') then return end + if skip_wanted('x-spam-status') then + return + end if not common['metric_score'] then common['metric_score'] = task:get_metric_score() end @@ -498,12 +535,13 @@ local function milter_headers(task) end routines['authentication-results'] = function() - if skip_wanted('authentication-results') then return end + if skip_wanted('authentication-results') then + return + end local ar = require "lua_auth_results" if settings.routines['authentication-results'].remove then - remove[settings.routines['authentication-results'].header] = - settings.routines['authentication-results'].remove + remove[settings.routines['authentication-results'].header] = settings.routines['authentication-results'].remove end local res = ar.gen_auth_results(task, @@ -516,10 +554,11 @@ local function milter_headers(task) end routines['stat-signature'] = function() - if skip_wanted('stat-signature') then return end + if skip_wanted('stat-signature') then + return + end if settings.routines['stat-signature'].remove then - remove[settings.routines['stat-signature'].header] = - settings.routines['stat-signature'].remove + remove[settings.routines['stat-signature'].header] = settings.routines['stat-signature'].remove end local res = task:get_mempool():get_variable("stat_signature") if res then @@ -531,7 +570,7 @@ local function milter_headers(task) local res = task:get_mempool():get_variable("fuzzy_hashes", "fstrings") if res and #res > 0 then - for _,h in ipairs(res) do + for _, h in ipairs(res) do add_header('fuzzy-hashes', h) end end @@ -583,8 +622,12 @@ local function milter_headers(task) end end - if not next(add) then add = nil end - if not next(remove) then remove = nil end + if not next(add) then + add = nil + end + if not next(remove) then + remove = nil + end if add or remove then lua_mime.modify_headers(task, { @@ -595,7 +638,9 @@ local function milter_headers(task) end local config_schema = ts.shape({ - use = ts.array_of(ts.string) + ts.string / function(s) return {s} end, + use = ts.array_of(ts.string) + ts.string / function(s) + return { s } + end, remove_upstream_spam_flag = ts.boolean:is_optional(), extended_spam_headers = ts.boolean:is_optional(), skip_local = ts.boolean:is_optional(), @@ -609,13 +654,15 @@ local config_schema = ts.shape({ }) local opts = rspamd_config:get_all_opt(N) or - rspamd_config:get_all_opt('rmilter_headers') + rspamd_config:get_all_opt('rmilter_headers') -if not opts then return end +if not opts then + return +end -- Process config do - local res,err = config_schema:transform(opts) + local res, err = config_schema:transform(opts) if not res then logger.errx(rspamd_config, 'invalid config for %s: %s', N, err) return diff --git a/src/plugins/lua/mime_types.lua b/src/plugins/lua/mime_types.lua index 428637d70..939251630 100644 --- a/src/plugins/lua/mime_types.lua +++ b/src/plugins/lua/mime_types.lua @@ -224,10 +224,10 @@ local function check_mime_type(task) local ext = {} for n = 1, 2 do - ext[n] = #parts > n and string.lower(parts[#parts + 1 - n]) or nil + ext[n] = #parts > n and string.lower(parts[#parts + 1 - n]) or nil end - return ext[1],ext[2],parts + return ext[1], ext[2], parts end local function check_filename(fname, ct, is_archive, part, detected_ext, nfiles) @@ -243,7 +243,9 @@ local function check_mime_type(task) -- Decode hex encoded characters fname = string.gsub(fname, '%%(%x%x)', - function (hex) return string.char(tonumber(hex,16)) end ) + function(hex) + return string.char(tonumber(hex, 16)) + end) -- Replace potentially bad characters with '?' fname = fname:gsub('[^%s%g]', '?') @@ -256,7 +258,7 @@ local function check_mime_type(task) return end - local ext,ext2,parts = gen_extension(fname) + local ext, ext2, parts = gen_extension(fname) -- ext is the last extension, LOWERCASED -- ext2 is the one before last extension LOWERCASED @@ -272,10 +274,14 @@ local function check_mime_type(task) false, part, nil, 1) end - if not ext then return end + if not ext then + return + end local function check_extension(badness_mult, badness_mult2) - if not badness_mult and not badness_mult2 then return end + if not badness_mult and not badness_mult2 then + return + end if #parts > 2 then -- We need to ensure that next-to-last extension is an extension, -- so we check for its length and if it is not a number or date @@ -317,7 +323,9 @@ local function check_mime_type(task) if user_settings.bad_extensions[1] then -- Convert to a key-value map extra_table = fun.tomap( - fun.map(function(e) return e,1.0 end, + fun.map(function(e) + return e, 1.0 + end, user_settings.bad_extensions)) else extra_table = user_settings.bad_extensions @@ -327,7 +335,9 @@ local function check_mime_type(task) if user_settings.bad_archive_extensions[1] then -- Convert to a key-value map extra_archive_table = fun.tomap(fun.map( - function(e) return e,1.0 end, + function(e) + return e, 1.0 + end, user_settings.bad_archive_extensions)) else extra_archive_table = user_settings.bad_archive_extensions @@ -386,7 +396,7 @@ local function check_mime_type(task) if mt and ct and ct ~= 'application/octet-stream' then local found local mult - for _,v in ipairs(mt) do + for _, v in ipairs(mt) do mult = v.mult if ct == v.ct then found = true @@ -404,8 +414,8 @@ local function check_mime_type(task) local parts = task:get_parts() if parts then - for _,p in ipairs(parts) do - local mtype,subtype = p:get_type() + for _, p in ipairs(parts) do + local mtype, subtype = p:get_type() if not mtype then lua_util.debugm(N, task, "no content type for part: %s", p:get_id()) @@ -476,7 +486,7 @@ local function check_mime_type(task) local nfiles = #fl - for _,f in ipairs(fl) do + for _, f in ipairs(fl) do if f['encrypted'] then task:insert_result(settings['symbol_encrypted_archive'], 1.0, f['name']) @@ -497,7 +507,7 @@ local function check_mime_type(task) if nfiles == 1 and fl[1].name then -- We check that extension of the file inside archive is -- the same as double extension of the file - local _,ext2 = gen_extension(filename) + local _, ext2 = gen_extension(filename) if ext2 and #ext2 > 0 then local enc_ext = gen_extension(fl[1].name) @@ -525,7 +535,9 @@ local function check_mime_type(task) if detected_type and detected_type.ct ~= ct then local v_detected = map:get_key(detected_type.ct) - if not v or v_detected and v_detected > v then v = v_detected end + if not v or v_detected and v_detected > v then + v = v_detected + end detected_different = true end if v then @@ -567,19 +579,19 @@ local function check_mime_type(task) end end -local opts = rspamd_config:get_all_opt('mime_types') +local opts = rspamd_config:get_all_opt('mime_types') if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end settings.filename_whitelist = lua_maps.rspamd_map_add('mime_types', 'filename_whitelist', 'regexp', - 'filename whitelist') + 'filename whitelist') local function change_extension_map_entry(ext, ct, mult) if type(ct) == 'table' then local tbl = {} - for _,elt in ipairs(ct) do + for _, elt in ipairs(ct) do table.insert(tbl, { ct = elt, mult = mult, @@ -595,22 +607,24 @@ if opts then end -- Transform extension_map - for ext,ct in pairs(settings.extension_map) do + for ext, ct in pairs(settings.extension_map) do change_extension_map_entry(ext, ct, 1.0) end -- Add all extensions - for _,pair in ipairs(lua_mime_types.full_extensions_map) do + for _, pair in ipairs(lua_mime_types.full_extensions_map) do local ext, ct = pair[1], pair[2] if not settings.extension_map[ext] then - change_extension_map_entry(ext, ct, settings.other_extensions_mult) + change_extension_map_entry(ext, ct, settings.other_extensions_mult) end end local map_type = 'map' - if settings['regexp'] then map_type = 'regexp' end + if settings['regexp'] then + map_type = 'regexp' + end map = lua_maps.rspamd_map_add('mime_types', 'file', map_type, - 'mime types map') + 'mime types map') if map then local id = rspamd_config:register_symbol({ name = 'MIME_TYPES_CALLBACK', diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 0e78a6e2c..b48f7bdff 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -43,27 +43,24 @@ local function parse_multimap_value(parse_rule, p_ret) local number = {} local digit = lpeg.R("09") - number.integer = - (lpeg.S("+-") ^ -1) * - (digit ^ 1) + number.integer = (lpeg.S("+-") ^ -1) * + (digit ^ 1) -- Matches: .6, .899, .9999873 - number.fractional = - (lpeg.P(".") ) * + number.fractional = (lpeg.P(".")) * (digit ^ 1) -- Matches: 55.97, -90.8, .9 - number.decimal = - (number.integer * -- Integer - (number.fractional ^ -1)) + -- Fractional + number.decimal = (number.integer * -- Integer + (number.fractional ^ -1)) + -- Fractional (lpeg.S("+-") * number.fractional) -- Completely fractional number local sym_start = lpeg.R("az", "AZ") + lpeg.S("_") local sym_elt = sym_start + lpeg.R("09") - local symbol = sym_start * sym_elt ^0 + local symbol = sym_start * sym_elt ^ 0 local symbol_cap = lpeg.Cg(symbol, 'symbol') local score_cap = lpeg.Cg(number.decimal, 'score') - local opts_cap = lpeg.Cg(lpeg.Ct(lpeg.C(symbol) * (lpeg.P(",") * lpeg.C(symbol))^0), 'opts') + local opts_cap = lpeg.Cg(lpeg.Ct(lpeg.C(symbol) * (lpeg.P(",") * lpeg.C(symbol)) ^ 0), 'opts') local symscore_cap = (symbol_cap * lpeg.P(":") * score_cap) local symscoreopt_cap = symscore_cap * lpeg.P(":") * opts_cap local grammar = symscoreopt_cap + symscore_cap + symbol_cap + score_cap @@ -86,74 +83,107 @@ local function parse_multimap_value(parse_rule, p_ret) opts = tbl.opts end - return true,sym,score,opts + return true, sym, score, opts else if p_ret ~= '' then rspamd_logger.infox(rspamd_config, '%s: cannot parse string "%s"', parse_rule.symbol, p_ret) end - return true,nil,1.0,{} + return true, nil, 1.0, {} end elseif type(p_ret) == 'boolean' then - return p_ret,nil,1.0,{} + return p_ret, nil, 1.0, {} end - return false,nil,0.0,{} + return false, nil, 0.0, {} end - local value_types = { ip = { - get_value = function(ip) return ip:to_string() end, + get_value = function(ip) + return ip:to_string() + end, }, from = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, helo = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, header = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, rcpt = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, user = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, url = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, dnsbl = { - get_value = function(ip) return ip:to_string() end, + get_value = function(ip) + return ip:to_string() + end, }, filename = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, content = { - get_value = function() return nil end, + get_value = function() + return nil + end, }, hostname = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, asn = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, country = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, received = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, mempool = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, selector = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, symbol_options = { - get_value = function(val) return val end, + get_value = function(val) + return val + end, }, } @@ -236,7 +266,7 @@ local function apply_url_filter(task, filter, url, r) return nil elseif string.find(filter, 'tld:regexp:') then if not r['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then r['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -254,7 +284,7 @@ local function apply_url_filter(task, filter, url, r) end elseif string.find(filter, 'full:regexp:') then if not r['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then r['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -272,7 +302,7 @@ local function apply_url_filter(task, filter, url, r) end elseif string.find(filter, 'regexp:') then if not r['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then r['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -305,27 +335,37 @@ local function apply_addr_filter(task, filter, input, rule) if filter == 'email:addr' or filter == 'email' then local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024) if addr and addr[1] then - return fun.totable(fun.map(function(a) return a.addr end, addr)) + return fun.totable(fun.map(function(a) + return a.addr + end, addr)) end elseif filter == 'email:user' then local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024) if addr and addr[1] then - return fun.totable(fun.map(function(a) return a.user end, addr)) + return fun.totable(fun.map(function(a) + return a.user + end, addr)) end elseif filter == 'email:domain' then local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024) if addr and addr[1] then - return fun.totable(fun.map(function(a) return a.domain end, addr)) + return fun.totable(fun.map(function(a) + return a.domain + end, addr)) end elseif filter == 'email:domain:tld' then local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024) if addr and addr[1] then - return fun.totable(fun.map(function(a) return rspamd_util.get_tld(a.domain) end, addr)) + return fun.totable(fun.map(function(a) + return rspamd_util.get_tld(a.domain) + end, addr)) end elseif filter == 'email:name' then local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024) if addr and addr[1] then - return fun.totable(fun.map(function(a) return a.name end, addr)) + return fun.totable(fun.map(function(a) + return a.name + end, addr)) end elseif filter == 'ip_addr' then local ip_addr = rspamd_ip.from_string(input) @@ -336,7 +376,7 @@ local function apply_addr_filter(task, filter, input, rule) else -- regexp case if not rule['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then rule['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -359,7 +399,7 @@ local function apply_filename_filter(task, filter, fn, r) return string.match(fn, '%.([^.]+)$') elseif string.find(filter, 'regexp:') then if not r['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then r['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -383,7 +423,7 @@ end local function apply_regexp_filter(task, filter, fn, r) if string.find(filter, 'regexp:') then if not r['re_filter'] then - local type,pat = string.match(filter, '(regexp:)(.+)') + local type, pat = string.match(filter, '(regexp:)(.+)') if type and pat then r['re_filter'] = rspamd_regexp.create_cached(pat) end @@ -406,26 +446,26 @@ end local function apply_content_filter(task, filter) if filter == 'body' then - return {task:get_rawbody()} + return { task:get_rawbody() } elseif filter == 'full' then - return {task:get_content()} + return { task:get_content() } elseif filter == 'headers' then - return {task:get_raw_headers()} + return { task:get_raw_headers() } elseif filter == 'text' then local ret = {} - for _,p in ipairs(task:get_text_parts()) do + for _, p in ipairs(task:get_text_parts()) do table.insert(ret, p:get_content()) end return ret elseif filter == 'rawtext' then local ret = {} - for _,p in ipairs(task:get_text_parts()) do + for _, p in ipairs(task:get_text_parts()) do table.insert(ret, p:get_raw_content()) end return ret elseif filter == 'oneline' then local ret = {} - for _,p in ipairs(task:get_text_parts()) do + for _, p in ipairs(task:get_text_parts()) do table.insert(ret, p:get_content_oneline()) end return ret @@ -456,7 +496,7 @@ local function multimap_query_redis(key, task, value, callback) cmd = 'HMGET' end - local srch = {key} + local srch = { key } -- Insert all ips for some mask :( if type(value) == 'userdata' and value.class == 'rspamd{ip}' then @@ -468,7 +508,7 @@ local function multimap_query_redis(key, task, value, callback) maxbits = 32 minbits = 8 end - for i=maxbits,minbits,-1 do + for i = maxbits, minbits, -1 do local nip = value:apply_mask(i):tostring() .. "/" .. i srch[#srch + 1] = nip end @@ -508,7 +548,7 @@ local function multimap_callback(task, rule) if ret then if type(err_or_data) == 'table' then - for _,elt in ipairs(err_or_data) do + for _, elt in ipairs(err_or_data) do callback(elt) end else @@ -516,7 +556,7 @@ local function multimap_callback(task, rule) end elseif err_code ~= 404 then rspamd_logger.infox(task, "map %s: get key returned error %s: %s", - rule.symbol, err_code, err_or_data) + rule.symbol, err_code, err_or_data) end end @@ -535,7 +575,7 @@ local function multimap_callback(task, rule) -- Here we need to spill this function into multiple queries if type(results) == 'table' then - for _,res in ipairs(results) do + for _, res in ipairs(results) do ret = multimap_query_redis(res, task, value, callback) if not ret then @@ -554,7 +594,7 @@ local function multimap_callback(task, rule) end local function insert_results(result, opt) - local _,symbol,score,opts = parse_multimap_value(rule, result) + local _, symbol, score, opts = parse_multimap_value(rule, result) local forced = false if symbol then if rule.symbols_set then @@ -569,9 +609,9 @@ local function multimap_callback(task, rule) if type(opt) == 'table' then table.insert(opt, result) elseif type(opt) ~= nil then - opt = {opt,result} + opt = { opt, result } else - opt = {result} + opt = { result } end else forced = not rule.dynamic_symbols @@ -614,7 +654,7 @@ local function multimap_callback(task, rule) local function rule_callback(result) if result then if type(result) == 'table' then - for _,rs in ipairs(result) do + for _, rs in ipairs(result) do if type(rs) ~= 'userdata' then rule_callback(rs) end @@ -639,7 +679,9 @@ local function multimap_callback(task, rule) end if type(value) == 'table' then - fun.each(function(elt) match_element(r, elt, rule_callback) end, value) + fun.each(function(elt) + match_element(r, elt, rule_callback) + end, value) else match_element(r, value, rule_callback) end @@ -659,17 +701,19 @@ local function multimap_callback(task, rule) end end, ls) else - fun.each(function(e) match_rule(r, e) end, ls) + fun.each(function(e) + match_rule(r, e) + end, ls) end end end local function match_addr(r, addr) - match_list(r, addr, {'addr'}) + match_list(r, addr, { 'addr' }) if not r.filter then - match_list(r, addr, {'domain'}) - match_list(r, addr, {'user'}) + match_list(r, addr, { 'domain' }) + match_list(r, addr, { 'user' }) end end @@ -703,7 +747,7 @@ local function multimap_callback(task, rule) return end else - if pos <= (total - (min_pos*-1)) then + if pos <= (total - (min_pos * -1)) then return end end @@ -713,7 +757,7 @@ local function multimap_callback(task, rule) end if max_pos then if max_pos < -1 then - if (total - (max_pos*-1)) >= pos then + if (total - (max_pos * -1)) >= pos then return end elseif max_pos > 0 then @@ -728,12 +772,16 @@ local function multimap_callback(task, rule) local got_flags = h['flags'] if match_flags then for _, flag in ipairs(match_flags) do - if not got_flags[flag] then return end + if not got_flags[flag] then + return + end end end if nmatch_flags then for _, flag in ipairs(nmatch_flags) do - if got_flags[flag] then return end + if got_flags[flag] then + return + end end end end @@ -759,16 +807,16 @@ local function multimap_callback(task, rule) if r['filter'] then data = apply_content_filter(task, r['filter'], r) else - data = {task:get_content()} + data = { task:get_content() } end - for _,v in ipairs(data) do + for _, v in ipairs(data) do match_rule(r, v) end end if rule.expression and not rule.combined then - local res,trace = rule['expression']:process_traced(task) + local res, trace = rule['expression']:process_traced(task) if not res or res == 0 then lua_util.debugm(N, task, 'condition is false for %s', @@ -811,7 +859,7 @@ local function multimap_callback(task, rule) end task:get_resolver():resolve_a({ - task= task, + task = task, name = to_resolve, callback = dns_cb, forced = true @@ -820,13 +868,13 @@ local function multimap_callback(task, rule) end, header = function() if type(rule['header']) == 'table' then - for _,rh in ipairs(rule['header']) do + for _, rh in ipairs(rule['header']) do local hv = task:get_header_full(rh) - match_list(rule, hv, {'decoded'}) + match_list(rule, hv, { 'decoded' }) end else local hv = task:get_header_full(rule['header']) - match_list(rule, hv, {'decoded'}) + match_list(rule, hv, { 'decoded' }) end end, rcpt = function() @@ -857,7 +905,7 @@ local function multimap_callback(task, rule) if task:has_urls() then local msg_urls = task:get_urls() - for _,url in ipairs(msg_urls) do + for _, url in ipairs(msg_urls) do match_url(rule, url) end end @@ -891,11 +939,11 @@ local function multimap_callback(task, rule) return p:is_archive() and det_type == 'archive' and not rule.skip_archives end - for _,p in fun.iter(fun.filter(filter_parts, parts)) do + for _, p in fun.iter(fun.filter(filter_parts, parts)) do if filter_archive(p) then local fnames = p:get_archive():get_files(1000) - for _,fn in ipairs(fnames) do + for _, fn in ipairs(fnames) do match_filename(rule, fn) end end @@ -971,7 +1019,7 @@ local function multimap_callback(task, rule) if elts then if type(elts) == 'table' then - for _,elt in ipairs(elts) do + for _, elt in ipairs(elts) do match_rule(rule, elt) end else @@ -980,9 +1028,9 @@ local function multimap_callback(task, rule) end end, combined = function() - local ret,trace = rule.combined:process(task) + local ret, trace = rule.combined:process(task) if ret and ret ~= 0 then - for n,t in pairs(trace) do + for n, t in pairs(trace) do insert_results(t.value, string.format("%s=%s", n, t.matched)) end @@ -999,7 +1047,6 @@ local function multimap_callback(task, rule) end end - local function gen_multimap_callback(rule) return function(task) multimap_callback(task, rule) @@ -1011,12 +1058,12 @@ local function multimap_on_load_gen(rule) lua_util.debugm(N, rspamd_config, "loaded map object for rule %s", rule['symbol']) local known_symbols = {} rule.map_obj:foreach(function(key, value) - local r,symbol,score,_ = parse_multimap_value(rule, value) + local r, symbol, score, _ = parse_multimap_value(rule, value) if r and symbol and not known_symbols[symbol] then lua_util.debugm(N, rspamd_config, "%s: adding new symbol %s (score = %s), triggered by %s", rule.symbol, symbol, score, key) - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = value, parent = rule.callback_id, type = 'virtual', @@ -1084,7 +1131,7 @@ local function add_multimap_rule(key, newrule) if newrule['url'] and not newrule['map'] then newrule['map'] = newrule['url'] end - if not (newrule.map or newrule.rules) then + if not (newrule.map or newrule.rules) then rspamd_logger.errx(rspamd_config, 'incomplete rule, missing map') return nil end @@ -1179,12 +1226,12 @@ local function add_multimap_rule(key, newrule) if type(newrule['flags']) == 'table' and newrule['flags'][1] then newrule['flags'] = newrule['flags'] elseif type(newrule['flags']) == 'string' then - newrule['flags'] = {newrule['flags']} + newrule['flags'] = { newrule['flags'] } end if type(newrule['nflags']) == 'table' and newrule['nflags'][1] then newrule['nflags'] = newrule['nflags'] elseif type(newrule['nflags']) == 'string' then - newrule['nflags'] = {newrule['nflags']} + newrule['nflags'] = { newrule['nflags'] } end local filter = newrule['filter'] or 'real_ip' if filter == 'real_ip' or filter == 'from_ip' then @@ -1259,7 +1306,7 @@ local function add_multimap_rule(key, newrule) end local expression = rspamd_expression.create(newrule['require_symbols'], - {parse_atom, process_atom}, rspamd_config:get_mempool()) + { parse_atom, process_atom }, rspamd_config:get_mempool()) if expression then newrule['expression'] = expression @@ -1280,11 +1327,11 @@ end local opts = rspamd_config:get_all_opt(N) if opts and type(opts) == 'table' then redis_params = rspamd_parse_redis_server(N) - for k,m in pairs(opts) do + for k, m in pairs(opts) do if type(m) == 'table' and m['type'] then local rule = add_multimap_rule(k, m) if not rule then - rspamd_logger.errx(rspamd_config, 'cannot add rule: "'..k..'"') + rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"') else rspamd_logger.infox(rspamd_config, 'added multimap rule: %s (%s)', k, rule.type) diff --git a/src/plugins/lua/mx_check.lua b/src/plugins/lua/mx_check.lua index 03d7f6087..71892b94b 100644 --- a/src/plugins/lua/mx_check.lua +++ b/src/plugins/lua/mx_check.lua @@ -80,7 +80,9 @@ local function mx_check(task) local valid = false local function check_results(mxes) - if fun.all(function(_, elt) return elt.checked end, mxes) then + if fun.all(function(_, elt) + return elt.checked + end, mxes) then -- Save cache local key = settings.key_prefix .. mx_domain local function redis_cache_cb(err) @@ -99,12 +101,12 @@ local function mx_check(task) task:insert_result(settings.symbol_bad_mx, 1.0) end local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_cache_cb, --callback - 'SETEX', -- command - {key, tostring(settings.expire_novalid), '0'} -- arguments + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_cache_cb, --callback + 'SETEX', -- command + { key, tostring(settings.expire_novalid), '0' } -- arguments ) lua_util.debugm(N, task, "set redis cache key: %s; invalid MX", key) if not ret then @@ -114,19 +116,21 @@ local function mx_check(task) local valid_mx = {} fun.each(function(k) table.insert(valid_mx, k) - end, fun.filter(function (_, elt) return elt.working end, mxes)) + end, fun.filter(function(_, elt) + return elt.working + end, mxes)) task:insert_result(settings.symbol_good_mx, 1.0, valid_mx) - local value = table.concat(valid_mx, ';') + local value = table.concat(valid_mx, ';') if mxes[mx_domain] and type(mxes[mx_domain]) == 'table' and mxes[mx_domain].mx_missing then value = mx_miss_cache_prefix .. value end local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_cache_cb, --callback - 'SETEX', -- command - {key, tostring(settings.expire), value} -- arguments + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_cache_cb, --callback + 'SETEX', -- command + { key, tostring(settings.expire), value } -- arguments ) lua_util.debugm(N, task, "set redis cache key: %s; %s", key, value) if not ret then @@ -230,11 +234,11 @@ local function mx_check(task) end) local max_mx_to_resolve = math.min(#results, settings.max_mx_a_records) - lua_util.debugm(N, task,'check %s MX records (%d actually returned)', + lua_util.debugm(N, task, 'check %s MX records (%d actually returned)', max_mx_to_resolve, #results) - for i=1,max_mx_to_resolve do + for i = 1, max_mx_to_resolve do local mx = results[i] - mxes[mx.name] = {checked = false, working = false, ips = {}} + mxes[mx.name] = { checked = false, working = false, ips = {} } local r = task:get_resolver() -- XXX: maybe add ipv6? r:resolve('a', { @@ -282,12 +286,12 @@ local function mx_check(task) local key = settings.key_prefix .. mx_domain local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_cache_get_cb, --callback - 'GET', -- command - {key} -- arguments + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_cache_get_cb, --callback + 'GET', -- command + { key } -- arguments ) if not ret then @@ -327,7 +331,7 @@ if opts then type = 'normal', callback = mx_check, flags = 'empty', - augmentations = {string.format("timeout=%f", settings.timeout + rspamd_config:get_dns_timeout() or 0.0)}, + augmentations = { string.format("timeout=%f", settings.timeout + rspamd_config:get_dns_timeout() or 0.0) }, }) rspamd_config:register_symbol({ name = settings.symbol_no_mx, @@ -379,7 +383,7 @@ if opts then }) if settings.exclude_domains then - exclude_domains = rspamd_config:add_map{ + exclude_domains = rspamd_config:add_map { type = 'set', description = 'Exclude specific domains from MX checks', url = settings.exclude_domains, diff --git a/src/plugins/lua/neural.lua b/src/plugins/lua/neural.lua index 33361b7df..f3b26f11a 100644 --- a/src/plugins/lua/neural.lua +++ b/src/plugins/lua/neural.lua @@ -35,7 +35,7 @@ local N = "neural" local settings = neural_common.settings -local redis_profile_schema = ts.shape{ +local redis_profile_schema = ts.shape { digest = ts.string, symbols = ts.array_of(ts.string), version = ts.number, @@ -77,7 +77,7 @@ local function new_ann_profile(task, rule, set, version) true, -- is write add_cb, --callback 'ZADD', -- command - {set.prefix, tostring(rspamd_util.get_time()), profile_serialized} + { set.prefix, tostring(rspamd_util.get_time()), profile_serialized } ) return profile @@ -87,7 +87,7 @@ end -- ANN filter function, used to insert scores based on the existing symbols local function ann_scores_filter(task) - for _,rule in pairs(settings.rules) do + for _, rule in pairs(settings.rules) do local sid = task:get_settings_id() or -1 local ann local profile @@ -228,14 +228,17 @@ local function ann_push_task_result(rule, task, verdict, score, set) end end - if learn_spam or learn_ham then local learn_type - if learn_spam then learn_type = 'spam' else learn_type = 'ham' end + if learn_spam then + learn_type = 'spam' + else + learn_type = 'ham' + end local function vectors_len_cb(err, data) if not err and type(data) == 'table' then - local nspam,nham = data[1],data[2] + local nspam, nham = data[1], data[2] if neural_common.can_push_train_vector(rule, task, learn_type, nspam, nham) then local vec = neural_common.result_to_vector(task, set) @@ -296,7 +299,7 @@ local function ann_push_task_result(rule, task, verdict, score, set) end lua_redis.exec_redis_script(neural_common.redis_script_id.vectors_len, - {task = task, is_write = false}, + { task = task, is_write = false }, vectors_len_cb, { set.ann.redis_key, @@ -318,7 +321,7 @@ end -- Utility to extract and split saved training vectors to a table of tables local function process_training_vectors(data) return fun.totable(fun.map(function(tok) - local _,str = rspamd_util.zstd_decompress(tok) + local _, str = rspamd_util.zstd_decompress(tok) return fun.totable(fun.map(tonumber, lua_util.str_split(tostring(str), ';'))) end, data)) end @@ -334,23 +337,23 @@ local function do_train_ann(worker, ev_base, rule, set, ann_key) local function redis_ham_cb(err, data) if err or type(data) ~= 'table' then rspamd_logger.errx(rspamd_config, 'cannot get ham tokens for ANN %s from redis: %s', - ann_key, err) + ann_key, err) -- Unlock on error lua_redis.redis_make_request_taskless(ev_base, - rspamd_config, - rule.redis, - nil, - true, -- is write + rspamd_config, + rule.redis, + nil, + true, -- is write neural_common.gen_unlock_cb(rule, set, ann_key), --callback - 'HDEL', -- command - {ann_key, 'lock'} + 'HDEL', -- command + { ann_key, 'lock' } ) else -- Decompress and convert to numbers each training vector ham_elts = process_training_vectors(data) - neural_common.spawn_train({worker = worker, ev_base = ev_base, - rule = rule, set = set, ann_key = ann_key, ham_vec = ham_elts, - spam_vec = spam_elts}) + neural_common.spawn_train({ worker = worker, ev_base = ev_base, + rule = rule, set = set, ann_key = ann_key, ham_vec = ham_elts, + spam_vec = spam_elts }) end end @@ -358,29 +361,29 @@ local function do_train_ann(worker, ev_base, rule, set, ann_key) local function redis_spam_cb(err, data) if err or type(data) ~= 'table' then rspamd_logger.errx(rspamd_config, 'cannot get spam tokens for ANN %s from redis: %s', - ann_key, err) + ann_key, err) -- Unlock ANN on error lua_redis.redis_make_request_taskless(ev_base, - rspamd_config, - rule.redis, - nil, - true, -- is write + rspamd_config, + rule.redis, + nil, + true, -- is write neural_common.gen_unlock_cb(rule, set, ann_key), --callback - 'HDEL', -- command - {ann_key, 'lock'} + 'HDEL', -- command + { ann_key, 'lock' } ) else -- Decompress and convert to numbers each training vector spam_elts = process_training_vectors(data) -- Now get ham vectors... lua_redis.redis_make_request_taskless(ev_base, - rspamd_config, - rule.redis, - nil, - false, -- is write - redis_ham_cb, --callback - 'SMEMBERS', -- command - {ann_key .. '_ham_set'} + rspamd_config, + rule.redis, + nil, + false, -- is write + redis_ham_cb, --callback + 'SMEMBERS', -- command + { ann_key .. '_ham_set' } ) end end @@ -388,21 +391,21 @@ local function do_train_ann(worker, ev_base, rule, set, ann_key) local function redis_lock_cb(err, data) if err then rspamd_logger.errx(rspamd_config, 'cannot call lock script for ANN %s from redis: %s', - ann_key, err) + ann_key, err) elseif type(data) == 'number' and data == 1 then -- ANN is locked, so we can extract SPAM and HAM vectors and spawn learning lua_redis.redis_make_request_taskless(ev_base, - rspamd_config, - rule.redis, - nil, - false, -- is write - redis_spam_cb, --callback - 'SMEMBERS', -- command - {ann_key .. '_spam_set'} + rspamd_config, + rule.redis, + nil, + false, -- is write + redis_spam_cb, --callback + 'SMEMBERS', -- command + { ann_key .. '_spam_set' } ) rspamd_logger.infox(rspamd_config, 'lock ANN %s:%s (key name %s) for learning', - rule.prefix, set.name, ann_key) + rule.prefix, set.name, ann_key) else local lock_tm = tonumber(data[1]) rspamd_logger.infox(rspamd_config, 'do not learn ANN %s:%s (key name %s), ' .. @@ -422,14 +425,14 @@ local function do_train_ann(worker, ev_base, rule, set, ann_key) -- This script returns either a boolean or a pair {'lock_time', 'hostname'} when -- ANN is locked by another host (or a process, meh) lua_redis.exec_redis_script(neural_common.redis_script_id.maybe_lock, - {ev_base = ev_base, is_write = true}, - redis_lock_cb, + { ev_base = ev_base, is_write = true }, + redis_lock_cb, { ann_key, tostring(os.time()), tostring(math.max(10.0, rule.watch_interval * 2)), rspamd_util.get_hostname() - }) + }) end -- This function loads new ann from Redis @@ -448,7 +451,7 @@ local function load_new_ann(rule, ev_base, set, profile, min_diff) else if type(data) == 'table' then if type(data[1]) == 'userdata' and data[1].cookie == text_cookie then - local _err,ann_data = rspamd_util.zstd_decompress(data[1]) + local _err, ann_data = rspamd_util.zstd_decompress(data[1]) local ann if _err or not ann_data then @@ -482,7 +485,7 @@ local function load_new_ann(rule, ev_base, set, profile, min_diff) true, -- is write rank_cb, --callback 'ZADD', -- command - {set.prefix, tostring(rspamd_util.get_time()), profile_serialized} + { set.prefix, tostring(rspamd_util.get_time()), profile_serialized } ) rspamd_logger.infox(rspamd_config, 'loaded ANN for %s:%s from %s; %s bytes compressed; version=%s', @@ -507,15 +510,15 @@ local function load_new_ann(rule, ev_base, set, profile, min_diff) local roc_thresholds = parser:get_object() set.ann.roc_thresholds = roc_thresholds rspamd_logger.infox(rspamd_config, - 'loaded ROC thresholds for %s:%s; version=%s', - rule.prefix, set.name, profile.version) + 'loaded ROC thresholds for %s:%s; version=%s', + rule.prefix, set.name, profile.version) rspamd_logger.debugx("ROC thresholds: %s", roc_thresholds) end end if set.ann and set.ann.ann and type(data[3]) == 'userdata' and data[3].cookie == text_cookie then -- PCA table - local _err,pca_data = rspamd_util.zstd_decompress(data[3]) + local _err, pca_data = rspamd_util.zstd_decompress(data[3]) if pca_data then if rule.max_inputs then -- We can use PCA @@ -555,8 +558,8 @@ local function load_new_ann(rule, ev_base, set, profile, min_diff) false, -- is write data_cb, --callback 'HMGET', -- command - {ann_key, 'ann', 'roc_thresholds', 'pca'}, -- arguments - {opaque_data = true} + { ann_key, 'ann', 'roc_thresholds', 'pca' }, -- arguments + { opaque_data = true } ) end @@ -570,7 +573,7 @@ local function process_existing_ann(_, ev_base, rule, set, profiles) local min_diff = math.huge local sel_elt - for _,elt in fun.iter(profiles) do + for _, elt in fun.iter(profiles) do if elt and elt.symbols then local dist = lua_util.distance_sorted(elt.symbols, my_symbols) -- Check distance @@ -641,7 +644,7 @@ local function maybe_train_existing_ann(worker, ev_base, rule, set, profiles) ham = 0, } - for _,elt in fun.iter(profiles) do + for _, elt in fun.iter(profiles) do if elt and elt.symbols then local dist = lua_util.distance_sorted(elt.symbols, my_symbols) -- Check distance @@ -734,7 +737,7 @@ local function maybe_train_existing_ann(worker, ev_base, rule, set, profiles) false, -- is write redis_len_cb_gen(initiate_train, 'ham', true), --callback 'SCARD', -- command - {ann_key .. '_ham_set'} + { ann_key .. '_ham_set' } ) end @@ -745,7 +748,7 @@ local function maybe_train_existing_ann(worker, ev_base, rule, set, profiles) false, -- is write redis_len_cb_gen(check_ham_len, 'spam', false), --callback 'SCARD', -- command - {ann_key .. '_spam_set'} + { ann_key .. '_spam_set' } ) end end @@ -755,14 +758,14 @@ local function load_ann_profile(element) local ucl = require "ucl" local parser = ucl.parser() - local res,ucl_err = parser:parse_string(element) + local res, ucl_err = parser:parse_string(element) if not res then rspamd_logger.warnx(rspamd_config, 'cannot parse ANN from redis: %s', ucl_err) return nil else local profile = parser:get_object() - local checked,schema_err = redis_profile_schema:transform(profile) + local checked, schema_err = redis_profile_schema:transform(profile) if not checked then rspamd_logger.errx(rspamd_config, "cannot parse profile schema: %s", schema_err) @@ -774,7 +777,7 @@ end -- Function to check or load ANNs from Redis local function check_anns(worker, cfg, ev_base, rule, process_callback, what) - for _,set in pairs(rule.settings) do + for _, set in pairs(rule.settings) do local function members_cb(err, data) if err then rspamd_logger.errx(cfg, 'cannot get ANNs list from redis: %s', @@ -800,7 +803,7 @@ local function check_anns(worker, cfg, ev_base, rule, process_callback, what) false, -- is write members_cb, --callback 'ZREVRANGE', -- command - {set.prefix, '0', tostring(settings.max_profiles)} -- arguments + { set.prefix, '0', tostring(settings.max_profiles) } -- arguments ) end end -- Cycle over all settings @@ -810,13 +813,13 @@ end -- Function to clean up old ANNs local function cleanup_anns(rule, cfg, ev_base) - for _,set in pairs(rule.settings) do + for _, set in pairs(rule.settings) do local function invalidate_cb(err, data) if err then rspamd_logger.errx(cfg, 'cannot exec invalidate script in redis: %s', err) elseif type(data) == 'table' then - for _,expired in ipairs(data) do + for _, expired in ipairs(data) do local profile = load_ann_profile(expired) rspamd_logger.infox(cfg, 'invalidated ANN for %s; redis key: %s; version=%s', rule.prefix .. ':' .. set.name, @@ -828,9 +831,9 @@ local function cleanup_anns(rule, cfg, ev_base) if type(set) == 'table' then lua_redis.exec_redis_script(neural_common.redis_script_id.maybe_invalidate, - {ev_base = ev_base, is_write = true}, + { ev_base = ev_base, is_write = true }, invalidate_cb, - {set.prefix, tostring(settings.max_profiles)}) + { set.prefix, tostring(settings.max_profiles) }) end end end @@ -845,7 +848,7 @@ local function ann_push_vector(task) return end - local verdict,score = lua_verdict.get_specific_verdict(N, task) + local verdict, score = lua_verdict.get_specific_verdict(N, task) if verdict == 'passthrough' then lua_util.debugm(N, task, 'ignore task as its verdict is %s(%s)', @@ -861,7 +864,7 @@ local function ann_push_vector(task) return end - for _,rule in pairs(settings.rules) do + for _, rule in pairs(settings.rules) do local set = neural_common.get_rule_settings(task, rule) if set then @@ -906,7 +909,7 @@ if settings.blacklisted_symbols and settings.blacklisted_symbols[1] then end -- Check all rules -for k,r in pairs(rules) do +for k, r in pairs(rules) do local rule_elt = lua_util.override_defaults(neural_common.default_options, r) rule_elt['redis'] = neural_common.redis_params rule_elt['anns'] = {} -- Store ANNs here @@ -921,7 +924,9 @@ for k,r in pairs(rules) do rule_elt.train.max_trains = rule_elt.train.max_train end - if not rule_elt.profile then rule_elt.profile = {} end + if not rule_elt.profile then + rule_elt.profile = {} + end if rule_elt.max_inputs and not has_blas then rspamd_logger.errx('cannot set max inputs to %s as BLAS is not compiled in', @@ -969,7 +974,7 @@ rspamd_config:register_symbol({ rspamd_config:add_post_init(neural_common.process_rules_settings) -- Add training scripts -for _,rule in pairs(settings.rules) do +for _, rule in pairs(settings.rules) do neural_common.load_scripts(rule.redis) -- This function will check ANNs in Redis when a worker is loaded rspamd_config:add_on_load(function(cfg, ev_base, worker) diff --git a/src/plugins/lua/once_received.lua b/src/plugins/lua/once_received.lua index 04e2168c1..2a5552ab9 100644 --- a/src/plugins/lua/once_received.lua +++ b/src/plugins/lua/once_received.lua @@ -71,11 +71,11 @@ local function check_quantity_received (task) else rspamd_logger.infox(task, 'source hostname has not been passed to Rspamd from MTA, ' .. 'but we could resolve source IP address PTR %s as "%s"', - to_resolve, results[1]) + to_resolve, results[1]) task:set_hostname(results[1]) if good_hosts then - for _,gh in ipairs(good_hosts) do + for _, gh in ipairs(good_hosts) do if string.find(results[1], gh) then return end @@ -84,7 +84,7 @@ local function check_quantity_received (task) if nreceived <= 1 then task:insert_result(symbol, 1) - for _,h in ipairs(bad_hosts) do + for _, h in ipairs(bad_hosts) do if string.find(results[1], h) then task:insert_result(symbol_strict, 1, h) @@ -104,17 +104,17 @@ local function check_quantity_received (task) end if whitelist and task_ip and whitelist:get_key(task_ip) then rspamd_logger.infox(task, 'whitelisted mail from %s', - task_ip:to_string()) + task_ip:to_string()) return end local hn = task:get_hostname() -- Here we don't care about received if (not hn) and task_ip and task_ip:is_valid() then - task:get_resolver():resolve_ptr({task = task, - name = task_ip:to_string(), - callback = recv_dns_cb, - forced = true + task:get_resolver():resolve_ptr({ task = task, + name = task_ip:to_string(), + callback = recv_dns_cb, + forced = true }) return end @@ -131,7 +131,7 @@ local function check_quantity_received (task) local rhn = string.lower(r['real_hostname']) -- Check for good hostname if rhn and good_hosts then - for _,gh in ipairs(good_hosts) do + for _, gh in ipairs(good_hosts) do if string.find(rhn, gh) then ret = false break @@ -146,8 +146,10 @@ local function check_quantity_received (task) -- Unresolved host task:insert_result(symbol, 1) - if not hn then return end - for _,h in ipairs(bad_hosts) do + if not hn then + return + end + for _, h in ipairs(bad_hosts) do if string.find(hn, h) then task:insert_result(symbol_strict, 1, h) return @@ -176,7 +178,7 @@ if opts then callback = check_quantity_received, }) - for n,v in pairs(opts) do + for n, v in pairs(opts) do if n == 'symbol_strict' then symbol_strict = v elseif n == 'symbol_rdns' then @@ -198,7 +200,7 @@ if opts then elseif n == 'whitelist' then local lua_maps = require "lua_maps" whitelist = lua_maps.map_add('once_received', 'whitelist', 'radix', - 'once received whitelist') + 'once received whitelist') elseif n == 'symbol_mx' then symbol_mx = v end @@ -214,7 +216,7 @@ if opts then type = 'virtual', parent = id }) - rspamd_config:register_symbol({ + rspamd_config:register_symbol({ name = symbol_strict, type = 'virtual', parent = id diff --git a/src/plugins/lua/p0f.lua b/src/plugins/lua/p0f.lua index d597ee25d..97757c23a 100644 --- a/src/plugins/lua/p0f.lua +++ b/src/plugins/lua/p0f.lua @@ -26,36 +26,36 @@ local N = 'p0f' if confighelp then rspamd_config:add_example(nil, N, - 'Detect remote OS via passive fingerprinting', - [[ -p0f { - # Enable module - enabled = true - - # Path to the unix socket that p0f listens on - socket = '/var/run/p0f.sock'; - - # Connection timeout - timeout = 5s; - - # If defined, insert symbol with lookup results - symbol = 'P0F'; - - # Patterns to match against results returned by p0f - # Symbol will be yielded on OS string, link type or distance matches - patterns = { - WINDOWS = '^Windows.*'; - #DSL = '^DSL$'; - #DISTANCE10 = '^distance:10$'; + 'Detect remote OS via passive fingerprinting', + [[ + p0f { + # Enable module + enabled = true + + # Path to the unix socket that p0f listens on + socket = '/var/run/p0f.sock'; + + # Connection timeout + timeout = 5s; + + # If defined, insert symbol with lookup results + symbol = 'P0F'; + + # Patterns to match against results returned by p0f + # Symbol will be yielded on OS string, link type or distance matches + patterns = { + WINDOWS = '^Windows.*'; + #DSL = '^DSL$'; + #DISTANCE10 = '^distance:10$'; + } + + # Cache lifetime in seconds (default - 2 hours) + expire = 7200; + + # Cache key prefix + prefix = 'p0f'; } - - # Cache lifetime in seconds (default - 2 hours) - expire = 7200; - - # Cache key prefix - prefix = 'p0f'; -} -]]) + ]]) return end @@ -90,7 +90,7 @@ if rule then priority = lua_util.symbols_priorities.medium, flags = 'empty,nostat', group = N, - augmentations = {string.format("timeout=%f", rule.timeout or 0.0)}, + augmentations = { string.format("timeout=%f", rule.timeout or 0.0) }, }) diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua index 628d6efa7..9dd03aa9d 100644 --- a/src/plugins/lua/phishing.lua +++ b/src/plugins/lua/phishing.lua @@ -48,7 +48,6 @@ local openphish_hash local generic_service_data = {} local openphish_data = {} - local opts = rspamd_config:get_all_opt(N) if not (opts and type(opts) == 'table') then rspamd_logger.infox(rspamd_config, 'Module is unconfigured') @@ -70,7 +69,7 @@ local function phishing_cb(task) local query = url:get_query() if path then - for _,d in ipairs(elt) do + for _, d in ipairs(elt) do if d['path'] == path then found_path = true data = d['data'] @@ -83,7 +82,7 @@ local function phishing_cb(task) end end else - for _,d in ipairs(elt) do + for _, d in ipairs(elt) do if not d['path'] then found_path = true end @@ -135,7 +134,9 @@ local function phishing_cb(task) local function compose_dns_query(elts) local cr = require "rspamd_cryptobox_hash" local h = cr.create() - for _,elt in ipairs(elts) do h:update(elt) end + for _, elt in ipairs(elts) do + h:update(elt) + end return string.format("%s.%s", h:base32():sub(1, 32), dns_suffix) end local r = task:get_resolver() @@ -154,14 +155,13 @@ local function phishing_cb(task) end end - local to_resolve_hp = compose_dns_query({host, path}) + local to_resolve_hp = compose_dns_query({ host, path }) rspamd_logger.debugm(N, task, 'try to resolve {%s, %s} -> %s', host, path, to_resolve_hp) r:resolve_txt({ task = task, name = to_resolve_hp, - callback = host_host_path_cb}) - + callback = host_host_path_cb }) if query then local function host_host_path_query_cb(_, _, results, err) @@ -170,13 +170,13 @@ local function phishing_cb(task) end end - local to_resolve_hpq = compose_dns_query({host, path, query}) + local to_resolve_hpq = compose_dns_query({ host, path, query }) rspamd_logger.debugm(N, task, 'try to resolve {%s, %s, %s} -> %s', host, path, query, to_resolve_hpq) r:resolve_txt({ task = task, name = to_resolve_hpq, - callback = host_host_path_query_cb}) + callback = host_host_path_query_cb }) end end @@ -193,8 +193,9 @@ local function phishing_cb(task) end local urls = task:get_urls() or {} - for _,url_iter in ipairs(urls) do - local function do_loop_iter() -- to emulate continue + for _, url_iter in ipairs(urls) do + local function do_loop_iter() + -- to emulate continue local url = url_iter if generic_service_hash then check_phishing_map(generic_service_data, url, generic_service_symbol) @@ -225,7 +226,6 @@ local function phishing_cb(task) purl = url:get_phished() end - if not purl then return end @@ -239,15 +239,15 @@ local function phishing_cb(task) if dmarc_dom and tld == dmarc_dom then lua_util.debugm(N, 'exclude phishing from %s -> %s by dmarc domain', tld, - ptld) + ptld) return end -- Now we can safely remove the last dot component if it is the same - local b,_ = string.find(tld, '%.[^%.]+$') - local b1,_ = string.find(ptld, '%.[^%.]+$') + local b, _ = string.find(tld, '%.[^%.]+$') + local b1, _ = string.find(ptld, '%.[^%.]+$') - local stripped_tld,stripped_ptld = tld, ptld + local stripped_tld, stripped_ptld = tld, ptld if b1 and b then if string.sub(tld, b) == string.sub(ptld, b1) then stripped_ptld = string.gsub(ptld, '%.[^%.]+$', '') @@ -260,7 +260,7 @@ local function phishing_cb(task) end local weight = 1.0 - local spoofed,why = util.is_utf_spoofed(tld, ptld) + local spoofed, why = util.is_utf_spoofed(tld, ptld) if spoofed then lua_util.debugm(N, task, "confusable: %1 -> %2: %3", tld, ptld, why) weight = 1.0 @@ -273,15 +273,19 @@ local function phishing_cb(task) weight = util.tanh(3 * (1 - dist + 0.1)) elseif dist > 1 then -- We also check if two labels are in the same ascii/non-ascii representation - local a1, a2 = false,false + local a1, a2 = false, false - if string.match(tld, '^[\001-\127]*$') then a1 = true end - if string.match(ptld, '^[\001-\127]*$') then a2 = true end + if string.match(tld, '^[\001-\127]*$') then + a1 = true + end + if string.match(ptld, '^[\001-\127]*$') then + a2 = true + end if a1 ~= a2 then weight = 1 lua_util.debugm(N, task, "confusable: %1 -> %2: different characters", - tld, ptld, why) + tld, ptld, why) else -- We have totally different strings in tld, so penalize it somehow weight = 0.5 @@ -292,20 +296,24 @@ local function phishing_cb(task) end local function is_url_in_map(map, furl) - for _,dn in ipairs({furl:get_tld(), furl:get_host()}) do + for _, dn in ipairs({ furl:get_tld(), furl:get_host() }) do if map:get_key(dn) then - return true,dn + return true, dn end end return false end local function found_in_map(map, furl, sweight) - if not furl then furl = url end - if not sweight then sweight = weight end + if not furl then + furl = url + end + if not sweight then + sweight = weight + end if #map > 0 then - for _,rule in ipairs(map) do - local found,dn = is_url_in_map(rule.map, furl) + for _, rule in ipairs(map) do + local found, dn = is_url_in_map(rule.map, furl) if found then task:insert_result(rule.symbol, sweight, string.format("%s->%s:%s", ptld, tld, dn)) return true @@ -342,13 +350,12 @@ local function phishing_map(mapname, phishmap, id) rspamd_logger.errx(rspamd_config, 'invalid exception table') end - - for sym,map_data in pairs(xd) do - local rmap = lua_maps.map_add_from_ucl (map_data, 'set', - 'Phishing ' .. mapname .. ' map') + for sym, map_data in pairs(xd) do + local rmap = lua_maps.map_add_from_ucl(map_data, 'set', + 'Phishing ' .. mapname .. ' map') if rmap then rspamd_config:register_virtual_symbol(sym, 1, id) - local rule = {symbol = sym, map = rmap} + local rule = { symbol = sym, map = rmap } table.insert(phishmap, rule) else rspamd_logger.infox(rspamd_config, 'cannot add map for symbol: %s', sym) @@ -360,8 +367,8 @@ end local function rspamd_str_split_fun(s, sep, func) local lpeg = require "lpeg" sep = lpeg.P(sep) - local elem = lpeg.P((1 - sep)^0 / func) - local p = lpeg.P(elem * (sep * elem)^0) + local elem = lpeg.P((1 - sep) ^ 0 / func) + local p = lpeg.P(elem * (sep * elem) ^ 0) return p:match(s) end @@ -382,7 +389,7 @@ local function insert_url_from_string(pool, tbl, str, data) if tbl[host] then table.insert(tbl[host], elt) else - tbl[host] = {elt} + tbl[host] = { elt } end return true @@ -408,7 +415,7 @@ local function generic_service_plain_cb(string) generic_service_data = new_data rspamd_logger.infox(generic_service_hash, "parsed %s elements from %s feed", - nelts, generic_service_name) + nelts, generic_service_name) pool:destroy() end @@ -424,7 +431,7 @@ local function openphish_json_cb(string) local function openphish_elt_parser(cap) if valid then local parser = ucl.parser() - local res,err = parser:parse_string(cap) + local res, err = parser:parse_string(cap) if not res then valid = false rspamd_logger.warnx(openphish_hash, 'cannot parse openphish map: ' .. err) @@ -445,7 +452,7 @@ local function openphish_json_cb(string) if valid then openphish_data = new_json_map rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed", - nelts) + nelts) end pool:destroy() @@ -467,7 +474,7 @@ local function openphish_plain_cb(s) openphish_data = new_data rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed", - nelts) + nelts) pool:destroy() end @@ -499,11 +506,11 @@ if opts then if opts['generic_service_enabled'] then generic_service_hash = rspamd_config:add_map({ - type = 'callback', - url = generic_service_map, - callback = generic_service_plain_cb, - description = 'Generic feed' - }) + type = 'callback', + url = generic_service_map, + callback = generic_service_plain_cb, + description = 'Generic feed' + }) end if opts['openphish_map'] then @@ -528,12 +535,12 @@ if opts then }) else openphish_hash = rspamd_config:add_map({ - type = 'callback', - url = openphish_map, - callback = openphish_json_cb, - opaque_data = true, - description = 'Open phishing premium feed map (see https://www.openphish.com for details)' - }) + type = 'callback', + url = openphish_map, + callback = openphish_json_cb, + opaque_data = true, + description = 'Open phishing premium feed map (see https://www.openphish.com for details)' + }) end end @@ -564,7 +571,7 @@ if opts then end if opts['domains'] and type(opts['domains']) == 'string' then domains = lua_maps.map_add_from_ucl(opts['domains'], 'set', - 'Phishing domains') + 'Phishing domains') end phishing_map('phishing_exceptions', phishing_exceptions_maps, id) phishing_map('exceptions', anchor_exceptions_maps, id) diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua index b225d0650..9f8292d6b 100644 --- a/src/plugins/lua/ratelimit.lua +++ b/src/plugins/lua/ratelimit.lua @@ -39,7 +39,7 @@ local redis_params -- Senders that are considered as bounce local settings = { bounce_senders = { 'postmaster', 'mailer-daemon', '', 'null', 'fetchmail-daemon', 'mdaemon' }, --- Do not check ratelimits for these recipients + -- Do not check ratelimits for these recipients whitelisted_rcpts = { 'postmaster', 'mailer-daemon' }, prefix = 'RL', ham_factor_rate = 1.01, @@ -54,11 +54,9 @@ local settings = { prefilter = true, } - local bucket_check_script = "ratelimit_check.lua" local bucket_check_id - local bucket_update_script = "ratelimit_update.lua" local bucket_update_id @@ -70,7 +68,6 @@ local message_func = function(_, limit_type, _, _, _) return string.format('Ratelimit "%s" exceeded', limit_type) end - local function load_scripts(_, _) bucket_check_id = lua_redis.load_redis_script_from_file(bucket_check_script, redis_params) bucket_update_id = lua_redis.load_redis_script_from_file(bucket_update_script, redis_params) @@ -106,27 +103,28 @@ local function parse_string_limit(lim, no_error) if not limit_parser then local digit = lpeg.R("09") limit_parser = {} - limit_parser.integer = - (lpeg.S("+-") ^ -1) * - (digit ^ 1) - limit_parser.fractional = - (lpeg.P(".") ) * - (digit ^ 1) - limit_parser.number = - (limit_parser.integer * - (limit_parser.fractional ^ -1)) + - (lpeg.S("+-") * limit_parser.fractional) + limit_parser.integer = (lpeg.S("+-") ^ -1) * + (digit ^ 1) + limit_parser.fractional = (lpeg.P(".")) * + (digit ^ 1) + limit_parser.number = (limit_parser.integer * + (limit_parser.fractional ^ -1)) + + (lpeg.S("+-") * limit_parser.fractional) limit_parser.time = lpeg.Cf(lpeg.Cc(1) * - (limit_parser.number / tonumber) * - ((lpeg.S("smhd") / parse_time_suffix) ^ -1), - function (acc, val) return acc * val end) + (limit_parser.number / tonumber) * + ((lpeg.S("smhd") / parse_time_suffix) ^ -1), + function(acc, val) + return acc * val + end) limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) * - (limit_parser.number / tonumber) * - ((lpeg.S("kmg") / parse_num_suffix) ^ -1), - function (acc, val) return acc * val end) + (limit_parser.number / tonumber) * + ((lpeg.S("kmg") / parse_num_suffix) ^ -1), + function(acc, val) + return acc * val + end) limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number * - (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) * - limit_parser.time) + (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) * + limit_parser.time) end local t = lpeg.match(limit_parser.limit, lim) @@ -142,7 +140,7 @@ local function parse_string_limit(lim, no_error) end local function str_to_rate(str) - local divider,divisor = parse_string_limit(str, false) + local divider, divisor = parse_string_limit(str, false) if not divisor then rspamd_logger.errx(rspamd_config, 'bad rate string: %s', str) @@ -153,7 +151,7 @@ local function str_to_rate(str) return divisor / divider end -local bucket_schema = ts.shape{ +local bucket_schema = ts.shape { burst = ts.number + ts.string / lua_util.dehumanize_number, rate = ts.number + ts.string / str_to_rate, skip_recipients = ts.boolean:is_optional(), @@ -184,7 +182,7 @@ local function parse_limit(name, data) return nil else - local parsed_bucket,err = bucket_schema:transform(data) + local parsed_bucket, err = bucket_schema:transform(data) if not parsed_bucket or err then rspamd_logger.errx(rspamd_config, 'cannot parse bucket for %s: %s; original value: %s', @@ -210,21 +208,27 @@ end --- Check whether this addr is bounce local function check_bounce(from) - return fun.any(function(b) return b == from end, settings.bounce_senders) + return fun.any(function(b) + return b == from + end, settings.bounce_senders) end local keywords = { ['ip'] = { ['get_value'] = function(task) local ip = task:get_ip() - if ip and ip:is_valid() then return tostring(ip) end + if ip and ip:is_valid() then + return tostring(ip) + end return nil end, }, ['rip'] = { ['get_value'] = function(task) local ip = task:get_ip() - if ip and ip:is_valid() and not ip:is_local() then return tostring(ip) end + if ip and ip:is_valid() and not ip:is_local() then + return tostring(ip) + end return nil end, }, @@ -243,7 +247,11 @@ local keywords = { if not ((from or E)[1] or E).user then return '_' end - if check_bounce(from[1]['user']) then return '_' else return nil end + if check_bounce(from[1]['user']) then + return '_' + else + return nil + end end, }, ['asn'] = { @@ -281,7 +289,7 @@ local keywords = { local parts = task:get_parts() or E local digests = {} - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do if p:get_filename() then table.insert(digests, p:get_digest()) end @@ -299,7 +307,7 @@ local keywords = { local parts = task:get_parts() or E local files = {} - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do local fname = p:get_filename() if fname then table.insert(files, fname) @@ -316,7 +324,7 @@ local keywords = { } local function gen_rate_key(task, rtype, bucket) - local key_t = {tostring(lua_util.round(100000.0 / bucket.burst))} + local key_t = { tostring(lua_util.round(100000.0 / bucket.burst)) } local key_keywords = lua_util.str_split(rtype, '_') local have_user = false @@ -326,9 +334,15 @@ local function gen_rate_key(task, rtype, bucket) if keywords[v] and type(keywords[v]['get_value']) == 'function' then ret = keywords[v]['get_value'](task) end - if not ret then return nil end - if v == 'user' then have_user = true end - if type(ret) ~= 'string' then ret = tostring(ret) end + if not ret then + return nil + end + if v == 'user' then + have_user = true + end + if type(ret) ~= 'string' then + ret = tostring(ret) + end table.insert(key_t, ret) end @@ -341,7 +355,9 @@ end local function make_prefix(redis_key, name, bucket) local hash_len = 24 - if hash_len > #redis_key then hash_len = #redis_key end + if hash_len > #redis_key then + hash_len = #redis_key + end local hash = settings.prefix .. string.sub(rspamd_hash.create(redis_key):base32(), 1, hash_len) -- Fill defaults @@ -367,7 +383,7 @@ end local function limit_to_prefixes(task, k, v, prefixes) local n = 0 - for _,bucket in ipairs(v.buckets) do + for _, bucket in ipairs(v.buckets) do if v.selector then local selectors = lua_selectors.process_selectors(task, v.selector) if selectors then @@ -403,7 +419,9 @@ end local function ratelimit_cb(task) if not settings.allow_local and - rspamd_lua_utils.is_rspamc_or_controller(task) then return end + rspamd_lua_utils.is_rspamc_or_controller(task) then + return + end -- Get initial task data local ip = task:get_from_ip() @@ -419,10 +437,14 @@ local function ratelimit_cb(task) local rcpts_user = {} if rcpts then fun.each(function(r) - fun.each(function(type) table.insert(rcpts_user, r[type]) end, {'user', 'addr'}) + fun.each(function(type) + table.insert(rcpts_user, r[type]) + end, { 'user', 'addr' }) end, rcpts) - if fun.any(function(r) return settings.whitelisted_rcpts:get_key(r) end, rcpts_user) then + if fun.any(function(r) + return settings.whitelisted_rcpts:get_key(r) + end, rcpts_user) then rspamd_logger.infox(task, 'skip ratelimit for whitelisted recipient') return end @@ -439,7 +461,7 @@ local function ratelimit_cb(task) local prefixes = {} local nprefixes = 0 - for k,v in pairs(settings.limits) do + for k, v in pairs(settings.limits) do nprefixes = nprefixes + limit_to_prefixes(task, k, v, prefixes) end @@ -504,9 +526,11 @@ local function ratelimit_cb(task) end -- Don't do anything if pre-result has been already set - if task:has_pre_result() then return end + if task:has_pre_result() then + return + end - local _,nrcpt = task:has_recipients('smtp') + local _, nrcpt = task:has_recipients('smtp') if not nrcpt or nrcpt <= 0 then nrcpt = 1 end @@ -518,19 +542,21 @@ local function ratelimit_cb(task) now = lua_util.round(now * 1000.0) -- Get milliseconds -- Now call check script for all defined prefixes - for pr,value in pairs(prefixes) do + for pr, value in pairs(prefixes) do local bucket = value.bucket local rate = (bucket.rate) / 1000.0 -- Leak rate in messages/ms local bincr = nrcpt - if bucket.skip_recipients then bincr = 1 end + if bucket.skip_recipients then + bincr = 1 + end lua_util.debugm(N, task, "check limit %s:%s -> %s (%s/%s)", value.name, pr, value.hash, bucket.burst, bucket.rate) lua_redis.exec_redis_script(bucket_check_id, - {key = value.hash, task = task, is_write = true}, + { key = value.hash, task = task, is_write = true }, gen_check_cb(pr, bucket, value.name, value.hash), - {value.hash, tostring(now), tostring(rate), tostring(bucket.burst), - tostring(settings.expire), tostring(bincr)}) + { value.hash, tostring(now), tostring(rate), tostring(bucket.burst), + tostring(settings.expire), tostring(bincr) }) end end end @@ -553,18 +579,20 @@ local function maybe_cleanup_pending(task) lua_util.debugm(N, task, 'cleaned pending bucked for %s: %s', k, data) end end - local _,nrcpt = task:has_recipients('smtp') + local _, nrcpt = task:has_recipients('smtp') if not nrcpt or nrcpt <= 0 then nrcpt = 1 end local bincr = nrcpt - if bucket.skip_recipients then bincr = 1 end + if bucket.skip_recipients then + bincr = 1 + end local now = task:get_timeval(true) now = lua_util.round(now * 1000.0) -- Get milliseconds lua_redis.exec_redis_script(bucket_cleanup_id, - {key = v.hash, task = task, is_write = true}, + { key = v.hash, task = task, is_write = true }, cleanup_cb, - {v.hash, tostring(now), tostring(settings.expire), tostring(bincr)}) + { v.hash, tostring(now), tostring(settings.expire), tostring(bincr) }) end end end @@ -590,7 +618,7 @@ local function ratelimit_update_cb(task) end local verdict = lua_verdict.get_specific_verdict(N, task) - local _,nrcpt = task:has_recipients('smtp') + local _, nrcpt = task:has_recipients('smtp') if not nrcpt or nrcpt <= 0 then nrcpt = 1 end @@ -624,14 +652,16 @@ local function ratelimit_update_cb(task) end local bincr = nrcpt - if bucket.skip_recipients then bincr = 1 end + if bucket.skip_recipients then + bincr = 1 + end lua_redis.exec_redis_script(bucket_update_id, - {key = v.hash, task = task, is_write = true}, + { key = v.hash, task = task, is_write = true }, update_bucket_cb, - {v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst), - tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult), - tostring(settings.expire), tostring(bincr)}) + { v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst), + tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult), + tostring(settings.expire), tostring(bincr) }) end end end @@ -653,7 +683,7 @@ if opts then if type(lim) == 'table' and lim.bucket then if lim.bucket[1] then - for _,bucket in ipairs(lim.bucket) do + for _, bucket in ipairs(lim.bucket) do local b = parse_limit(t, bucket) if not b then @@ -673,7 +703,7 @@ if opts then return end - buckets = {bucket} + buckets = { bucket } end settings.limits[t] = { @@ -696,7 +726,7 @@ if opts then buckets = parse_limit(t, lim) if buckets then settings.limits[t] = { - buckets = {buckets} + buckets = { buckets } } end end @@ -706,7 +736,7 @@ if opts then -- Display what's enabled fun.each(function(s) rspamd_logger.infox(rspamd_config, 'enabled ratelimit: %s', s) - end, fun.map(function(n,d) + end, fun.map(function(n, d) return string.format('%s [%s]', n, table.concat(fun.totable(fun.map(function(v) return string.format('symbol: %s, %s msgs burst, %s msgs/sec rate', @@ -723,14 +753,14 @@ if opts then if type(wrcpts) == 'string' then if string.find(wrcpts, ',') then settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl( - lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts') + lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts') else settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set', - 'Ratelimit whitelisted rcpts') + 'Ratelimit whitelisted rcpts') end elseif type(opts['whitelisted_rcpts']) == 'table' then settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set', - 'Ratelimit whitelisted rcpts') + 'Ratelimit whitelisted rcpts') else -- Stupid default... settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl( @@ -739,12 +769,12 @@ if opts then if opts['whitelisted_ip'] then settings.whitelisted_ip = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_ip', 'radix', - 'Ratelimit whitelist ip map') + 'Ratelimit whitelist ip map') end if opts['whitelisted_user'] then settings.whitelisted_user = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_user', 'set', - 'Ratelimit whitelist user map') + 'Ratelimit whitelist user map') end settings.custom_keywords = {} @@ -754,7 +784,7 @@ if opts then if ret then opts['custom_keywords'] = {} if type(res_or_err) == 'table' then - for k,hdl in pairs(res_or_err) do + for k, hdl in pairs(res_or_err) do settings['custom_keywords'][k] = hdl end elseif type(res_or_err) == 'function' then @@ -783,7 +813,7 @@ if opts then priority = lua_util.symbols_priorities.medium, callback = ratelimit_cb, flags = 'empty,nostat', - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}, + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }, } local id = rspamd_config:register_symbol(s) @@ -792,9 +822,9 @@ if opts then -- Display what's enabled fun.each(function(set, lim) if type(lim.buckets) == 'table' then - for _,b in ipairs(lim.buckets) do + for _, b in ipairs(lim.buckets) do if b.symbol then - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', name = b.symbol, score = 0.0, @@ -806,7 +836,7 @@ if opts then end, settings.limits) if settings.info_symbol then - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', name = settings.info_symbol, score = 0.0, @@ -814,7 +844,7 @@ if opts then } end if settings.symbol then - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', name = settings.symbol, score = 0.0, -- Might be overridden if needed @@ -827,7 +857,7 @@ if opts then name = 'RATELIMIT_UPDATE', flags = 'explicit_disable,ignore_passthrough', callback = ratelimit_update_cb, - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}, + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }, } end end diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua index 9d1097d94..b0ea8c63e 100644 --- a/src/plugins/lua/rbl.lua +++ b/src/plugins/lua/rbl.lua @@ -142,7 +142,9 @@ local function gen_check_rcvd_conditions(rbl, received_total) local nmatch_flags = rbl.received_nflags local function basic_received_check(rh) - if not (rh.real_ip and rh.real_ip:is_valid()) then return false end + if not (rh.real_ip and rh.real_ip:is_valid()) then + return false + end if ((rh.real_ip:get_version() == 6 and rbl.ipv6) or (rh.real_ip:get_version() == 4 and rbl.ipv4)) and ((rbl.exclude_private_ips and not rh.real_ip:is_local()) or @@ -155,7 +157,9 @@ local function gen_check_rcvd_conditions(rbl, received_total) end local function positioned_received_check(rh, pos) - if not rh or not basic_received_check(rh) then return false end + if not rh or not basic_received_check(rh) then + return false + end local got_flags = rh.flags or E if min_pos then if min_pos < 0 then @@ -200,7 +204,6 @@ local function gen_check_rcvd_conditions(rbl, received_total) return true end - if not (max_pos or min_pos or match_flags or nmatch_flags) then return basic_received_check else @@ -267,15 +270,15 @@ local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_tabl return end - for _,result in ipairs(results) do + for _, result in ipairs(results) do local ipstr = result:to_string() lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr) local foundrc = false -- Check return codes if rbl.returnbits then local ipnum = result:to_number() - for s,bits in pairs(rbl.returnbits) do - for _,check_bit in ipairs(bits) do + for s, bits in pairs(rbl.returnbits) do + for _, check_bit in ipairs(bits) do if bit.band(ipnum, check_bit) == check_bit then foundrc = true insert_results(s) @@ -285,7 +288,7 @@ local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_tabl end elseif rbl.returncodes then for s, codes in pairs(rbl.returncodes) do - for _,v in ipairs(codes) do + for _, v in ipairs(codes) do if string.find(ipstr, '^' .. v .. '$') then foundrc = true insert_results(s) @@ -370,7 +373,7 @@ local function gen_rbl_callback(rule) nreq.what[label] = true end - return true,nreq -- Duplicate + return true, nreq -- Duplicate else local nreq @@ -384,7 +387,7 @@ local function gen_rbl_callback(rule) n = processed, orig = req_str, resolve_ip = resolve_ip, - what = {[label] = true}, + what = { [label] = true }, } requests_table[req] = nreq end @@ -407,7 +410,7 @@ local function gen_rbl_callback(rule) n = to_resolve, orig = req_str, resolve_ip = resolve_ip, - what = {[label] = true}, + what = { [label] = true }, } requests_table[req] = nreq end @@ -431,7 +434,9 @@ local function gen_rbl_callback(rule) local function check_required_symbols(task, _) if rule.require_symbols then - return fun.all(function(sym) task:has_symbol(sym) end, rule.require_symbols) + return fun.all(function(sym) + task:has_symbol(sym) + end, rule.require_symbols) end return true @@ -484,7 +489,7 @@ local function gen_rbl_callback(rule) mime_from_domain = ((task:get_from('mime') or E)[1] or E).domain if mime_from_domain then local mime_from_domain_tld = rule.url_full_hostname and - mime_from_domain or rspamd_util.get_tld(mime_from_domain) + mime_from_domain or rspamd_util.get_tld(mime_from_domain) if rule.url_compose_map then mime_from_domain = rule.url_compose_map:process_url(task, mime_from_domain_tld, mime_from_domain) @@ -496,7 +501,7 @@ local function gen_rbl_callback(rule) for _, d in ipairs(das[1].options) do - local domain,result = d:match('^([^%:]*):([%+%-%~])$') + local domain, result = d:match('^([^%:]*):([%+%-%~])$') -- We must ignore bad signatures, omg if domain and result and result == '+' then @@ -516,7 +521,7 @@ local function gen_rbl_callback(rule) if mime_from_domain and mime_from_domain == domain_tld then add_dns_request(task, domain_tld, true, false, requests_table, - 'dkim', whitelist) + 'dkim', whitelist) end else if rule.dkim_domainonly then @@ -571,7 +576,7 @@ local function gen_rbl_callback(rule) local urls = lua_util.extract_specific_urls(ex_params) - for _,u in ipairs(urls) do + for _, u in ipairs(urls) do local flags = u:get_flags_num() if bit.band(flags, url_flag_bits.numeric) ~= 0 then @@ -616,14 +621,14 @@ local function gen_rbl_callback(rule) end local function check_received(task, requests_table, whitelist) - local received = fun.filter(function(h) + local received = fun .filter(function(h) return not h['flags']['artificial'] end, task:get_received_headers()):totable() local received_total = #received local check_conditions = gen_check_rcvd_conditions(rule, received_total) - for pos,rh in ipairs(received) do + for pos, rh in ipairs(received) do if check_conditions(rh, pos) then add_dns_request(task, rh.real_ip, false, true, requests_table, 'received', @@ -651,13 +656,13 @@ local function gen_rbl_callback(rule) local res = selector(task) if res and type(res) == 'table' then - for _,r in ipairs(res) do - add_dns_request(task, r, false, false, requests_table, - selector_label, whitelist) - end + for _, r in ipairs(res) do + add_dns_request(task, r, false, false, requests_table, + selector_label, whitelist) + end elseif res then add_dns_request(task, res, false, false, - requests_table, selector_label, whitelist) + requests_table, selector_label, whitelist) end end @@ -705,7 +710,9 @@ local function gen_rbl_callback(rule) local ex_params = { task = task, limit = rule.requests_limit, - filter = function(u) return u:get_protocol() == 'mailto' end, + filter = function(u) + return u:get_protocol() == 'mailto' + end, need_emails = true, prefix = 'rbl_email' } @@ -719,7 +726,7 @@ local function gen_rbl_callback(rule) local emails = lua_util.extract_specific_urls(ex_params) - for _,email in ipairs(emails) do + for _, email in ipairs(emails) do local domain if rule.emails_domainonly and not rule.url_full_hostname then if rule.url_compose_map then @@ -835,7 +842,7 @@ local function gen_rbl_callback(rule) end -- Execute functions pipeline - for i,f in ipairs(pipeline) do + for i, f in ipairs(pipeline) do if not f(task, dns_req, whitelist) then lua_util.debugm(N, task, "skip rbl check: %s; pipeline condition %s returned false", @@ -854,12 +861,12 @@ local function gen_rbl_callback(rule) local function gen_rbl_ip_dns_callback(orig_resolve_table_elt) return function(_, _, results, err) if not err then - for _,dns_res in ipairs(results) do + for _, dns_res in ipairs(results) do -- Check if we have rspamd{ip} userdata if type(dns_res) == 'userdata' then -- Add result as an actual RBL request local label = next(orig_resolve_table_elt.what) - local dup,nreq = add_dns_request(task, dns_res, false, true, + local dup, nreq = add_dns_request(task, dns_res, false, true, resolved_req, label) -- Add original name if not dup then @@ -932,7 +939,7 @@ local function gen_rbl_callback(rule) end end - return callback_f,string.format('checks: %s', table.concat(description, ',')) + return callback_f, string.format('checks: %s', table.concat(description, ',')) end local function add_rbl(key, rbl, global_opts) @@ -940,7 +947,7 @@ local function add_rbl(key, rbl, global_opts) rbl.symbol = key:upper() end - local flags_tbl = {'no_squeeze'} + local flags_tbl = { 'no_squeeze' } if rbl.is_whitelist then flags_tbl[#flags_tbl + 1] = 'nice' end @@ -955,7 +962,7 @@ local function add_rbl(key, rbl, global_opts) rbl.selectors = {} if type(rbl.selector) ~= 'table' then - rbl.selector = {['selector'] = rbl.selector} + rbl.selector = { ['selector'] = rbl.selector } end for selector_label, selector in pairs(rbl.selector) do @@ -970,7 +977,7 @@ local function add_rbl(key, rbl, global_opts) rbl.selector_flatten = true end local sel = selectors.create_selector_closure(rspamd_config, selector, '', - rbl.selector_flatten) + rbl.selector_flatten) if not sel then rspamd_logger.errx('invalid selector for rbl rule %s: %s', key, selector) @@ -1032,23 +1039,23 @@ local function add_rbl(key, rbl, global_opts) rbl.symbol) end - local callback,description = gen_rbl_callback(rbl) + local callback, description = gen_rbl_callback(rbl) if callback then local id if rbl.symbols_prefixes then - id = rspamd_config:register_symbol{ + id = rspamd_config:register_symbol { type = 'callback', callback = callback, - groups = {'rbl'}, + groups = { 'rbl' }, name = rbl.symbol .. '_CHECK', flags = table.concat(flags_tbl, ',') } - for _,prefix in pairs(rbl.symbols_prefixes) do + for _, prefix in pairs(rbl.symbols_prefixes) do -- For unknown results... - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', parent = id, group = 'rbl', @@ -1064,11 +1071,11 @@ local function add_rbl(key, rbl, global_opts) rbl.symbol, rbl.is_whitelist, rbl.ignore_whitelist) end else - id = rspamd_config:register_symbol{ + id = rspamd_config:register_symbol { type = 'callback', callback = callback, name = rbl.symbol, - groups = {'rbl'}, + groups = { 'rbl' }, group = 'rbl', score = 0, flags = table.concat(flags_tbl, ',') @@ -1082,7 +1089,6 @@ local function add_rbl(key, rbl, global_opts) end end - rspamd_logger.infox(rspamd_config, 'added rbl rule %s: %s', rbl.symbol, description) lua_util.debugm(N, rspamd_config, 'rule dump for %s: %s', @@ -1095,13 +1101,13 @@ local function add_rbl(key, rbl, global_opts) end if rbl.require_symbols then - for _,dep in ipairs(rbl.require_symbols) do + for _, dep in ipairs(rbl.require_symbols) do rspamd_config:register_dependency(check_sym, dep) end end -- Failure symbol - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', flags = 'nostat', name = rbl.symbol .. '_FAIL', @@ -1114,7 +1120,7 @@ local function add_rbl(key, rbl, global_opts) if s ~= rbl.symbol then -- hack - rspamd_config:register_symbol{ + rspamd_config:register_symbol { type = 'virtual', parent = id, name = s, @@ -1145,7 +1151,7 @@ local function add_rbl(key, rbl, global_opts) end if rbl.symbols_prefixes then - for _,prefix in pairs(rbl.symbols_prefixes) do + for _, prefix in pairs(rbl.symbols_prefixes) do process_specific_suffix(prefix .. '_' .. suffix) end else @@ -1155,13 +1161,13 @@ local function add_rbl(key, rbl, global_opts) end if rbl.returncodes then - for s,_ in pairs(rbl.returncodes) do + for s, _ in pairs(rbl.returncodes) do process_return_code(s) end end if rbl.returnbits then - for s,_ in pairs(rbl.returnbits) do + for s, _ in pairs(rbl.returnbits) do process_return_code(s) end end @@ -1200,16 +1206,16 @@ if opts.rules and opts.rbls then opts.rbls = lua_util.override_defaults(opts.rbls, opts.rules) end -if(opts['local_exclude_ip_map'] ~= nil) then +if (opts['local_exclude_ip_map'] ~= nil) then local_exclusions = lua_maps.map_add(N, 'local_exclude_ip_map', 'radix', - 'RBL exclusions map') + 'RBL exclusions map') end -- TODO: this code should be universal for all modules that use selectors to allow -- maps usage from selectors registered for a specific module if type(opts.attached_maps) == 'table' then opts.attached_maps_processed = {} - for i,map in ipairs(opts.attached_maps) do + for i, map in ipairs(opts.attached_maps) do -- Store maps in the configuration table to keep lifetime track opts.attached_maps_processed[i] = lua_maps.map_add_from_ucl(map) if opts.attached_maps_processed[i] == nil then @@ -1218,7 +1224,7 @@ if type(opts.attached_maps) == 'table' then end end -for key,rbl in pairs(opts.rbls) do +for key, rbl in pairs(opts.rbls) do if type(rbl) ~= 'table' or rbl.disabled == true or rbl.enabled == false then rspamd_logger.infox(rspamd_config, 'disable rbl "%s"', key) else @@ -1231,7 +1237,7 @@ for key,rbl in pairs(opts.rbls) do end -- Propagate default options from opts to rule if not rbl.ignore_defaults then - for default_opt_key,_ in pairs(rbl_common.default_options) do + for default_opt_key, _ in pairs(rbl_common.default_options) do local rbl_opt = default_opt_key:sub(#('default_') + 1) if rbl[rbl_opt] == nil then rbl[rbl_opt] = opts[default_opt_key] @@ -1243,15 +1249,19 @@ for key,rbl in pairs(opts.rbls) do rbl.requests_limit = rspamd_config:get_dns_max_requests() end - local res,err = rbl_common.rule_schema:transform(rbl) + local res, err = rbl_common.rule_schema:transform(rbl) if not res then rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED', key, err) else res = rbl_common.convert_checks(res) -- Aliases - if res.return_codes then res.returncodes = res.return_codes end - if res.return_bits then res.returnbits = res.return_bits end + if res.return_codes then + res.returncodes = res.return_codes + end + if res.return_bits then + res.returnbits = res.return_bits + end if not res then rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED', @@ -1272,10 +1282,12 @@ local function rbl_callback_white(task) local ws = task:get_symbol(w) if ws and ws[1] then ws = ws[1] - if not ws.options then ws.options = {} end - for _,opt in ipairs(ws.options) do - local elt,what = opt:match('^([^:]+):([^:]+)') - lua_util.debugm(N, task,'found whitelist from %s: %s(%s)', w, + if not ws.options then + ws.options = {} + end + for _, opt in ipairs(ws.options) do + local elt, what = opt:match('^([^:]+):([^:]+)') + lua_util.debugm(N, task, 'found whitelist from %s: %s(%s)', w, elt, what) if elt and what then whitelisted_elements[elt] = { @@ -1297,22 +1309,22 @@ local function rbl_callback_fin(task) lua_util.debugm(N, task, "finished rbl processing") end -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'callback', callback = rbl_callback_white, name = 'RBL_CALLBACK_WHITE', flags = 'nice,empty,no_squeeze', - groups = {'rbl'}, - augmentations = {string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0)}, + groups = { 'rbl' }, + augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0) }, } -rspamd_config:register_symbol{ +rspamd_config:register_symbol { type = 'callback', callback = rbl_callback_fin, name = 'RBL_CALLBACK', flags = 'empty,no_squeeze', - groups = {'rbl'}, - augmentations = {string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0)}, + groups = { 'rbl' }, + augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0) }, } for _, w in ipairs(white_symbols) do diff --git a/src/plugins/lua/replies.lua b/src/plugins/lua/replies.lua index 8debe3271..c4df9c97e 100644 --- a/src/plugins/lua/replies.lua +++ b/src/plugins/lua/replies.lua @@ -77,7 +77,9 @@ local function replies_check(task) return real_rcpt_h == stored_rcpt end - if fun.any(filter_predicate, fun.map(function(rcpt) return rcpt.addr or '' end, rcpts)) then + if fun.any(filter_predicate, fun.map(function(rcpt) + return rcpt.addr or '' + end, rcpts)) then lua_util.debugm(N, task, 'reply to %s validated', in_reply_to) return true end @@ -121,12 +123,12 @@ local function replies_check(task) local key = make_key(in_reply_to, settings.key_size, settings.key_prefix) local ret = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_get_cb, --callback - 'GET', -- command - {key} -- arguments + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_get_cb, --callback + 'GET', -- command + { key } -- arguments ) if not ret then @@ -136,7 +138,7 @@ end local function replies_set(task) local function redis_set_cb(err, _, addr) - if err ~=nil then + if err ~= nil then rspamd_logger.errx(task, 'redis_set_cb error when writing data to %s: %s', addr:get_addr(), err) end end @@ -162,14 +164,14 @@ local function replies_set(task) if sender then local sender_hash = make_key(sender:lower(), 8) lua_util.debugm(N, task, 'storing id: %s (%s), reply-to: %s (%s) for replies check', - msg_id, key, sender, sender_hash) + msg_id, key, sender, sender_hash) local ret = lua_redis.redis_make_request(task, redis_params, -- connect params key, -- hash key true, -- is write redis_set_cb, --callback 'PSETEX', -- command - {key, tostring(math.floor(settings['expire'] * 1000)), sender_hash} -- arguments + { key, tostring(math.floor(settings['expire'] * 1000)), sender_hash } -- arguments ) if not ret then rspamd_logger.errx(task, "redis request wasn't scheduled") @@ -181,7 +183,7 @@ end local function replies_check_cookie(task) local function cookie_matched(extra, ts) - local dt = task:get_date{format = 'connect', gmt = true} + local dt = task:get_date { format = 'connect', gmt = true } if dt < ts then rspamd_logger.infox(task, 'ignore cookie as its date is in future') @@ -232,7 +234,7 @@ local function replies_check_cookie(task) extracted_cookie = irt end - local dec_cookie,ts = cr.decrypt_cookie(settings.cookie_key, extracted_cookie) + local dec_cookie, ts = cr.decrypt_cookie(settings.cookie_key, extracted_cookie) if dec_cookie then -- We have something that looks like a cookie @@ -266,8 +268,10 @@ if opts then else -- Cookies mode -- Check key sanity: - local pattern = {'^'} - for i=1,32 do pattern[i + 1] = '[a-zA-Z0-9]' end + local pattern = { '^' } + for i = 1, 32 do + pattern[i + 1] = '[a-zA-Z0-9]' + end pattern[34] = '$' if not settings.cookie_key:match(table.concat(pattern, '')) then rspamd_logger.errx(rspamd_config, diff --git a/src/plugins/lua/rspamd_update.lua b/src/plugins/lua/rspamd_update.lua index 7800d419f..deda0381a 100644 --- a/src/plugins/lua/rspamd_update.lua +++ b/src/plugins/lua/rspamd_update.lua @@ -74,14 +74,14 @@ local function check_version(obj) if rspamd_version('cmp', obj['min_version']) > 0 then ret = false rspamd_logger.errx(rspamd_config, 'updates require at least %s version of rspamd', - obj['min_version']) + obj['min_version']) end end if obj['max_version'] then if rspamd_version('cmp', obj['max_version']) < 0 then ret = false rspamd_logger.errx(rspamd_config, 'updates require maximum %s version of rspamd', - obj['max_version']) + obj['max_version']) end end @@ -92,7 +92,7 @@ local function gen_callback() return function(data) local parser = ucl.parser() - local res,err = parser:parse_string(data) + local res, err = parser:parse_string(data) if not res then rspamd_logger.warnx(rspamd_config, 'cannot parse updates map: ' .. err) @@ -114,7 +114,7 @@ local function gen_callback() end rspamd_logger.infox(rspamd_config, 'loaded new rules with hash "%s"', - h:hex()) + h:hex()) end end @@ -131,7 +131,7 @@ if section and section.rules then end if type(section.rules) ~= 'table' then - section.rules = {section.rules} + section.rules = { section.rules } end fun.each(function(elt) diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua index db2f264e7..3027a541f 100644 --- a/src/plugins/lua/settings.lua +++ b/src/plugins/lua/settings.lua @@ -68,7 +68,7 @@ local function apply_settings(task, to_apply, id, name) end if to_apply.flags and type(to_apply.flags) == 'table' then - for _,fl in ipairs(to_apply.flags) do + for _, fl in ipairs(to_apply.flags) do task:set_flag(fl) end end @@ -77,12 +77,12 @@ local function apply_settings(task, to_apply, id, name) -- Add symbols, specified in the settings if #to_apply.symbols > 0 then -- Array like symbols - for _,val in ipairs(to_apply.symbols) do + for _, val in ipairs(to_apply.symbols) do task:insert_result(val, 1.0) end else -- Object like symbols - for k,v in pairs(to_apply.symbols) do + for k, v in pairs(to_apply.symbols) do if type(v) == 'table' then task:insert_result(k, v.score or 1.0, v.options or {}) elseif tonumber(v) then @@ -120,7 +120,7 @@ local function check_query_settings(task) if query_set then local parser = ucl.parser() - local res,err = parser:parse_text(query_set) + local res, err = parser:parse_text(query_set) if res then if settings_id then rspamd_logger.warnx(task, "both settings-id '%s' and settings headers are presented, ignore settings-id; ", @@ -129,7 +129,7 @@ local function check_query_settings(task) local settings_obj = parser:get_object() -- Treat as low priority - return settings_obj,nil,1 + return settings_obj, nil, 1 else rspamd_logger.errx(task, 'Parse error: %s', err) end @@ -141,8 +141,8 @@ local function check_query_settings(task) if query_maxscore then if settings_id then rspamd_logger.infox(task, "both settings id '%s' and maxscore '%s' headers are presented, merge them; " .. - "settings id has priority", - tostring(settings_id), tostring(query_maxscore)) + "settings id has priority", + tostring(settings_id), tostring(query_maxscore)) end -- We have score limits redefined by request local ms = tonumber(tostring(query_maxscore)) @@ -174,7 +174,7 @@ local function check_query_settings(task) if cached then local elt = cached.settings if elt['whitelist'] then - elt['apply'] = {whitelist = true} + elt['apply'] = { whitelist = true } end if elt.apply then @@ -290,7 +290,9 @@ local function check_settings(task) if elt then local input = elt.extract(task) - if not input then return false end + if not input then + return false + end if elt.check(input) then matched[#matched + 1] = atom @@ -307,7 +309,7 @@ local function check_settings(task) if res and res > 0 then if rule['whitelist'] then - rule['apply'] = {whitelist = true} + rule['apply'] = { whitelist = true } end return rule @@ -317,7 +319,7 @@ local function check_settings(task) end -- Check if we have override as query argument - local query_apply,id_elt,priority = check_query_settings(task) + local query_apply, id_elt, priority = check_query_settings(task) local function maybe_apply_query_settings() if query_apply then @@ -369,9 +371,9 @@ local function check_settings(task) -- Match rules according their order local applied = false - for pri = max_pri,min_pri,-1 do + for pri = max_pri, min_pri, -1 do if not applied and settings[pri] then - for _,s in ipairs(settings[pri]) do + for _, s in ipairs(settings[pri]) do local matched = {} local result = check_specific_setting(s.rule, matched) @@ -434,18 +436,18 @@ end local function convert_to_table(chk_elt, out) if type(chk_elt) == 'string' then - return {out} + return { out } end return out end local function gen_settings_external_cb(name) - return function (result, err_or_data, code, task) + return function(result, err_or_data, code, task) if result then local parser = ucl.parser() - local res,ucl_err = parser:parse_text(err_or_data) + local res, ucl_err = parser:parse_text(err_or_data) if not res then rspamd_logger.warnx(task, 'cannot parse settings from the external map %s: %s', name, ucl_err) @@ -467,7 +469,7 @@ local function process_ip_condition(ip) local out = {} if type(ip) == "table" then - for _,v in ipairs(ip) do + for _, v in ipairs(ip) do table.insert(out, process_ip_condition(v)) end elseif type(ip) == "string" then @@ -511,7 +513,7 @@ end local function process_email_condition(addr) local out = {} if type(addr) == "table" then - for _,v in ipairs(addr) do + for _, v in ipairs(addr) do table.insert(out, process_email_condition(v)) end elseif type(addr) == "string" then @@ -558,7 +560,7 @@ end local function process_string_condition(addr) local out = {} if type(addr) == "table" then - for _,v in ipairs(addr) do + for _, v in ipairs(addr) do table.insert(out, process_string_condition(v)) end elseif type(addr) == "string" then @@ -613,7 +615,9 @@ end -- Used to create a checking closure: if value matches expected somehow, return true local function gen_check_closure(expected, check_func) return function(value) - if not value then return false end + if not value then + return false + end if type(value) == 'function' then value = value() @@ -622,7 +626,9 @@ local function gen_check_closure(expected, check_func) if value then if not check_func then - check_func = function(a, b) return a == b end + check_func = function(a, b) + return a == b + end end local ret @@ -663,7 +669,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) check = gen_check_closure(convert_to_table(elt.ip, ips_table), check_ip_setting), extract = function(task) local ip = task:get_from_ip() - if ip and ip:is_valid() then return ip end + if ip and ip:is_valid() then + return ip + end return nil end, } @@ -680,7 +688,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) check = gen_check_closure(ips_map, check_map_setting), extract = function(task) local ip = task:get_from_ip() - if ip and ip:is_valid() then return ip end + if ip and ip:is_valid() then + return ip + end return nil end, } @@ -698,7 +708,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) check_ip_setting), extract = function(task) local ip = task:get_client_ip() - if ip:is_valid() then return ip end + if ip:is_valid() then + return ip + end return nil end, } @@ -715,7 +727,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) check = gen_check_closure(ips_map, check_map_setting), extract = function(task) local ip = task:get_client_ip() - if ip and ip:is_valid() then return ip end + if ip and ip:is_valid() then + return ip + end return nil end, } @@ -835,7 +849,12 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) lua_util.debugm(N, rspamd_config, 'added authenticated condition to "%s"', name) checks.authenticated = { - check = function(value) if value then return true end return false end, + check = function(value) + if value then + return true + end + return false + end, extract = function(task) return task:get_user() end @@ -846,7 +865,12 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) lua_util.debugm(N, rspamd_config, 'added local condition to "%s"', name) checks['local'] = { - check = function(value) if value then return true end return false end, + check = function(value) + if value then + return true + end + return false + end, extract = function(task) local ip = task:get_from_ip() if not ip or not ip:is_valid() then @@ -875,9 +899,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) checks[full_key] = cond -- Try numeric key - for i=1,1000 do + for i = 1, 1000 do local num_key = generic .. ':' .. tostring(i) - if not checks[num_key] then + if not checks[num_key] then checks[num_key] = cond aliases[num_key] = true break @@ -911,7 +935,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) if re then local cond = { check = function(values) - return fun.any(function(c) return re:match(c) end, values) + return fun.any(function(c) + return re:match(c) + end, values) end, extract = extractor_func(k), } @@ -923,7 +949,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) elseif type(v) == 'boolean' then local cond = { check = function(values) - if #values == 0 then return (not v) end + if #values == 0 then + return (not v) + end return v end, extract = extractor_func(k), @@ -943,7 +971,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) process_header_elt('request_header', function(hname) return function(task) local rh = task:get_request_header(hname) - if rh then return {rh} end + if rh then + return { rh } + end return {} end end) @@ -951,7 +981,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) return function(task) local rh = task:get_header_full(hname) if rh then - return fun.totable(fun.map(function(h) return h.decoded end, rh)) + return fun.totable(fun.map(function(h) + return h.decoded + end, rh)) end return {} end @@ -987,7 +1019,7 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) -- Count checks and create Rspamd expression from a set of rules local nchecks = 0 - for k,_ in pairs(checks) do + for k, _ in pairs(checks) do if not aliases[k] then nchecks = nchecks + 1 end @@ -1000,10 +1032,14 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) -- Here we get all keys and concatenate them with '&&' local s = ' && ' -- By De Morgan laws - if inverse then s = ' || ' end + if inverse then + s = ' || ' + end -- Exclude aliases and join all checks by key local expr_str = table.concat(lua_util.keys(fun.filter( - function(k, _) return not aliases[k] end, + function(k, _) + return not aliases[k] + end, checks)), s) if inverse then @@ -1030,7 +1066,9 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) rspamd_logger.errx(rspamd_config, 'use of undefined element "%s" when parsing settings expression, known checks: %s', - atom, table.concat(fun.totable(fun.map(function(k, _) return k end, checks)), ',')) + atom, table.concat(fun.totable(fun.map(function(k, _) + return k + end, checks)), ',')) return nil end @@ -1123,7 +1161,7 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) -- In fact, it is useless and evil but who cares... if elt.apply and elt.apply.symbols then -- Register virtual symbols - for k,v in pairs(elt.apply.symbols) do + for k, v in pairs(elt.apply.symbols) do local rtb = { type = 'virtual', parent = module_sym_id, @@ -1153,33 +1191,39 @@ local function process_settings_table(tbl, allow_ids, mempool, is_static) settings_initialized = false -- filter trash in the input local ft = fun.filter( - function(_, elt) - if type(elt) == "table" then - return true - end - return false - end, tbl) + function(_, elt) + if type(elt) == "table" then + return true + end + return false + end, tbl) -- clear all settings max_pri = 0 local nrules = 0 - for k in pairs(settings) do settings[k]={} end + for k in pairs(settings) do + settings[k] = {} + end -- fill new settings by priority fun.for_each(function(k, v) local pri = get_priority(v) - if pri > max_pri then max_pri = pri end + if pri > max_pri then + max_pri = pri + end if not settings[pri] then settings[pri] = {} end local s = process_setting_elt(k, v) if s then - table.insert(settings[pri], {name = k, rule = s}) + table.insert(settings[pri], { name = k, rule = s }) nrules = nrules + 1 end end, ft) -- sort settings with equal priorities in alphabetical order - for pri,_ in pairs(settings) do - table.sort(settings[pri], function(a,b) return a.name < b.name end) + for pri, _ in pairs(settings) do + table.sort(settings[pri], function(a, b) + return a.name < b.name + end) end settings_initialized = true @@ -1194,7 +1238,7 @@ local settings_map_pool local function process_settings_map(map_text) local parser = ucl.parser() - local res,err = parser:parse_text(map_text) + local res, err = parser:parse_text(map_text) if not res then rspamd_logger.warnx(rspamd_config, 'cannot parse settings map: ' .. err) @@ -1226,14 +1270,14 @@ local function gen_redis_callback(handler, id) for _, d in ipairs(data) do if type(d) == 'string' then local parser = ucl.parser() - local res,ucl_err = parser:parse_text(d) + local res, ucl_err = parser:parse_text(d) if not res then rspamd_logger.warnx(rspamd_config, 'cannot parse settings from redis: %s', - ucl_err) + ucl_err) else local obj = parser:get_object() rspamd_logger.infox(task, "<%1> apply settings according to redis rule %2", - task:get_message_id(), id) + task:get_message_id(), id) apply_settings(task, obj, nil, 'redis') break end @@ -1253,17 +1297,17 @@ local function gen_redis_callback(handler, id) if type(key) == 'table' then keys = key else - keys = {key} + keys = { key } end key = keys[1] - local ret,_,_ = rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_settings_cb, --callback - 'MGET', -- command - keys -- arguments + local ret, _, _ = rspamd_redis_make_request(task, + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_settings_cb, --callback + 'MGET', -- command + keys -- arguments ) if not ret then rspamd_logger.errx(task, 'Redis MGET failed: %s', ret) @@ -1279,17 +1323,17 @@ if redis_section then if redis_params then local handlers = redis_section.handlers - for id,h in pairs(handlers) do - local chunk,err = load(h) + for id, h in pairs(handlers) do + local chunk, err = load(h) if not chunk then rspamd_logger.errx(rspamd_config, 'Cannot load handler from string: %s', tostring(err)) else - local res,func = pcall(chunk) + local res, func = pcall(chunk) if not res then rspamd_logger.errx(rspamd_config, 'Cannot add handler from string: %s', - tostring(func)) + tostring(func)) else redis_key_handlers[id] = func end @@ -1304,7 +1348,7 @@ if redis_section then callback = gen_redis_callback(h, id), priority = lua_util.symbols_priorities.top, flags = 'empty,nostat', - augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}, + augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }, }) end, redis_key_handlers) end @@ -1337,7 +1381,7 @@ elseif set_section and type(set_section) == "table" then -- registered BEFORE settings plugin. Otherwise, we can have inconsistent settings expressions fun.each(function(_, elt) if elt.register_symbols then - for k,v in pairs(elt.register_symbols) do + for k, v in pairs(elt.register_symbols) do local rtb = { type = 'virtual', parent = module_sym_id, @@ -1347,7 +1391,7 @@ elseif set_section and type(set_section) == "table" then elseif type(k) == 'string' then rtb.name = k if type(v) == 'table' then - for kk,vv in pairs(v) do + for kk, vv in pairs(v) do -- Enrich table wih extra values rtb[kk] = vv end @@ -1358,7 +1402,7 @@ elseif set_section and type(set_section) == "table" then end if elt.apply and elt.apply.symbols then -- Register virtual symbols - for k,v in pairs(elt.apply.symbols) do + for k, v in pairs(elt.apply.symbols) do local rtb = { type = 'virtual', parent = module_sym_id, @@ -1372,7 +1416,7 @@ elseif set_section and type(set_section) == "table" then end end end, - -- Include only settings, exclude all maps + -- Include only settings, exclude all maps fun.filter( function(_, elt) if type(elt) == "table" then @@ -1382,7 +1426,7 @@ elseif set_section and type(set_section) == "table" then end, set_section) ) - rspamd_config:add_post_init(function () + rspamd_config:add_post_init(function() process_settings_table(set_section, true, settings_map_pool, true) end, 100) end diff --git a/src/plugins/lua/spamassassin.lua b/src/plugins/lua/spamassassin.lua index c98203fb6..3ea794495 100644 --- a/src/plugins/lua/spamassassin.lua +++ b/src/plugins/lua/spamassassin.lua @@ -159,7 +159,7 @@ end local ffi if type(jit) == 'table' then ffi = require("ffi") - ffi.cdef[[ + ffi.cdef [[ int rspamd_re_cache_type_from_string (const char *str); int rspamd_re_cache_process_ffi (void *ptask, void *pre, @@ -207,7 +207,7 @@ local function handle_header_def(hline, cur_rule) -- Check if an re is an ordinary re local ordinary = true - for _,h in ipairs(hdrs) do + for _, h in ipairs(hdrs) do if h == 'ALL' or h == 'ALL:raw' then ordinary = false cur_rule['type'] = 'function' @@ -239,71 +239,71 @@ local function handle_header_def(hline, cur_rule) end fun.each(function(func) - if func == 'addr' then - cur_param['function'] = function(str) - local addr_parsed = util.parse_mail_address(str) - local ret = {} - if addr_parsed then - for _,elt in ipairs(addr_parsed) do - if elt['addr'] then - table.insert(ret, elt['addr']) - end + if func == 'addr' then + cur_param['function'] = function(str) + local addr_parsed = util.parse_mail_address(str) + local ret = {} + if addr_parsed then + for _, elt in ipairs(addr_parsed) do + if elt['addr'] then + table.insert(ret, elt['addr']) end end - - return ret end - elseif func == 'name' then - cur_param['function'] = function(str) - local addr_parsed = util.parse_mail_address(str) - local ret = {} - if addr_parsed then - for _,elt in ipairs(addr_parsed) do - if elt['name'] then - table.insert(ret, elt['name']) - end - end - end - return ret - end - elseif func == 'raw' then - cur_param['raw'] = true - elseif func == 'case' then - cur_param['strong'] = true - else - rspamd_logger.warnx(rspamd_config, 'Function %1 is not supported in %2', - func, cur_rule['symbol']) + return ret end - end, fun.tail(args)) - - local function split_hdr_param(param, headers) - for _,hh in ipairs(headers) do - local nparam = {} - for k,v in pairs(param) do - if k ~= 'header' then - nparam[k] = v + elseif func == 'name' then + cur_param['function'] = function(str) + local addr_parsed = util.parse_mail_address(str) + local ret = {} + if addr_parsed then + for _, elt in ipairs(addr_parsed) do + if elt['name'] then + table.insert(ret, elt['name']) + end end end - nparam['header'] = hh - table.insert(hdr_params, nparam) + return ret end - end - -- Some header rules require splitting to check of multiple headers - if cur_param['header'] == 'MESSAGEID' then - -- Special case for spamassassin - ordinary = false - split_hdr_param(cur_param, { - 'Message-ID', - 'X-Message-ID', - 'Resent-Message-ID'}) - elseif cur_param['header'] == 'ToCc' then - ordinary = false - split_hdr_param(cur_param, { 'To', 'Cc', 'Bcc' }) + elseif func == 'raw' then + cur_param['raw'] = true + elseif func == 'case' then + cur_param['strong'] = true else - table.insert(hdr_params, cur_param) + rspamd_logger.warnx(rspamd_config, 'Function %1 is not supported in %2', + func, cur_rule['symbol']) end + end, fun.tail(args)) + + local function split_hdr_param(param, headers) + for _, hh in ipairs(headers) do + local nparam = {} + for k, v in pairs(param) do + if k ~= 'header' then + nparam[k] = v + end + end + + nparam['header'] = hh + table.insert(hdr_params, nparam) + end + end + -- Some header rules require splitting to check of multiple headers + if cur_param['header'] == 'MESSAGEID' then + -- Special case for spamassassin + ordinary = false + split_hdr_param(cur_param, { + 'Message-ID', + 'X-Message-ID', + 'Resent-Message-ID' }) + elseif cur_param['header'] == 'ToCc' then + ordinary = false + split_hdr_param(cur_param, { 'To', 'Cc', 'Bcc' }) + else + table.insert(hdr_params, cur_param) + end end cur_rule['ordinary'] = ordinary @@ -311,7 +311,6 @@ local function handle_header_def(hline, cur_rule) end end - local function freemail_search(input) local res = 0 local function trie_callback(number, pos) @@ -328,19 +327,19 @@ end local function gen_eval_rule(arg) local eval_funcs = { - {'check_freemail_from', function(task) - local from = task:get_from('mime') - if from and from[1] then - return freemail_search(string.lower(from[1]['addr'])) - end - return 0 - end}, - {'check_freemail_replyto', + { 'check_freemail_from', function(task) + local from = task:get_from('mime') + if from and from[1] then + return freemail_search(string.lower(from[1]['addr'])) + end + return 0 + end }, + { 'check_freemail_replyto', function(task) return freemail_search(task:get_header('Reply-To')) end }, - {'check_freemail_header', + { 'check_freemail_header', function(task, remain) -- Remain here contains one or two args: header and regexp to match local larg = string.match(remain, "^%(%s*['\"]([^%s]+)['\"]%s*%)$") @@ -353,7 +352,9 @@ local function gen_eval_rule(arg) local h if larg == 'EnvelopeFrom' then h = task:get_from('smtp') - if h then h = h[1]['addr'] end + if h then + h = h[1]['addr'] + end else h = task:get_header(larg) end @@ -382,7 +383,7 @@ local function gen_eval_rule(arg) }, { 'check_for_missing_to_header', - function (task) + function(task) local th = task:get_recipients('mime') if not th or #th == 0 then return 1 @@ -398,16 +399,20 @@ local function gen_eval_rule(arg) local rh_parsed = task:get_received_headers() local rh_cnt = 0 - if rh_mime then rh_cnt = #rh_mime end + if rh_mime then + rh_cnt = #rh_mime + end local parsed_cnt = 0 - if rh_parsed then parsed_cnt = #rh_parsed end + if rh_parsed then + parsed_cnt = #rh_parsed + end return rh_cnt - parsed_cnt end }, { 'check_for_shifted_date', - function (task, remain) + function(task, remain) -- Remain here contains two args: start and end hours shift local matches = internal_regexp['date_shift']:search(remain, true, true) if matches and matches[1] then @@ -426,8 +431,8 @@ local function gen_eval_rule(arg) end -- Now get the difference between Date and message received date - local dm = task:get_date { format = 'message', gmt = true} - local dt = task:get_date { format = 'connect', gmt = true} + local dm = task:get_date { format = 'message', gmt = true } + local dt = task:get_date { format = 'connect', gmt = true } local diff = dm - dt if (max_diff == 0 and diff >= min_diff) or @@ -449,7 +454,7 @@ local function gen_eval_rule(arg) if larg == 'mime_attachment' then local parts = task:get_parts() if parts then - for _,p in ipairs(parts) do + for _, p in ipairs(parts) do if p:get_filename() then return 1 end @@ -507,7 +512,7 @@ local function gen_eval_rule(arg) function(task) local rcpt = task:get_recipients('mime') if rcpt then - for _,r in ipairs(rcpt) do + for _, r in ipairs(rcpt) do if sa_lists['to_blacklist'][string.lower(r['addr'])] then return 1 end @@ -522,7 +527,7 @@ local function gen_eval_rule(arg) function(task) local rcpt = task:get_recipients('mime') if rcpt then - for _,r in ipairs(rcpt) do + for _, r in ipairs(rcpt) do if sa_lists['to_whitelist'][string.lower(r['addr'])] then return 1 end @@ -537,7 +542,7 @@ local function gen_eval_rule(arg) function(task, remain) local tp = task:get_text_parts() - for _,p in ipairs(tp) do + for _, p in ipairs(tp) do if p:is_html() then local hc = p:get_html() @@ -552,9 +557,9 @@ local function gen_eval_rule(arg) } } - for _,f in ipairs(eval_funcs) do + for _, f in ipairs(eval_funcs) do local pat = string.format('^%s', f[1]) - local first,last = string.find(arg, pat) + local first, last = string.find(arg, pat) if first then local func_arg = string.sub(arg, last + 1) @@ -572,10 +577,11 @@ local function maybe_parse_sa_function(line) arg = elts[2] lua_util.debugm(N, rspamd_config, 'trying to parse SA function %1 with args %2', - elts[1], elts[2]) + elts[1], elts[2]) local substitutions = { - {'^exists:', - function(task) -- filter + { '^exists:', + function(task) + -- filter local hdrs_check if arg == 'MESSAGEID' then hdrs_check = { @@ -586,10 +592,10 @@ local function maybe_parse_sa_function(line) elseif arg == 'ToCc' then hdrs_check = { 'To', 'Cc', 'Bcc' } else - hdrs_check = {arg} + hdrs_check = { arg } end - for _,h in ipairs(hdrs_check) do + for _, h in ipairs(hdrs_check) do if task:has_header(h) then return 1 end @@ -597,7 +603,7 @@ local function maybe_parse_sa_function(line) return 0 end, }, - {'^eval:', + { '^eval:', function(task) local func = func_cache[arg] if not func then @@ -607,7 +613,7 @@ local function maybe_parse_sa_function(line) if not func then rspamd_logger.errx(task, 'cannot find appropriate eval rule for function %1', - arg) + arg) else return func(task) end @@ -617,7 +623,7 @@ local function maybe_parse_sa_function(line) }, } - for _,s in ipairs(substitutions) do + for _, s in ipairs(substitutions) do if string.find(line, s[1]) then return s[2] end @@ -664,26 +670,26 @@ local function process_sa_conf(f) local valid_rule = false local function insert_cur_rule() - if cur_rule['type'] ~= 'meta' and cur_rule['publish'] then - -- Create meta rule from this rule - local nsym = '__fake' .. cur_rule['symbol'] - local nrule = { - type = 'meta', - symbol = cur_rule['symbol'], - score = cur_rule['score'], - meta = nsym, - description = cur_rule['description'], - } - rules[nrule['symbol']] = nrule - cur_rule['symbol'] = nsym - end - -- We have previous rule valid - if not cur_rule['symbol'] then - rspamd_logger.errx(rspamd_config, 'bad rule definition: %1', cur_rule) - end - rules[cur_rule['symbol']] = cur_rule - cur_rule = {} - valid_rule = false + if cur_rule['type'] ~= 'meta' and cur_rule['publish'] then + -- Create meta rule from this rule + local nsym = '__fake' .. cur_rule['symbol'] + local nrule = { + type = 'meta', + symbol = cur_rule['symbol'], + score = cur_rule['score'], + meta = nsym, + description = cur_rule['description'], + } + rules[nrule['symbol']] = nrule + cur_rule['symbol'] = nsym + end + -- We have previous rule valid + if not cur_rule['symbol'] then + rspamd_logger.errx(rspamd_config, 'bad rule definition: %1', cur_rule) + end + rules[cur_rule['symbol']] = cur_rule + cur_rule = {} + valid_rule = false end local function parse_score(words) @@ -706,359 +712,370 @@ local function process_sa_conf(f) local skip_to_endif = false local if_nested = 0 for l in f:lines() do - (function () - l = lua_util.rspamd_str_trim(l) - -- Replace bla=~/re/ with bla =~ /re/ (#2372) - l = l:gsub('([^%s])%s*([=!]~)%s*([^%s])', '%1 %2 %3') + (function() + l = lua_util.rspamd_str_trim(l) + -- Replace bla=~/re/ with bla =~ /re/ (#2372) + l = l:gsub('([^%s])%s*([=!]~)%s*([^%s])', '%1 %2 %3') - if string.len(l) == 0 or string.sub(l, 1, 1) == '#' then - return - end + if string.len(l) == 0 or string.sub(l, 1, 1) == '#' then + return + end - -- Unbalanced if/endif - if if_nested < 0 then if_nested = 0 end - if skip_to_endif then - if string.match(l, '^endif') then - if_nested = if_nested - 1 + -- Unbalanced if/endif + if if_nested < 0 then + if_nested = 0 + end + if skip_to_endif then + if string.match(l, '^endif') then + if_nested = if_nested - 1 - if if_nested == 0 then + if if_nested == 0 then + skip_to_endif = false + end + elseif string.match(l, '^if') then + if_nested = if_nested + 1 + elseif string.match(l, '^else') then + -- Else counterpart for if skip_to_endif = false end - elseif string.match(l, '^if') then - if_nested = if_nested + 1 - elseif string.match(l, '^else') then - -- Else counterpart for if - skip_to_endif = false - end - return - else - if string.match(l, '^ifplugin') then - local ls = split(l) + return + else + if string.match(l, '^ifplugin') then + local ls = split(l) - if not fun.any(function(pl) - if pl == ls[2] then return true end + if not fun.any(function(pl) + if pl == ls[2] then + return true + end + return false + end, known_plugins) then + skip_to_endif = true + end + if_nested = if_nested + 1 + elseif string.match(l, '^if !plugin%(') then + local pname = string.match(l, '^if !plugin%(([A-Za-z:]+)%)') + if fun.any(function(pl) + if pl == pname then + return true + end return false - end, known_plugins) then + end, known_plugins) then + skip_to_endif = true + end + if_nested = if_nested + 1 + elseif string.match(l, '^if') then + -- Unknown if + skip_to_endif = true + if_nested = if_nested + 1 + elseif string.match(l, '^else') then + -- Else counterpart for if skip_to_endif = true + elseif string.match(l, '^endif') then + if_nested = if_nested - 1 end - if_nested = if_nested + 1 - elseif string.match(l, '^if !plugin%(') then - local pname = string.match(l, '^if !plugin%(([A-Za-z:]+)%)') - if fun.any(function(pl) - if pl == pname then return true end - return false - end, known_plugins) then - skip_to_endif = true - end - if_nested = if_nested + 1 - elseif string.match(l, '^if') then - -- Unknown if - skip_to_endif = true - if_nested = if_nested + 1 - elseif string.match(l, '^else') then - -- Else counterpart for if - skip_to_endif = true - elseif string.match(l, '^endif') then - if_nested = if_nested - 1 - end - end - - -- Skip comments - local words = fun.totable(fun.take_while( - function(w) return string.sub(w, 1, 1) ~= '#' end, - fun.filter(function(w) - return w ~= "" end, - fun.iter(split(l))))) - - if words[1] == "header" or words[1] == 'mimeheader' then - -- header SYMBOL Header ~= /regexp/ - if valid_rule then - insert_cur_rule() end - if words[4] and (words[4] == '=~' or words[4] == '!~') then - cur_rule['type'] = 'header' - cur_rule['symbol'] = words[2] - if words[4] == '!~' then - cur_rule['not'] = true + -- Skip comments + local words = fun.totable(fun.take_while( + function(w) + return string.sub(w, 1, 1) ~= '#' + end, + fun.filter(function(w) + return w ~= "" + end, + fun.iter(split(l))))) + + if words[1] == "header" or words[1] == 'mimeheader' then + -- header SYMBOL Header ~= /regexp/ + if valid_rule then + insert_cur_rule() end + if words[4] and (words[4] == '=~' or words[4] == '!~') then + cur_rule['type'] = 'header' + cur_rule['symbol'] = words[2] - cur_rule['re_expr'] = words_to_re(words, 4) - local unset_comp = string.find(cur_rule['re_expr'], '%s+%[if%-unset:') - if unset_comp then - -- We have optional part that needs to be processed - local unset = string.match(string.sub(cur_rule['re_expr'], unset_comp), - '%[if%-unset:%s*([^%]%s]+)]') - cur_rule['unset'] = unset - -- Cut it down - cur_rule['re_expr'] = string.sub(cur_rule['re_expr'], 1, unset_comp - 1) - end + if words[4] == '!~' then + cur_rule['not'] = true + end - cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) + cur_rule['re_expr'] = words_to_re(words, 4) + local unset_comp = string.find(cur_rule['re_expr'], '%s+%[if%-unset:') + if unset_comp then + -- We have optional part that needs to be processed + local unset = string.match(string.sub(cur_rule['re_expr'], unset_comp), + '%[if%-unset:%s*([^%]%s]+)]') + cur_rule['unset'] = unset + -- Cut it down + cur_rule['re_expr'] = string.sub(cur_rule['re_expr'], 1, unset_comp - 1) + end - if not cur_rule['re'] then - rspamd_logger.warnx(rspamd_config, "Cannot parse regexp '%1' for %2", - cur_rule['re_expr'], cur_rule['symbol']) - else - cur_rule['re']:set_max_hits(1) - handle_header_def(words[3], cur_rule) - end + cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) - if cur_rule['unset'] then - cur_rule['ordinary'] = false - end + if not cur_rule['re'] then + rspamd_logger.warnx(rspamd_config, "Cannot parse regexp '%1' for %2", + cur_rule['re_expr'], cur_rule['symbol']) + else + cur_rule['re']:set_max_hits(1) + handle_header_def(words[3], cur_rule) + end - if words[1] == 'mimeheader' then - cur_rule['mime'] = true - else - cur_rule['mime'] = false - end + if cur_rule['unset'] then + cur_rule['ordinary'] = false + end - if cur_rule['re'] and cur_rule['symbol'] and - (cur_rule['header'] or cur_rule['function']) then - valid_rule = true - cur_rule['re']:set_max_hits(1) - if cur_rule['header'] and cur_rule['ordinary'] then - for _,h in ipairs(cur_rule['header']) do - if type(h) == 'string' then - if cur_rule['mime'] then - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'mimeheader', - header = h, - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - else - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'header', - header = h, - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - end - else - h['mime'] = cur_rule['mime'] - if cur_rule['mime'] then - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'mimeheader', - header = h['header'], - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - else - if h['raw'] then + if words[1] == 'mimeheader' then + cur_rule['mime'] = true + else + cur_rule['mime'] = false + end + + if cur_rule['re'] and cur_rule['symbol'] and + (cur_rule['header'] or cur_rule['function']) then + valid_rule = true + cur_rule['re']:set_max_hits(1) + if cur_rule['header'] and cur_rule['ordinary'] then + for _, h in ipairs(cur_rule['header']) do + if type(h) == 'string' then + if cur_rule['mime'] then rspamd_config:register_regexp({ re = cur_rule['re'], - type = 'rawheader', - header = h['header'], + type = 'mimeheader', + header = h, pcre_only = is_pcre_only(cur_rule['symbol']), }) else rspamd_config:register_regexp({ re = cur_rule['re'], type = 'header', + header = h, + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + end + else + h['mime'] = cur_rule['mime'] + if cur_rule['mime'] then + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'mimeheader', header = h['header'], pcre_only = is_pcre_only(cur_rule['symbol']), }) + else + if h['raw'] then + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'rawheader', + header = h['header'], + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + else + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'header', + header = h['header'], + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + end end end end + cur_rule['re']:set_limit(match_limit) + cur_rule['re']:set_max_hits(1) end - cur_rule['re']:set_limit(match_limit) - cur_rule['re']:set_max_hits(1) end - end - else - -- Maybe we know the function and can convert it - local args = words_to_re(words, 2) - local func = maybe_parse_sa_function(args) - - if func then - cur_rule['type'] = 'function' - cur_rule['symbol'] = words[2] - cur_rule['function'] = func - valid_rule = true else - rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + -- Maybe we know the function and can convert it + local args = words_to_re(words, 2) + local func = maybe_parse_sa_function(args) + + if func then + cur_rule['type'] = 'function' + cur_rule['symbol'] = words[2] + cur_rule['function'] = func + valid_rule = true + else + rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + end end - end - elseif words[1] == "body" then - -- body SYMBOL /regexp/ - if valid_rule then - insert_cur_rule() - end - - cur_rule['symbol'] = words[2] - if words[3] and (string.sub(words[3], 1, 1) == '/' - or string.sub(words[3], 1, 1) == 'm') then - cur_rule['type'] = 'sabody' - cur_rule['re_expr'] = words_to_re(words, 2) - cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) - if cur_rule['re'] then - - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'sabody', - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - valid_rule = true - cur_rule['re']:set_limit(match_limit) - cur_rule['re']:set_max_hits(1) + elseif words[1] == "body" then + -- body SYMBOL /regexp/ + if valid_rule then + insert_cur_rule() end - else - -- might be function - local args = words_to_re(words, 2) - local func = maybe_parse_sa_function(args) - if func then - cur_rule['type'] = 'function' - cur_rule['symbol'] = words[2] - cur_rule['function'] = func - valid_rule = true + cur_rule['symbol'] = words[2] + if words[3] and (string.sub(words[3], 1, 1) == '/' + or string.sub(words[3], 1, 1) == 'm') then + cur_rule['type'] = 'sabody' + cur_rule['re_expr'] = words_to_re(words, 2) + cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) + if cur_rule['re'] then + + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'sabody', + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + valid_rule = true + cur_rule['re']:set_limit(match_limit) + cur_rule['re']:set_max_hits(1) + end else - rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + -- might be function + local args = words_to_re(words, 2) + local func = maybe_parse_sa_function(args) + + if func then + cur_rule['type'] = 'function' + cur_rule['symbol'] = words[2] + cur_rule['function'] = func + valid_rule = true + else + rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + end end - end - elseif words[1] == "rawbody" then - -- body SYMBOL /regexp/ - if valid_rule then - insert_cur_rule() - end - - cur_rule['symbol'] = words[2] - if words[3] and (string.sub(words[3], 1, 1) == '/' - or string.sub(words[3], 1, 1) == 'm') then - cur_rule['type'] = 'sarawbody' - cur_rule['re_expr'] = words_to_re(words, 2) - cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) - if cur_rule['re'] then - - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'sarawbody', - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - valid_rule = true - cur_rule['re']:set_limit(match_limit) - cur_rule['re']:set_max_hits(1) + elseif words[1] == "rawbody" then + -- body SYMBOL /regexp/ + if valid_rule then + insert_cur_rule() end - else - -- might be function - local args = words_to_re(words, 2) - local func = maybe_parse_sa_function(args) - if func then - cur_rule['type'] = 'function' - cur_rule['symbol'] = words[2] - cur_rule['function'] = func - valid_rule = true + cur_rule['symbol'] = words[2] + if words[3] and (string.sub(words[3], 1, 1) == '/' + or string.sub(words[3], 1, 1) == 'm') then + cur_rule['type'] = 'sarawbody' + cur_rule['re_expr'] = words_to_re(words, 2) + cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) + if cur_rule['re'] then + + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'sarawbody', + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + valid_rule = true + cur_rule['re']:set_limit(match_limit) + cur_rule['re']:set_max_hits(1) + end else - rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + -- might be function + local args = words_to_re(words, 2) + local func = maybe_parse_sa_function(args) + + if func then + cur_rule['type'] = 'function' + cur_rule['symbol'] = words[2] + cur_rule['function'] = func + valid_rule = true + else + rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + end + end + elseif words[1] == "full" then + -- body SYMBOL /regexp/ + if valid_rule then + insert_cur_rule() end - end - elseif words[1] == "full" then - -- body SYMBOL /regexp/ - if valid_rule then - insert_cur_rule() - end - cur_rule['symbol'] = words[2] + cur_rule['symbol'] = words[2] - if words[3] and (string.sub(words[3], 1, 1) == '/' - or string.sub(words[3], 1, 1) == 'm') then - cur_rule['type'] = 'message' + if words[3] and (string.sub(words[3], 1, 1) == '/' + or string.sub(words[3], 1, 1) == 'm') then + cur_rule['type'] = 'message' + cur_rule['re_expr'] = words_to_re(words, 2) + cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) + cur_rule['raw'] = true + if cur_rule['re'] then + valid_rule = true + rspamd_config:register_regexp({ + re = cur_rule['re'], + type = 'body', + pcre_only = is_pcre_only(cur_rule['symbol']), + }) + cur_rule['re']:set_limit(match_limit) + cur_rule['re']:set_max_hits(1) + end + else + -- might be function + local args = words_to_re(words, 2) + local func = maybe_parse_sa_function(args) + + if func then + cur_rule['type'] = 'function' + cur_rule['symbol'] = words[2] + cur_rule['function'] = func + valid_rule = true + else + rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + end + end + elseif words[1] == "uri" then + -- uri SYMBOL /regexp/ + if valid_rule then + insert_cur_rule() + end + cur_rule['type'] = 'uri' + cur_rule['symbol'] = words[2] cur_rule['re_expr'] = words_to_re(words, 2) cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) - cur_rule['raw'] = true - if cur_rule['re'] then + if cur_rule['re'] and cur_rule['symbol'] then valid_rule = true rspamd_config:register_regexp({ re = cur_rule['re'], - type = 'body', + type = 'url', pcre_only = is_pcre_only(cur_rule['symbol']), }) cur_rule['re']:set_limit(match_limit) cur_rule['re']:set_max_hits(1) end - else - -- might be function - local args = words_to_re(words, 2) - local func = maybe_parse_sa_function(args) - - if func then - cur_rule['type'] = 'function' - cur_rule['symbol'] = words[2] - cur_rule['function'] = func - valid_rule = true - else - rspamd_logger.infox(rspamd_config, 'unknown function %1', args) + elseif words[1] == "meta" then + -- meta SYMBOL expression + if valid_rule then + insert_cur_rule() end - end - elseif words[1] == "uri" then - -- uri SYMBOL /regexp/ - if valid_rule then - insert_cur_rule() - end - cur_rule['type'] = 'uri' - cur_rule['symbol'] = words[2] - cur_rule['re_expr'] = words_to_re(words, 2) - cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr']) - if cur_rule['re'] and cur_rule['symbol'] then - valid_rule = true - rspamd_config:register_regexp({ - re = cur_rule['re'], - type = 'url', - pcre_only = is_pcre_only(cur_rule['symbol']), - }) - cur_rule['re']:set_limit(match_limit) - cur_rule['re']:set_max_hits(1) - end - elseif words[1] == "meta" then - -- meta SYMBOL expression - if valid_rule then - insert_cur_rule() - end - cur_rule['type'] = 'meta' - cur_rule['symbol'] = words[2] - cur_rule['meta'] = words_to_re(words, 2) - if cur_rule['meta'] and cur_rule['symbol'] - and cur_rule['meta'] ~= '0' then + cur_rule['type'] = 'meta' + cur_rule['symbol'] = words[2] + cur_rule['meta'] = words_to_re(words, 2) + if cur_rule['meta'] and cur_rule['symbol'] + and cur_rule['meta'] ~= '0' then valid_rule = true - end - elseif words[1] == "describe" and valid_rule then - cur_rule['description'] = words_to_re(words, 2) - elseif words[1] == "score" then - scores[words[2]] = parse_score(words) - elseif words[1] == 'freemail_domains' then - fun.each(function(dom) + end + elseif words[1] == "describe" and valid_rule then + cur_rule['description'] = words_to_re(words, 2) + elseif words[1] == "score" then + scores[words[2]] = parse_score(words) + elseif words[1] == 'freemail_domains' then + fun.each(function(dom) table.insert(freemail_domains, '@' .. dom) end, fun.drop_n(1, words)) - elseif words[1] == 'blacklist_from' then - sa_lists['from_blacklist'][words[2]] = 1 - sa_lists['elts'] = sa_lists['elts'] + 1 - elseif words[1] == 'whitelist_from' then - sa_lists['from_whitelist'][words[2]] = 1 - sa_lists['elts'] = sa_lists['elts'] + 1 - elseif words[1] == 'whitelist_to' then - sa_lists['to_whitelist'][words[2]] = 1 - sa_lists['elts'] = sa_lists['elts'] + 1 - elseif words[1] == 'blacklist_to' then - sa_lists['to_blacklist'][words[2]] = 1 - sa_lists['elts'] = sa_lists['elts'] + 1 - elseif words[1] == 'tflags' then - process_tflags(cur_rule, words) - elseif words[1] == 'replace_tag' then - process_replace(words, replace['tags']) - elseif words[1] == 'replace_pre' then - process_replace(words, replace['pre']) - elseif words[1] == 'replace_inter' then - process_replace(words, replace['inter']) - elseif words[1] == 'replace_post' then - process_replace(words, replace['post']) - elseif words[1] == 'replace_rules' then - fun.each(function(r) table.insert(replace['rules'], r) end, - fun.drop_n(1, words)) - end + elseif words[1] == 'blacklist_from' then + sa_lists['from_blacklist'][words[2]] = 1 + sa_lists['elts'] = sa_lists['elts'] + 1 + elseif words[1] == 'whitelist_from' then + sa_lists['from_whitelist'][words[2]] = 1 + sa_lists['elts'] = sa_lists['elts'] + 1 + elseif words[1] == 'whitelist_to' then + sa_lists['to_whitelist'][words[2]] = 1 + sa_lists['elts'] = sa_lists['elts'] + 1 + elseif words[1] == 'blacklist_to' then + sa_lists['to_blacklist'][words[2]] = 1 + sa_lists['elts'] = sa_lists['elts'] + 1 + elseif words[1] == 'tflags' then + process_tflags(cur_rule, words) + elseif words[1] == 'replace_tag' then + process_replace(words, replace['tags']) + elseif words[1] == 'replace_pre' then + process_replace(words, replace['pre']) + elseif words[1] == 'replace_inter' then + process_replace(words, replace['inter']) + elseif words[1] == 'replace_post' then + process_replace(words, replace['post']) + elseif words[1] == 'replace_rules' then + fun.each(function(r) + table.insert(replace['rules'], r) + end, + fun.drop_n(1, words)) + end end)() end if valid_rule then @@ -1069,7 +1086,9 @@ end -- Now check all valid rules and add the according rspamd rules local function calculate_score(sym, rule) - if fun.all(function(c) return c == '_' end, fun.take_n(2, fun.iter(sym))) then + if fun.all(function(c) + return c == '_' + end, fun.take_n(2, fun.iter(sym))) then return 0.0 end @@ -1102,7 +1121,9 @@ local function sa_regexp_match(data, re, raw, rule) end res = res + re:matchn(data, lim, raw) else - if re:match(data, raw) then res = 1 end + if re:match(data, raw) then + res = 1 + end end return res @@ -1117,26 +1138,26 @@ local function apply_replacements(str) local replacement = nil local ret = s fun.each(function(n, t) - local ns,matches = string.gsub(s, string.format("<%s%s>", prefix, n), "") + local ns, matches = string.gsub(s, string.format("<%s%s>", prefix, n), "") if matches > 0 then replacement = t ret = ns end end, tbl) - return ret,replacement + return ret, replacement end local repl - str,repl = check_specific_tag("pre ", str, replace['pre']) + str, repl = check_specific_tag("pre ", str, replace['pre']) if repl then pre = repl end - str,repl = check_specific_tag("inter ", str, replace['inter']) + str, repl = check_specific_tag("inter ", str, replace['inter']) if repl then inter = repl end - str,repl = check_specific_tag("post ", str, replace['post']) + str, repl = check_specific_tag("post ", str, replace['post']) if repl then post = repl end @@ -1160,12 +1181,11 @@ local function apply_replacements(str) local s = replace_all_tags(str) - if str ~= s then - return true,s + return true, s end - return false,str + return false, str end local function parse_atom(str) @@ -1180,7 +1200,7 @@ local function parse_atom(str) end local function gen_process_atom_cb(result_name, task) - return function (atom) + return function(atom) local atom_cb = atoms[atom] if atom_cb then @@ -1212,7 +1232,9 @@ local function post_process() -- Replace rule tags local ntags = {} local function rec_replace_tags(tag, tagv) - if ntags[tag] then return ntags[tag] end + if ntags[tag] then + return ntags[tag] + end fun.each(function(n, t) if n ~= tag then local s, matches = string.gsub(tagv, string.format("<%s>", n), t) @@ -1222,7 +1244,9 @@ local function post_process() end end, replace['tags']) - if not ntags[tag] then ntags[tag] = tagv end + if not ntags[tag] then + ntags[tag] = tagv + end return ntags[tag] end @@ -1283,7 +1307,7 @@ local function post_process() if not r['re'] then rspamd_logger.errx(task, 're is missing for rule %1 (%2 header)', k, - h['header']) + h['header']) return 0 end @@ -1333,7 +1357,9 @@ local function post_process() else str = rh['decoded'] end - if not str then return 0 end + if not str then + return 0 + end if h['function'] then str = h['function'](str) @@ -1353,7 +1379,9 @@ local function post_process() end, r['header']) if #check == 0 then - if r['not'] then return 1 end + if r['not'] then + return 1 + end return 0 end @@ -1375,10 +1403,10 @@ local function post_process() end atoms[k] = f end, - fun.filter(function(_, r) - return r['type'] == 'header' and r['header'] - end, - rules)) + fun.filter(function(_, r) + return r['type'] == 'header' and r['header'] + end, + rules)) -- Custom function rules fun.each(function(k, r) @@ -1397,10 +1425,10 @@ local function post_process() end atoms[k] = f end, - fun.filter(function(_, r) - return r['type'] == 'function' and r['function'] - end, - rules)) + fun.filter(function(_, r) + return r['type'] == 'function' and r['function'] + end, + rules)) -- Parts rules fun.each(function(k, r) @@ -1411,7 +1439,9 @@ local function post_process() end local t = 'mime' - if r['raw'] then t = 'rawmime' end + if r['raw'] then + t = 'rawmime' + end return process_regexp_opt(r.re, task, t) end @@ -1423,9 +1453,9 @@ local function post_process() end atoms[k] = f end, - fun.filter(function(_, r) - return r['type'] == 'part' - end, rules)) + fun.filter(function(_, r) + return r['type'] == 'part' + end, rules)) -- SA body rules fun.each(function(k, r) @@ -1448,9 +1478,9 @@ local function post_process() end atoms[k] = f end, - fun.filter(function(_, r) - return r['type'] == 'sabody' or r['type'] == 'message' or r['type'] == 'sarawbody' - end, rules)) + fun.filter(function(_, r) + return r['type'] == 'sabody' or r['type'] == 'message' or r['type'] == 'sarawbody' + end, rules)) -- URL rules fun.each(function(k, r) @@ -1470,156 +1500,156 @@ local function post_process() end atoms[k] = f end, - fun.filter(function(_, r) - return r['type'] == 'uri' - end, - rules)) + fun.filter(function(_, r) + return r['type'] == 'uri' + end, + rules)) -- Meta rules fun.each(function(k, r) - local expression = nil - -- Meta function callback - -- Here are dragons! - -- This function can be called from 2 DIFFERENT type of invocations: - -- 1) Invocation from Rspamd itself where `res_name` will be nil - -- 2) Invocation from other meta during expression:process_traced call - -- So we need to distinguish that and return different stuff to be able to deal with atoms - local meta_cb = function(task, res_name) - lua_util.debugm(N, task, 'meta callback for %s; result name: %s', k, res_name) - local cached = task:cache_get('sa_metas_processed') - - -- We avoid many task methods invocations here (likely) - if not cached then - cached = {} - task:cache_set('sa_metas_processed', cached) - end - - local already_processed = cached[k] - - -- Exclude elements that are named in the same way as the symbol itself - local function exclude_sym_filter(sopt) - return sopt ~= k - end + local expression = nil + -- Meta function callback + -- Here are dragons! + -- This function can be called from 2 DIFFERENT type of invocations: + -- 1) Invocation from Rspamd itself where `res_name` will be nil + -- 2) Invocation from other meta during expression:process_traced call + -- So we need to distinguish that and return different stuff to be able to deal with atoms + local meta_cb = function(task, res_name) + lua_util.debugm(N, task, 'meta callback for %s; result name: %s', k, res_name) + local cached = task:cache_get('sa_metas_processed') + + -- We avoid many task methods invocations here (likely) + if not cached then + cached = {} + task:cache_set('sa_metas_processed', cached) + end - if not (already_processed and already_processed[res_name or 'default']) then - -- Execute symbol - local function exec_symbol(cur_res) - local res,trace = expression:process_traced(gen_process_atom_cb(cur_res, task)) - lua_util.debugm(N, task, 'meta result for %s: %s; result name: %s', k, res, cur_res) - if res > 0 then - -- Symbol should be one shot to make it working properly - task:insert_result_named(cur_res, k, res, fun.totable(fun.filter(exclude_sym_filter, trace))) - end + local already_processed = cached[k] - if not cached[k] then - cached[k] = {} - end + -- Exclude elements that are named in the same way as the symbol itself + local function exclude_sym_filter(sopt) + return sopt ~= k + end - cached[k][cur_res] = res + if not (already_processed and already_processed[res_name or 'default']) then + -- Execute symbol + local function exec_symbol(cur_res) + local res, trace = expression:process_traced(gen_process_atom_cb(cur_res, task)) + lua_util.debugm(N, task, 'meta result for %s: %s; result name: %s', k, res, cur_res) + if res > 0 then + -- Symbol should be one shot to make it working properly + task:insert_result_named(cur_res, k, res, fun.totable(fun.filter(exclude_sym_filter, trace))) end - if not res_name then - -- Invoke for all named results - local named_results = task:get_all_named_results() - for _,cur_res in ipairs(named_results) do - exec_symbol(cur_res) - end - else - -- Invoked from another meta - exec_symbol(res_name) - return cached[k][res_name] or 0 + if not cached[k] then + cached[k] = {} end - else - -- We have cached the result - local res = already_processed[res_name or 'default'] or 0 - lua_util.debugm(N, task, 'cached meta result for %s: %s; result name: %s', - k, res, res_name) - if res_name then - return res + cached[k][cur_res] = res + end + + if not res_name then + -- Invoke for all named results + local named_results = task:get_all_named_results() + for _, cur_res in ipairs(named_results) do + exec_symbol(cur_res) end + else + -- Invoked from another meta + exec_symbol(res_name) + return cached[k][res_name] or 0 end + else + -- We have cached the result + local res = already_processed[res_name or 'default'] or 0 + lua_util.debugm(N, task, 'cached meta result for %s: %s; result name: %s', + k, res, res_name) - -- No return if invoked directly from Rspamd as we use task:insert_result_named directly + if res_name then + return res + end end - expression = rspamd_expression.create(r['meta'], parse_atom, rspamd_config:get_mempool()) - if not expression then - rspamd_logger.errx(rspamd_config, 'Cannot parse expression ' .. r['meta']) - else + -- No return if invoked directly from Rspamd as we use task:insert_result_named directly + end - if r['score'] then - rspamd_config:set_metric_symbol{ - name = k, score = r['score'], - description = r['description'], - priority = scores_priority, - one_shot = true - } - scores_added[k] = 1 - rspamd_config:register_symbol{ - name = k, - weight = calculate_score(k, r), - callback = meta_cb - } - else - -- Add 0 score to avoid issues - rspamd_config:register_symbol{ - name = k, - weight = calculate_score(k, r), - callback = meta_cb, - score = 0, - } - end + expression = rspamd_expression.create(r['meta'], parse_atom, rspamd_config:get_mempool()) + if not expression then + rspamd_logger.errx(rspamd_config, 'Cannot parse expression ' .. r['meta']) + else - r['expression'] = expression + if r['score'] then + rspamd_config:set_metric_symbol { + name = k, score = r['score'], + description = r['description'], + priority = scores_priority, + one_shot = true + } + scores_added[k] = 1 + rspamd_config:register_symbol { + name = k, + weight = calculate_score(k, r), + callback = meta_cb + } + else + -- Add 0 score to avoid issues + rspamd_config:register_symbol { + name = k, + weight = calculate_score(k, r), + callback = meta_cb, + score = 0, + } + end - if not atoms[k] then - atoms[k] = meta_cb - end + r['expression'] = expression + + if not atoms[k] then + atoms[k] = meta_cb end - end, - fun.filter(function(_, r) + end + end, + fun.filter(function(_, r) return r['type'] == 'meta' end, - rules)) + rules)) -- Check meta rules for foreign symbols and register dependencies -- First direct dependencies: fun.each(function(k, r) - if r['expression'] then - local expr_atoms = r['expression']:atoms() - - for _,a in ipairs(expr_atoms) do - if not atoms[a] then - local rspamd_symbol = replace_symbol(a) - if not external_deps[k] then - external_deps[k] = {} - end + if r['expression'] then + local expr_atoms = r['expression']:atoms() + + for _, a in ipairs(expr_atoms) do + if not atoms[a] then + local rspamd_symbol = replace_symbol(a) + if not external_deps[k] then + external_deps[k] = {} + end - if not external_deps[k][rspamd_symbol] then - rspamd_config:register_dependency(k, rspamd_symbol) - external_deps[k][rspamd_symbol] = true - lua_util.debugm(N, rspamd_config, + if not external_deps[k][rspamd_symbol] then + rspamd_config:register_dependency(k, rspamd_symbol) + external_deps[k][rspamd_symbol] = true + lua_util.debugm(N, rspamd_config, 'atom %1 is a direct foreign dependency, ' .. - 'register dependency for %2 on %3', + 'register dependency for %2 on %3', a, k, rspamd_symbol) - end end end end - end, - fun.filter(function(_, r) - return r['type'] == 'meta' - end, - rules)) + end + end, + fun.filter(function(_, r) + return r['type'] == 'meta' + end, + rules)) -- ... And then indirect ones ... local nchanges repeat - nchanges = 0 + nchanges = 0 fun.each(function(k, r) if r['expression'] then local expr_atoms = r['expression']:atoms() - for _,a in ipairs(expr_atoms) do + for _, a in ipairs(expr_atoms) do if type(external_deps[a]) == 'table' then for dep in pairs(external_deps[a]) do if not external_deps[k] then @@ -1629,16 +1659,16 @@ local function post_process() rspamd_config:register_dependency(k, dep) external_deps[k][dep] = true lua_util.debugm(N, rspamd_config, - 'atom %1 is an indirect foreign dependency, ' .. - 'register dependency for %2 on %3', - a, k, dep) - nchanges = nchanges + 1 + 'atom %1 is an indirect foreign dependency, ' .. + 'register dependency for %2 on %3', + a, k, dep) + nchanges = nchanges + 1 end end else local rspamd_symbol, replaced_symbol = replace_symbol(a) if replaced_symbol then - external_deps[a] = {[rspamd_symbol] = true} + external_deps[a] = { [rspamd_symbol] = true } else external_deps[a] = {} end @@ -1646,18 +1676,18 @@ local function post_process() end end end, - fun.filter(function(_, r) - return r['type'] == 'meta' - end, - rules)) + fun.filter(function(_, r) + return r['type'] == 'meta' + end, + rules)) until nchanges == 0 -- Set missing symbols fun.each(function(key, score) if not scores_added[key] then rspamd_config:set_metric_symbol({ - name = key, score = score, - priority = 2, flags = 'ignore'}) + name = key, score = score, + priority = 2, flags = 'ignore' }) end end, scores) @@ -1665,7 +1695,7 @@ local function post_process() if freemail_domains then freemail_trie = rspamd_trie.create(freemail_domains) rspamd_logger.infox(rspamd_config, 'loaded %1 freemail domains definitions', - #freemail_domains) + #freemail_domains) end rspamd_logger.infox(rspamd_config, 'loaded %1 blacklist/whitelist elements', sa_lists['elts']) @@ -1675,10 +1705,18 @@ local has_rules = false if type(section) == "table" then local keywords = { - pcre_only = {'table', function(v) pcre_only_regexps = lua_util.list_to_hash(v) end}, - alpha = {'number', function(v) meta_score_alpha = tonumber(v) end}, - match_limit = {'number', function(v) match_limit = tonumber(v) end}, - scores_priority = {'number', function(v) scores_priority = tonumber(v) end}, + pcre_only = { 'table', function(v) + pcre_only_regexps = lua_util.list_to_hash(v) + end }, + alpha = { 'number', function(v) + meta_score_alpha = tonumber(v) + end }, + match_limit = { 'number', function(v) + match_limit = tonumber(v) + end }, + scores_priority = { 'number', function(v) + scores_priority = tonumber(v) + end }, } for k, fn in pairs(section) do @@ -1694,7 +1732,7 @@ if type(section) == "table" then if not files or #files == 0 then rspamd_logger.errx(rspamd_config, "cannot find any files matching pattern %s", elt) else - for _,matched in ipairs(files) do + for _, matched in ipairs(files) do local f = io.open(matched, "r") if f then rspamd_logger.infox(rspamd_config, 'loading SA rules from %s', matched) @@ -1713,7 +1751,7 @@ if type(section) == "table" then if not files or #files == 0 then rspamd_logger.errx(rspamd_config, "cannot find any files matching pattern %s", fn) else - for _,matched in ipairs(files) do + for _, matched in ipairs(files) do local f = io.open(matched, "r") if f then rspamd_logger.infox(rspamd_config, 'loading SA rules from %s', matched) diff --git a/src/plugins/lua/spamtrap.lua b/src/plugins/lua/spamtrap.lua index 0f0bbe813..cd3b2968f 100644 --- a/src/plugins/lua/spamtrap.lua +++ b/src/plugins/lua/spamtrap.lua @@ -53,8 +53,8 @@ local function spamtrap_cb(task) local function do_action(rcpt) if settings['learn_fuzzy'] then rspamd_plugins.fuzzy_check.learn(task, - settings['fuzzy_flag'], - settings['fuzzy_weight']) + settings['fuzzy_flag'], + settings['fuzzy_weight']) end local act_flags = '' if settings['learn_spam'] then @@ -68,7 +68,7 @@ local function spamtrap_cb(task) rspamd_logger.infox(task, 'spamtrap found: <%s>', rcpt) local smtp_message if settings.smtp_message then - smtp_message = lua_util.template(settings.smtp_message, { rcpt = rcpt}) + smtp_message = lua_util.template(settings.smtp_message, { rcpt = rcpt }) else smtp_message = 'unknown error' if settings.action == 'no action' then @@ -77,10 +77,10 @@ local function spamtrap_cb(task) smtp_message = 'message rejected' end end - task:set_pre_result{action = settings.action, - message = smtp_message, - module = 'spamtrap', - flags = act_flags} + task:set_pre_result { action = settings.action, + message = smtp_message, + module = 'spamtrap', + flags = act_flags } end return true @@ -106,7 +106,7 @@ local function spamtrap_cb(task) false, -- is write gen_redis_spamtrap_cb(target), -- callback 'GET', -- command - {key} -- arguments + { key } -- arguments ) if not ret then rspamd_logger.errx(task, "redis request wasn't scheduled") @@ -120,8 +120,10 @@ local function spamtrap_cb(task) end -- Do not risk a FP by checking for more than one recipient - if rcpts and (#rcpts == 1 or (#rcpts > 0 and settings.allow_multiple_rcpts)) then - local targets = fun.map(function(r) return r['addr']:lower() end, rcpts) + if rcpts and (#rcpts == 1 or (#rcpts > 0 and settings.allow_multiple_rcpts)) then + local targets = fun.map(function(r) + return r['addr']:lower() + end, rcpts) if use_redis then fun.each(function(target) local key = settings['key_prefix'] .. target @@ -131,7 +133,7 @@ local function spamtrap_cb(task) false, -- is write gen_redis_spamtrap_cb(target), -- callback 'GET', -- command - {key} -- arguments + { key } -- arguments ) if not ret then rspamd_logger.errx(task, "redis request wasn't scheduled") @@ -159,18 +161,17 @@ if not (opts and type(opts) == 'table') then return end - local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, 'spamtrap', false, false) check_local = auth_and_local_conf[1] check_authed = auth_and_local_conf[2] if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do settings[k] = v end if settings['map'] then - settings['map'] = rspamd_config:add_map{ + settings['map'] = rspamd_config:add_map { url = settings['map'], description = string.format("Spamtrap map for %s", settings['symbol']), type = "regexp" @@ -179,7 +180,7 @@ if opts then redis_params = rspamd_parse_redis_server('spamtrap') if not redis_params then rspamd_logger.errx( - rspamd_config, 'no redis servers are specified, disabling module') + rspamd_config, 'no redis servers are specified, disabling module') return end use_redis = true; diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua index 997c07bca..5e151281a 100644 --- a/src/plugins/lua/spf.lua +++ b/src/plugins/lua/spf.lua @@ -88,7 +88,7 @@ local function spf_check_callback(task) local rh = task:get_received_headers() or {} local found = false - for i,hdr in ipairs(rh) do + for i, hdr in ipairs(rh) do if hdr.real_ip and local_config.external_relay:get_key(hdr.real_ip) then -- We can use the next header as a source of IP address if rh[i + 1] then @@ -129,16 +129,16 @@ local function spf_check_callback(task) local function policy_decode(res) if res == rspamd_spf.policy.fail then - return local_config.symbols.fail,'-' + return local_config.symbols.fail, '-' elseif res == rspamd_spf.policy.pass then - return local_config.symbols.allow,'+' + return local_config.symbols.allow, '+' elseif res == rspamd_spf.policy.soft_fail then - return local_config.symbols.softfail,'~' + return local_config.symbols.softfail, '~' elseif res == rspamd_spf.policy.neutral then - return local_config.symbols.neutral,'?' + return local_config.symbols.neutral, '?' end - return 'SPF_UNKNOWN','?' + return 'SPF_UNKNOWN', '?' end local function spf_resolved_cb(record, flags, err) @@ -153,7 +153,7 @@ local function spf_check_callback(task) ip, flags, err, error_or_addr) if result then - local sym,code = policy_decode(flag_or_policy) + local sym, code = policy_decode(flag_or_policy) local opt = string.format('%s%s', code, error_or_addr.str or '???') if bit.band(flags, rspamd_spf.flags.cached) ~= 0 then opt = opt .. ':c' @@ -203,17 +203,17 @@ end -- Register all symbols and init rspamd_spf library rspamd_spf.config(local_config) -local sym_id = rspamd_config:register_symbol{ +local sym_id = rspamd_config:register_symbol { name = 'SPF_CHECK', type = 'callback', flags = 'fine,empty', - groups = {'policies','spf'}, + groups = { 'policies', 'spf' }, score = 0.0, callback = spf_check_callback, -- We can merely estimate timeout here, as it is possible to construct an SPF record that would cause -- many DNS requests. But we won't like to set the maximum value for that all the time, as -- the majority of requests will typically have 1-4 subrequests - augmentations = {string.format("timeout=%f", rspamd_config:get_dns_timeout() * 4 or 0.0)}, + augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() * 4 or 0.0) }, } if local_config.whitelist then @@ -227,15 +227,15 @@ if local_config.external_relay then local lua_maps = require "lua_maps" local_config.external_relay = lua_maps.map_add_from_ucl(local_config.external_relay, - "radix", "External IP SPF map") + "radix", "External IP SPF map") end -for _,sym in pairs(local_config.symbols) do - rspamd_config:register_symbol{ +for _, sym in pairs(local_config.symbols) do + rspamd_config:register_symbol { name = sym, type = 'virtual', parent = sym_id, - groups = {'policies', 'spf'}, + groups = { 'policies', 'spf' }, } end diff --git a/src/plugins/lua/trie.lua b/src/plugins/lua/trie.lua index 0adf12dc1..7ba455289 100644 --- a/src/plugins/lua/trie.lua +++ b/src/plugins/lua/trie.lua @@ -56,14 +56,14 @@ local function tries_callback(task) params = body_params end - return function (idx, pos) + return function(idx, pos) local param = params[idx] local pattern = patterns[idx] local pattern_idx = pattern .. tostring(idx) .. type if param['multi'] or not matched[pattern_idx] then lua_util.debugm(N, task, "<%1> matched pattern %2 at pos %3", - task:get_message_id(), pattern, pos) + task:get_message_id(), pattern, pos) task:insert_result(param['symbol'], 1.0, type) if not param['multi'] then matched[pattern_idx] = true @@ -86,17 +86,19 @@ end local function process_single_pattern(pat, symbol, cf) if pat then local multi = false - if cf['multi'] then multi = true end + if cf['multi'] then + multi = true + end if cf['raw'] then table.insert(raw_patterns, pat) - table.insert(raw_params, {symbol=symbol, multi=multi}) + table.insert(raw_params, { symbol = symbol, multi = multi }) elseif cf['body'] then table.insert(body_patterns, pat) - table.insert(body_params, {symbol=symbol, multi=multi}) + table.insert(body_params, { symbol = symbol, multi = multi }) else table.insert(mime_patterns, pat) - table.insert(mime_params, {symbol=symbol, multi=multi}) + table.insert(mime_params, { symbol = symbol, multi = multi }) end end end @@ -109,7 +111,7 @@ local function process_trie_file(symbol, cf) else if cf['binary'] then rspamd_logger.errx(rspamd_config, 'binary trie patterns are not implemented yet: %1', - cf['file']) + cf['file']) else for line in file:lines() do local pat = string.match(line, '^([^#].*[^%s])%s*$') @@ -122,7 +124,7 @@ end local function process_trie_conf(symbol, cf) if type(cf) ~= 'table' then rspamd_logger.errx(rspamd_config, 'invalid value for symbol %1: "%2", expected table', - symbol, cf) + symbol, cf) return end @@ -135,16 +137,16 @@ local function process_trie_conf(symbol, cf) end end -local opts = rspamd_config:get_all_opt("trie") +local opts = rspamd_config:get_all_opt("trie") if opts then for sym, opt in pairs(opts) do - process_trie_conf(sym, opt) + process_trie_conf(sym, opt) end if #raw_patterns > 0 then raw_trie = rspamd_trie.create(raw_patterns) rspamd_logger.infox(rspamd_config, 'registered raw search trie from %1 patterns', #raw_patterns) - end + end if #mime_patterns > 0 then mime_trie = rspamd_trie.create(mime_patterns) diff --git a/src/plugins/lua/url_redirector.lua b/src/plugins/lua/url_redirector.lua index c5748647d..19189a5ee 100644 --- a/src/plugins/lua/url_redirector.lua +++ b/src/plugins/lua/url_redirector.lua @@ -60,7 +60,7 @@ local settings = { local function adjust_url(task, orig_url, redir_url) if type(redir_url) == 'string' then - redir_url = rspamd_url.create(task:get_mempool(), redir_url, {'redirect_target'}) + redir_url = rspamd_url.create(task:get_mempool(), redir_url, { 'redirect_target' }) end if redir_url then @@ -90,7 +90,7 @@ local function cache_url(task, orig_url, url, key, prefix) rspamd_logger.errx(task, 'got error while getting top urls count: %s', err) else rspamd_logger.infox(task, 'trimmed url set to %s elements', - settings.top_urls_count) + settings.top_urls_count) end end @@ -102,20 +102,20 @@ local function cache_url(task, orig_url, url, key, prefix) if data then if tonumber(data) > settings.top_urls_count * 2 then local ret = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_trim_cb, --callback - 'ZREMRANGEBYRANK', -- command - {settings.top_urls_key, '0', - tostring(-(settings.top_urls_count + 1))} -- arguments + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_trim_cb, --callback + 'ZREMRANGEBYRANK', -- command + { settings.top_urls_key, '0', + tostring(-(settings.top_urls_count + 1)) } -- arguments ) if not ret then rspamd_logger.errx(task, 'cannot trim top urls set') else rspamd_logger.infox(task, 'need to trim urls set from %s to %s elements', - data, - settings.top_urls_count) + data, + settings.top_urls_count) return end end @@ -128,12 +128,12 @@ local function cache_url(task, orig_url, url, key, prefix) rspamd_logger.errx(task, 'got error while setting redirect keys: %s', err) else local ret = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_card_cb, --callback - 'ZCARD', -- command - {settings.top_urls_key} -- arguments + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_card_cb, --callback + 'ZCARD', -- command + { settings.top_urls_key } -- arguments ) if not ret then rspamd_logger.errx(task, 'cannot make redis request to cache results') @@ -145,25 +145,27 @@ local function cache_url(task, orig_url, url, key, prefix) -- Save url with prefix str_url = string.format('^%s:%s', prefix, str_url) end - local ret,conn,_ = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_set_cb, --callback - 'SETEX', -- command - {key, tostring(settings.expire), str_url} -- arguments + local ret, conn, _ = lua_redis.redis_make_request(task, + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_set_cb, --callback + 'SETEX', -- command + { key, tostring(settings.expire), str_url } -- arguments ) if not ret then rspamd_logger.errx(task, 'cannot make redis request to cache results') else - conn:add_cmd('ZINCRBY', {settings.top_urls_key, '1', str_url}) + conn:add_cmd('ZINCRBY', { settings.top_urls_key, '1', str_url }) end end -- Reduce length of a string to a given length (16 by default) local function maybe_trim_url(url, limit) - if not limit then limit = 16 end + if not limit then + limit = 16 + end if #url > limit then return string.sub(url, 1, limit) .. '...' else @@ -179,7 +181,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) if ntries > settings.nested_limit then -- We cannot resolve more, stop rspamd_logger.debugm(N, task, 'cannot get more requests to resolve %s, stop on %s after %s attempts', - orig_url, url, ntries) + orig_url, url, ntries) cache_url(task, orig_url, url, key, 'nested') task:insert_result(settings.redirector_symbol_nested, 1.0, string.format('%s->%s:%d', maybe_trim_url(orig_url), maybe_trim_url(url), ntries)) @@ -198,16 +200,16 @@ local function resolve_cached(task, orig_url, url, key, ntries) local function http_callback(err, code, _, headers) if err then rspamd_logger.infox(task, 'found redirect error from %s to %s, err message: %s', - orig_url, url, err) + orig_url, url, err) cache_url(task, orig_url, url, key) else if code == 200 then if orig_url == url then rspamd_logger.infox(task, 'direct url %s, err code 200', - url) + url) else rspamd_logger.infox(task, 'found redirect from %s to %s, err code 200', - orig_url, url) + orig_url, url) end cache_url(task, orig_url, url, key) @@ -219,7 +221,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) redir_url = rspamd_url.create(task:get_mempool(), loc) end rspamd_logger.debugm(N, task, 'found redirect from %s to %s, err code %s', - orig_url, loc, code) + orig_url, loc, code) if redir_url then if settings.redirectors_only then @@ -227,7 +229,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) resolve_cached(task, orig_url, redir_url, key, ntries + 1) else lua_util.debugm(N, task, - "stop resolving redirects as %s is not a redirector", loc) + "stop resolving redirects as %s is not a redirector", loc) cache_url(task, orig_url, redir_url, key) end else @@ -239,7 +241,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) end else rspamd_logger.debugm(N, task, 'found redirect error from %s to %s, err code: %s', - orig_url, url, code) + orig_url, url, code) cache_url(task, orig_url, url, key) end end @@ -254,7 +256,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) lua_util.debugm(N, task, 'select user agent %s', ua) - rspamd_http.request{ + rspamd_http.request { headers = { ['User-Agent'] = ua, }, @@ -274,7 +276,7 @@ local function resolve_cached(task, orig_url, url, key, ntries) if data ~= 'processing' then -- Got cached result rspamd_logger.debugm(N, task, 'found cached redirect from %s to %s', - url, data) + url, data) if data.sub(1, 1) == '^' then -- Prefixed url stored local prefix, new_url = data:match('^%^(%a+):(.+)$') @@ -302,12 +304,12 @@ local function resolve_cached(task, orig_url, url, key, ntries) if ntries == 1 then -- Reserve key in Redis that we are processing this redirection local ret = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - true, -- is write - redis_reserve_cb, --callback - 'SET', -- command - {key, 'processing', 'EX', tostring(settings.timeout * 2), 'NX'} -- arguments + redis_params, -- connect params + key, -- hash key + true, -- is write + redis_reserve_cb, --callback + 'SET', -- command + { key, 'processing', 'EX', tostring(settings.timeout * 2), 'NX' } -- arguments ) if not ret then rspamd_logger.errx(task, 'Couldn\'t schedule SET') @@ -319,12 +321,12 @@ local function resolve_cached(task, orig_url, url, key, ntries) end local ret = lua_redis.redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_get_cb, --callback - 'GET', -- command - {key} -- arguments + redis_params, -- connect params + key, -- hash key + false, -- is write + redis_get_cb, --callback + 'GET', -- command + { key } -- arguments ) if not ret then rspamd_logger.errx(task, 'cannot make redis request to check results') @@ -354,13 +356,13 @@ local function url_redirector_handler(task) }) if sp_urls then - for _,u in ipairs(sp_urls) do + for _, u in ipairs(sp_urls) do url_redirector_process_url(task, u) end end end -local opts = rspamd_config:get_all_opt('url_redirector') +local opts = rspamd_config:get_all_opt('url_redirector') if opts then settings = lua_util.override_defaults(settings, opts) redis_params = lua_redis.parse_redis_server('url_redirector', settings) @@ -388,16 +390,16 @@ if opts then type = 'zlist', }) end - local id = rspamd_config:register_symbol{ + local id = rspamd_config:register_symbol { name = 'URL_REDIRECTOR_CHECK', type = 'callback,prefilter', priority = lua_util.symbols_priorities.medium, callback = url_redirector_handler, -- In fact, the real timeout is nested_limit * timeout... - augmentations = {string.format("timeout=%f", settings.timeout)} + augmentations = { string.format("timeout=%f", settings.timeout) } } - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = settings.redirector_symbol_nested, type = 'virtual', parent = id, @@ -405,7 +407,7 @@ if opts then } if settings.redirector_symbol then - rspamd_config:register_symbol{ + rspamd_config:register_symbol { name = settings.redirector_symbol, type = 'virtual', parent = id, diff --git a/src/plugins/lua/whitelist.lua b/src/plugins/lua/whitelist.lua index fc339c6ce..fa76da8d1 100644 --- a/src/plugins/lua/whitelist.lua +++ b/src/plugins/lua/whitelist.lua @@ -45,66 +45,70 @@ local function whitelist_cb(symbol, rule, task) local how = 'wl' -- Can be overridden - if rule.blacklist then how = 'bl' end + if rule.blacklist then + how = 'bl' + end local function parse_val(val) local how_override -- Strict is 'special' - if rule.strict then how_override = 'both' end + if rule.strict then + how_override = 'both' + end if val then lua_util.debugm(N, task, "found whitelist key: %s=%s", dom, val) if val == '' then - return (how_override or how),1.0 + return (how_override or how), 1.0 elseif val:match('^bl:') then - return (how_override or 'bl'),(tonumber(val:sub(4)) or 1.0) + return (how_override or 'bl'), (tonumber(val:sub(4)) or 1.0) elseif val:match('^wl:') then - return (how_override or 'wl'),(tonumber(val:sub(4)) or 1.0) + return (how_override or 'wl'), (tonumber(val:sub(4)) or 1.0) elseif val:match('^both:') then - return (how_override or 'both'),(tonumber(val:sub(6)) or 1.0) + return (how_override or 'both'), (tonumber(val:sub(6)) or 1.0) else - return (how_override or how),(tonumber(val) or 1.0) + return (how_override or how), (tonumber(val) or 1.0) end end - return (how_override or how),1.0 + return (how_override or how), 1.0 end if rule['map'] then local val = rule['map']:get_key(dom) if val then - how,mult = parse_val(val) + how, mult = parse_val(val) if not domains[check] then domains[check] = {} end domains[check] = { - [dom] = {how, mult} + [dom] = { how, mult } } lua_util.debugm(N, task, "final result: %s: %s->%s", dom, how, mult) - return true,mult,how + return true, mult, how end elseif rule['maps'] then - for _,v in pairs(rule['maps']) do + for _, v in pairs(rule['maps']) do local map = v.map if map then local val = map:get_key(dom) if val then - how,mult = parse_val(val) + how, mult = parse_val(val) if not domains[check] then domains[check] = {} end domains[check] = { - [dom] = {how, mult} + [dom] = { how, mult } } lua_util.debugm(N, task, "final result: %s: %s->%s", dom, how, mult) - return true,mult,how + return true, mult, how end end end @@ -116,14 +120,14 @@ local function whitelist_cb(symbol, rule, task) end domains[check] = { - [dom] = {how, mult} + [dom] = { how, mult } } - return true, mult,how + return true, mult, how end end - return false,0.0,how + return false, 0.0, how end local spf_violated = false @@ -164,14 +168,14 @@ local function whitelist_cb(symbol, rule, task) local dkim_opts = sym[1]['options'] if dkim_opts then fun.each(function(val) - if val[2] == '+' then - local tld = rspamd_util.get_tld(val[1]) - find_domain(tld, 'dkim_success') - elseif val[2] == '-' then - local tld = rspamd_util.get_tld(val[1]) - find_domain(tld, 'dkim_fail') - end - end, + if val[2] == '+' then + local tld = rspamd_util.get_tld(val[1]) + find_domain(tld, 'dkim_success') + elseif val[2] == '-' then + local tld = rspamd_util.get_tld(val[1]) + find_domain(tld, 'dkim_fail') + end + end, fun.map(function(s) return lua_util.rspamd_str_split(s, ':') end, dkim_opts)) @@ -198,7 +202,6 @@ local function whitelist_cb(symbol, rule, task) end end - local final_mult = 1.0 local found_wl, found_bl = false, false local opts = {} @@ -206,7 +209,7 @@ local function whitelist_cb(symbol, rule, task) if rule.valid_dkim then dkim_violated = true - for dom,val in pairs(domains.dkim_success or E) do + for dom, val in pairs(domains.dkim_success or E) do if val[1] == 'wl' or val[1] == 'both' then -- We have valid and whitelisted signature table.insert(opts, dom .. ':d:+') @@ -220,7 +223,7 @@ local function whitelist_cb(symbol, rule, task) end -- Blacklist counterpart - for dom,val in pairs(domains.dkim_fail or E) do + for dom, val in pairs(domains.dkim_fail or E) do if val[1] == 'bl' or val[1] == 'both' then -- We have valid and whitelisted signature table.insert(opts, dom .. ':d:-') @@ -255,7 +258,7 @@ local function whitelist_cb(symbol, rule, task) found_wl = false - for dom,val in pairs(domains.dmarc or E) do + for dom, val in pairs(domains.dmarc or E) do check_domain_violation('D', dom, val, (dmarc_violated or dkim_violated)) end @@ -264,7 +267,7 @@ local function whitelist_cb(symbol, rule, task) if rule.valid_spf then found_wl = false - for dom,val in pairs(domains.spf or E) do + for dom, val in pairs(domains.spf or E) do check_domain_violation('s', dom, val, (spf_violated or dkim_violated)) end @@ -318,7 +321,7 @@ end local configure_whitelist_module = function() local opts = rspamd_config:get_all_opt('whitelist') if opts then - for k,v in pairs(opts) do + for k, v in pairs(opts) do options[k] = v end @@ -335,7 +338,7 @@ local configure_whitelist_module = function() fun.each(function(symbol, rule) if rule['domains'] then if type(rule['domains']) == 'string' then - rule['map'] = rspamd_config:add_map{ + rule['map'] = rspamd_config:add_map { url = rule['domains'], description = "Whitelist map for " .. symbol, type = 'map' @@ -347,7 +350,7 @@ local configure_whitelist_module = function() if type(v) == 'table' then return true elseif type(v) == 'string' and not (string.match(v, '^https?://') or - string.match(v, '^ftp://') or string.match(v, '^[./]')) then + string.match(v, '^ftp://') or string.match(v, '^[./]')) then return true end @@ -357,13 +360,13 @@ local configure_whitelist_module = function() if is_domains_list then rule['domains'] = fun.tomap(fun.map(function(d) if type(d) == 'table' then - return d[1],d[2] + return d[1], d[2] end - return d,1.0 + return d, 1.0 end, rule['domains'])) else - rule['map'] = rspamd_config:add_map{ + rule['map'] = rspamd_config:add_map { url = rule['domains'], description = "Whitelist map for " .. symbol, type = 'map' @@ -371,7 +374,7 @@ local configure_whitelist_module = function() end else rspamd_logger.errx(rspamd_config, 'whitelist %s has bad "domains" value', - symbol) + symbol) return end |