diff options
author | Vsevolod Stakhov <vsevolod@rspamd.com> | 2024-09-05 16:20:46 +0600 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-09-05 16:20:46 +0600 |
commit | 737a2ce03b826f86851d021d628274ab8c8ea7fb (patch) | |
tree | 7918e548808c5981fab465726f545a5c6cc9fe49 /src/plugins | |
parent | dd47f82a317ad2ed9a9270c9779bf866ff5989fd (diff) | |
parent | 40a6ddd69be80e6a4ad8a29053bbfa18d24b3bd8 (diff) | |
download | rspamd-737a2ce03b826f86851d021d628274ab8c8ea7fb.tar.gz rspamd-737a2ce03b826f86851d021d628274ab8c8ea7fb.zip |
Merge branch 'master' into vstakhov-utf8-mime
Diffstat (limited to 'src/plugins')
-rw-r--r-- | src/plugins/fuzzy_check.c | 14 | ||||
-rw-r--r-- | src/plugins/lua/arc.lua | 12 | ||||
-rw-r--r-- | src/plugins/lua/gpt.lua | 41 | ||||
-rw-r--r-- | src/plugins/lua/history_redis.lua | 4 | ||||
-rw-r--r-- | src/plugins/lua/known_senders.lua | 3 | ||||
-rw-r--r-- | src/plugins/lua/once_received.lua | 103 | ||||
-rw-r--r-- | src/plugins/lua/spf.lua | 3 |
7 files changed, 82 insertions, 98 deletions
diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index a035eeaae..91b77c702 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -543,15 +543,13 @@ fuzzy_parse_rule(struct rspamd_config *cfg, const ucl_object_t *obj, k = ucl_object_tostring(value); if (k == NULL || (rule->peer_key = - rspamd_pubkey_from_base32(k, 0, RSPAMD_KEYPAIR_KEX, - RSPAMD_CRYPTOBOX_MODE_25519)) == NULL) { + rspamd_pubkey_from_base32(k, 0, RSPAMD_KEYPAIR_KEX)) == NULL) { msg_err_config("bad encryption key value: %s", k); return -1; } - rule->local_key = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX, - RSPAMD_CRYPTOBOX_MODE_25519); + rule->local_key = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX); } if ((value = ucl_object_lookup(obj, "learn_condition")) != NULL) { @@ -1334,8 +1332,7 @@ fuzzy_encrypt_cmd(struct fuzzy_rule *rule, rule->local_key, rule->peer_key); rspamd_cryptobox_encrypt_nm_inplace(data, datalen, hdr->nonce, rspamd_pubkey_get_nm(rule->peer_key, rule->local_key), - hdr->mac, - rspamd_pubkey_alg(rule->peer_key)); + hdr->mac); } static struct fuzzy_cmd_io * @@ -2209,8 +2206,7 @@ fuzzy_process_reply(unsigned char **pos, int *r, GPtrArray *req, sizeof(encrep.rep), encrep.hdr.nonce, rspamd_pubkey_get_nm(rule->peer_key, rule->local_key), - encrep.hdr.mac, - rspamd_pubkey_alg(rule->peer_key))) { + encrep.hdr.mac)) { msg_info("cannot decrypt reply"); return NULL; } @@ -2299,6 +2295,8 @@ fuzzy_insert_result(struct fuzzy_client_session *session, * Otherwise `value` means error code */ + msg_debug_fuzzy_check("got reply with probability %.2f and value %.2f", + (double) rep->v1.prob, (double) rep->v1.value); nval = fuzzy_normalize(rep->v1.value, weight); if (io) { diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua index ff19aef4c..90e254e78 100644 --- a/src/plugins/lua/arc.lua +++ b/src/plugins/lua/arc.lua @@ -635,11 +635,21 @@ local function prepare_arc_selector(task, sel) end end + local function arc_result_from_ar(ar_header) + ar_header = ar_header or "" + for k, v in string.gmatch(ar_header, "(%w+)=(%w+)") do + if k == 'arc' then + return v + end + end + return nil + end + if settings.reuse_auth_results then local ar_header = task:get_header('Authentication-Results') if ar_header then - local arc_match = string.match(ar_header, 'arc=(%w+)') + local arc_match = arc_result_from_ar(ar_header) if arc_match then if arc_match == 'none' or arc_match == 'pass' then diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua index 6adbce3bf..823dbd045 100644 --- a/src/plugins/lua/gpt.lua +++ b/src/plugins/lua/gpt.lua @@ -27,13 +27,11 @@ gpt { # Your key to access the API api_key = "xxx"; # Model name - model = "gpt-3.5-turbo"; + model = "gpt-4o-mini"; # Maximum tokens to generate max_tokens = 1000; # Temperature for sampling - temperature = 0.7; - # Top p for sampling - top_p = 0.9; + temperature = 0.0; # Timeout for requests timeout = 10s; # Prompt for the model (use default if not set) @@ -71,10 +69,9 @@ local default_symbols_to_except = { local settings = { type = 'openai', api_key = nil, - model = 'gpt-3.5-turbo', + model = 'gpt-4o-mini', max_tokens = 1000, - temperature = 0.7, - top_p = 0.9, + temperature = 0.0, timeout = 10, prompt = nil, condition = nil, @@ -97,11 +94,11 @@ local function default_condition(task) local action = result.action if action == 'reject' and result.npositive > 1 then - return true, 'already decided as spam' + return false, 'already decided as spam' end if action == 'no action' and score < 0 then - return true, 'negative score, already decided as ham' + return false, 'negative score, already decided as ham' end end -- We also exclude some symbols @@ -109,10 +106,12 @@ local function default_condition(task) if task:has_symbol(s) then if required_weight > 0 then -- Also check score - local sym = task:get_symbol(s) + local sym = task:get_symbol(s) or E -- Must exist as we checked it before with `has_symbol` - if math.abs(sym.weight) >= required_weight then - return false, 'skip as "' .. s .. '" is found (weight: ' .. sym.weight .. ')' + if sym.weight then + if math.abs(sym.weight) >= required_weight then + return false, 'skip as "' .. s .. '" is found (weight: ' .. sym.weight .. ')' + end end lua_util.debugm(N, task, 'symbol %s has weight %s, but required %s', s, sym.weight, required_weight) @@ -195,6 +194,18 @@ local function default_conversion(task, input) if type(reply) == 'table' and reply.probability then local spam_score = tonumber(reply.probability) + + if not spam_score then + -- Maybe we need GPT to convert GPT reply here? + if reply.probability == "high" then + spam_score = 0.9 + elseif reply.probability == "low" then + spam_score = 0.1 + else + rspamd_logger.infox("cannot convert to spam probability: %s", reply.probability) + end + end + if type(reply.usage) == 'table' then rspamd_logger.infox(task, 'usage: %s tokens', reply.usage.total_tokens) end @@ -276,7 +287,7 @@ local function openai_gpt_check(task) model = settings.model, max_tokens = settings.max_tokens, temperature = settings.temperature, - top_p = settings.top_p, + response_format = { type = "json_object" }, messages = { { role = 'system', @@ -348,8 +359,8 @@ if opts then end if not settings.prompt then - settings.prompt = "You will be provided with the email message, " .. - "and your task is to classify its probability to be spam, " .. + settings.prompt = "You will be provided with the email message, subject, from and url domains, " .. + "and your task is to evaluate the probability to be spam as number from 0 to 1, " .. "output result as JSON with 'probability' field." end diff --git a/src/plugins/lua/history_redis.lua b/src/plugins/lua/history_redis.lua index 3365b30cd..fff9f46b3 100644 --- a/src/plugins/lua/history_redis.lua +++ b/src/plugins/lua/history_redis.lua @@ -21,8 +21,8 @@ if confighelp then redis_history { # History key name key_prefix = 'rs_history{{HOSTNAME}}{{COMPRESS}}'; - # History expire in seconds - expire = 0; + # Expire in seconds for inactive keys, default to 5 days + expire = 432000; # History rows limit nrows = 200; # Use zstd compression when storing data in redis diff --git a/src/plugins/lua/known_senders.lua b/src/plugins/lua/known_senders.lua index 6d57acea3..5cb2ddcf5 100644 --- a/src/plugins/lua/known_senders.lua +++ b/src/plugins/lua/known_senders.lua @@ -18,6 +18,7 @@ limitations under the License. local rspamd_logger = require "rspamd_logger" local N = 'known_senders' +local E = {} local lua_util = require "lua_util" local lua_redis = require "lua_redis" local lua_maps = require "lua_maps" @@ -258,7 +259,7 @@ local function verify_local_replies_set(task) return nil end - local replies_recipients = task:get_recipients('mime') + local replies_recipients = task:get_recipients('mime') or E local replies_sender_string = lua_util.maybe_obfuscate_string(tostring(replies_sender), settings, settings.sender_prefix) diff --git a/src/plugins/lua/once_received.lua b/src/plugins/lua/once_received.lua index 2a5552ab9..5c5ff7986 100644 --- a/src/plugins/lua/once_received.lua +++ b/src/plugins/lua/once_received.lua @@ -19,10 +19,7 @@ if confighelp then end -- 0 or 1 received: = spam - local symbol = 'ONCE_RECEIVED' -local symbol_rdns = 'RDNS_NONE' -local symbol_rdns_dnsfail = 'RDNS_DNSFAIL' local symbol_mx = 'DIRECT_TO_MX' -- Symbol for strict checks local symbol_strict = nil @@ -47,54 +44,6 @@ local function check_quantity_received (task) return not h['flags']['artificial'] end, recvh)) - local function recv_dns_cb(_, to_resolve, results, err) - if err and (err ~= 'requested record is not found' and err ~= 'no records with this name') then - rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, err) - task:insert_result(symbol_rdns_dnsfail, 1.0) - end - - if not results then - if nreceived <= 1 then - task:insert_result(symbol, 1) - -- Avoid strict symbol inserting as the remaining symbols have already - -- quote a significant weight, so a message could be rejected by just - -- this property. - --task:insert_result(symbol_strict, 1) - -- Check for MUAs - local ua = task:get_header('User-Agent') - local xm = task:get_header('X-Mailer') - if (ua or xm) then - task:insert_result(symbol_mx, 1, (ua or xm)) - end - end - task:insert_result(symbol_rdns, 1) - else - rspamd_logger.infox(task, 'source hostname has not been passed to Rspamd from MTA, ' .. - 'but we could resolve source IP address PTR %s as "%s"', - to_resolve, results[1]) - task:set_hostname(results[1]) - - if good_hosts then - for _, gh in ipairs(good_hosts) do - if string.find(results[1], gh) then - return - end - end - end - - if nreceived <= 1 then - task:insert_result(symbol, 1) - for _, h in ipairs(bad_hosts) do - if string.find(results[1], h) then - - task:insert_result(symbol_strict, 1, h) - return - end - end - end - end - end - local task_ip = task:get_ip() if ((not check_authed and task:get_user()) or @@ -110,13 +59,39 @@ local function check_quantity_received (task) local hn = task:get_hostname() -- Here we don't care about received - if (not hn) and task_ip and task_ip:is_valid() then - task:get_resolver():resolve_ptr({ task = task, - name = task_ip:to_string(), - callback = recv_dns_cb, - forced = true - }) + if not hn then + if nreceived <= 1 then + task:insert_result(symbol, 1) + -- Avoid strict symbol inserting as the remaining symbols have already + -- quote a significant weight, so a message could be rejected by just + -- this property. + --task:insert_result(symbol_strict, 1) + -- Check for MUAs + local ua = task:get_header('User-Agent') + local xm = task:get_header('X-Mailer') + if (ua or xm) then + task:insert_result(symbol_mx, 1, (ua or xm)) + end + end return + else + if good_hosts then + for _, gh in ipairs(good_hosts) do + if string.find(hn, gh) then + return + end + end + end + + if nreceived <= 1 then + task:insert_result(symbol, 1) + for _, h in ipairs(bad_hosts) do + if string.find(hn, h) then + task:insert_result(symbol_strict, 1, h) + break + end + end + end end if nreceived <= 1 then @@ -181,10 +156,6 @@ if opts then for n, v in pairs(opts) do if n == 'symbol_strict' then symbol_strict = v - elseif n == 'symbol_rdns' then - symbol_rdns = v - elseif n == 'symbol_rdns_dnsfail' then - symbol_rdns_dnsfail = v elseif n == 'bad_host' then if type(v) == 'string' then bad_hosts[1] = v @@ -207,16 +178,6 @@ if opts then end rspamd_config:register_symbol({ - name = symbol_rdns, - type = 'virtual', - parent = id - }) - rspamd_config:register_symbol({ - name = symbol_rdns_dnsfail, - type = 'virtual', - parent = id - }) - rspamd_config:register_symbol({ name = symbol_strict, type = 'virtual', parent = id diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua index 48f3c17be..356507250 100644 --- a/src/plugins/lua/spf.lua +++ b/src/plugins/lua/spf.lua @@ -56,6 +56,7 @@ local symbols = { dnsfail = "R_SPF_DNSFAIL", permfail = "R_SPF_PERMFAIL", na = "R_SPF_NA", + plusall = "R_SPF_PLUSALL", } local default_config = { @@ -118,6 +119,8 @@ local function spf_check_callback(task) local function flag_to_symbol(fl) if bit.band(fl, rspamd_spf.flags.temp_fail) ~= 0 then return local_config.symbols.dnsfail + elseif bit.band(fl, rspamd_spf.flags.plusall) ~= 0 then + return local_config.symbols.plusall elseif bit.band(fl, rspamd_spf.flags.perm_fail) ~= 0 then return local_config.symbols.permfail elseif bit.band(fl, rspamd_spf.flags.na) ~= 0 then |