aboutsummaryrefslogtreecommitdiffstats
path: root/src/plugins
diff options
context:
space:
mode:
authorVsevolod Stakhov <vsevolod@rspamd.com>2024-09-05 16:20:46 +0600
committerGitHub <noreply@github.com>2024-09-05 16:20:46 +0600
commit737a2ce03b826f86851d021d628274ab8c8ea7fb (patch)
tree7918e548808c5981fab465726f545a5c6cc9fe49 /src/plugins
parentdd47f82a317ad2ed9a9270c9779bf866ff5989fd (diff)
parent40a6ddd69be80e6a4ad8a29053bbfa18d24b3bd8 (diff)
downloadrspamd-737a2ce03b826f86851d021d628274ab8c8ea7fb.tar.gz
rspamd-737a2ce03b826f86851d021d628274ab8c8ea7fb.zip
Merge branch 'master' into vstakhov-utf8-mime
Diffstat (limited to 'src/plugins')
-rw-r--r--src/plugins/fuzzy_check.c14
-rw-r--r--src/plugins/lua/arc.lua12
-rw-r--r--src/plugins/lua/gpt.lua41
-rw-r--r--src/plugins/lua/history_redis.lua4
-rw-r--r--src/plugins/lua/known_senders.lua3
-rw-r--r--src/plugins/lua/once_received.lua103
-rw-r--r--src/plugins/lua/spf.lua3
7 files changed, 82 insertions, 98 deletions
diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index a035eeaae..91b77c702 100644
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -543,15 +543,13 @@ fuzzy_parse_rule(struct rspamd_config *cfg, const ucl_object_t *obj,
k = ucl_object_tostring(value);
if (k == NULL || (rule->peer_key =
- rspamd_pubkey_from_base32(k, 0, RSPAMD_KEYPAIR_KEX,
- RSPAMD_CRYPTOBOX_MODE_25519)) == NULL) {
+ rspamd_pubkey_from_base32(k, 0, RSPAMD_KEYPAIR_KEX)) == NULL) {
msg_err_config("bad encryption key value: %s",
k);
return -1;
}
- rule->local_key = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX,
- RSPAMD_CRYPTOBOX_MODE_25519);
+ rule->local_key = rspamd_keypair_new(RSPAMD_KEYPAIR_KEX);
}
if ((value = ucl_object_lookup(obj, "learn_condition")) != NULL) {
@@ -1334,8 +1332,7 @@ fuzzy_encrypt_cmd(struct fuzzy_rule *rule,
rule->local_key, rule->peer_key);
rspamd_cryptobox_encrypt_nm_inplace(data, datalen,
hdr->nonce, rspamd_pubkey_get_nm(rule->peer_key, rule->local_key),
- hdr->mac,
- rspamd_pubkey_alg(rule->peer_key));
+ hdr->mac);
}
static struct fuzzy_cmd_io *
@@ -2209,8 +2206,7 @@ fuzzy_process_reply(unsigned char **pos, int *r, GPtrArray *req,
sizeof(encrep.rep),
encrep.hdr.nonce,
rspamd_pubkey_get_nm(rule->peer_key, rule->local_key),
- encrep.hdr.mac,
- rspamd_pubkey_alg(rule->peer_key))) {
+ encrep.hdr.mac)) {
msg_info("cannot decrypt reply");
return NULL;
}
@@ -2299,6 +2295,8 @@ fuzzy_insert_result(struct fuzzy_client_session *session,
* Otherwise `value` means error code
*/
+ msg_debug_fuzzy_check("got reply with probability %.2f and value %.2f",
+ (double) rep->v1.prob, (double) rep->v1.value);
nval = fuzzy_normalize(rep->v1.value, weight);
if (io) {
diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua
index ff19aef4c..90e254e78 100644
--- a/src/plugins/lua/arc.lua
+++ b/src/plugins/lua/arc.lua
@@ -635,11 +635,21 @@ local function prepare_arc_selector(task, sel)
end
end
+ local function arc_result_from_ar(ar_header)
+ ar_header = ar_header or ""
+ for k, v in string.gmatch(ar_header, "(%w+)=(%w+)") do
+ if k == 'arc' then
+ return v
+ end
+ end
+ return nil
+ end
+
if settings.reuse_auth_results then
local ar_header = task:get_header('Authentication-Results')
if ar_header then
- local arc_match = string.match(ar_header, 'arc=(%w+)')
+ local arc_match = arc_result_from_ar(ar_header)
if arc_match then
if arc_match == 'none' or arc_match == 'pass' then
diff --git a/src/plugins/lua/gpt.lua b/src/plugins/lua/gpt.lua
index 6adbce3bf..823dbd045 100644
--- a/src/plugins/lua/gpt.lua
+++ b/src/plugins/lua/gpt.lua
@@ -27,13 +27,11 @@ gpt {
# Your key to access the API
api_key = "xxx";
# Model name
- model = "gpt-3.5-turbo";
+ model = "gpt-4o-mini";
# Maximum tokens to generate
max_tokens = 1000;
# Temperature for sampling
- temperature = 0.7;
- # Top p for sampling
- top_p = 0.9;
+ temperature = 0.0;
# Timeout for requests
timeout = 10s;
# Prompt for the model (use default if not set)
@@ -71,10 +69,9 @@ local default_symbols_to_except = {
local settings = {
type = 'openai',
api_key = nil,
- model = 'gpt-3.5-turbo',
+ model = 'gpt-4o-mini',
max_tokens = 1000,
- temperature = 0.7,
- top_p = 0.9,
+ temperature = 0.0,
timeout = 10,
prompt = nil,
condition = nil,
@@ -97,11 +94,11 @@ local function default_condition(task)
local action = result.action
if action == 'reject' and result.npositive > 1 then
- return true, 'already decided as spam'
+ return false, 'already decided as spam'
end
if action == 'no action' and score < 0 then
- return true, 'negative score, already decided as ham'
+ return false, 'negative score, already decided as ham'
end
end
-- We also exclude some symbols
@@ -109,10 +106,12 @@ local function default_condition(task)
if task:has_symbol(s) then
if required_weight > 0 then
-- Also check score
- local sym = task:get_symbol(s)
+ local sym = task:get_symbol(s) or E
-- Must exist as we checked it before with `has_symbol`
- if math.abs(sym.weight) >= required_weight then
- return false, 'skip as "' .. s .. '" is found (weight: ' .. sym.weight .. ')'
+ if sym.weight then
+ if math.abs(sym.weight) >= required_weight then
+ return false, 'skip as "' .. s .. '" is found (weight: ' .. sym.weight .. ')'
+ end
end
lua_util.debugm(N, task, 'symbol %s has weight %s, but required %s', s,
sym.weight, required_weight)
@@ -195,6 +194,18 @@ local function default_conversion(task, input)
if type(reply) == 'table' and reply.probability then
local spam_score = tonumber(reply.probability)
+
+ if not spam_score then
+ -- Maybe we need GPT to convert GPT reply here?
+ if reply.probability == "high" then
+ spam_score = 0.9
+ elseif reply.probability == "low" then
+ spam_score = 0.1
+ else
+ rspamd_logger.infox("cannot convert to spam probability: %s", reply.probability)
+ end
+ end
+
if type(reply.usage) == 'table' then
rspamd_logger.infox(task, 'usage: %s tokens', reply.usage.total_tokens)
end
@@ -276,7 +287,7 @@ local function openai_gpt_check(task)
model = settings.model,
max_tokens = settings.max_tokens,
temperature = settings.temperature,
- top_p = settings.top_p,
+ response_format = { type = "json_object" },
messages = {
{
role = 'system',
@@ -348,8 +359,8 @@ if opts then
end
if not settings.prompt then
- settings.prompt = "You will be provided with the email message, " ..
- "and your task is to classify its probability to be spam, " ..
+ settings.prompt = "You will be provided with the email message, subject, from and url domains, " ..
+ "and your task is to evaluate the probability to be spam as number from 0 to 1, " ..
"output result as JSON with 'probability' field."
end
diff --git a/src/plugins/lua/history_redis.lua b/src/plugins/lua/history_redis.lua
index 3365b30cd..fff9f46b3 100644
--- a/src/plugins/lua/history_redis.lua
+++ b/src/plugins/lua/history_redis.lua
@@ -21,8 +21,8 @@ if confighelp then
redis_history {
# History key name
key_prefix = 'rs_history{{HOSTNAME}}{{COMPRESS}}';
- # History expire in seconds
- expire = 0;
+ # Expire in seconds for inactive keys, default to 5 days
+ expire = 432000;
# History rows limit
nrows = 200;
# Use zstd compression when storing data in redis
diff --git a/src/plugins/lua/known_senders.lua b/src/plugins/lua/known_senders.lua
index 6d57acea3..5cb2ddcf5 100644
--- a/src/plugins/lua/known_senders.lua
+++ b/src/plugins/lua/known_senders.lua
@@ -18,6 +18,7 @@ limitations under the License.
local rspamd_logger = require "rspamd_logger"
local N = 'known_senders'
+local E = {}
local lua_util = require "lua_util"
local lua_redis = require "lua_redis"
local lua_maps = require "lua_maps"
@@ -258,7 +259,7 @@ local function verify_local_replies_set(task)
return nil
end
- local replies_recipients = task:get_recipients('mime')
+ local replies_recipients = task:get_recipients('mime') or E
local replies_sender_string = lua_util.maybe_obfuscate_string(tostring(replies_sender), settings,
settings.sender_prefix)
diff --git a/src/plugins/lua/once_received.lua b/src/plugins/lua/once_received.lua
index 2a5552ab9..5c5ff7986 100644
--- a/src/plugins/lua/once_received.lua
+++ b/src/plugins/lua/once_received.lua
@@ -19,10 +19,7 @@ if confighelp then
end
-- 0 or 1 received: = spam
-
local symbol = 'ONCE_RECEIVED'
-local symbol_rdns = 'RDNS_NONE'
-local symbol_rdns_dnsfail = 'RDNS_DNSFAIL'
local symbol_mx = 'DIRECT_TO_MX'
-- Symbol for strict checks
local symbol_strict = nil
@@ -47,54 +44,6 @@ local function check_quantity_received (task)
return not h['flags']['artificial']
end, recvh))
- local function recv_dns_cb(_, to_resolve, results, err)
- if err and (err ~= 'requested record is not found' and err ~= 'no records with this name') then
- rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, err)
- task:insert_result(symbol_rdns_dnsfail, 1.0)
- end
-
- if not results then
- if nreceived <= 1 then
- task:insert_result(symbol, 1)
- -- Avoid strict symbol inserting as the remaining symbols have already
- -- quote a significant weight, so a message could be rejected by just
- -- this property.
- --task:insert_result(symbol_strict, 1)
- -- Check for MUAs
- local ua = task:get_header('User-Agent')
- local xm = task:get_header('X-Mailer')
- if (ua or xm) then
- task:insert_result(symbol_mx, 1, (ua or xm))
- end
- end
- task:insert_result(symbol_rdns, 1)
- else
- rspamd_logger.infox(task, 'source hostname has not been passed to Rspamd from MTA, ' ..
- 'but we could resolve source IP address PTR %s as "%s"',
- to_resolve, results[1])
- task:set_hostname(results[1])
-
- if good_hosts then
- for _, gh in ipairs(good_hosts) do
- if string.find(results[1], gh) then
- return
- end
- end
- end
-
- if nreceived <= 1 then
- task:insert_result(symbol, 1)
- for _, h in ipairs(bad_hosts) do
- if string.find(results[1], h) then
-
- task:insert_result(symbol_strict, 1, h)
- return
- end
- end
- end
- end
- end
-
local task_ip = task:get_ip()
if ((not check_authed and task:get_user()) or
@@ -110,13 +59,39 @@ local function check_quantity_received (task)
local hn = task:get_hostname()
-- Here we don't care about received
- if (not hn) and task_ip and task_ip:is_valid() then
- task:get_resolver():resolve_ptr({ task = task,
- name = task_ip:to_string(),
- callback = recv_dns_cb,
- forced = true
- })
+ if not hn then
+ if nreceived <= 1 then
+ task:insert_result(symbol, 1)
+ -- Avoid strict symbol inserting as the remaining symbols have already
+ -- quote a significant weight, so a message could be rejected by just
+ -- this property.
+ --task:insert_result(symbol_strict, 1)
+ -- Check for MUAs
+ local ua = task:get_header('User-Agent')
+ local xm = task:get_header('X-Mailer')
+ if (ua or xm) then
+ task:insert_result(symbol_mx, 1, (ua or xm))
+ end
+ end
return
+ else
+ if good_hosts then
+ for _, gh in ipairs(good_hosts) do
+ if string.find(hn, gh) then
+ return
+ end
+ end
+ end
+
+ if nreceived <= 1 then
+ task:insert_result(symbol, 1)
+ for _, h in ipairs(bad_hosts) do
+ if string.find(hn, h) then
+ task:insert_result(symbol_strict, 1, h)
+ break
+ end
+ end
+ end
end
if nreceived <= 1 then
@@ -181,10 +156,6 @@ if opts then
for n, v in pairs(opts) do
if n == 'symbol_strict' then
symbol_strict = v
- elseif n == 'symbol_rdns' then
- symbol_rdns = v
- elseif n == 'symbol_rdns_dnsfail' then
- symbol_rdns_dnsfail = v
elseif n == 'bad_host' then
if type(v) == 'string' then
bad_hosts[1] = v
@@ -207,16 +178,6 @@ if opts then
end
rspamd_config:register_symbol({
- name = symbol_rdns,
- type = 'virtual',
- parent = id
- })
- rspamd_config:register_symbol({
- name = symbol_rdns_dnsfail,
- type = 'virtual',
- parent = id
- })
- rspamd_config:register_symbol({
name = symbol_strict,
type = 'virtual',
parent = id
diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua
index 48f3c17be..356507250 100644
--- a/src/plugins/lua/spf.lua
+++ b/src/plugins/lua/spf.lua
@@ -56,6 +56,7 @@ local symbols = {
dnsfail = "R_SPF_DNSFAIL",
permfail = "R_SPF_PERMFAIL",
na = "R_SPF_NA",
+ plusall = "R_SPF_PLUSALL",
}
local default_config = {
@@ -118,6 +119,8 @@ local function spf_check_callback(task)
local function flag_to_symbol(fl)
if bit.band(fl, rspamd_spf.flags.temp_fail) ~= 0 then
return local_config.symbols.dnsfail
+ elseif bit.band(fl, rspamd_spf.flags.plusall) ~= 0 then
+ return local_config.symbols.plusall
elseif bit.band(fl, rspamd_spf.flags.perm_fail) ~= 0 then
return local_config.symbols.permfail
elseif bit.band(fl, rspamd_spf.flags.na) ~= 0 then