From e4ac34be79aa9bbbfe2926e37f443c32221cbfe0 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Mon, 4 Nov 2019 17:53:58 +0000 Subject: [PATCH] [Feature] Add verdict library in lua --- lualib/lua_bayes_learn.lua | 10 +- lualib/lua_util.lua | 30 +----- lualib/lua_verdict.lua | 189 +++++++++++++++++++++++++++++++++ src/plugins/lua/clustering.lua | 3 +- src/plugins/lua/neural.lua | 3 +- src/plugins/lua/ratelimit.lua | 3 +- src/plugins/lua/reputation.lua | 5 +- 7 files changed, 208 insertions(+), 35 deletions(-) create mode 100644 lualib/lua_verdict.lua diff --git a/lualib/lua_bayes_learn.lua b/lualib/lua_bayes_learn.lua index 066e86a4d..ae8d901f8 100644 --- a/lualib/lua_bayes_learn.lua +++ b/lualib/lua_bayes_learn.lua @@ -17,7 +17,7 @@ limitations under the License. -- This file contains functions to simplify bayes classifier auto-learning local lua_util = require "lua_util" - +local lua_verdict = require "lua_verdict" local N = "lua_bayes" local exports = {} @@ -76,7 +76,7 @@ exports.autolearn = function(task, conf) end -- We have autolearn config so let's figure out what is requested - local verdict,score = lua_util.get_task_verdict(task) + local verdict,score = lua_verdict.get_specific_verdict("bayes", task) local learn_spam,learn_ham = false, false if verdict == 'passthrough' then @@ -98,6 +98,12 @@ exports.autolearn = function(task, conf) learn_ham = true end end + elseif conf.learn_verdict then + if verdict == 'spam' or verdict == 'junk' then + learn_spam = true + elseif verdict == 'ham' then + learn_ham = true + end end if conf.check_balance then diff --git a/lualib/lua_util.lua b/lualib/lua_util.lua index 842f079b2..bda8b0c02 100644 --- a/lualib/lua_util.lua +++ b/lualib/lua_util.lua @@ -1026,35 +1026,9 @@ end -- * `uncertain`: all other cases --]] exports.get_task_verdict = function(task) - local result = task:get_metric_result() + local lua_verdict = require "lua_verdict" - if result then - - if result.passthrough then - return 'passthrough',nil - end - - local score = result.score - - local action = result.action - - if action == 'reject' and result.npositive > 1 then - return 'spam',score - elseif action == 'no action' then - if score < 0 or result.nnegative > 3 then - return 'ham',score - end - else - -- All colors of junk - if action == 'add header' or action == 'rewrite subject' then - if result.npositive > 2 then - return 'junk',score - end - end - end - - return 'uncertain',score - end + return lua_verdict.get_default_verdict(task) end ---[[[ diff --git a/lualib/lua_verdict.lua b/lualib/lua_verdict.lua new file mode 100644 index 000000000..d6a1634a7 --- /dev/null +++ b/lualib/lua_verdict.lua @@ -0,0 +1,189 @@ +--[[ +Copyright (c) 2019, Vsevolod Stakhov + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +local exports = {} + +---[[[ +-- @function lua_verdict.get_default_verdict(task) +-- Returns verdict for a task + score if certain, must be called from idempotent filters only +-- Returns string: +-- * `spam`: if message have over reject threshold and has more than one positive rule +-- * `junk`: if a message has between score between [add_header/rewrite subject] to reject thresholds and has more than two positive rules +-- * `passthrough`: if a message has been passed through some short-circuit rule +-- * `ham`: if a message has overall score below junk level **and** more than three negative rule, or negative total score +-- * `uncertain`: all other cases +--]] +local function default_verdict_function(task) + local result = task:get_metric_result() + + if result then + + if result.passthrough then + return 'passthrough',nil + end + + local score = result.score + + local action = result.action + + if action == 'reject' and result.npositive > 1 then + return 'spam',score + elseif action == 'no action' then + if score < 0 or result.nnegative > 3 then + return 'ham',score + end + else + -- All colors of junk + if action == 'add header' or action == 'rewrite subject' then + if result.npositive > 2 then + return 'junk',score + end + end + end + + return 'uncertain',score + end +end + +local default_possible_verdicts = { + passthrough = { + can_learn = false, + description = 'message has passthrough result', + }, + spam = { + can_learn = 'spam', + description = 'message is likely spam', + }, + junk = { + can_learn = 'spam', + description = 'message is likely possible spam', + }, + ham = { + can_learn = 'ham', + description = 'message is likely ham', + }, + uncertain = { + can_learn = false, + description = 'not certainity in verdict' + } +} + +-- Verdict functions specific for modules +local specific_verdicts = { + default = { + callback = default_verdict_function, + possible_verdicts = default_possible_verdicts + } +} + +local default_verdict = specific_verdicts.default + +exports.get_default_verdict = default_verdict.callback +exports.set_verdict_function = function(func, what) + assert(type(func) == 'function') + if not what then + -- Default verdict + local existing = specific_verdicts.default.callback + specific_verdicts.default.callback = func + exports.get_default_verdict = func + + return existing + else + local existing = specific_verdicts[what] + + if not existing then + specific_verdicts[what] = { + callback = func, + possible_verdicts = default_possible_verdicts + } + else + existing = existing.callback + end + + specific_verdicts[what].callback = func + return existing + end +end + +exports.set_verdict_table = function(verdict_tbl, what) + assert(type(verdict_tbl) == 'table' and + type(verdict_tbl.callback) == 'function' and + type(verdict_tbl.possible_verdicts) == 'table') + + if not what then + -- Default verdict + local existing = specific_verdicts.default + specific_verdicts.default = verdict_tbl + exports.get_default_verdict = specific_verdicts.default.callback + + return existing + else + local existing = specific_verdicts[what] + specific_verdicts[what] = verdict_tbl + return existing + end +end + +exports.get_specific_verdict = function(what, task) + if specific_verdicts[what] then + return specific_verdicts[what].callback(task) + end + + return exports.get_default_verdict(task) +end + +exports.get_possible_verdicts = function(what) + local lua_util = require "lua_util" + if what then + if specific_verdicts[what] then + return lua_util.keys(specific_verdicts[what].possible_verdicts) + end + else + return lua_util.keys(specific_verdicts.default.possible_verdicts) + end + + return nil +end + +exports.can_learn = function(verdict, what) + if what then + if specific_verdicts[what] and specific_verdicts[what].possible_verdicts[verdict] then + return specific_verdicts[what].possible_verdicts[verdict].can_learn + end + else + if specific_verdicts.default.possible_verdicts[verdict] then + return specific_verdicts.default.possible_verdicts[verdict].can_learn + end + end + + return nil -- To distinguish from `false` that could happen in can_learn +end + +exports.describe = function(verdict, what) + if what then + if specific_verdicts[what] and specific_verdicts[what].possible_verdicts[verdict] then + return specific_verdicts[what].possible_verdicts[verdict].description + end + else + if specific_verdicts.default.possible_verdicts[verdict] then + return specific_verdicts.default.possible_verdicts[verdict].description + end + end + + return nil +end + +return exports \ No newline at end of file diff --git a/src/plugins/lua/clustering.lua b/src/plugins/lua/clustering.lua index 0b514476c..45e5d4bc1 100644 --- a/src/plugins/lua/clustering.lua +++ b/src/plugins/lua/clustering.lua @@ -24,6 +24,7 @@ local N = 'clustering' local rspamd_logger = require "rspamd_logger" local lua_util = require "lua_util" +local lua_verdict = require "lua_verdict" local lua_redis = require "lua_redis" local lua_selectors = require "lua_selectors" local ts = require("tableshape").types @@ -190,7 +191,7 @@ local function clusterting_idempotent_cb(task, rule) if task:has_flag('skip') then return end if not rule.allow_local and lua_util.is_rspamc_or_controller(task) then return end - local verdict = lua_util.get_task_verdict(task) + local verdict = lua_verdict.get_specific_verdict(N, task) local score if verdict == 'ham' then diff --git a/src/plugins/lua/neural.lua b/src/plugins/lua/neural.lua index cb3a51300..d96ca896a 100644 --- a/src/plugins/lua/neural.lua +++ b/src/plugins/lua/neural.lua @@ -28,6 +28,7 @@ local fun = require "fun" local lua_settings = require "lua_settings" local meta_functions = require "lua_meta" local ts = require("tableshape").types +local lua_verdict = require "lua_verdict" local N = "neural" -- Module vars @@ -1164,7 +1165,7 @@ local function ann_push_vector(task) return end - local verdict,score = lua_util.get_task_verdict(task) + local verdict,score = lua_verdict.get_specific_verdict(N, task) if verdict == 'passthrough' then lua_util.debugm(N, task, 'ignore task as its verdict is %s(%s)', diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua index bd77b1ad6..731bd7ac3 100644 --- a/src/plugins/lua/ratelimit.lua +++ b/src/plugins/lua/ratelimit.lua @@ -26,6 +26,7 @@ local lua_redis = require "lua_redis" local fun = require "fun" local lua_maps = require "lua_maps" local lua_util = require "lua_util" +local lua_verdict = require "lua_verdict" local rspamd_hash = require "rspamd_cryptobox_hash" local lua_selectors = require "lua_selectors" local ts = require("tableshape").types @@ -666,7 +667,7 @@ local function ratelimit_update_cb(task) return end - local verdict = lua_util.get_task_verdict(task) + local verdict = lua_verdict.get_specific_verdict(N, task) local _,nrcpt = task:has_recipients('smtp') if not nrcpt or nrcpt <= 0 then nrcpt = 1 diff --git a/src/plugins/lua/reputation.lua b/src/plugins/lua/reputation.lua index 8b2b8c5a4..f88973d71 100644 --- a/src/plugins/lua/reputation.lua +++ b/src/plugins/lua/reputation.lua @@ -109,9 +109,10 @@ end -- Extracts task score and subtracts score of the rule itself local function extract_task_score(task, rule) - local _,score = lua_util.get_task_verdict(task) + local lua_verdict = require "lua_verdict" + local verdict,score = lua_verdict.get_specific_verdict(N, task) - if not score then return nil end + if not score or verdict == 'passthrough' then return nil end return sub_symbol_score(task, rule, score) end -- 2.39.5