1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
--[[
Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
]]
-- This file contains functions to support Bayes statistics in Redis
local exports = {}
local lua_redis = require "lua_redis"
local logger = require "rspamd_logger"
local lua_util = require "lua_util"
local N = "bayes"
local function gen_classify_functor(redis_params, classify_script_id)
return function(task, expanded_key, id, is_spam, stat_tokens, callback)
local function classify_redis_cb(err, data)
lua_util.debugm(N, task, 'classify redis cb: %s, %s', err, data)
if err then
callback(task, false, err)
else
callback(task, true, data[1], data[2], data[3], data[4])
end
end
lua_redis.exec_redis_script(classify_script_id,
{ task = task, is_write = false, key = expanded_key },
classify_redis_cb, { expanded_key, stat_tokens })
end
end
local function gen_learn_functor(redis_params, learn_script_id)
return function(task, expanded_key, id, is_spam, symbol, is_unlearn, stat_tokens, callback)
local function learn_redis_cb(err, data)
lua_util.debugm(N, task, 'learn redis cb: %s, %s', err, data)
if err then
callback(task, false, err)
else
callback(task, true)
end
end
lua_redis.exec_redis_script(learn_script_id,
{ task = task, is_write = false, key = expanded_key },
learn_redis_cb, { expanded_key, tostring(is_spam), symbol, tostring(is_unlearn), stat_tokens })
end
end
---
--- Init bayes classifier
--- @param classifier_ucl ucl of the classifier config
--- @param statfile_ucl ucl of the statfile config
--- @return a pair of (classify_functor, learn_functor) or `nil` in case of error
exports.lua_bayes_init_statfile = function(classifier_ucl, statfile_ucl, symbol, is_spam, ev_base, stat_periodic_cb)
local redis_params
-- Try load from statfile options
if statfile_ucl.redis then
redis_params = lua_redis.try_load_redis_servers(statfile_ucl.redis, rspamd_config, true)
end
if not redis_params then
if statfile_ucl then
redis_params = lua_redis.try_load_redis_servers(statfile_ucl, rspamd_config, true)
end
end
-- Try load from classifier config
if not redis_params and classifier_ucl.backend then
redis_params = lua_redis.try_load_redis_servers(classifier_ucl.backend, rspamd_config, true)
end
if not redis_params and classifier_ucl.redis then
redis_params = lua_redis.try_load_redis_servers(classifier_ucl.redis, rspamd_config, true)
end
if not redis_params then
redis_params = lua_redis.try_load_redis_servers(classifier_ucl, rspamd_config, true)
end
-- Try load global options
if not redis_params then
redis_params = lua_redis.try_load_redis_servers(rspamd_config:get_all_opt('redis'), rspamd_config, true)
end
if not redis_params then
logger.err(rspamd_config, "cannot load Redis parameters for the classifier")
return nil
end
local classify_script_id = lua_redis.load_redis_script_from_file("bayes_classify.lua", redis_params)
local learn_script_id = lua_redis.load_redis_script_from_file("bayes_learn.lua", redis_params)
local stat_script_id = lua_redis.load_redis_script_from_file("bayes_stat.lua", redis_params)
local max_users = classifier_ucl.max_users or 1000
local current_data = {
users = 0,
revision = 0,
}
local final_data = {
users = 0,
revision = 0, -- number of learns
}
local cursor = 0
rspamd_config:add_periodic(ev_base, 0.0, function(cfg, _)
local function stat_redis_cb(err, data)
-- TODO: write this function
lua_util.debugm(N, cfg, 'stat redis cb: %s, %s', err, data)
if err then
logger.warn(cfg, 'cannot get bayes statistics for %s: %s', symbol, err)
else
local new_cursor = data[1]
current_data.users = current_data.users + data[2]
current_data.revision = current_data.revision + data[3]
if new_cursor == 0 then
-- Done iteration
final_data = lua_util.shallowcopy(current_data)
current_data = {
users = 0,
revision = 0,
}
lua_util.debugm(N, cfg, 'final data: %s', final_data)
stat_periodic_cb(cfg, final_data)
end
cursor = new_cursor
end
end
lua_redis.exec_redis_script(stat_script_id,
{ ev_base = ev_base, cfg = cfg, is_write = false },
stat_redis_cb, { tostring(cursor),
symbol,
is_spam and "learns_spam" or "learns_ham",
tostring(max_users) })
return statfile_ucl.monitor_timeout or classifier_ucl.monitor_timeout or 30.0
end)
return gen_classify_functor(redis_params, classify_script_id), gen_learn_functor(redis_params, learn_script_id)
end
return exports
|
