1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
--[[
Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
]]
-- This file contains functions to support Bayes statistics in Redis
local exports = {}
local lua_redis = require "lua_redis"
local logger = require "rspamd_logger"
local lua_util = require "lua_util"
local N = "bayes"
local function gen_classify_functor(redis_params, classify_script_id)
return function(task, expanded_key, id, is_spam, stat_tokens, callback)
local function classify_redis_cb(err, data)
lua_util.debugm(N, task, 'classify redis cb: %s, %s', err, data)
if err then
callback(task, false, err)
else
callback(task, true, data[1], data[2], data[3], data[4])
end
end
lua_redis.exec_redis_script(classify_script_id,
{ task = task, is_write = false, key = expanded_key },
classify_redis_cb, { expanded_key, stat_tokens })
end
end
local function gen_learn_functor(redis_params, learn_script_id)
return function(task, expanded_key, id, is_spam, symbol, is_unlearn, stat_tokens, callback, maybe_text_tokens)
local function learn_redis_cb(err, data)
lua_util.debugm(N, task, 'learn redis cb: %s, %s', err, data)
if err then
callback(task, false, err)
else
callback(task, true)
end
end
if maybe_text_tokens then
lua_redis.exec_redis_script(learn_script_id,
{ task = task, is_write = false, key = expanded_key },
learn_redis_cb,
{ expanded_key, tostring(is_spam), symbol, tostring(is_unlearn), stat_tokens, maybe_text_tokens })
else
lua_redis.exec_redis_script(learn_script_id,
{ task = task, is_write = false, key = expanded_key },
learn_redis_cb, { expanded_key, tostring(is_spam), symbol, tostring(is_unlearn), stat_tokens })
end
end
end
local function load_redis_params(classifier_ucl, statfile_ucl)
local redis_params
-- Try load from statfile options
if statfile_ucl.redis then
redis_params = lua_redis.try_load_redis_servers(statfile_ucl.redis, rspamd_config, true)
end
if not redis_params then
if statfile_ucl then
redis_params = lua_redis.try_load_redis_servers(statfile_ucl, rspamd_config, true)
end
end
-- Try load from classifier config
if not redis_params and classifier_ucl.backend then
redis_params = lua_redis.try_load_redis_servers(classifier_ucl.backend, rspamd_config, true)
end
if not redis_params and classifier_ucl.redis then
redis_params = lua_redis.try_load_redis_servers(classifier_ucl.redis, rspamd_config, true)
end
if not redis_params then
redis_params = lua_redis.try_load_redis_servers(classifier_ucl, rspamd_config, true)
end
-- Try load global options
if not redis_params then
redis_params = lua_redis.try_load_redis_servers(rspamd_config:get_all_opt('redis'), rspamd_config, true)
end
if not redis_params then
logger.err(rspamd_config, "cannot load Redis parameters for the classifier")
return nil
end
return redis_params
end
---
--- Init bayes classifier
--- @param classifier_ucl ucl of the classifier config
--- @param statfile_ucl ucl of the statfile config
--- @return a pair of (classify_functor, learn_functor) or `nil` in case of error
exports.lua_bayes_init_statfile = function(classifier_ucl, statfile_ucl, symbol, is_spam, ev_base, stat_periodic_cb)
local redis_params = load_redis_params(classifier_ucl, statfile_ucl)
if not redis_params then
return nil
end
local classify_script_id = lua_redis.load_redis_script_from_file("bayes_classify.lua", redis_params)
local learn_script_id = lua_redis.load_redis_script_from_file("bayes_learn.lua", redis_params)
local stat_script_id = lua_redis.load_redis_script_from_file("bayes_stat.lua", redis_params)
local max_users = classifier_ucl.max_users or 1000
local current_data = {
users = 0,
revision = 0,
}
local final_data = {
users = 0,
revision = 0, -- number of learns
}
local cursor = 0
rspamd_config:add_periodic(ev_base, 0.0, function(cfg, _)
local function stat_redis_cb(err, data)
-- TODO: write this function
lua_util.debugm(N, cfg, 'stat redis cb: %s, %s', err, data)
if err then
logger.warn(cfg, 'cannot get bayes statistics for %s: %s', symbol, err)
else
local new_cursor = data[1]
current_data.users = current_data.users + data[2]
current_data.revision = current_data.revision + data[3]
if new_cursor == 0 then
-- Done iteration
final_data = lua_util.shallowcopy(current_data)
current_data = {
users = 0,
revision = 0,
}
lua_util.debugm(N, cfg, 'final data: %s', final_data)
stat_periodic_cb(cfg, final_data)
end
cursor = new_cursor
end
end
lua_redis.exec_redis_script(stat_script_id,
{ ev_base = ev_base, cfg = cfg, is_write = false },
stat_redis_cb, { tostring(cursor),
symbol,
is_spam and "learns_spam" or "learns_ham",
tostring(max_users) })
return statfile_ucl.monitor_timeout or classifier_ucl.monitor_timeout or 30.0
end)
return gen_classify_functor(redis_params, classify_script_id), gen_learn_functor(redis_params, learn_script_id)
end
exports.lua_bayes_init_cache = function(classifier_ucl, statfile_ucl)
local redis_params = load_redis_params(classifier_ucl, statfile_ucl)
if not redis_params then
return nil
end
end
return exports
|