aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAndrew Lewis <nerf@judo.za.org>2016-11-15 17:13:31 +0200
committerAndrew Lewis <nerf@judo.za.org>2016-11-15 17:13:31 +0200
commit0e92776b992527838a49e4a2193e33330c931f08 (patch)
tree6ef1cf82499b8bbf117de096664d11239a6e6130
parent8f8e4d9fa99128d6f69872640b3e17d159b3977d (diff)
downloadrspamd-0e92776b992527838a49e4a2193e33330c931f08.tar.gz
rspamd-0e92776b992527838a49e4a2193e33330c931f08.zip
[Minor] Lint fann_redis
-rw-r--r--.luacheckrc1
-rw-r--r--src/plugins/lua/fann_redis.lua108
2 files changed, 50 insertions, 59 deletions
diff --git a/.luacheckrc b/.luacheckrc
index 5269a4dcc..419d9e301 100644
--- a/.luacheckrc
+++ b/.luacheckrc
@@ -4,7 +4,6 @@ std = 'min'
exclude_files = {
'/**/src/plugins/lua/fann_classifier.lua',
'/**/src/plugins/lua/fann_scores.lua',
- '/**/src/plugins/lua/fann_redis.lua',
}
globals = {
diff --git a/src/plugins/lua/fann_redis.lua b/src/plugins/lua/fann_redis.lua
index ad8a0f79a..08216fe5e 100644
--- a/src/plugins/lua/fann_redis.lua
+++ b/src/plugins/lua/fann_redis.lua
@@ -22,8 +22,7 @@ local rspamd_fann = require "rspamd_fann"
local rspamd_util = require "rspamd_util"
local fann_symbol_spam = 'FANNR_SPAM'
local fann_symbol_ham = 'FANNR_HAM'
-require "fun" ()
-local ucl = require "ucl"
+local fun = require "fun"
local module_log_id = 0x200
-- Module vars
@@ -169,7 +168,7 @@ local function redis_make_request(ev_base, cfg, key, is_write, callback, command
end
if not addr then
- logger.errx(task, 'cannot select server to make redis request')
+ rspamd_logger.errx(cfg, 'cannot select server to make redis request')
end
local options = {
@@ -202,9 +201,9 @@ local function symbols_to_fann_vector(syms, scores)
local matched_symbols = {}
local n = rspamd_config:get_symbols_count()
- each(function(s, score)
+ fun.each(function(s, score)
matched_symbols[s + 1] = rspamd_util.tanh(score)
- end, zip(syms, scores))
+ end, fun.zip(syms, scores))
for i=1,n do
if matched_symbols[i] then
@@ -303,11 +302,11 @@ local function load_or_invalidate_fann(data, id, ev_base)
fanns[id].fann = ann
rspamd_logger.infox(rspamd_config, 'loaded ann %s from redis', id)
else
- local function redis_invalidate_cb(err, data)
- if err then
- rspamd_logger.errx(rspamd_config, 'cannot invalidate ANN %s from redis: %s', id, err)
- elseif type(data) == 'string' then
- rspamd_logger.info(rspamd_config, 'invalidated ANN %s from redis: %s', id, err)
+ local function redis_invalidate_cb(_err, _data)
+ if _err then
+ rspamd_logger.errx(rspamd_config, 'cannot invalidate ANN %s from redis: %s', id, _err)
+ elseif type(_data) == 'string' then
+ rspamd_logger.info(rspamd_config, 'invalidated ANN %s from redis: %s', id, _err)
end
end
-- Invalidate ANN
@@ -323,10 +322,10 @@ local function load_or_invalidate_fann(data, id, ev_base)
end
end
-local function fann_train_callback(score, required_score, results, cf, id, opts, extra, ev_base)
+local function fann_train_callback(score, required_score, results, _, id, opts, extra, ev_base)
local fname,suffix = gen_fann_prefix(id)
- local learn_spam, learn_ham = false, false
+ local learn_spam, learn_ham
if opts['spam_score'] then
learn_spam = score >= opts['spam_score']
@@ -343,7 +342,7 @@ local function fann_train_callback(score, required_score, results, cf, id, opts,
local k
if learn_spam then k = 'spam' else k = 'ham' end
- local function learn_vec_cb(err, data)
+ local function learn_vec_cb(err)
if err then
rspamd_logger.errx(rspamd_config, 'cannot store train vector: %s', err)
end
@@ -352,11 +351,11 @@ local function fann_train_callback(score, required_score, results, cf, id, opts,
local function can_train_cb(err, data)
if not err and tonumber(data) > 0 then
local learn_data = symbols_to_fann_vector(
- map(function(r) return r[1] end, results),
- map(function(r) return r[2] end, results)
+ fun.map(function(r) return r[1] end, results),
+ fun.map(function(r) return r[2] end, results)
)
-- Add filtered meta tokens
- each(function(e) table.insert(learn_data, e) end, extra)
+ fun.each(function(e) table.insert(learn_data, e) end, extra)
local str = rspamd_util.zstd_compress(table.concat(learn_data, ';'))
redis_make_request(ev_base,
@@ -385,19 +384,19 @@ local function fann_train_callback(score, required_score, results, cf, id, opts,
end
end
-local function train_fann(cfg, ev_base, elt)
+local function train_fann(_, ev_base, elt)
local spam_elts = {}
local ham_elts = {}
elt = tostring(elt)
- local function redis_unlock_cb(err, data)
+ local function redis_unlock_cb(err)
if err then
rspamd_logger.errx(rspamd_config, 'cannot unlock ANN %s from redis: %s',
fann_prefix .. elt, err)
end
end
- local function redis_save_cb(err, data)
+ local function redis_save_cb(err)
if err then
rspamd_logger.errx(rspamd_config, 'cannot save ANN %s to redis: %s',
fann_prefix .. elt, err)
@@ -449,21 +448,21 @@ local function train_fann(cfg, ev_base, elt)
)
else
-- Decompress and convert to numbers each training vector
- ham_elts = map(function(tok)
+ ham_elts = fun.map(function(tok)
local _,str = rspamd_util.zstd_decompress(tok)
- return map(tonumber, rspamd_str_split(tostring(str), ';'))
+ return fun.map(tonumber, rspamd_str_split(tostring(str), ';'))
end, data)
-- Now we need to join inputs and create the appropriate test vectors
local inputs = {}
local outputs = {}
- each(function(sample)
- table.insert(inputs, totable(sample[1]))
+ fun.each(function(sample)
+ table.insert(inputs, fun.totable(sample[1]))
table.insert(outputs, {1.0})
- table.insert(inputs, totable(sample[2]))
+ table.insert(inputs, fun.totable(sample[2]))
table.insert(outputs, {-1.0})
- end, zip(spam_elts, ham_elts))
+ end, fun.zip(spam_elts, ham_elts))
-- Now we can train fann
local n = rspamd_config:get_symbols_count() + rspamd_count_metatokens()
@@ -492,9 +491,9 @@ local function train_fann(cfg, ev_base, elt)
)
else
-- Decompress and convert to numbers each training vector
- spam_elts = map(function(tok)
+ spam_elts = fun.map(function(tok)
local _,str = rspamd_util.zstd_decompress(tok)
- return map(tonumber, rspamd_str_split(tostring(str), ';'))
+ return fun.map(tonumber, rspamd_str_split(tostring(str), ';'))
end, data)
redis_make_request(ev_base,
rspamd_config,
@@ -540,27 +539,20 @@ local function maybe_train_fanns(cfg, ev_base)
if err then
rspamd_logger.errx(rspamd_config, 'cannot get FANNS list from redis: %s', err)
elseif type(data) == 'table' then
- each(function(elt)
+ fun.each(function(elt)
elt = tostring(elt)
- local redis_len_cb = function(err, data)
- if err then
- rspamd_logger.errx(rspamd_config, 'cannot get FANN trains %s from redis: %s', elt, err)
- elseif data and type(data) == 'number' or type(data) == 'string' then
- if tonumber(data) and tonumber(data) > max_trains then
+ local redis_len_cb = function(_err, _data)
+ if _err then
+ rspamd_logger.errx(rspamd_config, 'cannot get FANN trains %s from redis: %s', elt, _err)
+ elseif _data and type(_data) == 'number' or type(_data) == 'string' then
+ if tonumber(_data) and tonumber(_data) > max_trains then
rspamd_logger.infox(rspamd_config, 'need to learn ANN %s after %s learn vectors (%s required)',
- tonumber(data), max_trains)
+ tonumber(_data), max_trains)
train_fann(cfg, ev_base, elt)
end
end
end
- local local_ver = 0
- local numelt = tonumber(elt)
- if fanns[numelt] then
- if fanns[numelt].version then
- local_ver = fanns[numelt].version
- end
- end
redis_make_request(ev_base,
rspamd_config,
nil,
@@ -591,18 +583,18 @@ local function maybe_train_fanns(cfg, ev_base)
return watch_interval
end
-local function check_fanns(cfg, ev_base)
+local function check_fanns(_, ev_base)
local function members_cb(err, data)
if err then
rspamd_logger.errx(rspamd_config, 'cannot get FANNS list from redis: %s', err)
elseif type(data) == 'table' then
- each(function(elt)
+ fun.each(function(elt)
elt = tostring(elt)
- local redis_update_cb = function(err, data)
- if err then
- rspamd_logger.errx(rspamd_config, 'cannot get FANN version %s from redis: %s', elt, err)
- elseif data and type(data) == 'string' then
- load_or_invalidate_fann(data, elt, ev_base)
+ local redis_update_cb = function(_err, _data)
+ if _err then
+ rspamd_logger.errx(rspamd_config, 'cannot get FANN version %s from redis: %s', elt, _err)
+ elseif _data and type(_data) == 'string' then
+ load_or_invalidate_fann(_data, elt, ev_base)
end
end
@@ -695,13 +687,13 @@ else
max_epoch = opts['train']['max_epoch']
end
local ret = cfg:register_worker_script("log_helper",
- function(score, req_score, results, cf, id, extra, ev_base)
- -- map (snd x) (filter (fst x == module_id) extra)
- local extra_fann = map(function(e) return e[2] end,
- filter(function(e) return e[1] == module_log_id end, extra))
+ function(score, req_score, results, cf, _id, extra, ev_base)
+ -- fun.map (snd x) (fun.filter (fst x == module_id) extra)
+ local extra_fann = fun.map(function(e) return e[2] end,
+ fun.filter(function(e) return e[1] == module_log_id end, extra))
if use_settings then
fann_train_callback(score, req_score, results, cf,
- tostring(id), opts['train'], extra_fann, ev_base)
+ tostring(_id), opts['train'], extra_fann, ev_base)
else
fann_train_callback(score, req_score, results, cf, '0',
opts['train'], extra_fann, ev_base)
@@ -715,7 +707,7 @@ else
-- This is needed to pass extra tokens from worker to log_helper
rspamd_plugins["fann_redis"] = {
log_callback = function(task)
- return totable(map(
+ return fun.totable(fun.map(
function(tok) return {module_log_id, tok} end,
rspamd_gen_metatokens(task)))
end
@@ -746,8 +738,8 @@ else
redis_maybe_load_sha = tostring(data)
rspamd_config:add_periodic(ev_base, 0.0,
- function(cfg, ev_base)
- return check_fanns(cfg, ev_base)
+ function(_cfg, _ev_base)
+ return check_fanns(_cfg, _ev_base)
end)
end
end
@@ -811,8 +803,8 @@ else
if worker:get_name() == 'normal' then
-- We also want to train neural nets when they have enough data
rspamd_config:add_periodic(ev_base, 0.0,
- function(cfg, ev_base)
- return maybe_train_fanns(cfg, ev_base)
+ function(_cfg, _ev_base)
+ return maybe_train_fanns(_cfg, _ev_base)
end)
end
end)