From 3cf2f3130034e46ff0224cc576626599422e8401 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Sat, 4 Nov 2017 12:42:14 +0000 Subject: [Rework] Stop embedding rspamadm scripts into C --- lualib/rspamadm/ansicolors.lua | 58 ++++++++ lualib/rspamadm/confighelp.lua | 111 +++++++++++++++ lualib/rspamadm/fuzzy_convert.lua | 198 +++++++++++++++++++++++++++ lualib/rspamadm/fuzzy_stat.lua | 274 ++++++++++++++++++++++++++++++++++++++ lualib/rspamadm/getopt.lua | 34 +++++ lualib/rspamadm/grep.lua | 112 ++++++++++++++++ lualib/rspamadm/stat_convert.lua | 225 +++++++++++++++++++++++++++++++ src/rspamadm/CMakeLists.txt | 9 -- src/rspamadm/ansicolors.lua.in | 56 -------- src/rspamadm/confighelp.c | 3 +- src/rspamadm/confighelp.lua | 112 ---------------- src/rspamadm/fuzzy_convert.c | 4 +- src/rspamadm/fuzzy_convert.lua | 198 --------------------------- src/rspamadm/fuzzy_stat.lua | 274 -------------------------------------- src/rspamadm/getopt.lua.in | 31 ----- src/rspamadm/grep.c | 4 +- src/rspamadm/grep.lua | 112 ---------------- src/rspamadm/rspamadm.c | 20 +-- src/rspamadm/rspamadm.h | 2 +- src/rspamadm/stat_convert.c | 4 +- src/rspamadm/stat_convert.lua | 225 ------------------------------- 21 files changed, 1032 insertions(+), 1034 deletions(-) create mode 100644 lualib/rspamadm/ansicolors.lua create mode 100644 lualib/rspamadm/confighelp.lua create mode 100644 lualib/rspamadm/fuzzy_convert.lua create mode 100644 lualib/rspamadm/fuzzy_stat.lua create mode 100644 lualib/rspamadm/getopt.lua create mode 100644 lualib/rspamadm/grep.lua create mode 100644 lualib/rspamadm/stat_convert.lua delete mode 100644 src/rspamadm/ansicolors.lua.in delete mode 100644 src/rspamadm/confighelp.lua delete mode 100644 src/rspamadm/fuzzy_convert.lua delete mode 100644 src/rspamadm/fuzzy_stat.lua delete mode 100644 src/rspamadm/getopt.lua.in delete mode 100644 src/rspamadm/grep.lua delete mode 100644 src/rspamadm/stat_convert.lua diff --git a/lualib/rspamadm/ansicolors.lua b/lualib/rspamadm/ansicolors.lua new file mode 100644 index 000000000..739cf427c --- /dev/null +++ b/lualib/rspamadm/ansicolors.lua @@ -0,0 +1,58 @@ +local colormt = {} +local ansicolors = {} + +function colormt:__tostring() + return self.value +end + +function colormt:__concat(other) + return tostring(self) .. tostring(other) +end + +function colormt:__call(s) + return self .. s .. ansicolors.reset +end + +colormt.__metatable = {} + +local function makecolor(value) + return setmetatable({ value = string.char(27) .. '[' .. tostring(value) .. 'm' }, colormt) +end + +local colors = { + -- attributes + reset = 0, + clear = 0, + bright = 1, + dim = 2, + underscore = 4, + blink = 5, + reverse = 7, + hidden = 8, + + -- foreground + black = 30, + red = 31, + green = 32, + yellow = 33, + blue = 34, + magenta = 35, + cyan = 36, + white = 37, + + -- background + onblack = 40, + onred = 41, + ongreen = 42, + onyellow = 43, + onblue = 44, + onmagenta = 45, + oncyan = 46, + onwhite = 47, +} + +for c, v in pairs(colors) do + ansicolors[c] = makecolor(v) +end + +return ansicolors \ No newline at end of file diff --git a/lualib/rspamadm/confighelp.lua b/lualib/rspamadm/confighelp.lua new file mode 100644 index 000000000..a03578b6e --- /dev/null +++ b/lualib/rspamadm/confighelp.lua @@ -0,0 +1,111 @@ +local opts = {} +local known_attrs = { + data = 1, + example = 1, + type = 1, + required = 1, + default = 1, +} + +local getopt = require "rspamadm/getopt" +local ansicolors = require "rspamadm/ansicolors" + +local function maybe_print_color(key) + if not opts['no-color'] then + return ansicolors.white .. key .. ansicolors.reset + else + return key + end +end + +local function sort_values(tbl) + local res = {} + for k, v in pairs(tbl) do + table.insert(res, { key = k, value = v }) + end + + -- Sort order + local order = { + options = 1, + dns = 2, + upstream = 3, + logging = 4, + metric = 5, + composite = 6, + classifier = 7, + modules = 8, + lua = 9, + worker = 10, + workers = 11, + } + + table.sort(res, function(a, b) + local oa = order[a['key']] + local ob = order[b['key']] + + if oa and ob then + return oa < ob + elseif oa then + return -1 < 0 + elseif ob then + return 1 < 0 + else + return a['key'] < b['key'] + end + + end) + + return res +end + +local function print_help(key, value, tabs) + print(string.format('%sConfiguration element: %s', tabs, maybe_print_color(key))) + + if not opts['short'] then + if value['data'] then + local nv = string.match(value['data'], '^#%s*(.*)%s*$') or value.data + print(string.format('%s\tDescription: %s', tabs, nv)) + end + if value['type'] then + print(string.format('%s\tType: %s', tabs, value['type'])) + end + if type(value['required']) == 'boolean' then + if value['required'] then + print(string.format('%s\tRequired: %s', tabs, + maybe_print_color(tostring(value['required'])))) + else + print(string.format('%s\tRequired: %s', tabs, + tostring(value['required']))) + end + end + if value['default'] then + print(string.format('%s\tDefault: %s', tabs, value['default'])) + end + if not opts['no-examples'] and value['example'] then + local nv = string.match(value['example'], '^%s*(.*[^%s])%s*$') or value.example + print(string.format('%s\tExample:\n%s', tabs, nv)) + end + if value.type and value.type == 'object' then + print('') + end + end + + local sorted = sort_values(value) + for _, v in ipairs(sorted) do + if not known_attrs[v['key']] then + -- We need to go deeper + print_help(v['key'], v['value'], tabs .. '\t') + end + end +end + +return function(args, res) + opts = getopt.getopt(args, '') + + local sorted = sort_values(res) + + for _,v in ipairs(sorted) do + print_help(v['key'], v['value'], '') + print('') + end +end diff --git a/lualib/rspamadm/fuzzy_convert.lua b/lualib/rspamadm/fuzzy_convert.lua new file mode 100644 index 000000000..2d473ca46 --- /dev/null +++ b/lualib/rspamadm/fuzzy_convert.lua @@ -0,0 +1,198 @@ +local sqlite3 = require "rspamd_sqlite3" +local redis = require "rspamd_redis" +local util = require "rspamd_util" + +local function connect_redis(server, password, db) + local ret + local conn, err = redis.connect_sync({ + host = server, + }) + + if not conn then + return nil, 'Cannot connect: ' .. err + end + + if password then + ret = conn:add_cmd('AUTH', {password}) + if not ret then + return nil, 'Cannot queue command' + end + end + if db then + ret = conn:add_cmd('SELECT', {db}) + if not ret then + return nil, 'Cannot queue command' + end + end + + return conn, nil +end + +local function send_digests(digests, redis_host, redis_password, redis_db) + local conn, err = connect_redis(redis_host, redis_password, redis_db) + if err then + print(err) + return false + end + local ret + for _, v in ipairs(digests) do + ret = conn:add_cmd('HMSET', { + 'fuzzy' .. v[1], + 'F', v[2], + 'V', v[3], + }) + if not ret then + print('Cannot batch command') + return false + end + ret = conn:add_cmd('EXPIRE', { + 'fuzzy' .. v[1], + tostring(v[4]), + }) + if not ret then + print('Cannot batch command') + return false + end + end + ret, err = conn:exec() + if not ret then + print('Cannot execute batched commands: ' .. err) + return false + end + return true +end + +local function send_shingles(shingles, redis_host, redis_password, redis_db) + local conn, err = connect_redis(redis_host, redis_password, redis_db) + if err then + print("Redis error: " .. err) + return false + end + local ret + for _, v in ipairs(shingles) do + ret = conn:add_cmd('SET', { + 'fuzzy_' .. v[2] .. '_' .. v[1], + v[4], + }) + if not ret then + print('Cannot batch SET command: ' .. err) + return false + end + ret = conn:add_cmd('EXPIRE', { + 'fuzzy_' .. v[2] .. '_' .. v[1], + tostring(v[3]), + }) + if not ret then + print('Cannot batch command') + return false + end + end + ret, err = conn:exec() + if not ret then + print('Cannot execute batched commands: ' .. err) + return false + end + return true +end + +local function update_counters(total, redis_host, redis_password, redis_db) + local conn, err = connect_redis(redis_host, redis_password, redis_db) + if err then + print(err) + return false + end + local ret + ret = conn:add_cmd('SET', { + 'fuzzylocal', + total, + }) + if not ret then + print('Cannot batch command') + return false + end + ret = conn:add_cmd('SET', { + 'fuzzy_count', + total, + }) + if not ret then + print('Cannot batch command') + return false + end + ret, err = conn:exec() + if not ret then + print('Cannot execute batched commands: ' .. err) + return false + end + return true +end + +return function (_, res) + local db = sqlite3.open(res['source_db']) + local shingles = {} + local digests = {} + local num_batch_digests = 0 + local num_batch_shingles = 0 + local total_digests = 0 + local total_shingles = 0 + local lim_batch = 1000 -- Update each 1000 entries + local redis_password = res['redis_password'] + local redis_db = nil + + if res['redis_db'] then + redis_db = tostring(res['redis_db']) + end + + if not db then + print('Cannot open source db: ' .. res['source_db']) + return + end + + local now = util.get_time() + for row in db:rows('SELECT id, flag, digest, value, time FROM digests') do + + local expire_in = math.floor(now - row.time + res['expiry']) + if expire_in >= 1 then + table.insert(digests, {row.digest, row.flag, row.value, expire_in}) + num_batch_digests = num_batch_digests + 1 + total_digests = total_digests + 1 + for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do + table.insert(shingles, {srow.value, srow.number, expire_in, row.digest}) + total_shingles = total_shingles + 1 + num_batch_shingles = num_batch_shingles + 1 + end + end + if num_batch_digests >= lim_batch then + if not send_digests(digests, res['redis_host'], redis_password, redis_db) then + return + end + num_batch_digests = 0 + digests = {} + end + if num_batch_shingles >= lim_batch then + if not send_shingles(shingles, res['redis_host'], redis_password, redis_db) then + return + end + num_batch_shingles = 0 + shingles = {} + end + end + if digests[1] then + if not send_digests(digests, res['redis_host'], redis_password, redis_db) then + return + end + end + if shingles[1] then + if not send_shingles(shingles, res['redis_host'], redis_password, redis_db) then + return + end + end + + local message = string.format( + 'Migrated %d digests and %d shingles', + total_digests, total_shingles + ) + if not update_counters(total_digests, res['redis_host'], redis_password, redis_db) then + message = message .. ' but failed to update counters' + end + print(message) +end diff --git a/lualib/rspamadm/fuzzy_stat.lua b/lualib/rspamadm/fuzzy_stat.lua new file mode 100644 index 000000000..748dbda20 --- /dev/null +++ b/lualib/rspamadm/fuzzy_stat.lua @@ -0,0 +1,274 @@ +local util = require "rspamd_util" +local opts = {} + +local function add_data(target, src) + for k,v in pairs(src) do + if k ~= 'ips' then + if target[k] then + target[k] = target[k] + v + else + target[k] = v + end + else + if not target['ips'] then target['ips'] = {} end + -- Iterate over IPs + for ip,st in pairs(v) do + if not target['ips'][ip] then target['ips'][ip] = {} end + add_data(target['ips'][ip], st) + end + end + end +end + +local function print_num(num) + if opts['n'] or opts['number'] then + return tostring(num) + else + return util.humanize_number(num) + end +end + +local function print_stat(st, tabs) + if st['checked'] then + print(string.format('%sChecked: %s', tabs, print_num(st['checked']))) + end + if st['matched'] then + print(string.format('%sMatched: %s', tabs, print_num(st['matched']))) + end + if st['errors'] then + print(string.format('%sErrors: %s', tabs, print_num(st['errors']))) + end + if st['added'] then + print(string.format('%sAdded: %s', tabs, print_num(st['added']))) + end + if st['deleted'] then + print(string.format('%sDeleted: %s', tabs, print_num(st['deleted']))) + end +end + +-- Sort by checked +local function sort_ips(tbl, _opts) + local res = {} + for k,v in pairs(tbl) do + table.insert(res, {ip = k, data = v}) + end + + local function sort_order(elt) + local key = 'checked' + local _res = 0 + + if _opts['sort'] then + if _opts['sort'] == 'matched' then + key = 'matched' + elseif _opts['sort'] == 'errors' then + key = 'errors' + elseif _opts['sort'] == 'ip' then + return elt['ip'] + end + end + + if elt['data'][key] then + _res = elt['data'][key] + end + + return _res + end + + table.sort(res, function(a, b) + return sort_order(a) > sort_order(b) + end) + + return res +end + +local function add_result(dst, src, k) + if type(src) == 'table' then + if type(dst) == 'number' then + -- Convert dst to table + dst = {dst} + elseif type(dst) == 'nil' then + dst = {} + end + + for i,v in ipairs(src) do + if dst[i] and k ~= 'fuzzy_stored' then + dst[i] = dst[i] + v + else + dst[i] = v + end + end + else + if type(dst) == 'table' then + if k ~= 'fuzzy_stored' then + dst[1] = dst[1] + src + else + dst[1] = src + end + else + if dst and k ~= 'fuzzy_stored' then + dst = dst + src + else + dst = src + end + end + end + + return dst +end + +local function print_result(r) + local function num_to_epoch(num) + if num == 1 then + return 'v0.6' + elseif num == 2 then + return 'v0.8' + elseif num == 3 then + return 'v0.9' + elseif num == 4 then + return 'v1.0+' + end + return '???' + end + if type(r) == 'table' then + local res = {} + for i,num in ipairs(r) do + res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num)) + end + + return table.concat(res, ', ') + end + + return print_num(r) +end + +local getopt = require "rspamadm/getopt" + +return function(args, res) + local res_ips = {} + local res_databases = {} + local wrk = res['workers'] + opts = getopt.getopt(args, '') + + if wrk then + for _,pr in pairs(wrk) do + -- processes cycle + if pr['data'] then + local id = pr['id'] + + if id then + local res_db = res_databases[id] + if not res_db then + res_db = { + keys = {} + } + res_databases[id] = res_db + end + + -- General stats + for k,v in pairs(pr['data']) do + if k ~= 'keys' and k ~= 'errors_ips' then + res_db[k] = add_result(res_db[k], v, k) + elseif k == 'errors_ips' then + -- Errors ips + if not res_db['errors_ips'] then + res_db['errors_ips'] = {} + end + for ip,nerrors in pairs(v) do + if not res_db['errors_ips'][ip] then + res_db['errors_ips'][ip] = nerrors + else + res_db['errors_ips'][ip] = nerrors + res_db['errors_ips'][ip] + end + end + end + end + + if pr['data']['keys'] then + local res_keys = res_db['keys'] + if not res_keys then + res_keys = {} + res_db['keys'] = res_keys + end + -- Go through keys in input + for k,elts in pairs(pr['data']['keys']) do + -- keys cycle + if not res_keys[k] then + res_keys[k] = {} + end + + add_data(res_keys[k], elts) + + if elts['ips'] then + for ip,v in pairs(elts['ips']) do + if not res_ips[ip] then + res_ips[ip] = {} + end + add_data(res_ips[ip], v) + end + end + end + end + end + end + end + end + + -- General stats + for db,st in pairs(res_databases) do + print(string.format('Statistics for storage %s', db)) + + for k,v in pairs(st) do + if k ~= 'keys' and k ~= 'errors_ips' then + print(string.format('%s: %s', k, print_result(v))) + end + end + print('') + + local res_keys = st['keys'] + if res_keys and not opts['no-keys'] and not opts['short'] then + print('Keys statistics:') + for k,_st in pairs(res_keys) do + print(string.format('Key id: %s', k)) + print_stat(_st, '\t') + + if _st['ips'] and not opts['no-ips'] then + print('') + print('\tIPs stat:') + local sorted_ips = sort_ips(_st['ips'], opts) + + for _,v in ipairs(sorted_ips) do + print(string.format('\t%s', v['ip'])) + print_stat(v['data'], '\t\t') + print('') + end + end + + print('') + end + end + if st['errors_ips'] and not opts['no-ips'] and not opts['short'] then + print('') + print('Errors IPs statistics:') + table.sort(st['errors_ips'], function(a, b) + return a > b + end) + for i, v in pairs(st['errors_ips']) do + print(string.format('%s: %s', i, print_result(v))) + end + print('') + end + end + + if not opts['no-ips'] and not opts['short'] then + print('') + print('IPs statistics:') + + local sorted_ips = sort_ips(res_ips, opts) + for _, v in ipairs(sorted_ips) do + print(string.format('%s', v['ip'])) + print_stat(v['data'], '\t') + print('') + end + end +end + diff --git a/lualib/rspamadm/getopt.lua b/lualib/rspamadm/getopt.lua new file mode 100644 index 000000000..bd0a2f67e --- /dev/null +++ b/lualib/rspamadm/getopt.lua @@ -0,0 +1,34 @@ +local function getopt(arg, options) + local tab = {} + for k, v in ipairs(arg) do + if string.sub(v, 1, 2) == "--" then + local x = string.find(v, "=", 1, true) + if x then tab[string.sub(v, 3, x - 1)] = string.sub(v, x + 1) + else tab[string.sub(v, 3)] = true + end + elseif string.sub(v, 1, 1) == "-" then + local y = 2 + local l = string.len(v) + local jopt + while (y <= l) do + jopt = string.sub(v, y, y) + if string.find(options, jopt, 1, true) then + if y < l then + tab[jopt] = string.sub(v, y + 1) + y = l + else + tab[jopt] = arg[k + 1] + end + else + tab[jopt] = true + end + y = y + 1 + end + end + end + return tab +end + +return { + getopt = getopt +} diff --git a/lualib/rspamadm/grep.lua b/lualib/rspamadm/grep.lua new file mode 100644 index 000000000..a9d4b084a --- /dev/null +++ b/lualib/rspamadm/grep.lua @@ -0,0 +1,112 @@ +return function(_, res) + + local rspamd_regexp = require 'rspamd_regexp' + + local buffer = {} + local matches = {} + + local pattern = res['pattern'] + local re + if pattern then + re = rspamd_regexp.create(pattern) + if not re then + io.stderr:write("Couldn't compile regex: " .. pattern .. '\n') + os.exit(1) + end + end + + local plainm = true + if res['luapat'] then + plainm = false + end + local orphans = res['orphans'] + local search_str = res['string'] + local sensitive = res['sensitive'] + local partial = res['partial'] + if search_str and not sensitive then + search_str = string.lower(search_str) + end + local inputs = res['inputs'] + + for _, n in ipairs(inputs) do + local h, err + if string.match(n, '%.xz$') then + h, err = io.popen('xzcat ' .. n, 'r') + elseif string.match(n, '%.bz2$') then + h, err = io.popen('bzcat ' .. n, 'r') + elseif string.match(n, '%.gz$') then + h, err = io.popen('zcat ' .. n, 'r') + elseif n == 'stdin' then + h = io.input() + else + h, err = io.open(n, 'r') + end + if not h then + if err then + io.stderr:write("Couldn't open file (" .. n .. '): ' .. err .. '\n') + else + io.stderr:write("Couldn't open file (" .. n .. '): no error\n') + end + else + for line in h:lines() do + local hash = string.match(line, '<(%x+)>') + local already_matching = false + if hash then + if matches[hash] then + table.insert(matches[hash], line) + already_matching = true + else + if buffer[hash] then + table.insert(buffer[hash], line) + else + buffer[hash] = {line} + end + end + end + local ismatch = false + if re then + ismatch = re:match(line) + elseif sensitive and search_str then + ismatch = string.find(line, search_str, 1, plainm) + elseif search_str then + local lwr = string.lower(line) + ismatch = string.find(lwr, search_str, 1, plainm) + end + if ismatch then + if not hash then + if orphans then + print('*** orphaned ***') + print(line) + print() + end + elseif not already_matching then + matches[hash] = buffer[hash] + end + end + local is_end = string.match(line, '<%x+>; task; rspamd_protocol_http_reply:') + if is_end then + buffer[hash] = nil + if matches[hash] then + for _, v in ipairs(matches[hash]) do + print(v) + end + print() + matches[hash] = nil + end + end + end + if partial then + for k, v in pairs(matches) do + print('*** partial ***') + for _, vv in ipairs(v) do + print(vv) + end + print() + matches[k] = nil + end + else + matches = {} + end + end + end +end diff --git a/lualib/rspamadm/stat_convert.lua b/lualib/rspamadm/stat_convert.lua new file mode 100644 index 000000000..7b6de9836 --- /dev/null +++ b/lualib/rspamadm/stat_convert.lua @@ -0,0 +1,225 @@ +local sqlite3 = require "rspamd_sqlite3" +local redis = require "rspamd_redis" +local util = require "rspamd_util" + +local function send_redis(server, symbol, tokens, password, db, cmd) + local ret = true + local conn,err = redis.connect_sync({ + host = server, + }) + + local err_str + + if not conn then + print('Cannot connect to ' .. server .. ' error: ' .. err) + return false, err + end + + if password then + conn:add_cmd('AUTH', {password}) + end + if db then + conn:add_cmd('SELECT', {db}) + end + + for _,t in ipairs(tokens) do + if not conn:add_cmd(cmd, {symbol .. t[3], t[1], t[2]}) then + ret = false + err_str = 'add command failure' .. string.format('%s %s', + cmd, table.concat({symbol .. t[3], t[1], t[2]}, ' ')) + end + end + + if ret then + ret,err_str = conn:exec() + end + + return ret,err_str +end + +local function convert_learned(cache, server, password, redis_db) + local converted = 0 + local db = sqlite3.open(cache) + local ret = true + local err_str + + if not db then + print('Cannot open cache database: ' .. cache) + return false + end + + db:sql('BEGIN;') + + local conn,err = redis.connect_sync({ + host = server, + }) + + if not conn then + print('Cannot connect to ' .. server .. ' error: ' .. err) + return false + end + + if password then + conn:add_cmd('AUTH', {password}) + end + if redis_db then + conn:add_cmd('SELECT', {redis_db}) + end + + for row in db:rows('SELECT * FROM learns;') do + local is_spam + local digest = tostring(util.encode_base32(row.digest)) + + if row.flag == '0' then + is_spam = '-1' + else + is_spam = '1' + end + + if not conn:add_cmd('HSET', {'learned_ids', digest, is_spam}) then + print('Cannot add hash: ' .. digest) + ret = false + else + converted = converted + 1 + end + end + db:sql('COMMIT;') + + if ret then + ret,err_str = conn:exec() + end + + if ret then + print(string.format('Converted %d cached items from sqlite3 learned cache to redis', + converted)) + else + print('Error occurred during sending data to redis: ' .. err_str) + end + + return ret +end + +return function (_, res) + local db = sqlite3.open(res['source_db']) + local tokens = {} + local num = 0 + local total = 0 + local nusers = 0 + local lim = 1000 -- Update each 1000 tokens + local users_map = {} + local learns = {} + local redis_password = res['redis_password'] + local redis_db = nil + local cmd = 'HINCRBY' + local ret, err_str + + if res['redis_db'] then + redis_db = tostring(res['redis_db']) + end + if res['reset_previous'] then + cmd = 'HSET' + end + + if res['cache_db'] then + if not convert_learned(res['cache_db'], res['redis_host'], + redis_password, redis_db) then + print('Cannot convert learned cache to redis') + return + end + end + + if not db then + print('Cannot open source db: ' .. res['source_db']) + return + end + + db:sql('BEGIN;') + -- Fill users mapping + for row in db:rows('SELECT * FROM users;') do + if row.id == '0' then + users_map[row.id] = '' + else + users_map[row.id] = row.name + end + learns[row.id] = row.learns + nusers = nusers + 1 + end + + -- Workaround for old databases + for row in db:rows('SELECT * FROM languages') do + if learns['0'] then + learns['0'] = learns['0'] + row.learns + else + learns['0'] = row.learns + end + end + + -- Fill tokens, sending data to redis each `lim` records + for row in db:rows('SELECT token,value,user FROM tokens;') do + local user = '' + if row.user ~= 0 and users_map[row.user] then + user = users_map[row.user] + end + + table.insert(tokens, {row.token, row.value, user}) + + num = num + 1 + total = total + 1 + if num > lim then + ret,err_str = send_redis(res['redis_host'], res['symbol'], + tokens, redis_password, redis_db, cmd) + if not ret then + print('Cannot send tokens to the redis server: ' .. err_str) + return + end + + num = 0 + tokens = {} + end + end + if #tokens > 0 then + ret, err_str = send_redis(res['redis_host'], res['symbol'], tokens, + redis_password, redis_db, cmd) + + if not ret then + print('Cannot send tokens to the redis server: ' .. err_str) + return + end + end + -- Now update all users + local conn,err = redis.connect_sync({ + host = res['redis_host'], + }) + + if not conn then + print('Cannot connect to ' .. res['redis_host'] .. ' error: ' .. err) + return false + end + + if redis_password then + conn:add_cmd('AUTH', {redis_password}) + end + if redis_db then + conn:add_cmd('SELECT', {redis_db}) + end + + for id,learned in pairs(learns) do + local user = users_map[id] + if not conn:add_cmd(cmd, {res['symbol'] .. user, 'learns', learned}) then + print('Cannot update learns for user: ' .. user) + end + if not conn:add_cmd('SADD', {res['symbol'] .. '_keys', res['symbol'] .. user}) then + print('Cannot update learns for user: ' .. user) + end + end + db:sql('COMMIT;') + + ret = conn:exec() + + if ret then + print(string.format('Migrated %d tokens for %d users for symbol %s', + total, nusers, res['symbol'])) + else + print('Error occurred during sending data to redis') + end +end diff --git a/src/rspamadm/CMakeLists.txt b/src/rspamadm/CMakeLists.txt index 752e23d8b..7e991372a 100644 --- a/src/rspamadm/CMakeLists.txt +++ b/src/rspamadm/CMakeLists.txt @@ -21,16 +21,7 @@ SET(RSPAMADMSRC rspamadm.c ${CMAKE_SOURCE_DIR}/src/worker.c ${CMAKE_SOURCE_DIR}/src/rspamd_proxy.c ${CMAKE_SOURCE_DIR}/src/log_helper.c) -SET(RSPAMADMLUASRC - ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_stat.lua - ${CMAKE_CURRENT_SOURCE_DIR}/confighelp.lua) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -ADD_CUSTOM_TARGET(rspamadm_lua_preprocess - ${PERL_EXECUTABLE} - "${CMAKE_SOURCE_DIR}/lua_preprocess.pl" - "${CMAKE_CURRENT_SOURCE_DIR}" - "${CMAKE_CURRENT_BINARY_DIR}" - SOURCES ${RSPAMADMLUASRC} ${CMAKE_SOURCE_DIR}/lua_preprocess.pl) IF (ENABLE_HYPERSCAN MATCHES "ON") LIST(APPEND RSPAMADMSRC "${CMAKE_SOURCE_DIR}/src/hs_helper.c") ENDIF() diff --git a/src/rspamadm/ansicolors.lua.in b/src/rspamadm/ansicolors.lua.in deleted file mode 100644 index 1e9ca2065..000000000 --- a/src/rspamadm/ansicolors.lua.in +++ /dev/null @@ -1,56 +0,0 @@ -local colormt = {} -local ansicolors = {} - -function colormt:__tostring() - return self.value -end - -function colormt:__concat(other) - return tostring(self) .. tostring(other) -end - -function colormt:__call(s) - return self .. s .. _M.reset -end - -colormt.__metatable = {} - -local function makecolor(value) - return setmetatable({ value = string.char(27) .. '[' .. tostring(value) .. 'm' }, colormt) -end - -local colors = { - -- attributes - reset = 0, - clear = 0, - bright = 1, - dim = 2, - underscore = 4, - blink = 5, - reverse = 7, - hidden = 8, - - -- foreground - black = 30, - red = 31, - green = 32, - yellow = 33, - blue = 34, - magenta = 35, - cyan = 36, - white = 37, - - -- background - onblack = 40, - onred = 41, - ongreen = 42, - onyellow = 43, - onblue = 44, - onmagenta = 45, - oncyan = 46, - onwhite = 47, -} - -for c, v in pairs(colors) do - ansicolors[c] = makecolor(v) -end diff --git a/src/rspamadm/confighelp.c b/src/rspamadm/confighelp.c index a64afe4b0..f3da9422d 100644 --- a/src/rspamadm/confighelp.c +++ b/src/rspamadm/confighelp.c @@ -85,6 +85,7 @@ rspamadm_confighelp_show (struct rspamd_config *cfg, gint argc, gchar **argv, { rspamd_fstring_t *out; + rspamd_lua_set_path (cfg->lua_state, NULL, NULL); out = rspamd_fstring_new (); if (json) { @@ -107,7 +108,7 @@ rspamadm_confighelp_show (struct rspamd_config *cfg, gint argc, gchar **argv, argc, argv, obj, - rspamadm_script_confighelp); + "confighelp"); rspamd_fstring_free (out); return; diff --git a/src/rspamadm/confighelp.lua b/src/rspamadm/confighelp.lua deleted file mode 100644 index e95dd0b36..000000000 --- a/src/rspamadm/confighelp.lua +++ /dev/null @@ -1,112 +0,0 @@ -local opts = {} -local known_attrs = { - data = 1, - example = 1, - type = 1, - required = 1, - default = 1, -} - ---.USE "getopt" ---.USE "ansicolors" - - -local function maybe_print_color(key) - if not opts['no-color'] then - return ansicolors.white .. key .. ansicolors.reset - else - return key - end -end - -local function sort_values(tbl) - local res = {} - for k, v in pairs(tbl) do - table.insert(res, { key = k, value = v }) - end - - -- Sort order - local order = { - options = 1, - dns = 2, - upstream = 3, - logging = 4, - metric = 5, - composite = 6, - classifier = 7, - modules = 8, - lua = 9, - worker = 10, - workers = 11, - } - - table.sort(res, function(a, b) - local oa = order[a['key']] - local ob = order[b['key']] - - if oa and ob then - return oa < ob - elseif oa then - return -1 < 0 - elseif ob then - return 1 < 0 - else - return a['key'] < b['key'] - end - - end) - - return res -end - -local function print_help(key, value, tabs) - print(string.format('%sConfiguration element: %s', tabs, maybe_print_color(key))) - - if not opts['short'] then - if value['data'] then - local nv = string.match(value['data'], '^#%s*(.*)%s*$') or value.data - print(string.format('%s\tDescription: %s', tabs, nv)) - end - if value['type'] then - print(string.format('%s\tType: %s', tabs, value['type'])) - end - if type(value['required']) == 'boolean' then - if value['required'] then - print(string.format('%s\tRequired: %s', tabs, - maybe_print_color(tostring(value['required'])))) - else - print(string.format('%s\tRequired: %s', tabs, - tostring(value['required']))) - end - end - if value['default'] then - print(string.format('%s\tDefault: %s', tabs, value['default'])) - end - if not opts['no-examples'] and value['example'] then - local nv = string.match(value['example'], '^%s*(.*[^%s])%s*$') or value.example - print(string.format('%s\tExample:\n%s', tabs, nv)) - end - if value.type and value.type == 'object' then - print('') - end - end - - local sorted = sort_values(value) - for _, v in ipairs(sorted) do - if not known_attrs[v['key']] then - -- We need to go deeper - print_help(v['key'], v['value'], tabs .. '\t') - end - end -end - -return function(args, res) - opts = getopt(args, '') - - local sorted = sort_values(res) - - for _,v in ipairs(sorted) do - print_help(v['key'], v['value'], '') - print('') - end -end diff --git a/src/rspamadm/fuzzy_convert.c b/src/rspamadm/fuzzy_convert.c index dc15b7e38..a143b89d9 100644 --- a/src/rspamadm/fuzzy_convert.c +++ b/src/rspamadm/fuzzy_convert.c @@ -17,7 +17,6 @@ #include "config.h" #include "rspamadm.h" #include "lua/lua_common.h" -#include "fuzzy_convert.lua.h" static gchar *source_db = NULL; static gchar *redis_host = NULL; @@ -109,6 +108,7 @@ rspamadm_fuzzyconvert (gint argc, gchar **argv) } L = rspamd_lua_init (); + rspamd_lua_set_path (L, NULL, NULL); obj = ucl_object_typed_new (UCL_OBJECT); ucl_object_insert_key (obj, ucl_object_fromstring (source_db), @@ -132,7 +132,7 @@ rspamadm_fuzzyconvert (gint argc, gchar **argv) argc, argv, obj, - rspamadm_script_fuzzy_convert); + "fuzzy_convert"); lua_close (L); ucl_object_unref (obj); diff --git a/src/rspamadm/fuzzy_convert.lua b/src/rspamadm/fuzzy_convert.lua deleted file mode 100644 index 2d473ca46..000000000 --- a/src/rspamadm/fuzzy_convert.lua +++ /dev/null @@ -1,198 +0,0 @@ -local sqlite3 = require "rspamd_sqlite3" -local redis = require "rspamd_redis" -local util = require "rspamd_util" - -local function connect_redis(server, password, db) - local ret - local conn, err = redis.connect_sync({ - host = server, - }) - - if not conn then - return nil, 'Cannot connect: ' .. err - end - - if password then - ret = conn:add_cmd('AUTH', {password}) - if not ret then - return nil, 'Cannot queue command' - end - end - if db then - ret = conn:add_cmd('SELECT', {db}) - if not ret then - return nil, 'Cannot queue command' - end - end - - return conn, nil -end - -local function send_digests(digests, redis_host, redis_password, redis_db) - local conn, err = connect_redis(redis_host, redis_password, redis_db) - if err then - print(err) - return false - end - local ret - for _, v in ipairs(digests) do - ret = conn:add_cmd('HMSET', { - 'fuzzy' .. v[1], - 'F', v[2], - 'V', v[3], - }) - if not ret then - print('Cannot batch command') - return false - end - ret = conn:add_cmd('EXPIRE', { - 'fuzzy' .. v[1], - tostring(v[4]), - }) - if not ret then - print('Cannot batch command') - return false - end - end - ret, err = conn:exec() - if not ret then - print('Cannot execute batched commands: ' .. err) - return false - end - return true -end - -local function send_shingles(shingles, redis_host, redis_password, redis_db) - local conn, err = connect_redis(redis_host, redis_password, redis_db) - if err then - print("Redis error: " .. err) - return false - end - local ret - for _, v in ipairs(shingles) do - ret = conn:add_cmd('SET', { - 'fuzzy_' .. v[2] .. '_' .. v[1], - v[4], - }) - if not ret then - print('Cannot batch SET command: ' .. err) - return false - end - ret = conn:add_cmd('EXPIRE', { - 'fuzzy_' .. v[2] .. '_' .. v[1], - tostring(v[3]), - }) - if not ret then - print('Cannot batch command') - return false - end - end - ret, err = conn:exec() - if not ret then - print('Cannot execute batched commands: ' .. err) - return false - end - return true -end - -local function update_counters(total, redis_host, redis_password, redis_db) - local conn, err = connect_redis(redis_host, redis_password, redis_db) - if err then - print(err) - return false - end - local ret - ret = conn:add_cmd('SET', { - 'fuzzylocal', - total, - }) - if not ret then - print('Cannot batch command') - return false - end - ret = conn:add_cmd('SET', { - 'fuzzy_count', - total, - }) - if not ret then - print('Cannot batch command') - return false - end - ret, err = conn:exec() - if not ret then - print('Cannot execute batched commands: ' .. err) - return false - end - return true -end - -return function (_, res) - local db = sqlite3.open(res['source_db']) - local shingles = {} - local digests = {} - local num_batch_digests = 0 - local num_batch_shingles = 0 - local total_digests = 0 - local total_shingles = 0 - local lim_batch = 1000 -- Update each 1000 entries - local redis_password = res['redis_password'] - local redis_db = nil - - if res['redis_db'] then - redis_db = tostring(res['redis_db']) - end - - if not db then - print('Cannot open source db: ' .. res['source_db']) - return - end - - local now = util.get_time() - for row in db:rows('SELECT id, flag, digest, value, time FROM digests') do - - local expire_in = math.floor(now - row.time + res['expiry']) - if expire_in >= 1 then - table.insert(digests, {row.digest, row.flag, row.value, expire_in}) - num_batch_digests = num_batch_digests + 1 - total_digests = total_digests + 1 - for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do - table.insert(shingles, {srow.value, srow.number, expire_in, row.digest}) - total_shingles = total_shingles + 1 - num_batch_shingles = num_batch_shingles + 1 - end - end - if num_batch_digests >= lim_batch then - if not send_digests(digests, res['redis_host'], redis_password, redis_db) then - return - end - num_batch_digests = 0 - digests = {} - end - if num_batch_shingles >= lim_batch then - if not send_shingles(shingles, res['redis_host'], redis_password, redis_db) then - return - end - num_batch_shingles = 0 - shingles = {} - end - end - if digests[1] then - if not send_digests(digests, res['redis_host'], redis_password, redis_db) then - return - end - end - if shingles[1] then - if not send_shingles(shingles, res['redis_host'], redis_password, redis_db) then - return - end - end - - local message = string.format( - 'Migrated %d digests and %d shingles', - total_digests, total_shingles - ) - if not update_counters(total_digests, res['redis_host'], redis_password, redis_db) then - message = message .. ' but failed to update counters' - end - print(message) -end diff --git a/src/rspamadm/fuzzy_stat.lua b/src/rspamadm/fuzzy_stat.lua deleted file mode 100644 index 401b297d1..000000000 --- a/src/rspamadm/fuzzy_stat.lua +++ /dev/null @@ -1,274 +0,0 @@ -local util = require "rspamd_util" -local opts = {} - -local function add_data(target, src) - for k,v in pairs(src) do - if k ~= 'ips' then - if target[k] then - target[k] = target[k] + v - else - target[k] = v - end - else - if not target['ips'] then target['ips'] = {} end - -- Iterate over IPs - for ip,st in pairs(v) do - if not target['ips'][ip] then target['ips'][ip] = {} end - add_data(target['ips'][ip], st) - end - end - end -end - -local function print_num(num) - if opts['n'] or opts['number'] then - return tostring(num) - else - return util.humanize_number(num) - end -end - -local function print_stat(st, tabs) - if st['checked'] then - print(string.format('%sChecked: %s', tabs, print_num(st['checked']))) - end - if st['matched'] then - print(string.format('%sMatched: %s', tabs, print_num(st['matched']))) - end - if st['errors'] then - print(string.format('%sErrors: %s', tabs, print_num(st['errors']))) - end - if st['added'] then - print(string.format('%sAdded: %s', tabs, print_num(st['added']))) - end - if st['deleted'] then - print(string.format('%sDeleted: %s', tabs, print_num(st['deleted']))) - end -end - --- Sort by checked -local function sort_ips(tbl, _opts) - local res = {} - for k,v in pairs(tbl) do - table.insert(res, {ip = k, data = v}) - end - - local function sort_order(elt) - local key = 'checked' - local _res = 0 - - if _opts['sort'] then - if _opts['sort'] == 'matched' then - key = 'matched' - elseif _opts['sort'] == 'errors' then - key = 'errors' - elseif _opts['sort'] == 'ip' then - return elt['ip'] - end - end - - if elt['data'][key] then - _res = elt['data'][key] - end - - return _res - end - - table.sort(res, function(a, b) - return sort_order(a) > sort_order(b) - end) - - return res -end - -local function add_result(dst, src, k) - if type(src) == 'table' then - if type(dst) == 'number' then - -- Convert dst to table - dst = {dst} - elseif type(dst) == 'nil' then - dst = {} - end - - for i,v in ipairs(src) do - if dst[i] and k ~= 'fuzzy_stored' then - dst[i] = dst[i] + v - else - dst[i] = v - end - end - else - if type(dst) == 'table' then - if k ~= 'fuzzy_stored' then - dst[1] = dst[1] + src - else - dst[1] = src - end - else - if dst and k ~= 'fuzzy_stored' then - dst = dst + src - else - dst = src - end - end - end - - return dst -end - -local function print_result(r) - local function num_to_epoch(num) - if num == 1 then - return 'v0.6' - elseif num == 2 then - return 'v0.8' - elseif num == 3 then - return 'v0.9' - elseif num == 4 then - return 'v1.0+' - end - return '???' - end - if type(r) == 'table' then - local res = {} - for i,num in ipairs(r) do - res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num)) - end - - return table.concat(res, ', ') - end - - return print_num(r) -end - ---.USE "getopt" - -return function(args, res) - local res_ips = {} - local res_databases = {} - local wrk = res['workers'] - opts = getopt(args, '') - - if wrk then - for _,pr in pairs(wrk) do - -- processes cycle - if pr['data'] then - local id = pr['id'] - - if id then - local res_db = res_databases[id] - if not res_db then - res_db = { - keys = {} - } - res_databases[id] = res_db - end - - -- General stats - for k,v in pairs(pr['data']) do - if k ~= 'keys' and k ~= 'errors_ips' then - res_db[k] = add_result(res_db[k], v, k) - elseif k == 'errors_ips' then - -- Errors ips - if not res_db['errors_ips'] then - res_db['errors_ips'] = {} - end - for ip,nerrors in pairs(v) do - if not res_db['errors_ips'][ip] then - res_db['errors_ips'][ip] = nerrors - else - res_db['errors_ips'][ip] = nerrors + res_db['errors_ips'][ip] - end - end - end - end - - if pr['data']['keys'] then - local res_keys = res_db['keys'] - if not res_keys then - res_keys = {} - res_db['keys'] = res_keys - end - -- Go through keys in input - for k,elts in pairs(pr['data']['keys']) do - -- keys cycle - if not res_keys[k] then - res_keys[k] = {} - end - - add_data(res_keys[k], elts) - - if elts['ips'] then - for ip,v in pairs(elts['ips']) do - if not res_ips[ip] then - res_ips[ip] = {} - end - add_data(res_ips[ip], v) - end - end - end - end - end - end - end - end - - -- General stats - for db,st in pairs(res_databases) do - print(string.format('Statistics for storage %s', db)) - - for k,v in pairs(st) do - if k ~= 'keys' and k ~= 'errors_ips' then - print(string.format('%s: %s', k, print_result(v))) - end - end - print('') - - local res_keys = st['keys'] - if res_keys and not opts['no-keys'] and not opts['short'] then - print('Keys statistics:') - for k,_st in pairs(res_keys) do - print(string.format('Key id: %s', k)) - print_stat(_st, '\t') - - if _st['ips'] and not opts['no-ips'] then - print('') - print('\tIPs stat:') - local sorted_ips = sort_ips(_st['ips'], opts) - - for _,v in ipairs(sorted_ips) do - print(string.format('\t%s', v['ip'])) - print_stat(v['data'], '\t\t') - print('') - end - end - - print('') - end - end - if st['errors_ips'] and not opts['no-ips'] and not opts['short'] then - print('') - print('Errors IPs statistics:') - table.sort(st['errors_ips'], function(a, b) - return a > b - end) - for i, v in pairs(st['errors_ips']) do - print(string.format('%s: %s', i, print_result(v))) - end - print('') - end - end - - if not opts['no-ips'] and not opts['short'] then - print('') - print('IPs statistics:') - - local sorted_ips = sort_ips(res_ips, opts) - for _, v in ipairs(sorted_ips) do - print(string.format('%s', v['ip'])) - print_stat(v['data'], '\t') - print('') - end - end -end - diff --git a/src/rspamadm/getopt.lua.in b/src/rspamadm/getopt.lua.in deleted file mode 100644 index d069d2d5f..000000000 --- a/src/rspamadm/getopt.lua.in +++ /dev/null @@ -1,31 +0,0 @@ -local function getopt(arg, options) - local tab = {} - for k, v in ipairs(arg) do - if string.sub(v, 1, 2) == "--" then - local x = string.find(v, "=", 1, true) - if x then tab[string.sub(v, 3, x - 1)] = string.sub(v, x + 1) - else tab[string.sub(v, 3)] = true - end - elseif string.sub(v, 1, 1) == "-" then - local y = 2 - local l = string.len(v) - local jopt - while (y <= l) do - jopt = string.sub(v, y, y) - if string.find(options, jopt, 1, true) then - if y < l then - tab[jopt] = string.sub(v, y + 1) - y = l - else - tab[jopt] = arg[k + 1] - end - else - tab[jopt] = true - end - y = y + 1 - end - end - end - return tab -end - diff --git a/src/rspamadm/grep.c b/src/rspamadm/grep.c index 9665b40e8..2ce4f2c76 100644 --- a/src/rspamadm/grep.c +++ b/src/rspamadm/grep.c @@ -17,7 +17,6 @@ #include "config.h" #include "rspamadm.h" #include "lua/lua_common.h" -#include "grep.lua.h" static gchar *string = NULL; static gchar *pattern = NULL; @@ -115,6 +114,7 @@ rspamadm_grep (gint argc, gchar **argv) } L = rspamd_lua_init (); + rspamd_lua_set_path (L, NULL, NULL); obj = ucl_object_typed_new (UCL_OBJECT); if (string) { @@ -148,7 +148,7 @@ rspamadm_grep (gint argc, gchar **argv) argc, argv, obj, - rspamadm_script_grep); + "grep"); lua_close (L); ucl_object_unref (obj); diff --git a/src/rspamadm/grep.lua b/src/rspamadm/grep.lua deleted file mode 100644 index a9d4b084a..000000000 --- a/src/rspamadm/grep.lua +++ /dev/null @@ -1,112 +0,0 @@ -return function(_, res) - - local rspamd_regexp = require 'rspamd_regexp' - - local buffer = {} - local matches = {} - - local pattern = res['pattern'] - local re - if pattern then - re = rspamd_regexp.create(pattern) - if not re then - io.stderr:write("Couldn't compile regex: " .. pattern .. '\n') - os.exit(1) - end - end - - local plainm = true - if res['luapat'] then - plainm = false - end - local orphans = res['orphans'] - local search_str = res['string'] - local sensitive = res['sensitive'] - local partial = res['partial'] - if search_str and not sensitive then - search_str = string.lower(search_str) - end - local inputs = res['inputs'] - - for _, n in ipairs(inputs) do - local h, err - if string.match(n, '%.xz$') then - h, err = io.popen('xzcat ' .. n, 'r') - elseif string.match(n, '%.bz2$') then - h, err = io.popen('bzcat ' .. n, 'r') - elseif string.match(n, '%.gz$') then - h, err = io.popen('zcat ' .. n, 'r') - elseif n == 'stdin' then - h = io.input() - else - h, err = io.open(n, 'r') - end - if not h then - if err then - io.stderr:write("Couldn't open file (" .. n .. '): ' .. err .. '\n') - else - io.stderr:write("Couldn't open file (" .. n .. '): no error\n') - end - else - for line in h:lines() do - local hash = string.match(line, '<(%x+)>') - local already_matching = false - if hash then - if matches[hash] then - table.insert(matches[hash], line) - already_matching = true - else - if buffer[hash] then - table.insert(buffer[hash], line) - else - buffer[hash] = {line} - end - end - end - local ismatch = false - if re then - ismatch = re:match(line) - elseif sensitive and search_str then - ismatch = string.find(line, search_str, 1, plainm) - elseif search_str then - local lwr = string.lower(line) - ismatch = string.find(lwr, search_str, 1, plainm) - end - if ismatch then - if not hash then - if orphans then - print('*** orphaned ***') - print(line) - print() - end - elseif not already_matching then - matches[hash] = buffer[hash] - end - end - local is_end = string.match(line, '<%x+>; task; rspamd_protocol_http_reply:') - if is_end then - buffer[hash] = nil - if matches[hash] then - for _, v in ipairs(matches[hash]) do - print(v) - end - print() - matches[hash] = nil - end - end - end - if partial then - for k, v in pairs(matches) do - print('*** partial ***') - for _, vv in ipairs(v) do - print(vv) - end - print() - matches[k] = nil - end - else - matches = {} - end - end - end -end diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index d524a8f61..f58da84af 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -187,28 +187,30 @@ rspamadm_parse_ucl_var (const gchar *option_name, gboolean rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv, - const ucl_object_t *res, const gchar *script) + const ucl_object_t *res, const gchar *script_name) { lua_State *L = pL; gint err_idx, i, ret; GString *tb; + gchar str[PATH_MAX]; - g_assert (script != NULL); + g_assert (script_name != NULL); g_assert (res != NULL); g_assert (L != NULL); - if (luaL_dostring (L, script) != 0) { - msg_err ("cannot execute lua script: %s", - lua_tostring (L, -1)); + rspamd_snprintf (str, sizeof (str), "return require \"%s.%s\"", "rspamadm", + script_name); + + if (luaL_dostring (L, str) != 0) { + msg_err ("cannot execute lua script %s: %s", + str, lua_tostring (L, -1)); return FALSE; } else { if (lua_type (L, -1) != LUA_TFUNCTION) { msg_err ("lua script must return " "function and not %s", - lua_typename (L, - lua_type (L, -1))); - lua_settop (L, 0); + lua_typename (L, lua_type (L, -1))); return FALSE; } @@ -233,7 +235,7 @@ rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv, if ((ret = lua_pcall (L, 2, 0, err_idx)) != 0) { tb = lua_touserdata (L, -1); - msg_err ("call to adm lua script failed (%d): %v", ret, tb); + msg_err ("call to rspamadm lua script failed (%d): %v", ret, tb); if (tb) { g_string_free (tb, TRUE); diff --git a/src/rspamadm/rspamadm.h b/src/rspamadm/rspamadm.h index 3c4da9aca..a778457bc 100644 --- a/src/rspamadm/rspamadm.h +++ b/src/rspamadm/rspamadm.h @@ -41,6 +41,6 @@ extern struct rspamadm_command help_command; const struct rspamadm_command *rspamadm_search_command (const gchar *name); gboolean rspamadm_execute_lua_ucl_subr (gpointer L, gint argc, gchar **argv, - const ucl_object_t *res, const gchar *script); + const ucl_object_t *res, const gchar *script_name); #endif diff --git a/src/rspamadm/stat_convert.c b/src/rspamadm/stat_convert.c index 48acbc9bd..98eb33700 100644 --- a/src/rspamadm/stat_convert.c +++ b/src/rspamadm/stat_convert.c @@ -16,7 +16,6 @@ #include "config.h" #include "rspamadm.h" #include "lua/lua_common.h" -#include "stat_convert.lua.h" static gchar *source_db = NULL; static gchar *redis_host = NULL; @@ -117,6 +116,7 @@ rspamadm_statconvert (gint argc, gchar **argv) } L = rspamd_lua_init (); + rspamd_lua_set_path (L, NULL, NULL); obj = ucl_object_typed_new (UCL_OBJECT); ucl_object_insert_key (obj, ucl_object_fromstring (source_db), @@ -147,7 +147,7 @@ rspamadm_statconvert (gint argc, gchar **argv) argc, argv, obj, - rspamadm_script_stat_convert); + "stat_convert"); lua_close (L); ucl_object_unref (obj); diff --git a/src/rspamadm/stat_convert.lua b/src/rspamadm/stat_convert.lua deleted file mode 100644 index 7b6de9836..000000000 --- a/src/rspamadm/stat_convert.lua +++ /dev/null @@ -1,225 +0,0 @@ -local sqlite3 = require "rspamd_sqlite3" -local redis = require "rspamd_redis" -local util = require "rspamd_util" - -local function send_redis(server, symbol, tokens, password, db, cmd) - local ret = true - local conn,err = redis.connect_sync({ - host = server, - }) - - local err_str - - if not conn then - print('Cannot connect to ' .. server .. ' error: ' .. err) - return false, err - end - - if password then - conn:add_cmd('AUTH', {password}) - end - if db then - conn:add_cmd('SELECT', {db}) - end - - for _,t in ipairs(tokens) do - if not conn:add_cmd(cmd, {symbol .. t[3], t[1], t[2]}) then - ret = false - err_str = 'add command failure' .. string.format('%s %s', - cmd, table.concat({symbol .. t[3], t[1], t[2]}, ' ')) - end - end - - if ret then - ret,err_str = conn:exec() - end - - return ret,err_str -end - -local function convert_learned(cache, server, password, redis_db) - local converted = 0 - local db = sqlite3.open(cache) - local ret = true - local err_str - - if not db then - print('Cannot open cache database: ' .. cache) - return false - end - - db:sql('BEGIN;') - - local conn,err = redis.connect_sync({ - host = server, - }) - - if not conn then - print('Cannot connect to ' .. server .. ' error: ' .. err) - return false - end - - if password then - conn:add_cmd('AUTH', {password}) - end - if redis_db then - conn:add_cmd('SELECT', {redis_db}) - end - - for row in db:rows('SELECT * FROM learns;') do - local is_spam - local digest = tostring(util.encode_base32(row.digest)) - - if row.flag == '0' then - is_spam = '-1' - else - is_spam = '1' - end - - if not conn:add_cmd('HSET', {'learned_ids', digest, is_spam}) then - print('Cannot add hash: ' .. digest) - ret = false - else - converted = converted + 1 - end - end - db:sql('COMMIT;') - - if ret then - ret,err_str = conn:exec() - end - - if ret then - print(string.format('Converted %d cached items from sqlite3 learned cache to redis', - converted)) - else - print('Error occurred during sending data to redis: ' .. err_str) - end - - return ret -end - -return function (_, res) - local db = sqlite3.open(res['source_db']) - local tokens = {} - local num = 0 - local total = 0 - local nusers = 0 - local lim = 1000 -- Update each 1000 tokens - local users_map = {} - local learns = {} - local redis_password = res['redis_password'] - local redis_db = nil - local cmd = 'HINCRBY' - local ret, err_str - - if res['redis_db'] then - redis_db = tostring(res['redis_db']) - end - if res['reset_previous'] then - cmd = 'HSET' - end - - if res['cache_db'] then - if not convert_learned(res['cache_db'], res['redis_host'], - redis_password, redis_db) then - print('Cannot convert learned cache to redis') - return - end - end - - if not db then - print('Cannot open source db: ' .. res['source_db']) - return - end - - db:sql('BEGIN;') - -- Fill users mapping - for row in db:rows('SELECT * FROM users;') do - if row.id == '0' then - users_map[row.id] = '' - else - users_map[row.id] = row.name - end - learns[row.id] = row.learns - nusers = nusers + 1 - end - - -- Workaround for old databases - for row in db:rows('SELECT * FROM languages') do - if learns['0'] then - learns['0'] = learns['0'] + row.learns - else - learns['0'] = row.learns - end - end - - -- Fill tokens, sending data to redis each `lim` records - for row in db:rows('SELECT token,value,user FROM tokens;') do - local user = '' - if row.user ~= 0 and users_map[row.user] then - user = users_map[row.user] - end - - table.insert(tokens, {row.token, row.value, user}) - - num = num + 1 - total = total + 1 - if num > lim then - ret,err_str = send_redis(res['redis_host'], res['symbol'], - tokens, redis_password, redis_db, cmd) - if not ret then - print('Cannot send tokens to the redis server: ' .. err_str) - return - end - - num = 0 - tokens = {} - end - end - if #tokens > 0 then - ret, err_str = send_redis(res['redis_host'], res['symbol'], tokens, - redis_password, redis_db, cmd) - - if not ret then - print('Cannot send tokens to the redis server: ' .. err_str) - return - end - end - -- Now update all users - local conn,err = redis.connect_sync({ - host = res['redis_host'], - }) - - if not conn then - print('Cannot connect to ' .. res['redis_host'] .. ' error: ' .. err) - return false - end - - if redis_password then - conn:add_cmd('AUTH', {redis_password}) - end - if redis_db then - conn:add_cmd('SELECT', {redis_db}) - end - - for id,learned in pairs(learns) do - local user = users_map[id] - if not conn:add_cmd(cmd, {res['symbol'] .. user, 'learns', learned}) then - print('Cannot update learns for user: ' .. user) - end - if not conn:add_cmd('SADD', {res['symbol'] .. '_keys', res['symbol'] .. user}) then - print('Cannot update learns for user: ' .. user) - end - end - db:sql('COMMIT;') - - ret = conn:exec() - - if ret then - print(string.format('Migrated %d tokens for %d users for symbol %s', - total, nusers, res['symbol'])) - else - print('Error occurred during sending data to redis') - end -end -- cgit v1.2.3