--- /dev/null
+local colormt = {}
+local ansicolors = {}
+
+function colormt:__tostring()
+ return self.value
+end
+
+function colormt:__concat(other)
+ return tostring(self) .. tostring(other)
+end
+
+function colormt:__call(s)
+ return self .. s .. ansicolors.reset
+end
+
+colormt.__metatable = {}
+
+local function makecolor(value)
+ return setmetatable({ value = string.char(27) .. '[' .. tostring(value) .. 'm' }, colormt)
+end
+
+local colors = {
+ -- attributes
+ reset = 0,
+ clear = 0,
+ bright = 1,
+ dim = 2,
+ underscore = 4,
+ blink = 5,
+ reverse = 7,
+ hidden = 8,
+
+ -- foreground
+ black = 30,
+ red = 31,
+ green = 32,
+ yellow = 33,
+ blue = 34,
+ magenta = 35,
+ cyan = 36,
+ white = 37,
+
+ -- background
+ onblack = 40,
+ onred = 41,
+ ongreen = 42,
+ onyellow = 43,
+ onblue = 44,
+ onmagenta = 45,
+ oncyan = 46,
+ onwhite = 47,
+}
+
+for c, v in pairs(colors) do
+ ansicolors[c] = makecolor(v)
+end
+
+return ansicolors
\ No newline at end of file
--- /dev/null
+local opts = {}
+local known_attrs = {
+ data = 1,
+ example = 1,
+ type = 1,
+ required = 1,
+ default = 1,
+}
+
+local getopt = require "rspamadm/getopt"
+local ansicolors = require "rspamadm/ansicolors"
+
+local function maybe_print_color(key)
+ if not opts['no-color'] then
+ return ansicolors.white .. key .. ansicolors.reset
+ else
+ return key
+ end
+end
+
+local function sort_values(tbl)
+ local res = {}
+ for k, v in pairs(tbl) do
+ table.insert(res, { key = k, value = v })
+ end
+
+ -- Sort order
+ local order = {
+ options = 1,
+ dns = 2,
+ upstream = 3,
+ logging = 4,
+ metric = 5,
+ composite = 6,
+ classifier = 7,
+ modules = 8,
+ lua = 9,
+ worker = 10,
+ workers = 11,
+ }
+
+ table.sort(res, function(a, b)
+ local oa = order[a['key']]
+ local ob = order[b['key']]
+
+ if oa and ob then
+ return oa < ob
+ elseif oa then
+ return -1 < 0
+ elseif ob then
+ return 1 < 0
+ else
+ return a['key'] < b['key']
+ end
+
+ end)
+
+ return res
+end
+
+local function print_help(key, value, tabs)
+ print(string.format('%sConfiguration element: %s', tabs, maybe_print_color(key)))
+
+ if not opts['short'] then
+ if value['data'] then
+ local nv = string.match(value['data'], '^#%s*(.*)%s*$') or value.data
+ print(string.format('%s\tDescription: %s', tabs, nv))
+ end
+ if value['type'] then
+ print(string.format('%s\tType: %s', tabs, value['type']))
+ end
+ if type(value['required']) == 'boolean' then
+ if value['required'] then
+ print(string.format('%s\tRequired: %s', tabs,
+ maybe_print_color(tostring(value['required']))))
+ else
+ print(string.format('%s\tRequired: %s', tabs,
+ tostring(value['required'])))
+ end
+ end
+ if value['default'] then
+ print(string.format('%s\tDefault: %s', tabs, value['default']))
+ end
+ if not opts['no-examples'] and value['example'] then
+ local nv = string.match(value['example'], '^%s*(.*[^%s])%s*$') or value.example
+ print(string.format('%s\tExample:\n%s', tabs, nv))
+ end
+ if value.type and value.type == 'object' then
+ print('')
+ end
+ end
+
+ local sorted = sort_values(value)
+ for _, v in ipairs(sorted) do
+ if not known_attrs[v['key']] then
+ -- We need to go deeper
+ print_help(v['key'], v['value'], tabs .. '\t')
+ end
+ end
+end
+
+return function(args, res)
+ opts = getopt.getopt(args, '')
+
+ local sorted = sort_values(res)
+
+ for _,v in ipairs(sorted) do
+ print_help(v['key'], v['value'], '')
+ print('')
+ end
+end
--- /dev/null
+local sqlite3 = require "rspamd_sqlite3"
+local redis = require "rspamd_redis"
+local util = require "rspamd_util"
+
+local function connect_redis(server, password, db)
+ local ret
+ local conn, err = redis.connect_sync({
+ host = server,
+ })
+
+ if not conn then
+ return nil, 'Cannot connect: ' .. err
+ end
+
+ if password then
+ ret = conn:add_cmd('AUTH', {password})
+ if not ret then
+ return nil, 'Cannot queue command'
+ end
+ end
+ if db then
+ ret = conn:add_cmd('SELECT', {db})
+ if not ret then
+ return nil, 'Cannot queue command'
+ end
+ end
+
+ return conn, nil
+end
+
+local function send_digests(digests, redis_host, redis_password, redis_db)
+ local conn, err = connect_redis(redis_host, redis_password, redis_db)
+ if err then
+ print(err)
+ return false
+ end
+ local ret
+ for _, v in ipairs(digests) do
+ ret = conn:add_cmd('HMSET', {
+ 'fuzzy' .. v[1],
+ 'F', v[2],
+ 'V', v[3],
+ })
+ if not ret then
+ print('Cannot batch command')
+ return false
+ end
+ ret = conn:add_cmd('EXPIRE', {
+ 'fuzzy' .. v[1],
+ tostring(v[4]),
+ })
+ if not ret then
+ print('Cannot batch command')
+ return false
+ end
+ end
+ ret, err = conn:exec()
+ if not ret then
+ print('Cannot execute batched commands: ' .. err)
+ return false
+ end
+ return true
+end
+
+local function send_shingles(shingles, redis_host, redis_password, redis_db)
+ local conn, err = connect_redis(redis_host, redis_password, redis_db)
+ if err then
+ print("Redis error: " .. err)
+ return false
+ end
+ local ret
+ for _, v in ipairs(shingles) do
+ ret = conn:add_cmd('SET', {
+ 'fuzzy_' .. v[2] .. '_' .. v[1],
+ v[4],
+ })
+ if not ret then
+ print('Cannot batch SET command: ' .. err)
+ return false
+ end
+ ret = conn:add_cmd('EXPIRE', {
+ 'fuzzy_' .. v[2] .. '_' .. v[1],
+ tostring(v[3]),
+ })
+ if not ret then
+ print('Cannot batch command')
+ return false
+ end
+ end
+ ret, err = conn:exec()
+ if not ret then
+ print('Cannot execute batched commands: ' .. err)
+ return false
+ end
+ return true
+end
+
+local function update_counters(total, redis_host, redis_password, redis_db)
+ local conn, err = connect_redis(redis_host, redis_password, redis_db)
+ if err then
+ print(err)
+ return false
+ end
+ local ret
+ ret = conn:add_cmd('SET', {
+ 'fuzzylocal',
+ total,
+ })
+ if not ret then
+ print('Cannot batch command')
+ return false
+ end
+ ret = conn:add_cmd('SET', {
+ 'fuzzy_count',
+ total,
+ })
+ if not ret then
+ print('Cannot batch command')
+ return false
+ end
+ ret, err = conn:exec()
+ if not ret then
+ print('Cannot execute batched commands: ' .. err)
+ return false
+ end
+ return true
+end
+
+return function (_, res)
+ local db = sqlite3.open(res['source_db'])
+ local shingles = {}
+ local digests = {}
+ local num_batch_digests = 0
+ local num_batch_shingles = 0
+ local total_digests = 0
+ local total_shingles = 0
+ local lim_batch = 1000 -- Update each 1000 entries
+ local redis_password = res['redis_password']
+ local redis_db = nil
+
+ if res['redis_db'] then
+ redis_db = tostring(res['redis_db'])
+ end
+
+ if not db then
+ print('Cannot open source db: ' .. res['source_db'])
+ return
+ end
+
+ local now = util.get_time()
+ for row in db:rows('SELECT id, flag, digest, value, time FROM digests') do
+
+ local expire_in = math.floor(now - row.time + res['expiry'])
+ if expire_in >= 1 then
+ table.insert(digests, {row.digest, row.flag, row.value, expire_in})
+ num_batch_digests = num_batch_digests + 1
+ total_digests = total_digests + 1
+ for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do
+ table.insert(shingles, {srow.value, srow.number, expire_in, row.digest})
+ total_shingles = total_shingles + 1
+ num_batch_shingles = num_batch_shingles + 1
+ end
+ end
+ if num_batch_digests >= lim_batch then
+ if not send_digests(digests, res['redis_host'], redis_password, redis_db) then
+ return
+ end
+ num_batch_digests = 0
+ digests = {}
+ end
+ if num_batch_shingles >= lim_batch then
+ if not send_shingles(shingles, res['redis_host'], redis_password, redis_db) then
+ return
+ end
+ num_batch_shingles = 0
+ shingles = {}
+ end
+ end
+ if digests[1] then
+ if not send_digests(digests, res['redis_host'], redis_password, redis_db) then
+ return
+ end
+ end
+ if shingles[1] then
+ if not send_shingles(shingles, res['redis_host'], redis_password, redis_db) then
+ return
+ end
+ end
+
+ local message = string.format(
+ 'Migrated %d digests and %d shingles',
+ total_digests, total_shingles
+ )
+ if not update_counters(total_digests, res['redis_host'], redis_password, redis_db) then
+ message = message .. ' but failed to update counters'
+ end
+ print(message)
+end
--- /dev/null
+local util = require "rspamd_util"
+local opts = {}
+
+local function add_data(target, src)
+ for k,v in pairs(src) do
+ if k ~= 'ips' then
+ if target[k] then
+ target[k] = target[k] + v
+ else
+ target[k] = v
+ end
+ else
+ if not target['ips'] then target['ips'] = {} end
+ -- Iterate over IPs
+ for ip,st in pairs(v) do
+ if not target['ips'][ip] then target['ips'][ip] = {} end
+ add_data(target['ips'][ip], st)
+ end
+ end
+ end
+end
+
+local function print_num(num)
+ if opts['n'] or opts['number'] then
+ return tostring(num)
+ else
+ return util.humanize_number(num)
+ end
+end
+
+local function print_stat(st, tabs)
+ if st['checked'] then
+ print(string.format('%sChecked: %s', tabs, print_num(st['checked'])))
+ end
+ if st['matched'] then
+ print(string.format('%sMatched: %s', tabs, print_num(st['matched'])))
+ end
+ if st['errors'] then
+ print(string.format('%sErrors: %s', tabs, print_num(st['errors'])))
+ end
+ if st['added'] then
+ print(string.format('%sAdded: %s', tabs, print_num(st['added'])))
+ end
+ if st['deleted'] then
+ print(string.format('%sDeleted: %s', tabs, print_num(st['deleted'])))
+ end
+end
+
+-- Sort by checked
+local function sort_ips(tbl, _opts)
+ local res = {}
+ for k,v in pairs(tbl) do
+ table.insert(res, {ip = k, data = v})
+ end
+
+ local function sort_order(elt)
+ local key = 'checked'
+ local _res = 0
+
+ if _opts['sort'] then
+ if _opts['sort'] == 'matched' then
+ key = 'matched'
+ elseif _opts['sort'] == 'errors' then
+ key = 'errors'
+ elseif _opts['sort'] == 'ip' then
+ return elt['ip']
+ end
+ end
+
+ if elt['data'][key] then
+ _res = elt['data'][key]
+ end
+
+ return _res
+ end
+
+ table.sort(res, function(a, b)
+ return sort_order(a) > sort_order(b)
+ end)
+
+ return res
+end
+
+local function add_result(dst, src, k)
+ if type(src) == 'table' then
+ if type(dst) == 'number' then
+ -- Convert dst to table
+ dst = {dst}
+ elseif type(dst) == 'nil' then
+ dst = {}
+ end
+
+ for i,v in ipairs(src) do
+ if dst[i] and k ~= 'fuzzy_stored' then
+ dst[i] = dst[i] + v
+ else
+ dst[i] = v
+ end
+ end
+ else
+ if type(dst) == 'table' then
+ if k ~= 'fuzzy_stored' then
+ dst[1] = dst[1] + src
+ else
+ dst[1] = src
+ end
+ else
+ if dst and k ~= 'fuzzy_stored' then
+ dst = dst + src
+ else
+ dst = src
+ end
+ end
+ end
+
+ return dst
+end
+
+local function print_result(r)
+ local function num_to_epoch(num)
+ if num == 1 then
+ return 'v0.6'
+ elseif num == 2 then
+ return 'v0.8'
+ elseif num == 3 then
+ return 'v0.9'
+ elseif num == 4 then
+ return 'v1.0+'
+ end
+ return '???'
+ end
+ if type(r) == 'table' then
+ local res = {}
+ for i,num in ipairs(r) do
+ res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num))
+ end
+
+ return table.concat(res, ', ')
+ end
+
+ return print_num(r)
+end
+
+local getopt = require "rspamadm/getopt"
+
+return function(args, res)
+ local res_ips = {}
+ local res_databases = {}
+ local wrk = res['workers']
+ opts = getopt.getopt(args, '')
+
+ if wrk then
+ for _,pr in pairs(wrk) do
+ -- processes cycle
+ if pr['data'] then
+ local id = pr['id']
+
+ if id then
+ local res_db = res_databases[id]
+ if not res_db then
+ res_db = {
+ keys = {}
+ }
+ res_databases[id] = res_db
+ end
+
+ -- General stats
+ for k,v in pairs(pr['data']) do
+ if k ~= 'keys' and k ~= 'errors_ips' then
+ res_db[k] = add_result(res_db[k], v, k)
+ elseif k == 'errors_ips' then
+ -- Errors ips
+ if not res_db['errors_ips'] then
+ res_db['errors_ips'] = {}
+ end
+ for ip,nerrors in pairs(v) do
+ if not res_db['errors_ips'][ip] then
+ res_db['errors_ips'][ip] = nerrors
+ else
+ res_db['errors_ips'][ip] = nerrors + res_db['errors_ips'][ip]
+ end
+ end
+ end
+ end
+
+ if pr['data']['keys'] then
+ local res_keys = res_db['keys']
+ if not res_keys then
+ res_keys = {}
+ res_db['keys'] = res_keys
+ end
+ -- Go through keys in input
+ for k,elts in pairs(pr['data']['keys']) do
+ -- keys cycle
+ if not res_keys[k] then
+ res_keys[k] = {}
+ end
+
+ add_data(res_keys[k], elts)
+
+ if elts['ips'] then
+ for ip,v in pairs(elts['ips']) do
+ if not res_ips[ip] then
+ res_ips[ip] = {}
+ end
+ add_data(res_ips[ip], v)
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ -- General stats
+ for db,st in pairs(res_databases) do
+ print(string.format('Statistics for storage %s', db))
+
+ for k,v in pairs(st) do
+ if k ~= 'keys' and k ~= 'errors_ips' then
+ print(string.format('%s: %s', k, print_result(v)))
+ end
+ end
+ print('')
+
+ local res_keys = st['keys']
+ if res_keys and not opts['no-keys'] and not opts['short'] then
+ print('Keys statistics:')
+ for k,_st in pairs(res_keys) do
+ print(string.format('Key id: %s', k))
+ print_stat(_st, '\t')
+
+ if _st['ips'] and not opts['no-ips'] then
+ print('')
+ print('\tIPs stat:')
+ local sorted_ips = sort_ips(_st['ips'], opts)
+
+ for _,v in ipairs(sorted_ips) do
+ print(string.format('\t%s', v['ip']))
+ print_stat(v['data'], '\t\t')
+ print('')
+ end
+ end
+
+ print('')
+ end
+ end
+ if st['errors_ips'] and not opts['no-ips'] and not opts['short'] then
+ print('')
+ print('Errors IPs statistics:')
+ table.sort(st['errors_ips'], function(a, b)
+ return a > b
+ end)
+ for i, v in pairs(st['errors_ips']) do
+ print(string.format('%s: %s', i, print_result(v)))
+ end
+ print('')
+ end
+ end
+
+ if not opts['no-ips'] and not opts['short'] then
+ print('')
+ print('IPs statistics:')
+
+ local sorted_ips = sort_ips(res_ips, opts)
+ for _, v in ipairs(sorted_ips) do
+ print(string.format('%s', v['ip']))
+ print_stat(v['data'], '\t')
+ print('')
+ end
+ end
+end
+
--- /dev/null
+local function getopt(arg, options)
+ local tab = {}
+ for k, v in ipairs(arg) do
+ if string.sub(v, 1, 2) == "--" then
+ local x = string.find(v, "=", 1, true)
+ if x then tab[string.sub(v, 3, x - 1)] = string.sub(v, x + 1)
+ else tab[string.sub(v, 3)] = true
+ end
+ elseif string.sub(v, 1, 1) == "-" then
+ local y = 2
+ local l = string.len(v)
+ local jopt
+ while (y <= l) do
+ jopt = string.sub(v, y, y)
+ if string.find(options, jopt, 1, true) then
+ if y < l then
+ tab[jopt] = string.sub(v, y + 1)
+ y = l
+ else
+ tab[jopt] = arg[k + 1]
+ end
+ else
+ tab[jopt] = true
+ end
+ y = y + 1
+ end
+ end
+ end
+ return tab
+end
+
+return {
+ getopt = getopt
+}
--- /dev/null
+return function(_, res)
+
+ local rspamd_regexp = require 'rspamd_regexp'
+
+ local buffer = {}
+ local matches = {}
+
+ local pattern = res['pattern']
+ local re
+ if pattern then
+ re = rspamd_regexp.create(pattern)
+ if not re then
+ io.stderr:write("Couldn't compile regex: " .. pattern .. '\n')
+ os.exit(1)
+ end
+ end
+
+ local plainm = true
+ if res['luapat'] then
+ plainm = false
+ end
+ local orphans = res['orphans']
+ local search_str = res['string']
+ local sensitive = res['sensitive']
+ local partial = res['partial']
+ if search_str and not sensitive then
+ search_str = string.lower(search_str)
+ end
+ local inputs = res['inputs']
+
+ for _, n in ipairs(inputs) do
+ local h, err
+ if string.match(n, '%.xz$') then
+ h, err = io.popen('xzcat ' .. n, 'r')
+ elseif string.match(n, '%.bz2$') then
+ h, err = io.popen('bzcat ' .. n, 'r')
+ elseif string.match(n, '%.gz$') then
+ h, err = io.popen('zcat ' .. n, 'r')
+ elseif n == 'stdin' then
+ h = io.input()
+ else
+ h, err = io.open(n, 'r')
+ end
+ if not h then
+ if err then
+ io.stderr:write("Couldn't open file (" .. n .. '): ' .. err .. '\n')
+ else
+ io.stderr:write("Couldn't open file (" .. n .. '): no error\n')
+ end
+ else
+ for line in h:lines() do
+ local hash = string.match(line, '<(%x+)>')
+ local already_matching = false
+ if hash then
+ if matches[hash] then
+ table.insert(matches[hash], line)
+ already_matching = true
+ else
+ if buffer[hash] then
+ table.insert(buffer[hash], line)
+ else
+ buffer[hash] = {line}
+ end
+ end
+ end
+ local ismatch = false
+ if re then
+ ismatch = re:match(line)
+ elseif sensitive and search_str then
+ ismatch = string.find(line, search_str, 1, plainm)
+ elseif search_str then
+ local lwr = string.lower(line)
+ ismatch = string.find(lwr, search_str, 1, plainm)
+ end
+ if ismatch then
+ if not hash then
+ if orphans then
+ print('*** orphaned ***')
+ print(line)
+ print()
+ end
+ elseif not already_matching then
+ matches[hash] = buffer[hash]
+ end
+ end
+ local is_end = string.match(line, '<%x+>; task; rspamd_protocol_http_reply:')
+ if is_end then
+ buffer[hash] = nil
+ if matches[hash] then
+ for _, v in ipairs(matches[hash]) do
+ print(v)
+ end
+ print()
+ matches[hash] = nil
+ end
+ end
+ end
+ if partial then
+ for k, v in pairs(matches) do
+ print('*** partial ***')
+ for _, vv in ipairs(v) do
+ print(vv)
+ end
+ print()
+ matches[k] = nil
+ end
+ else
+ matches = {}
+ end
+ end
+ end
+end
--- /dev/null
+local sqlite3 = require "rspamd_sqlite3"
+local redis = require "rspamd_redis"
+local util = require "rspamd_util"
+
+local function send_redis(server, symbol, tokens, password, db, cmd)
+ local ret = true
+ local conn,err = redis.connect_sync({
+ host = server,
+ })
+
+ local err_str
+
+ if not conn then
+ print('Cannot connect to ' .. server .. ' error: ' .. err)
+ return false, err
+ end
+
+ if password then
+ conn:add_cmd('AUTH', {password})
+ end
+ if db then
+ conn:add_cmd('SELECT', {db})
+ end
+
+ for _,t in ipairs(tokens) do
+ if not conn:add_cmd(cmd, {symbol .. t[3], t[1], t[2]}) then
+ ret = false
+ err_str = 'add command failure' .. string.format('%s %s',
+ cmd, table.concat({symbol .. t[3], t[1], t[2]}, ' '))
+ end
+ end
+
+ if ret then
+ ret,err_str = conn:exec()
+ end
+
+ return ret,err_str
+end
+
+local function convert_learned(cache, server, password, redis_db)
+ local converted = 0
+ local db = sqlite3.open(cache)
+ local ret = true
+ local err_str
+
+ if not db then
+ print('Cannot open cache database: ' .. cache)
+ return false
+ end
+
+ db:sql('BEGIN;')
+
+ local conn,err = redis.connect_sync({
+ host = server,
+ })
+
+ if not conn then
+ print('Cannot connect to ' .. server .. ' error: ' .. err)
+ return false
+ end
+
+ if password then
+ conn:add_cmd('AUTH', {password})
+ end
+ if redis_db then
+ conn:add_cmd('SELECT', {redis_db})
+ end
+
+ for row in db:rows('SELECT * FROM learns;') do
+ local is_spam
+ local digest = tostring(util.encode_base32(row.digest))
+
+ if row.flag == '0' then
+ is_spam = '-1'
+ else
+ is_spam = '1'
+ end
+
+ if not conn:add_cmd('HSET', {'learned_ids', digest, is_spam}) then
+ print('Cannot add hash: ' .. digest)
+ ret = false
+ else
+ converted = converted + 1
+ end
+ end
+ db:sql('COMMIT;')
+
+ if ret then
+ ret,err_str = conn:exec()
+ end
+
+ if ret then
+ print(string.format('Converted %d cached items from sqlite3 learned cache to redis',
+ converted))
+ else
+ print('Error occurred during sending data to redis: ' .. err_str)
+ end
+
+ return ret
+end
+
+return function (_, res)
+ local db = sqlite3.open(res['source_db'])
+ local tokens = {}
+ local num = 0
+ local total = 0
+ local nusers = 0
+ local lim = 1000 -- Update each 1000 tokens
+ local users_map = {}
+ local learns = {}
+ local redis_password = res['redis_password']
+ local redis_db = nil
+ local cmd = 'HINCRBY'
+ local ret, err_str
+
+ if res['redis_db'] then
+ redis_db = tostring(res['redis_db'])
+ end
+ if res['reset_previous'] then
+ cmd = 'HSET'
+ end
+
+ if res['cache_db'] then
+ if not convert_learned(res['cache_db'], res['redis_host'],
+ redis_password, redis_db) then
+ print('Cannot convert learned cache to redis')
+ return
+ end
+ end
+
+ if not db then
+ print('Cannot open source db: ' .. res['source_db'])
+ return
+ end
+
+ db:sql('BEGIN;')
+ -- Fill users mapping
+ for row in db:rows('SELECT * FROM users;') do
+ if row.id == '0' then
+ users_map[row.id] = ''
+ else
+ users_map[row.id] = row.name
+ end
+ learns[row.id] = row.learns
+ nusers = nusers + 1
+ end
+
+ -- Workaround for old databases
+ for row in db:rows('SELECT * FROM languages') do
+ if learns['0'] then
+ learns['0'] = learns['0'] + row.learns
+ else
+ learns['0'] = row.learns
+ end
+ end
+
+ -- Fill tokens, sending data to redis each `lim` records
+ for row in db:rows('SELECT token,value,user FROM tokens;') do
+ local user = ''
+ if row.user ~= 0 and users_map[row.user] then
+ user = users_map[row.user]
+ end
+
+ table.insert(tokens, {row.token, row.value, user})
+
+ num = num + 1
+ total = total + 1
+ if num > lim then
+ ret,err_str = send_redis(res['redis_host'], res['symbol'],
+ tokens, redis_password, redis_db, cmd)
+ if not ret then
+ print('Cannot send tokens to the redis server: ' .. err_str)
+ return
+ end
+
+ num = 0
+ tokens = {}
+ end
+ end
+ if #tokens > 0 then
+ ret, err_str = send_redis(res['redis_host'], res['symbol'], tokens,
+ redis_password, redis_db, cmd)
+
+ if not ret then
+ print('Cannot send tokens to the redis server: ' .. err_str)
+ return
+ end
+ end
+ -- Now update all users
+ local conn,err = redis.connect_sync({
+ host = res['redis_host'],
+ })
+
+ if not conn then
+ print('Cannot connect to ' .. res['redis_host'] .. ' error: ' .. err)
+ return false
+ end
+
+ if redis_password then
+ conn:add_cmd('AUTH', {redis_password})
+ end
+ if redis_db then
+ conn:add_cmd('SELECT', {redis_db})
+ end
+
+ for id,learned in pairs(learns) do
+ local user = users_map[id]
+ if not conn:add_cmd(cmd, {res['symbol'] .. user, 'learns', learned}) then
+ print('Cannot update learns for user: ' .. user)
+ end
+ if not conn:add_cmd('SADD', {res['symbol'] .. '_keys', res['symbol'] .. user}) then
+ print('Cannot update learns for user: ' .. user)
+ end
+ end
+ db:sql('COMMIT;')
+
+ ret = conn:exec()
+
+ if ret then
+ print(string.format('Migrated %d tokens for %d users for symbol %s',
+ total, nusers, res['symbol']))
+ else
+ print('Error occurred during sending data to redis')
+ end
+end
${CMAKE_SOURCE_DIR}/src/worker.c
${CMAKE_SOURCE_DIR}/src/rspamd_proxy.c
${CMAKE_SOURCE_DIR}/src/log_helper.c)
-SET(RSPAMADMLUASRC
- ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_stat.lua
- ${CMAKE_CURRENT_SOURCE_DIR}/confighelp.lua)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
-ADD_CUSTOM_TARGET(rspamadm_lua_preprocess
- ${PERL_EXECUTABLE}
- "${CMAKE_SOURCE_DIR}/lua_preprocess.pl"
- "${CMAKE_CURRENT_SOURCE_DIR}"
- "${CMAKE_CURRENT_BINARY_DIR}"
- SOURCES ${RSPAMADMLUASRC} ${CMAKE_SOURCE_DIR}/lua_preprocess.pl)
IF (ENABLE_HYPERSCAN MATCHES "ON")
LIST(APPEND RSPAMADMSRC "${CMAKE_SOURCE_DIR}/src/hs_helper.c")
ENDIF()
+++ /dev/null
-local colormt = {}
-local ansicolors = {}
-
-function colormt:__tostring()
- return self.value
-end
-
-function colormt:__concat(other)
- return tostring(self) .. tostring(other)
-end
-
-function colormt:__call(s)
- return self .. s .. _M.reset
-end
-
-colormt.__metatable = {}
-
-local function makecolor(value)
- return setmetatable({ value = string.char(27) .. '[' .. tostring(value) .. 'm' }, colormt)
-end
-
-local colors = {
- -- attributes
- reset = 0,
- clear = 0,
- bright = 1,
- dim = 2,
- underscore = 4,
- blink = 5,
- reverse = 7,
- hidden = 8,
-
- -- foreground
- black = 30,
- red = 31,
- green = 32,
- yellow = 33,
- blue = 34,
- magenta = 35,
- cyan = 36,
- white = 37,
-
- -- background
- onblack = 40,
- onred = 41,
- ongreen = 42,
- onyellow = 43,
- onblue = 44,
- onmagenta = 45,
- oncyan = 46,
- onwhite = 47,
-}
-
-for c, v in pairs(colors) do
- ansicolors[c] = makecolor(v)
-end
{
rspamd_fstring_t *out;
+ rspamd_lua_set_path (cfg->lua_state, NULL, NULL);
out = rspamd_fstring_new ();
if (json) {
argc,
argv,
obj,
- rspamadm_script_confighelp);
+ "confighelp");
rspamd_fstring_free (out);
return;
+++ /dev/null
-local opts = {}
-local known_attrs = {
- data = 1,
- example = 1,
- type = 1,
- required = 1,
- default = 1,
-}
-
---.USE "getopt"
---.USE "ansicolors"
-
-
-local function maybe_print_color(key)
- if not opts['no-color'] then
- return ansicolors.white .. key .. ansicolors.reset
- else
- return key
- end
-end
-
-local function sort_values(tbl)
- local res = {}
- for k, v in pairs(tbl) do
- table.insert(res, { key = k, value = v })
- end
-
- -- Sort order
- local order = {
- options = 1,
- dns = 2,
- upstream = 3,
- logging = 4,
- metric = 5,
- composite = 6,
- classifier = 7,
- modules = 8,
- lua = 9,
- worker = 10,
- workers = 11,
- }
-
- table.sort(res, function(a, b)
- local oa = order[a['key']]
- local ob = order[b['key']]
-
- if oa and ob then
- return oa < ob
- elseif oa then
- return -1 < 0
- elseif ob then
- return 1 < 0
- else
- return a['key'] < b['key']
- end
-
- end)
-
- return res
-end
-
-local function print_help(key, value, tabs)
- print(string.format('%sConfiguration element: %s', tabs, maybe_print_color(key)))
-
- if not opts['short'] then
- if value['data'] then
- local nv = string.match(value['data'], '^#%s*(.*)%s*$') or value.data
- print(string.format('%s\tDescription: %s', tabs, nv))
- end
- if value['type'] then
- print(string.format('%s\tType: %s', tabs, value['type']))
- end
- if type(value['required']) == 'boolean' then
- if value['required'] then
- print(string.format('%s\tRequired: %s', tabs,
- maybe_print_color(tostring(value['required']))))
- else
- print(string.format('%s\tRequired: %s', tabs,
- tostring(value['required'])))
- end
- end
- if value['default'] then
- print(string.format('%s\tDefault: %s', tabs, value['default']))
- end
- if not opts['no-examples'] and value['example'] then
- local nv = string.match(value['example'], '^%s*(.*[^%s])%s*$') or value.example
- print(string.format('%s\tExample:\n%s', tabs, nv))
- end
- if value.type and value.type == 'object' then
- print('')
- end
- end
-
- local sorted = sort_values(value)
- for _, v in ipairs(sorted) do
- if not known_attrs[v['key']] then
- -- We need to go deeper
- print_help(v['key'], v['value'], tabs .. '\t')
- end
- end
-end
-
-return function(args, res)
- opts = getopt(args, '')
-
- local sorted = sort_values(res)
-
- for _,v in ipairs(sorted) do
- print_help(v['key'], v['value'], '')
- print('')
- end
-end
#include "config.h"
#include "rspamadm.h"
#include "lua/lua_common.h"
-#include "fuzzy_convert.lua.h"
static gchar *source_db = NULL;
static gchar *redis_host = NULL;
}
L = rspamd_lua_init ();
+ rspamd_lua_set_path (L, NULL, NULL);
obj = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (obj, ucl_object_fromstring (source_db),
argc,
argv,
obj,
- rspamadm_script_fuzzy_convert);
+ "fuzzy_convert");
lua_close (L);
ucl_object_unref (obj);
+++ /dev/null
-local sqlite3 = require "rspamd_sqlite3"
-local redis = require "rspamd_redis"
-local util = require "rspamd_util"
-
-local function connect_redis(server, password, db)
- local ret
- local conn, err = redis.connect_sync({
- host = server,
- })
-
- if not conn then
- return nil, 'Cannot connect: ' .. err
- end
-
- if password then
- ret = conn:add_cmd('AUTH', {password})
- if not ret then
- return nil, 'Cannot queue command'
- end
- end
- if db then
- ret = conn:add_cmd('SELECT', {db})
- if not ret then
- return nil, 'Cannot queue command'
- end
- end
-
- return conn, nil
-end
-
-local function send_digests(digests, redis_host, redis_password, redis_db)
- local conn, err = connect_redis(redis_host, redis_password, redis_db)
- if err then
- print(err)
- return false
- end
- local ret
- for _, v in ipairs(digests) do
- ret = conn:add_cmd('HMSET', {
- 'fuzzy' .. v[1],
- 'F', v[2],
- 'V', v[3],
- })
- if not ret then
- print('Cannot batch command')
- return false
- end
- ret = conn:add_cmd('EXPIRE', {
- 'fuzzy' .. v[1],
- tostring(v[4]),
- })
- if not ret then
- print('Cannot batch command')
- return false
- end
- end
- ret, err = conn:exec()
- if not ret then
- print('Cannot execute batched commands: ' .. err)
- return false
- end
- return true
-end
-
-local function send_shingles(shingles, redis_host, redis_password, redis_db)
- local conn, err = connect_redis(redis_host, redis_password, redis_db)
- if err then
- print("Redis error: " .. err)
- return false
- end
- local ret
- for _, v in ipairs(shingles) do
- ret = conn:add_cmd('SET', {
- 'fuzzy_' .. v[2] .. '_' .. v[1],
- v[4],
- })
- if not ret then
- print('Cannot batch SET command: ' .. err)
- return false
- end
- ret = conn:add_cmd('EXPIRE', {
- 'fuzzy_' .. v[2] .. '_' .. v[1],
- tostring(v[3]),
- })
- if not ret then
- print('Cannot batch command')
- return false
- end
- end
- ret, err = conn:exec()
- if not ret then
- print('Cannot execute batched commands: ' .. err)
- return false
- end
- return true
-end
-
-local function update_counters(total, redis_host, redis_password, redis_db)
- local conn, err = connect_redis(redis_host, redis_password, redis_db)
- if err then
- print(err)
- return false
- end
- local ret
- ret = conn:add_cmd('SET', {
- 'fuzzylocal',
- total,
- })
- if not ret then
- print('Cannot batch command')
- return false
- end
- ret = conn:add_cmd('SET', {
- 'fuzzy_count',
- total,
- })
- if not ret then
- print('Cannot batch command')
- return false
- end
- ret, err = conn:exec()
- if not ret then
- print('Cannot execute batched commands: ' .. err)
- return false
- end
- return true
-end
-
-return function (_, res)
- local db = sqlite3.open(res['source_db'])
- local shingles = {}
- local digests = {}
- local num_batch_digests = 0
- local num_batch_shingles = 0
- local total_digests = 0
- local total_shingles = 0
- local lim_batch = 1000 -- Update each 1000 entries
- local redis_password = res['redis_password']
- local redis_db = nil
-
- if res['redis_db'] then
- redis_db = tostring(res['redis_db'])
- end
-
- if not db then
- print('Cannot open source db: ' .. res['source_db'])
- return
- end
-
- local now = util.get_time()
- for row in db:rows('SELECT id, flag, digest, value, time FROM digests') do
-
- local expire_in = math.floor(now - row.time + res['expiry'])
- if expire_in >= 1 then
- table.insert(digests, {row.digest, row.flag, row.value, expire_in})
- num_batch_digests = num_batch_digests + 1
- total_digests = total_digests + 1
- for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do
- table.insert(shingles, {srow.value, srow.number, expire_in, row.digest})
- total_shingles = total_shingles + 1
- num_batch_shingles = num_batch_shingles + 1
- end
- end
- if num_batch_digests >= lim_batch then
- if not send_digests(digests, res['redis_host'], redis_password, redis_db) then
- return
- end
- num_batch_digests = 0
- digests = {}
- end
- if num_batch_shingles >= lim_batch then
- if not send_shingles(shingles, res['redis_host'], redis_password, redis_db) then
- return
- end
- num_batch_shingles = 0
- shingles = {}
- end
- end
- if digests[1] then
- if not send_digests(digests, res['redis_host'], redis_password, redis_db) then
- return
- end
- end
- if shingles[1] then
- if not send_shingles(shingles, res['redis_host'], redis_password, redis_db) then
- return
- end
- end
-
- local message = string.format(
- 'Migrated %d digests and %d shingles',
- total_digests, total_shingles
- )
- if not update_counters(total_digests, res['redis_host'], redis_password, redis_db) then
- message = message .. ' but failed to update counters'
- end
- print(message)
-end
+++ /dev/null
-local util = require "rspamd_util"
-local opts = {}
-
-local function add_data(target, src)
- for k,v in pairs(src) do
- if k ~= 'ips' then
- if target[k] then
- target[k] = target[k] + v
- else
- target[k] = v
- end
- else
- if not target['ips'] then target['ips'] = {} end
- -- Iterate over IPs
- for ip,st in pairs(v) do
- if not target['ips'][ip] then target['ips'][ip] = {} end
- add_data(target['ips'][ip], st)
- end
- end
- end
-end
-
-local function print_num(num)
- if opts['n'] or opts['number'] then
- return tostring(num)
- else
- return util.humanize_number(num)
- end
-end
-
-local function print_stat(st, tabs)
- if st['checked'] then
- print(string.format('%sChecked: %s', tabs, print_num(st['checked'])))
- end
- if st['matched'] then
- print(string.format('%sMatched: %s', tabs, print_num(st['matched'])))
- end
- if st['errors'] then
- print(string.format('%sErrors: %s', tabs, print_num(st['errors'])))
- end
- if st['added'] then
- print(string.format('%sAdded: %s', tabs, print_num(st['added'])))
- end
- if st['deleted'] then
- print(string.format('%sDeleted: %s', tabs, print_num(st['deleted'])))
- end
-end
-
--- Sort by checked
-local function sort_ips(tbl, _opts)
- local res = {}
- for k,v in pairs(tbl) do
- table.insert(res, {ip = k, data = v})
- end
-
- local function sort_order(elt)
- local key = 'checked'
- local _res = 0
-
- if _opts['sort'] then
- if _opts['sort'] == 'matched' then
- key = 'matched'
- elseif _opts['sort'] == 'errors' then
- key = 'errors'
- elseif _opts['sort'] == 'ip' then
- return elt['ip']
- end
- end
-
- if elt['data'][key] then
- _res = elt['data'][key]
- end
-
- return _res
- end
-
- table.sort(res, function(a, b)
- return sort_order(a) > sort_order(b)
- end)
-
- return res
-end
-
-local function add_result(dst, src, k)
- if type(src) == 'table' then
- if type(dst) == 'number' then
- -- Convert dst to table
- dst = {dst}
- elseif type(dst) == 'nil' then
- dst = {}
- end
-
- for i,v in ipairs(src) do
- if dst[i] and k ~= 'fuzzy_stored' then
- dst[i] = dst[i] + v
- else
- dst[i] = v
- end
- end
- else
- if type(dst) == 'table' then
- if k ~= 'fuzzy_stored' then
- dst[1] = dst[1] + src
- else
- dst[1] = src
- end
- else
- if dst and k ~= 'fuzzy_stored' then
- dst = dst + src
- else
- dst = src
- end
- end
- end
-
- return dst
-end
-
-local function print_result(r)
- local function num_to_epoch(num)
- if num == 1 then
- return 'v0.6'
- elseif num == 2 then
- return 'v0.8'
- elseif num == 3 then
- return 'v0.9'
- elseif num == 4 then
- return 'v1.0+'
- end
- return '???'
- end
- if type(r) == 'table' then
- local res = {}
- for i,num in ipairs(r) do
- res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num))
- end
-
- return table.concat(res, ', ')
- end
-
- return print_num(r)
-end
-
---.USE "getopt"
-
-return function(args, res)
- local res_ips = {}
- local res_databases = {}
- local wrk = res['workers']
- opts = getopt(args, '')
-
- if wrk then
- for _,pr in pairs(wrk) do
- -- processes cycle
- if pr['data'] then
- local id = pr['id']
-
- if id then
- local res_db = res_databases[id]
- if not res_db then
- res_db = {
- keys = {}
- }
- res_databases[id] = res_db
- end
-
- -- General stats
- for k,v in pairs(pr['data']) do
- if k ~= 'keys' and k ~= 'errors_ips' then
- res_db[k] = add_result(res_db[k], v, k)
- elseif k == 'errors_ips' then
- -- Errors ips
- if not res_db['errors_ips'] then
- res_db['errors_ips'] = {}
- end
- for ip,nerrors in pairs(v) do
- if not res_db['errors_ips'][ip] then
- res_db['errors_ips'][ip] = nerrors
- else
- res_db['errors_ips'][ip] = nerrors + res_db['errors_ips'][ip]
- end
- end
- end
- end
-
- if pr['data']['keys'] then
- local res_keys = res_db['keys']
- if not res_keys then
- res_keys = {}
- res_db['keys'] = res_keys
- end
- -- Go through keys in input
- for k,elts in pairs(pr['data']['keys']) do
- -- keys cycle
- if not res_keys[k] then
- res_keys[k] = {}
- end
-
- add_data(res_keys[k], elts)
-
- if elts['ips'] then
- for ip,v in pairs(elts['ips']) do
- if not res_ips[ip] then
- res_ips[ip] = {}
- end
- add_data(res_ips[ip], v)
- end
- end
- end
- end
- end
- end
- end
- end
-
- -- General stats
- for db,st in pairs(res_databases) do
- print(string.format('Statistics for storage %s', db))
-
- for k,v in pairs(st) do
- if k ~= 'keys' and k ~= 'errors_ips' then
- print(string.format('%s: %s', k, print_result(v)))
- end
- end
- print('')
-
- local res_keys = st['keys']
- if res_keys and not opts['no-keys'] and not opts['short'] then
- print('Keys statistics:')
- for k,_st in pairs(res_keys) do
- print(string.format('Key id: %s', k))
- print_stat(_st, '\t')
-
- if _st['ips'] and not opts['no-ips'] then
- print('')
- print('\tIPs stat:')
- local sorted_ips = sort_ips(_st['ips'], opts)
-
- for _,v in ipairs(sorted_ips) do
- print(string.format('\t%s', v['ip']))
- print_stat(v['data'], '\t\t')
- print('')
- end
- end
-
- print('')
- end
- end
- if st['errors_ips'] and not opts['no-ips'] and not opts['short'] then
- print('')
- print('Errors IPs statistics:')
- table.sort(st['errors_ips'], function(a, b)
- return a > b
- end)
- for i, v in pairs(st['errors_ips']) do
- print(string.format('%s: %s', i, print_result(v)))
- end
- print('')
- end
- end
-
- if not opts['no-ips'] and not opts['short'] then
- print('')
- print('IPs statistics:')
-
- local sorted_ips = sort_ips(res_ips, opts)
- for _, v in ipairs(sorted_ips) do
- print(string.format('%s', v['ip']))
- print_stat(v['data'], '\t')
- print('')
- end
- end
-end
-
+++ /dev/null
-local function getopt(arg, options)
- local tab = {}
- for k, v in ipairs(arg) do
- if string.sub(v, 1, 2) == "--" then
- local x = string.find(v, "=", 1, true)
- if x then tab[string.sub(v, 3, x - 1)] = string.sub(v, x + 1)
- else tab[string.sub(v, 3)] = true
- end
- elseif string.sub(v, 1, 1) == "-" then
- local y = 2
- local l = string.len(v)
- local jopt
- while (y <= l) do
- jopt = string.sub(v, y, y)
- if string.find(options, jopt, 1, true) then
- if y < l then
- tab[jopt] = string.sub(v, y + 1)
- y = l
- else
- tab[jopt] = arg[k + 1]
- end
- else
- tab[jopt] = true
- end
- y = y + 1
- end
- end
- end
- return tab
-end
-
#include "config.h"
#include "rspamadm.h"
#include "lua/lua_common.h"
-#include "grep.lua.h"
static gchar *string = NULL;
static gchar *pattern = NULL;
}
L = rspamd_lua_init ();
+ rspamd_lua_set_path (L, NULL, NULL);
obj = ucl_object_typed_new (UCL_OBJECT);
if (string) {
argc,
argv,
obj,
- rspamadm_script_grep);
+ "grep");
lua_close (L);
ucl_object_unref (obj);
+++ /dev/null
-return function(_, res)
-
- local rspamd_regexp = require 'rspamd_regexp'
-
- local buffer = {}
- local matches = {}
-
- local pattern = res['pattern']
- local re
- if pattern then
- re = rspamd_regexp.create(pattern)
- if not re then
- io.stderr:write("Couldn't compile regex: " .. pattern .. '\n')
- os.exit(1)
- end
- end
-
- local plainm = true
- if res['luapat'] then
- plainm = false
- end
- local orphans = res['orphans']
- local search_str = res['string']
- local sensitive = res['sensitive']
- local partial = res['partial']
- if search_str and not sensitive then
- search_str = string.lower(search_str)
- end
- local inputs = res['inputs']
-
- for _, n in ipairs(inputs) do
- local h, err
- if string.match(n, '%.xz$') then
- h, err = io.popen('xzcat ' .. n, 'r')
- elseif string.match(n, '%.bz2$') then
- h, err = io.popen('bzcat ' .. n, 'r')
- elseif string.match(n, '%.gz$') then
- h, err = io.popen('zcat ' .. n, 'r')
- elseif n == 'stdin' then
- h = io.input()
- else
- h, err = io.open(n, 'r')
- end
- if not h then
- if err then
- io.stderr:write("Couldn't open file (" .. n .. '): ' .. err .. '\n')
- else
- io.stderr:write("Couldn't open file (" .. n .. '): no error\n')
- end
- else
- for line in h:lines() do
- local hash = string.match(line, '<(%x+)>')
- local already_matching = false
- if hash then
- if matches[hash] then
- table.insert(matches[hash], line)
- already_matching = true
- else
- if buffer[hash] then
- table.insert(buffer[hash], line)
- else
- buffer[hash] = {line}
- end
- end
- end
- local ismatch = false
- if re then
- ismatch = re:match(line)
- elseif sensitive and search_str then
- ismatch = string.find(line, search_str, 1, plainm)
- elseif search_str then
- local lwr = string.lower(line)
- ismatch = string.find(lwr, search_str, 1, plainm)
- end
- if ismatch then
- if not hash then
- if orphans then
- print('*** orphaned ***')
- print(line)
- print()
- end
- elseif not already_matching then
- matches[hash] = buffer[hash]
- end
- end
- local is_end = string.match(line, '<%x+>; task; rspamd_protocol_http_reply:')
- if is_end then
- buffer[hash] = nil
- if matches[hash] then
- for _, v in ipairs(matches[hash]) do
- print(v)
- end
- print()
- matches[hash] = nil
- end
- end
- end
- if partial then
- for k, v in pairs(matches) do
- print('*** partial ***')
- for _, vv in ipairs(v) do
- print(vv)
- end
- print()
- matches[k] = nil
- end
- else
- matches = {}
- end
- end
- end
-end
gboolean
rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv,
- const ucl_object_t *res, const gchar *script)
+ const ucl_object_t *res, const gchar *script_name)
{
lua_State *L = pL;
gint err_idx, i, ret;
GString *tb;
+ gchar str[PATH_MAX];
- g_assert (script != NULL);
+ g_assert (script_name != NULL);
g_assert (res != NULL);
g_assert (L != NULL);
- if (luaL_dostring (L, script) != 0) {
- msg_err ("cannot execute lua script: %s",
- lua_tostring (L, -1));
+ rspamd_snprintf (str, sizeof (str), "return require \"%s.%s\"", "rspamadm",
+ script_name);
+
+ if (luaL_dostring (L, str) != 0) {
+ msg_err ("cannot execute lua script %s: %s",
+ str, lua_tostring (L, -1));
return FALSE;
}
else {
if (lua_type (L, -1) != LUA_TFUNCTION) {
msg_err ("lua script must return "
"function and not %s",
- lua_typename (L,
- lua_type (L, -1)));
- lua_settop (L, 0);
+ lua_typename (L, lua_type (L, -1)));
return FALSE;
}
if ((ret = lua_pcall (L, 2, 0, err_idx)) != 0) {
tb = lua_touserdata (L, -1);
- msg_err ("call to adm lua script failed (%d): %v", ret, tb);
+ msg_err ("call to rspamadm lua script failed (%d): %v", ret, tb);
if (tb) {
g_string_free (tb, TRUE);
const struct rspamadm_command *rspamadm_search_command (const gchar *name);
gboolean rspamadm_execute_lua_ucl_subr (gpointer L, gint argc, gchar **argv,
- const ucl_object_t *res, const gchar *script);
+ const ucl_object_t *res, const gchar *script_name);
#endif
#include "config.h"
#include "rspamadm.h"
#include "lua/lua_common.h"
-#include "stat_convert.lua.h"
static gchar *source_db = NULL;
static gchar *redis_host = NULL;
}
L = rspamd_lua_init ();
+ rspamd_lua_set_path (L, NULL, NULL);
obj = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (obj, ucl_object_fromstring (source_db),
argc,
argv,
obj,
- rspamadm_script_stat_convert);
+ "stat_convert");
lua_close (L);
ucl_object_unref (obj);
+++ /dev/null
-local sqlite3 = require "rspamd_sqlite3"
-local redis = require "rspamd_redis"
-local util = require "rspamd_util"
-
-local function send_redis(server, symbol, tokens, password, db, cmd)
- local ret = true
- local conn,err = redis.connect_sync({
- host = server,
- })
-
- local err_str
-
- if not conn then
- print('Cannot connect to ' .. server .. ' error: ' .. err)
- return false, err
- end
-
- if password then
- conn:add_cmd('AUTH', {password})
- end
- if db then
- conn:add_cmd('SELECT', {db})
- end
-
- for _,t in ipairs(tokens) do
- if not conn:add_cmd(cmd, {symbol .. t[3], t[1], t[2]}) then
- ret = false
- err_str = 'add command failure' .. string.format('%s %s',
- cmd, table.concat({symbol .. t[3], t[1], t[2]}, ' '))
- end
- end
-
- if ret then
- ret,err_str = conn:exec()
- end
-
- return ret,err_str
-end
-
-local function convert_learned(cache, server, password, redis_db)
- local converted = 0
- local db = sqlite3.open(cache)
- local ret = true
- local err_str
-
- if not db then
- print('Cannot open cache database: ' .. cache)
- return false
- end
-
- db:sql('BEGIN;')
-
- local conn,err = redis.connect_sync({
- host = server,
- })
-
- if not conn then
- print('Cannot connect to ' .. server .. ' error: ' .. err)
- return false
- end
-
- if password then
- conn:add_cmd('AUTH', {password})
- end
- if redis_db then
- conn:add_cmd('SELECT', {redis_db})
- end
-
- for row in db:rows('SELECT * FROM learns;') do
- local is_spam
- local digest = tostring(util.encode_base32(row.digest))
-
- if row.flag == '0' then
- is_spam = '-1'
- else
- is_spam = '1'
- end
-
- if not conn:add_cmd('HSET', {'learned_ids', digest, is_spam}) then
- print('Cannot add hash: ' .. digest)
- ret = false
- else
- converted = converted + 1
- end
- end
- db:sql('COMMIT;')
-
- if ret then
- ret,err_str = conn:exec()
- end
-
- if ret then
- print(string.format('Converted %d cached items from sqlite3 learned cache to redis',
- converted))
- else
- print('Error occurred during sending data to redis: ' .. err_str)
- end
-
- return ret
-end
-
-return function (_, res)
- local db = sqlite3.open(res['source_db'])
- local tokens = {}
- local num = 0
- local total = 0
- local nusers = 0
- local lim = 1000 -- Update each 1000 tokens
- local users_map = {}
- local learns = {}
- local redis_password = res['redis_password']
- local redis_db = nil
- local cmd = 'HINCRBY'
- local ret, err_str
-
- if res['redis_db'] then
- redis_db = tostring(res['redis_db'])
- end
- if res['reset_previous'] then
- cmd = 'HSET'
- end
-
- if res['cache_db'] then
- if not convert_learned(res['cache_db'], res['redis_host'],
- redis_password, redis_db) then
- print('Cannot convert learned cache to redis')
- return
- end
- end
-
- if not db then
- print('Cannot open source db: ' .. res['source_db'])
- return
- end
-
- db:sql('BEGIN;')
- -- Fill users mapping
- for row in db:rows('SELECT * FROM users;') do
- if row.id == '0' then
- users_map[row.id] = ''
- else
- users_map[row.id] = row.name
- end
- learns[row.id] = row.learns
- nusers = nusers + 1
- end
-
- -- Workaround for old databases
- for row in db:rows('SELECT * FROM languages') do
- if learns['0'] then
- learns['0'] = learns['0'] + row.learns
- else
- learns['0'] = row.learns
- end
- end
-
- -- Fill tokens, sending data to redis each `lim` records
- for row in db:rows('SELECT token,value,user FROM tokens;') do
- local user = ''
- if row.user ~= 0 and users_map[row.user] then
- user = users_map[row.user]
- end
-
- table.insert(tokens, {row.token, row.value, user})
-
- num = num + 1
- total = total + 1
- if num > lim then
- ret,err_str = send_redis(res['redis_host'], res['symbol'],
- tokens, redis_password, redis_db, cmd)
- if not ret then
- print('Cannot send tokens to the redis server: ' .. err_str)
- return
- end
-
- num = 0
- tokens = {}
- end
- end
- if #tokens > 0 then
- ret, err_str = send_redis(res['redis_host'], res['symbol'], tokens,
- redis_password, redis_db, cmd)
-
- if not ret then
- print('Cannot send tokens to the redis server: ' .. err_str)
- return
- end
- end
- -- Now update all users
- local conn,err = redis.connect_sync({
- host = res['redis_host'],
- })
-
- if not conn then
- print('Cannot connect to ' .. res['redis_host'] .. ' error: ' .. err)
- return false
- end
-
- if redis_password then
- conn:add_cmd('AUTH', {redis_password})
- end
- if redis_db then
- conn:add_cmd('SELECT', {redis_db})
- end
-
- for id,learned in pairs(learns) do
- local user = users_map[id]
- if not conn:add_cmd(cmd, {res['symbol'] .. user, 'learns', learned}) then
- print('Cannot update learns for user: ' .. user)
- end
- if not conn:add_cmd('SADD', {res['symbol'] .. '_keys', res['symbol'] .. user}) then
- print('Cannot update learns for user: ' .. user)
- end
- end
- db:sql('COMMIT;')
-
- ret = conn:exec()
-
- if ret then
- print(string.format('Migrated %d tokens for %d users for symbol %s',
- total, nusers, res['symbol']))
- else
- print('Error occurred during sending data to redis')
- end
-end
projects / rspamd.git / commitdiff