[Feature] Add statistic convertation module to configwizard

2 files changed, 138 insertions, 10 deletions

diff --git a/lualib/rspamadm/configwizard.lua b/lualib/rspamadm/configwizard.lua
index 278202ffd..a86bb78ec 100644
--- a/lualib/rspamadm/configwizard.lua
+++ b/lualib/rspamadm/configwizard.lua
@@ -1,5 +1,5 @@
 --[[
-Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,6 +19,8 @@ local local_conf = rspamd_paths['CONFDIR']
 local rspamd_util = require "rspamd_util"
 local rspamd_logger = require "rspamd_logger"
 local lua_util = require "lua_util"
+local lua_stat_tools = require "stat_tools"
+local lua_redis = require "lua_redis"
 local ucl = require "ucl"
 
 local plugins_stat = require "rspamadm/plugins_stats"
@@ -32,6 +34,8 @@ local rspamd_logo = [[
              |_|
 ]]
 
+local redis_params
+
 local function printf(fmt, ...)
   io.write(string.format(fmt, ...))
   io.write('\n')
@@ -73,7 +77,7 @@ end
 local function print_changes(changes)
   local function print_change(k, c, where)
     printf('File: %s, changes list:', highlight(local_conf .. '/'
-        .. where .. '/'.. k))
+        .. where .. '/'.. k .. '.conf'))
 
     for ek,ev in pairs(c) do
       printf("%s => %s", highlight(ek), rspamd_logger.slog("%s", ev))
@@ -185,9 +189,32 @@ local function setup_redis(cfg, changes)
       write_servers = read_servers
     end
 
+    redis_params = {
+      read_servers = rs,
+    }
+
     local ws = parse_servers(write_servers)
     if ws and #ws > 0 then
       changes.l['redis.conf']['write_servers'] = table.concat(ws, ",")
+      redis_params['write_servers'] = ws
+    end
+
+    if ask_yes_no('Do you have any password set for your Redis?') then
+      local passwd = readline_default("Enter Redis password:", nil)
+
+      if passwd then
+        changes.l['redis.conf']['password'] = passwd
+        redis_params['password'] = passwd
+      end
+    end
+
+    if ask_yes_no('Do you have any specific database for your Redis?') then
+      local db = readline_default("Enter Redis database:", nil)
+
+      if db then
+        changes.l['redis.conf']['db'] = db
+        redis_params['db'] = db
+      end
     end
   end
 end
@@ -261,7 +288,96 @@ local function setup_dkim_signing(cfg, changes)
     }
   until not ask_yes_no("Do you wish to add another DKIM domain?")
 
-  changes.l.dkim_signing= {domain = domains}
+  changes.l.dkim_signing = {domain = domains}
+end
+
+local function parse_time_interval(str)
+  local function parse_time_suffix(s)
+    if s == 's' then
+      return 1
+    elseif s == 'm' then
+      return 60
+    elseif s == 'h' then
+      return 3600
+    elseif s == 'd' then
+      return 86400
+    elseif s == 'y' then
+      return 365 * 86400;
+    end
+  end
+
+  local lpeg = require "lpeg"
+
+  local digit = lpeg.R("09")
+  local parser = {}
+  parser.integer =
+  (lpeg.S("+-") ^ -1) *
+      (digit   ^  1)
+  parser.fractional =
+  (lpeg.P(".")   ) *
+      (digit ^ 1)
+  parser.number =
+  (parser.integer *
+      (parser.fractional ^ -1)) +
+      (lpeg.S("+-") * parser.fractional)
+  parser.time = lpeg.Cf(lpeg.Cc(1) *
+      (parser.number / tonumber) *
+      ((lpeg.S("smhdy") / parse_time_suffix) ^ -1),
+    function (acc, val) return acc * val end)
+
+  local t = lpeg.match(parser.time, str)
+
+  return t
+end
+
+local function setup_statistic(cfg, changes)
+  local sqlite_configs = lua_stat_tools.load_sqlite_config(cfg)
+
+  if #sqlite_configs > 0 then
+
+    if not redis_params then
+      printf('You have %d sqlite classifiers, but you have no Redis servers being set',
+        #sqlite_configs)
+      return false
+    end
+
+    local parsed_redis = {}
+    if lua_redis.try_load_redis_servers(redis_params, nil, parsed_redis) then
+      printf('You have %d sqlite classifiers', #sqlite_configs)
+      local expire = readline_default("Expire time for new tokens  [default: 100d]: ",
+        '100d')
+      expire = parse_time_interval(expire)
+
+      local reset_previous = ask_yes_no("Reset previuous data?")
+      if ask_yes_no('Do you wish to convert them to Redis?', true) then
+
+        for _,cls in ipairs(sqlite_configs) do
+          if not lua_stat_tools.convert_sqlite_to_redis(parsed_redis, cls.db_spam,
+            cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, expire,
+            reset_previous) then
+            rspamd_logger.errx('conversion failed')
+
+            return false
+          end
+          rspamd_logger.messagex('Converted classifier to the from sqlite to redis')
+          changes.l['classifier_bayes'] = {
+            backend = 'redis',
+            new_schema = true,
+          }
+
+          if expire then
+            changes.l['classifier_bayes'].expire = expire
+          end
+
+          if cls.learn_cache then
+            changes.l['classifier_bayes'].cache = {
+              backend = 'redis'
+            }
+          end
+        end
+      end
+    end
+  end
 end
 
 local function find_worker(cfg, wtype)
@@ -350,7 +466,11 @@ return function(args, cfg)
     if has_check('redis') then
       if not cfg.redis or (not cfg.redis.servers and not cfg.redis.read_servers) then
         setup_redis(cfg, changes)
+      else
+        redis_params = cfg.redis
       end
+    else
+      redis_params = cfg.redis
     end
 
     if has_check('dkim') then
@@ -361,6 +481,10 @@ return function(args, cfg)
       end
     end
 
+    if has_check('statistic') then
+      setup_statistic(cfg, changes)
+    end
+
     local nchanges = 0
     for _,_ in pairs(changes.l) do nchanges = nchanges + 1 end
     for _,_ in pairs(changes.o) do nchanges = nchanges + 1 end
diff --git a/lualib/stat_tools.lua b/lualib/stat_tools.lua
index 47f954235..4f5cafe3c 100644
--- a/lualib/stat_tools.lua
+++ b/lualib/stat_tools.lua
@@ -192,6 +192,11 @@ end
 
   local function convert_db(db, is_spam)
     -- Map users and languages
+    local what = 'ham'
+    if is_spam then
+      what = 'spam'
+    end
+
     local learns = {}
     db:sql('BEGIN;')
     -- Fill users mapping
@@ -266,7 +271,7 @@ end
         tokens = {}
       end
 
-      io.write(string.format('Processed batch: %s/%s\r', total, ntokens))
+      io.write(string.format('Processed batch %s: %s/%s\r', what, total, ntokens))
     end
     -- Last batch
     if #tokens > 0 then
@@ -277,7 +282,7 @@ end
         return false
       end
 
-      io.write(string.format('Processed batch: %s/%s\r', total, ntokens))
+      io.write(string.format('Processed batch %s: %s/%s\r', what, total, ntokens))
     end
     io.write('\n')
 
@@ -325,8 +330,7 @@ end
     logger.messagex('Convert learned ids from %s', learn_cache_db)
     local db = sqlite3.open(learn_cache_db)
     local ret = true
-    local err_str
-    local converted = 0
+    local total = 0
 
     if not db then
       logger.errx('Cannot open cache database: ' .. learn_cache_db)
@@ -349,7 +353,7 @@ end
         logger.errx('Cannot add hash: ' .. digest)
         ret = false
       else
-        converted = converted + 1
+        total = total + 1
       end
     end
     db:sql('COMMIT;')
@@ -360,9 +364,9 @@ end
 
     if ret then
       logger.messagex('Converted %s cached items from sqlite3 learned cache to redis',
-        converted)
+        total)
     else
-      logger.errx('Error occurred during sending data to redis: ' .. err_str)
+      logger.errx('Error occurred during sending data to redis')
     end
   end