[Project] Preliminary version of the new stat_convert

lualib/lua_redis.lua

@@ -22,81 +22,101 @@ local exports = {}
 
 local E = {}
 
-local function rspamd_parse_redis_server(module_name, module_opts, no_fallback)
-
-  local result = {}
+local function try_load_redis_servers(options, rspamd_config, result)
   local default_port = 6379
   local default_timeout = 1.0
   local default_expand_keys = false
   local upstream_list = require "rspamd_upstream_list"
 
-  local function try_load_redis_servers(options)
-    -- Try to get read servers:
-    local upstreams_read, upstreams_write
+  -- Try to get read servers:
+  local upstreams_read, upstreams_write
 
-    if options['read_servers'] then
+  if options['read_servers'] then
+    if rspamd_config then
       upstreams_read = upstream_list.create(rspamd_config,
         options['read_servers'], default_port)
-    elseif options['servers'] then
+    else
+      upstreams_read = upstream_list.create(options['read_servers'],
+        default_port)
+    end
+  elseif options['servers'] then
+    if rspamd_config then
       upstreams_read = upstream_list.create(rspamd_config,
         options['servers'], default_port)
-    elseif options['server'] then
+    else
+      upstreams_read = upstream_list.create(options['servers'], default_port)
+    end
+  elseif options['server'] then
+    if rspamd_config then
       upstreams_read = upstream_list.create(rspamd_config,
         options['server'], default_port)
+    else
+      upstreams_read = upstream_list.create(options['server'], default_port)
     end
+  end
 
-    if upstreams_read then
-      if options['write_servers'] then
+  if upstreams_read then
+    if options['write_servers'] then
+      if rspamd_config then
         upstreams_write = upstream_list.create(rspamd_config,
           options['write_servers'], default_port)
       else
-        upstreams_write = upstreams_read
-      end
-    end
-
-    -- Store options
-    if not result['timeout'] or result['timeout'] == default_timeout then
-      if options['timeout'] then
-        result['timeout'] = tonumber(options['timeout'])
-      else
-        result['timeout'] = default_timeout
+        upstreams_write = upstream_list.create(options['write_servers'],
+          default_port)
       end
+    else
+      upstreams_write = upstreams_read
     end
+  end
 
-    if options['prefix'] and not result['prefix'] then
-      result['prefix'] = options['prefix']
-    end
-
-    if type(options['expand_keys']) == 'boolean' then
-      result['expand_keys'] = options['expand_keys']
+  -- Store options
+  if not result['timeout'] or result['timeout'] == default_timeout then
+    if options['timeout'] then
+      result['timeout'] = tonumber(options['timeout'])
     else
-      result['expand_keys'] = default_expand_keys
+      result['timeout'] = default_timeout
     end
+  end
 
-    if not result['db'] then
-      if options['db'] then
-        result['db'] = tostring(options['db'])
-      elseif options['dbname'] then
-        result['db'] = tostring(options['dbname'])
-      end
-    end
-    if options['password'] and not result['password'] then
-      result['password'] = options['password']
-    end
+  if options['prefix'] and not result['prefix'] then
+    result['prefix'] = options['prefix']
+  end
 
-    if upstreams_write and upstreams_read then
-      result.read_servers = upstreams_read
-      result.write_servers = upstreams_write
+  if type(options['expand_keys']) == 'boolean' then
+    result['expand_keys'] = options['expand_keys']
+  else
+    result['expand_keys'] = default_expand_keys
+  end
 
-      return true
+  if not result['db'] then
+    if options['db'] then
+      result['db'] = tostring(options['db'])
+    elseif options['dbname'] then
+      result['db'] = tostring(options['dbname'])
     end
+  end
+  if options['password'] and not result['password'] then
+    result['password'] = options['password']
+  end
 
-    return false
+  if upstreams_write and upstreams_read then
+    result.read_servers = upstreams_read
+    result.write_servers = upstreams_write
+
+    return true
   end
 
+  return false
+end
+
+exports.try_load_redis_servers = try_load_redis_servers
+
+-- This function parses redis server definition using either
+-- specific server string for this module or global
+-- redis section
+local function rspamd_parse_redis_server(module_name, module_opts, no_fallback)
+  local result = {}
+
   -- Try local options
   local opts
   if not module_opts then
@@ -109,14 +129,14 @@ local function rspamd_parse_redis_server(module_name, module_opts, no_fallback)
     local ret
 
     if opts.redis then
-      ret = try_load_redis_servers(opts.redis, result)
+      ret = try_load_redis_servers(opts.redis, rspamd_config, result)
 
       if ret then
         return result
       end
     end
 
-    ret = try_load_redis_servers(opts, result)
+    ret = try_load_redis_servers(opts, rspamd_config, result)
 
     if ret then
       return result
@@ -132,12 +152,12 @@ local function rspamd_parse_redis_server(module_name, module_opts, no_fallback)
     local ret
 
     if opts[module_name] then
-      ret = try_load_redis_servers(opts[module_name], result)
+      ret = try_load_redis_servers(opts[module_name], rspamd_config, result)
       if ret then
         return result
       end
     else
-      ret = try_load_redis_servers(opts, result)
+      ret = try_load_redis_servers(opts, rspamd_config, result)
 
       -- Exclude disabled
       if opts['disabled_modules'] then

lualib/rspamadm/stat_convert.lua

@@ -1,7 +1,35 @@
 local lua_redis = require "rspamd_redis"
 local stat_tools = require "stat_tools"
+local ucl = require "ucl"
+local logger = require "rspamd_logger"
 
 
 return function (_, res)
-  local redis_params =
+  local redis_params = {}
+  if not lua_redis.try_load_redis_servers(res.redis, nil, redis_params) then
+    logger.errx('cannot load redis server definition')
+
+    return false
+  end
+
+  local sqlite_params = stat_tools.load_sqlite_config(res)
+
+  if #sqlite_params == 0 then
+    logger.errx('cannot load sqlite classifiers')
+    return false
+  end
+
+  for _,cls in ipairs(sqlite_params) do
+    if not stat_tools.convert_sqlite_to_redis(redis_params, cls.db_spam,
+        cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, res.expire,
+        res.reset_previous) then
+      logger.errx('conversion failed')
+
+      return false
+    end
+    logger.infox('Converted classifier to the from sqlite to redis')
+    logger.infox('Suggested configuration:')
+    logger.infox(ucl.to_format(stat_tools.redis_classifier_from_sqlite(cls),
+      'config'))
+  end
 end

src/rspamadm/stat_convert.c

@@ -24,6 +24,8 @@ static gchar *config_file = NULL;
 static gchar *symbol_ham = NULL;
 static gchar *symbol_spam = NULL;
 
+static gdouble expire = 0.0;
+
 /* Inputs */
 static gchar *spam_db = NULL;
 static gchar *ham_db = NULL;
@@ -51,6 +53,8 @@ static GOptionEntry entries[] = {
 				"Config file to read data from",      NULL},
 		{"reset", 'r', 0, G_OPTION_ARG_NONE, &reset_previous,
 				"Reset previous data instead of appending values", NULL},
+		{"expire", 'e', 0, G_OPTION_ARG_DOUBLE, &expire,
+				"Set expiration in seconds (can be fractional)", NULL},
 
 		{"symbol-spam", 0, 0, G_OPTION_ARG_STRING, &symbol_spam,
 				"Symbol for spam (e.g. BAYES_SPAM)", NULL},
@@ -83,6 +87,7 @@ rspamadm_statconvert_help (gboolean full_help)
 				"Where options are:\n\n"
 				"-c: config file to read data from\n"
 				"-r: reset previous data instead of increasing values\n"
+				"-e: set expire to that amount of seconds\n"
 				"** Or specify options directly **\n"
 				"--redis-host: output redis ip (in format ip:port)\n"
 				"--redis-db: output redis database\n"
@@ -235,6 +240,10 @@ rspamadm_statconvert (gint argc, gchar **argv)
 	ucl_object_insert_key (obj, ucl_object_frombool (reset_previous),
 			"reset_previous", 0, false);
 
+	if (expire != 0) {
+		ucl_object_insert_key (obj, ucl_object_fromdouble (expire),
+				"expire", 0, false);
+	}
 
 	rspamadm_execute_lua_ucl_subr (L,
 			argc,