diff options
author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2018-02-15 15:28:47 +0000 |
---|---|---|
committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2018-02-15 15:28:47 +0000 |
commit | 5a9b7ee710a43901b7cc6605506be035bfe335b1 (patch) | |
tree | f3a5cdb82ca3f6ef68e9e77fa35f264da022567b | |
parent | 4b7e987b91a8782d5558f24cbd6bda7a70a53da1 (diff) | |
download | rspamd-5a9b7ee710a43901b7cc6605506be035bfe335b1.tar.gz rspamd-5a9b7ee710a43901b7cc6605506be035bfe335b1.zip |
[Project] Preliminary version of the new stat_convert
-rw-r--r-- | lualib/lua_redis.lua | 120 | ||||
-rw-r--r-- | lualib/rspamadm/stat_convert.lua | 30 | ||||
-rw-r--r-- | src/rspamadm/stat_convert.c | 9 |
3 files changed, 108 insertions, 51 deletions
diff --git a/lualib/lua_redis.lua b/lualib/lua_redis.lua index 10c333da0..1ab876109 100644 --- a/lualib/lua_redis.lua +++ b/lualib/lua_redis.lua @@ -22,81 +22,101 @@ local exports = {} local E = {} --- This function parses redis server definition using either --- specific server string for this module or global --- redis section -local function rspamd_parse_redis_server(module_name, module_opts, no_fallback) - - local result = {} +local function try_load_redis_servers(options, rspamd_config, result) local default_port = 6379 local default_timeout = 1.0 local default_expand_keys = false local upstream_list = require "rspamd_upstream_list" - local function try_load_redis_servers(options) - -- Try to get read servers: - local upstreams_read, upstreams_write + -- Try to get read servers: + local upstreams_read, upstreams_write - if options['read_servers'] then + if options['read_servers'] then + if rspamd_config then upstreams_read = upstream_list.create(rspamd_config, options['read_servers'], default_port) - elseif options['servers'] then + else + upstreams_read = upstream_list.create(options['read_servers'], + default_port) + end + elseif options['servers'] then + if rspamd_config then upstreams_read = upstream_list.create(rspamd_config, options['servers'], default_port) - elseif options['server'] then + else + upstreams_read = upstream_list.create(options['servers'], default_port) + end + elseif options['server'] then + if rspamd_config then upstreams_read = upstream_list.create(rspamd_config, options['server'], default_port) + else + upstreams_read = upstream_list.create(options['server'], default_port) end + end - if upstreams_read then - if options['write_servers'] then + if upstreams_read then + if options['write_servers'] then + if rspamd_config then upstreams_write = upstream_list.create(rspamd_config, options['write_servers'], default_port) else - upstreams_write = upstreams_read - end - end - - -- Store options - if not result['timeout'] or result['timeout'] == default_timeout then - if options['timeout'] then - result['timeout'] = tonumber(options['timeout']) - else - result['timeout'] = default_timeout + upstreams_write = upstream_list.create(options['write_servers'], + default_port) end + else + upstreams_write = upstreams_read end + end - if options['prefix'] and not result['prefix'] then - result['prefix'] = options['prefix'] - end - - if type(options['expand_keys']) == 'boolean' then - result['expand_keys'] = options['expand_keys'] + -- Store options + if not result['timeout'] or result['timeout'] == default_timeout then + if options['timeout'] then + result['timeout'] = tonumber(options['timeout']) else - result['expand_keys'] = default_expand_keys + result['timeout'] = default_timeout end + end - if not result['db'] then - if options['db'] then - result['db'] = tostring(options['db']) - elseif options['dbname'] then - result['db'] = tostring(options['dbname']) - end - end - if options['password'] and not result['password'] then - result['password'] = options['password'] - end + if options['prefix'] and not result['prefix'] then + result['prefix'] = options['prefix'] + end - if upstreams_write and upstreams_read then - result.read_servers = upstreams_read - result.write_servers = upstreams_write + if type(options['expand_keys']) == 'boolean' then + result['expand_keys'] = options['expand_keys'] + else + result['expand_keys'] = default_expand_keys + end - return true + if not result['db'] then + if options['db'] then + result['db'] = tostring(options['db']) + elseif options['dbname'] then + result['db'] = tostring(options['dbname']) end + end + if options['password'] and not result['password'] then + result['password'] = options['password'] + end - return false + if upstreams_write and upstreams_read then + result.read_servers = upstreams_read + result.write_servers = upstreams_write + + return true end + return false +end + +exports.try_load_redis_servers = try_load_redis_servers + +-- This function parses redis server definition using either +-- specific server string for this module or global +-- redis section +local function rspamd_parse_redis_server(module_name, module_opts, no_fallback) + local result = {} + -- Try local options local opts if not module_opts then @@ -109,14 +129,14 @@ local function rspamd_parse_redis_server(module_name, module_opts, no_fallback) local ret if opts.redis then - ret = try_load_redis_servers(opts.redis, result) + ret = try_load_redis_servers(opts.redis, rspamd_config, result) if ret then return result end end - ret = try_load_redis_servers(opts, result) + ret = try_load_redis_servers(opts, rspamd_config, result) if ret then return result @@ -132,12 +152,12 @@ local function rspamd_parse_redis_server(module_name, module_opts, no_fallback) local ret if opts[module_name] then - ret = try_load_redis_servers(opts[module_name], result) + ret = try_load_redis_servers(opts[module_name], rspamd_config, result) if ret then return result end else - ret = try_load_redis_servers(opts, result) + ret = try_load_redis_servers(opts, rspamd_config, result) -- Exclude disabled if opts['disabled_modules'] then diff --git a/lualib/rspamadm/stat_convert.lua b/lualib/rspamadm/stat_convert.lua index 3845702da..7c902dd99 100644 --- a/lualib/rspamadm/stat_convert.lua +++ b/lualib/rspamadm/stat_convert.lua @@ -1,7 +1,35 @@ local lua_redis = require "rspamd_redis" local stat_tools = require "stat_tools" +local ucl = require "ucl" +local logger = require "rspamd_logger" return function (_, res) - local redis_params = + local redis_params = {} + if not lua_redis.try_load_redis_servers(res.redis, nil, redis_params) then + logger.errx('cannot load redis server definition') + + return false + end + + local sqlite_params = stat_tools.load_sqlite_config(res) + + if #sqlite_params == 0 then + logger.errx('cannot load sqlite classifiers') + return false + end + + for _,cls in ipairs(sqlite_params) do + if not stat_tools.convert_sqlite_to_redis(redis_params, cls.db_spam, + cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, res.expire, + res.reset_previous) then + logger.errx('conversion failed') + + return false + end + logger.infox('Converted classifier to the from sqlite to redis') + logger.infox('Suggested configuration:') + logger.infox(ucl.to_format(stat_tools.redis_classifier_from_sqlite(cls), + 'config')) + end end diff --git a/src/rspamadm/stat_convert.c b/src/rspamadm/stat_convert.c index c88bc54bb..67d943f7b 100644 --- a/src/rspamadm/stat_convert.c +++ b/src/rspamadm/stat_convert.c @@ -24,6 +24,8 @@ static gchar *config_file = NULL; static gchar *symbol_ham = NULL; static gchar *symbol_spam = NULL; +static gdouble expire = 0.0; + /* Inputs */ static gchar *spam_db = NULL; static gchar *ham_db = NULL; @@ -51,6 +53,8 @@ static GOptionEntry entries[] = { "Config file to read data from", NULL}, {"reset", 'r', 0, G_OPTION_ARG_NONE, &reset_previous, "Reset previous data instead of appending values", NULL}, + {"expire", 'e', 0, G_OPTION_ARG_DOUBLE, &expire, + "Set expiration in seconds (can be fractional)", NULL}, {"symbol-spam", 0, 0, G_OPTION_ARG_STRING, &symbol_spam, "Symbol for spam (e.g. BAYES_SPAM)", NULL}, @@ -83,6 +87,7 @@ rspamadm_statconvert_help (gboolean full_help) "Where options are:\n\n" "-c: config file to read data from\n" "-r: reset previous data instead of increasing values\n" + "-e: set expire to that amount of seconds\n" "** Or specify options directly **\n" "--redis-host: output redis ip (in format ip:port)\n" "--redis-db: output redis database\n" @@ -235,6 +240,10 @@ rspamadm_statconvert (gint argc, gchar **argv) ucl_object_insert_key (obj, ucl_object_frombool (reset_previous), "reset_previous", 0, false); + if (expire != 0) { + ucl_object_insert_key (obj, ucl_object_fromdouble (expire), + "expire", 0, false); + } rspamadm_execute_lua_ucl_subr (L, argc, |