diff options
| -rw-r--r-- | circle.yml | 2 | ||||
| -rw-r--r-- | src/libserver/CMakeLists.txt | 7 | ||||
| -rw-r--r-- | src/libserver/cfg_utils.c | 2 | ||||
| -rw-r--r-- | src/libserver/fuzzy_backend.c | 6 | ||||
| -rw-r--r-- | src/libserver/worker_util.c | 2 | ||||
| -rw-r--r-- | src/plugins/lua/fann_redis.lua | 222 |
6 files changed, 131 insertions, 110 deletions
diff --git a/circle.yml b/circle.yml index a115d0c7b..83ddbdff0 100644 --- a/circle.yml +++ b/circle.yml @@ -1,7 +1,7 @@ dependencies: pre: - sudo apt-get update -qq || true - - sudo apt-get install -qq cmake gcc lcov libevent-dev libglib2.0-dev libgmime-2.6-dev libluajit-5.1-dev liblua5.1-0-dev libmagic-dev libpcre3-dev libsqlite3-dev libssl-dev luarocks make python-dev ragel redis-server + - sudo apt-get install -qq cmake gcc lcov libevent-dev libglib2.0-dev libgmime-2.6-dev libluajit-5.1-dev liblua5.1-0-dev libmagic-dev libpcre3-dev libsqlite3-dev libssl-dev luarocks make python-dev ragel redis-server libgd-dev - gem install coveralls-lcov - sudo pip install demjson ipaddress psutil robotframework - cd .. && wget http://download.redis.io/redis-stable.tar.gz && tar xvzf redis-stable.tar.gz && cd redis-stable && make && sudo cp src/redis-cli /usr/bin/ diff --git a/src/libserver/CMakeLists.txt b/src/libserver/CMakeLists.txt index 14f82e482..474e695d1 100644 --- a/src/libserver/CMakeLists.txt +++ b/src/libserver/CMakeLists.txt @@ -8,12 +8,10 @@ SET(LIBRSPAMDSERVERSRC ${CMAKE_CURRENT_SOURCE_DIR}/dynamic_cfg.c ${CMAKE_CURRENT_SOURCE_DIR}/events.c ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend.c - ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend_redis.c ${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend_sqlite.c ${CMAKE_CURRENT_SOURCE_DIR}/html.c ${CMAKE_CURRENT_SOURCE_DIR}/monitored.c ${CMAKE_CURRENT_SOURCE_DIR}/protocol.c - ${CMAKE_CURRENT_SOURCE_DIR}/redis_pool.c ${CMAKE_CURRENT_SOURCE_DIR}/re_cache.c ${CMAKE_CURRENT_SOURCE_DIR}/roll_history.c ${CMAKE_CURRENT_SOURCE_DIR}/spf.c @@ -22,5 +20,10 @@ SET(LIBRSPAMDSERVERSRC ${CMAKE_CURRENT_SOURCE_DIR}/url.c ${CMAKE_CURRENT_SOURCE_DIR}/worker_util.c) +IF (ENABLE_HIREDIS MATCHES "ON") + LIST(APPEND LIBRSPAMDSERVERSRC "${CMAKE_CURRENT_SOURCE_DIR}/fuzzy_backend_redis.c") + LIST(APPEND LIBRSPAMDSERVERSRC "${CMAKE_CURRENT_SOURCE_DIR}/redis_pool.c") +ENDIF () + # Librspamd-server SET(RSPAMD_SERVER ${LIBRSPAMDSERVERSRC} PARENT_SCOPE) diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index 24b5568b9..33b438037 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -172,7 +172,9 @@ rspamd_config_new (void) cfg->ssl_ciphers = "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4"; cfg->max_message = DEFAULT_MAX_MESSAGE; cfg->monitored_ctx = rspamd_monitored_ctx_init (); +#ifdef ENABLE_HIREDIS cfg->redis_pool = rspamd_redis_pool_init (); +#endif REF_INIT_RETAIN (cfg, rspamd_config_free); diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c index 3dec60c57..0f89ad196 100644 --- a/src/libserver/fuzzy_backend.c +++ b/src/libserver/fuzzy_backend.c @@ -85,8 +85,9 @@ static const struct rspamd_fuzzy_backend_subr fuzzy_subrs[] = { .id = rspamd_fuzzy_backend_id_sqlite, .periodic = rspamd_fuzzy_backend_expire_sqlite, .close = rspamd_fuzzy_backend_close_sqlite, - }, - [RSPAMD_FUZZY_BACKEND_REDIS] = { + } +#ifdef ENABLE_HIREDIS + ,[RSPAMD_FUZZY_BACKEND_REDIS] = { .init = rspamd_fuzzy_backend_init_redis, .check = rspamd_fuzzy_backend_check_redis, .update = rspamd_fuzzy_backend_update_redis, @@ -96,6 +97,7 @@ static const struct rspamd_fuzzy_backend_subr fuzzy_subrs[] = { .periodic = rspamd_fuzzy_backend_expire_redis, .close = rspamd_fuzzy_backend_close_redis, } +#endif }; struct rspamd_fuzzy_backend { diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c index 5d0747a66..0f7c67579 100644 --- a/src/libserver/worker_util.c +++ b/src/libserver/worker_util.c @@ -289,8 +289,10 @@ rspamd_prepare_worker (struct rspamd_worker *worker, const char *name, rspamd_worker_init_signals (worker, ev_base); rspamd_control_worker_add_default_handler (worker, ev_base); +#ifdef ENABLE_HIREDIS rspamd_redis_pool_config (worker->srv->cfg->redis_pool, worker->srv->cfg, ev_base); +#endif /* Accept all sockets */ if (accept_handler) { diff --git a/src/plugins/lua/fann_redis.lua b/src/plugins/lua/fann_redis.lua index 468e9b466..743860208 100644 --- a/src/plugins/lua/fann_redis.lua +++ b/src/plugins/lua/fann_redis.lua @@ -209,6 +209,111 @@ local function redis_make_request(ev_base, cfg, key, is_write, callback, command return ret,conn,addr end +local function load_scripts(cfg, ev_base, on_load_cb) + local function can_train_sha_cb(err, data) + if err or not data or type(data) ~= 'string' then + rspamd_logger.errx(cfg, 'cannot save redis train script: %s', err) + else + redis_can_train_sha = tostring(data) + end + end + redis_make_request(ev_base, + rspamd_config, + nil, + true, -- is write + can_train_sha_cb, --callback + 'SCRIPT', -- command + {'LOAD', redis_lua_script_can_train} -- arguments + ) + + local function maybe_load_sha_cb(err, data) + if err or not data or type(data) ~= 'string' then + rspamd_logger.errx(cfg, 'cannot save redis load script: %s', err) + else + redis_maybe_load_sha = tostring(data) + + if on_load_cb then + rspamd_config:add_periodic(ev_base, 0.0, + function(_cfg, _ev_base) + return on_load_cb(_cfg, _ev_base) + end) + end + end + end + redis_make_request(ev_base, + rspamd_config, + nil, + true, -- is write + maybe_load_sha_cb, --callback + 'SCRIPT', -- command + {'LOAD', redis_lua_script_maybe_load} -- arguments + ) + + local function maybe_invalidate_sha_cb(err, data) + if err or not data or type(data) ~= 'string' then + rspamd_logger.errx(cfg, 'cannot save redis invalidate script: %s', err) + else + redis_maybe_invalidate_sha = tostring(data) + end + end + redis_make_request(ev_base, + rspamd_config, + nil, + true, -- is write + maybe_invalidate_sha_cb, --callback + 'SCRIPT', -- command + {'LOAD', redis_lua_script_maybe_invalidate} -- arguments + ) + + local function locked_invalidate_sha_cb(err, data) + if err or not data or type(data) ~= 'string' then + rspamd_logger.errx(cfg, 'cannot save redis locked invalidate script: %s', err) + else + redis_locked_invalidate_sha = tostring(data) + end + end + redis_make_request(ev_base, + rspamd_config, + nil, + true, -- is write + locked_invalidate_sha_cb, --callback + 'SCRIPT', -- command + {'LOAD', redis_lua_script_locked_invalidate} -- arguments + ) + + local function maybe_lock_sha_cb(err, data) + if err or not data or type(data) ~= 'string' then + rspamd_logger.errx(cfg, 'cannot save redis lock script: %s', err) + else + redis_maybe_lock_sha = tostring(data) + end + end + redis_make_request(ev_base, + rspamd_config, + nil, + true, -- is write + maybe_lock_sha_cb, --callback + 'SCRIPT', -- command + {'LOAD', redis_lua_script_maybe_lock} -- arguments + ) + + local function save_unlock_sha_cb(err, data) + if err or not data or type(data) ~= 'string' then + rspamd_logger.errx(cfg, 'cannot save redis save script: %s', err) + else + redis_save_unlock_sha = tostring(data) + end + end + redis_make_request(ev_base, + rspamd_config, + nil, + true, -- is write + save_unlock_sha_cb, --callback + 'SCRIPT', -- command + {'LOAD', redis_lua_script_save_unlock} -- arguments + ) +end + local function symbols_to_fann_vector(syms, scores) local learn_data = {} local matched_symbols = {} @@ -339,6 +444,9 @@ local function load_or_invalidate_fann(data, id, ev_base) local function redis_invalidate_cb(_err, _data) if _err then rspamd_logger.errx(rspamd_config, 'cannot invalidate ANN %s from redis: %s', id, _err) + if string.match(_err, 'NOSCRIPT') then + load_scripts(rspamd_config, ev_base, nil) + end elseif type(_data) == 'string' then rspamd_logger.infox(rspamd_config, 'invalidated ANN %s from redis: %s', id, _err) fanns[id].version = 0 @@ -404,6 +512,9 @@ local function fann_train_callback(score, required_score, results, _, id, opts, else if err then rspamd_logger.errx(rspamd_config, 'cannot check if we can train: %s', err) + if string.match(err, 'NOSCRIPT') then + load_scripts(rspamd_config, ev_base, nil) + end end end end @@ -443,6 +554,9 @@ local function train_fann(_, ev_base, elt) 'DEL', -- command {fann_prefix .. elt .. '_lock'} ) + if string.match(err, 'NOSCRIPT') then + load_scripts(rspamd_config, ev_base, nil) + end end end @@ -581,6 +695,9 @@ local function train_fann(_, ev_base, elt) if err then rspamd_logger.errx(rspamd_config, 'cannot lock ANN %s from redis: %s', fann_prefix .. elt, err) + if string.match(err, 'NOSCRIPT') then + load_scripts(rspamd_config, ev_base, nil) + end elseif type(data) == 'number' then -- Can train ANN redis_make_request(ev_base, @@ -688,111 +805,6 @@ local function maybe_train_fanns(cfg, ev_base) return watch_interval end -local function load_scripts(cfg, ev_base, on_load_cb) - local function can_train_sha_cb(err, data) - if err or not data or type(data) ~= 'string' then - rspamd_logger.errx(cfg, 'cannot save redis train script: %s', err) - else - redis_can_train_sha = tostring(data) - end - end - redis_make_request(ev_base, - rspamd_config, - nil, - true, -- is write - can_train_sha_cb, --callback - 'SCRIPT', -- command - {'LOAD', redis_lua_script_can_train} -- arguments - ) - - local function maybe_load_sha_cb(err, data) - if err or not data or type(data) ~= 'string' then - rspamd_logger.errx(cfg, 'cannot save redis load script: %s', err) - else - redis_maybe_load_sha = tostring(data) - - if on_load_cb then - rspamd_config:add_periodic(ev_base, 0.0, - function(_cfg, _ev_base) - return on_load_cb(_cfg, _ev_base) - end) - end - end - end - redis_make_request(ev_base, - rspamd_config, - nil, - true, -- is write - maybe_load_sha_cb, --callback - 'SCRIPT', -- command - {'LOAD', redis_lua_script_maybe_load} -- arguments - ) - - local function maybe_invalidate_sha_cb(err, data) - if err or not data or type(data) ~= 'string' then - rspamd_logger.errx(cfg, 'cannot save redis invalidate script: %s', err) - else - redis_maybe_invalidate_sha = tostring(data) - end - end - redis_make_request(ev_base, - rspamd_config, - nil, - true, -- is write - maybe_invalidate_sha_cb, --callback - 'SCRIPT', -- command - {'LOAD', redis_lua_script_maybe_invalidate} -- arguments - ) - - local function locked_invalidate_sha_cb(err, data) - if err or not data or type(data) ~= 'string' then - rspamd_logger.errx(cfg, 'cannot save redis locked invalidate script: %s', err) - else - redis_locked_invalidate_sha = tostring(data) - end - end - redis_make_request(ev_base, - rspamd_config, - nil, - true, -- is write - locked_invalidate_sha_cb, --callback - 'SCRIPT', -- command - {'LOAD', redis_lua_script_locked_invalidate} -- arguments - ) - - local function maybe_lock_sha_cb(err, data) - if err or not data or type(data) ~= 'string' then - rspamd_logger.errx(cfg, 'cannot save redis lock script: %s', err) - else - redis_maybe_lock_sha = tostring(data) - end - end - redis_make_request(ev_base, - rspamd_config, - nil, - true, -- is write - maybe_lock_sha_cb, --callback - 'SCRIPT', -- command - {'LOAD', redis_lua_script_maybe_lock} -- arguments - ) - - local function save_unlock_sha_cb(err, data) - if err or not data or type(data) ~= 'string' then - rspamd_logger.errx(cfg, 'cannot save redis save script: %s', err) - else - redis_save_unlock_sha = tostring(data) - end - end - redis_make_request(ev_base, - rspamd_config, - nil, - true, -- is write - save_unlock_sha_cb, --callback - 'SCRIPT', -- command - {'LOAD', redis_lua_script_save_unlock} -- arguments - ) -end - local function check_fanns(_, ev_base) local function members_cb(err, data) if err then |