From da01d43036c2f4756eab943bd48561840b2178ce Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 16 Mar 2016 14:01:16 +0000 Subject: [PATCH] [Feature] More sane configuration for redis statistics --- src/libstat/backends/redis_backend.c | 91 ++++++++++------ src/libstat/learn_cache/redis_cache.c | 151 ++++++++++++++------------ 2 files changed, 139 insertions(+), 103 deletions(-) diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c index 5fbb7f2da..3b1667e1c 100644 --- a/src/libstat/backends/redis_backend.c +++ b/src/libstat/backends/redis_backend.c @@ -874,43 +874,32 @@ rspamd_redis_learned (redisAsyncContext *c, gpointer r, gpointer priv) rt->conn_state = RSPAMD_REDIS_DISCONNECTED; } -gpointer -rspamd_redis_init (struct rspamd_stat_ctx *ctx, - struct rspamd_config *cfg, struct rspamd_statfile *st) +static gboolean +rspamd_redis_try_ucl (struct redis_stat_ctx *backend, + const ucl_object_t *obj, + struct rspamd_config *cfg, + const gchar *symbol) { - struct redis_stat_ctx *backend; - struct rspamd_statfile_config *stf = st->stcf; - struct rspamd_redis_stat_elt *st_elt; - const ucl_object_t *elt, *relt = NULL, *users_enabled; + const ucl_object_t *elt, *relt, *users_enabled; const gchar *lua_script; - backend = g_slice_alloc0 (sizeof (*backend)); + elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); - elt = ucl_object_lookup_any (stf->opts, "read_servers", "servers", NULL); if (elt == NULL) { - - if (st->classifier->cfg->opts) { - elt = ucl_object_lookup_any (st->classifier->cfg->opts, - "read_servers", "servers", NULL); - } - - if (elt == NULL) { - msg_err ("statfile %s has no redis servers", stf->symbol); - - return NULL; - } + return FALSE; } - relt = elt; backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->read_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get read servers configuration", - stf->symbol); - return NULL; + symbol); + return FALSE; } - elt = ucl_object_lookup (stf->opts, "write_servers"); + relt = elt; + + elt = ucl_object_lookup (obj, "write_servers"); if (elt == NULL) { /* Use read servers as write ones */ g_assert (relt != NULL); @@ -918,8 +907,8 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, if (!rspamd_upstreams_from_ucl (backend->write_servers, relt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", - stf->symbol); - return NULL; + symbol); + return FALSE; } } else { @@ -927,13 +916,13 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, if (!rspamd_upstreams_from_ucl (backend->write_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", - stf->symbol); + symbol); rspamd_upstreams_destroy (backend->write_servers); backend->write_servers = NULL; } } - elt = ucl_object_lookup (stf->opts, "prefix"); + elt = ucl_object_lookup (obj, "prefix"); if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { /* Default non-users statistics */ backend->redis_object = REDIS_DEFAULT_OBJECT; @@ -941,7 +930,7 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, /* * Make redis backend compatible with sqlite3 backend in users settings */ - users_enabled = ucl_object_lookup_any (stf->clcf->opts, "per_user", + users_enabled = ucl_object_lookup_any (obj, "per_user", "users_enabled", NULL); if (users_enabled != NULL) { @@ -984,7 +973,7 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, backend->redis_object = ucl_object_tostring (elt); } - elt = ucl_object_lookup (stf->opts, "timeout"); + elt = ucl_object_lookup (obj, "timeout"); if (elt) { backend->timeout = ucl_object_todouble (elt); } @@ -992,7 +981,7 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, backend->timeout = REDIS_DEFAULT_TIMEOUT; } - elt = ucl_object_lookup (stf->opts, "password"); + elt = ucl_object_lookup (obj, "password"); if (elt) { backend->password = ucl_object_tostring (elt); } @@ -1000,7 +989,7 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, backend->password = NULL; } - elt = ucl_object_lookup_any (stf->opts, "db", "database", NULL); + elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); if (elt) { backend->dbname = ucl_object_tostring (elt); } @@ -1008,6 +997,44 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, backend->dbname = NULL; } + return TRUE; +} + +gpointer +rspamd_redis_init (struct rspamd_stat_ctx *ctx, + struct rspamd_config *cfg, struct rspamd_statfile *st) +{ + struct redis_stat_ctx *backend; + struct rspamd_statfile_config *stf = st->stcf; + struct rspamd_redis_stat_elt *st_elt; + const ucl_object_t *obj; + gboolean ret = FALSE; + + backend = g_slice_alloc0 (sizeof (*backend)); + + /* First search in backend configuration */ + obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); + if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) { + ret = rspamd_redis_try_ucl (backend, obj, cfg, stf->symbol); + } + + /* Now try statfiles config */ + if (!ret) { + ret = rspamd_redis_try_ucl (backend, stf->opts, cfg, stf->symbol); + } + + /* Now try classifier config */ + if (!ret) { + ret = rspamd_redis_try_ucl (backend, st->classifier->cfg->opts, cfg, + stf->symbol); + } + + if (!ret) { + msg_err_config ("cannot init redis backend for %s", stf->symbol); + g_slice_free1 (sizeof (*backend), backend); + return NULL; + } + stf->clcf->flags |= RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND; backend->stcf = stf; diff --git a/src/libstat/learn_cache/redis_cache.c b/src/libstat/learn_cache/redis_cache.c index b48aeecea..af44ab7a0 100644 --- a/src/libstat/learn_cache/redis_cache.c +++ b/src/libstat/learn_cache/redis_cache.c @@ -187,101 +187,79 @@ rspamd_stat_cache_redis_generate_id (struct rspamd_task *task) rspamd_mempool_set_variable (task->task_pool, "words_hash", b32out, g_free); } -gpointer -rspamd_stat_cache_redis_init (struct rspamd_stat_ctx *ctx, +static gboolean +rspamd_redis_cache_try_ucl (struct rspamd_redis_cache_ctx *cache_ctx, + const ucl_object_t *obj, struct rspamd_config *cfg, - struct rspamd_statfile *st, - const ucl_object_t *cf) + const gchar *symbol) { - struct rspamd_redis_cache_ctx *cache_ctx; - struct rspamd_statfile_config *stf = st->stcf; - const ucl_object_t *elt, *relt, *telt; + const ucl_object_t *elt, *relt; - cache_ctx = g_slice_alloc0 (sizeof (*cache_ctx)); + elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); - elt = ucl_object_lookup_any (stf->opts, "read_servers", "servers", NULL); if (elt == NULL) { - - if (st->classifier->cfg->opts) { - elt = ucl_object_lookup_any (st->classifier->cfg->opts, - "read_servers", "servers", NULL); - } - - if (elt == NULL) { - msg_err ("statfile %s has no redis servers needed by cache", stf->symbol); - - return NULL; - } - - telt = ucl_object_lookup (st->classifier->cfg->opts, "password"); - if (telt) { - cache_ctx->password = ucl_object_tostring (telt); - } - else { - cache_ctx->password = NULL; - } - - telt = ucl_object_lookup_any (st->classifier->cfg->opts, - "db", "database", NULL); - if (telt) { - cache_ctx->dbname = ucl_object_tostring (telt); - } - else { - cache_ctx->dbname = NULL; - } - } - else { - telt = ucl_object_lookup (stf->opts, "password"); - if (telt) { - cache_ctx->password = ucl_object_tostring (telt); - } - else { - cache_ctx->password = NULL; - } - - telt = ucl_object_lookup_any (stf->opts, - "db", "database", NULL); - if (telt) { - cache_ctx->dbname = ucl_object_tostring (telt); - } - else { - cache_ctx->dbname = NULL; - } + return FALSE; } - relt = elt; cache_ctx->read_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (cache_ctx->read_servers, elt, REDIS_DEFAULT_PORT, NULL)) { - msg_err ("statfile %s cannot get read servers configuration for the cache", - stf->symbol); - return NULL; + msg_err ("statfile %s cannot get read servers configuration", + symbol); + return FALSE; } - elt = ucl_object_lookup (stf->opts, "write_servers"); + relt = elt; + + elt = ucl_object_lookup (obj, "write_servers"); if (elt == NULL) { /* Use read servers as write ones */ g_assert (relt != NULL); cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, relt, REDIS_DEFAULT_PORT, NULL)) { - msg_err ("statfile %s cannot get write servers configuration for the cache", - stf->symbol); - return NULL; + msg_err ("statfile %s cannot get write servers configuration", + symbol); + return FALSE; } } else { cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, elt, REDIS_DEFAULT_PORT, NULL)) { - msg_err ("statfile %s cannot get write servers configuration for the cache", - stf->symbol); + msg_err ("statfile %s cannot get write servers configuration", + symbol); rspamd_upstreams_destroy (cache_ctx->write_servers); cache_ctx->write_servers = NULL; } } - elt = ucl_object_lookup (stf->opts, "key"); + + elt = ucl_object_lookup (obj, "timeout"); + if (elt) { + cache_ctx->timeout = ucl_object_todouble (elt); + } + else { + cache_ctx->timeout = REDIS_DEFAULT_TIMEOUT; + } + + elt = ucl_object_lookup (obj, "password"); + if (elt) { + cache_ctx->password = ucl_object_tostring (elt); + } + else { + cache_ctx->password = NULL; + } + + elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); + if (elt) { + cache_ctx->dbname = ucl_object_tostring (elt); + } + else { + cache_ctx->dbname = NULL; + } + + elt = ucl_object_lookup_any (obj, "cache_key", "key", NULL); if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { cache_ctx->redis_object = DEFAULT_REDIS_KEY; } @@ -289,12 +267,43 @@ rspamd_stat_cache_redis_init (struct rspamd_stat_ctx *ctx, cache_ctx->redis_object = ucl_object_tostring (elt); } - elt = ucl_object_lookup (stf->opts, "timeout"); - if (elt) { - cache_ctx->timeout = ucl_object_todouble (elt); + return TRUE; +} + +gpointer +rspamd_stat_cache_redis_init (struct rspamd_stat_ctx *ctx, + struct rspamd_config *cfg, + struct rspamd_statfile *st, + const ucl_object_t *cf) +{ + struct rspamd_redis_cache_ctx *cache_ctx; + struct rspamd_statfile_config *stf = st->stcf; + const ucl_object_t *obj; + gboolean ret = FALSE; + + cache_ctx = g_slice_alloc0 (sizeof (*cache_ctx)); + + /* First search in backend configuration */ + obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); + if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) { + ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg, stf->symbol); } - else { - cache_ctx->timeout = REDIS_DEFAULT_TIMEOUT; + + /* Now try statfiles config */ + if (!ret) { + ret = rspamd_redis_cache_try_ucl (cache_ctx, stf->opts, cfg, stf->symbol); + } + + /* Now try classifier config */ + if (!ret) { + ret = rspamd_redis_cache_try_ucl (cache_ctx, st->classifier->cfg->opts, cfg, + stf->symbol); + } + + if (!ret) { + msg_err_config ("cannot init redis cache for %s", stf->symbol); + g_slice_free1 (sizeof (*cache_ctx), cache_ctx); + return NULL; } cache_ctx->stcf = stf; -- 2.39.5