12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922 |
- /*-
- * Copyright 2016 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- #include "config.h"
- #include "rspamd.h"
- #include "stat_internal.h"
- #include "upstream.h"
- #include "lua/lua_common.h"
- #include "libserver/mempool_vars_internal.h"
-
- #ifdef WITH_HIREDIS
- #include "hiredis.h"
- #include "adapters/libevent.h"
- #include "ref.h"
-
- #define msg_debug_stat_redis(...) rspamd_conditional_debug_fast (NULL, NULL, \
- rspamd_stat_redis_log_id, "stat_redis", task->task_pool->tag.uid, \
- G_STRFUNC, \
- __VA_ARGS__)
-
- INIT_LOG_MODULE(stat_redis)
-
- #define REDIS_CTX(p) (struct redis_stat_ctx *)(p)
- #define REDIS_RUNTIME(p) (struct redis_stat_runtime *)(p)
- #define REDIS_BACKEND_TYPE "redis"
- #define REDIS_DEFAULT_PORT 6379
- #define REDIS_DEFAULT_OBJECT "%s%l"
- #define REDIS_DEFAULT_USERS_OBJECT "%s%l%r"
- #define REDIS_DEFAULT_TIMEOUT 0.5
- #define REDIS_STAT_TIMEOUT 30
-
- struct redis_stat_ctx {
- lua_State *L;
- struct rspamd_statfile_config *stcf;
- gint conf_ref;
- struct rspamd_stat_async_elt *stat_elt;
- const gchar *redis_object;
- const gchar *password;
- const gchar *dbname;
- gdouble timeout;
- gboolean enable_users;
- gboolean store_tokens;
- gboolean new_schema;
- gboolean enable_signatures;
- guint expiry;
- gint cbref_user;
- };
-
- enum rspamd_redis_connection_state {
- RSPAMD_REDIS_DISCONNECTED = 0,
- RSPAMD_REDIS_CONNECTED,
- RSPAMD_REDIS_REQUEST_SENT,
- RSPAMD_REDIS_TIMEDOUT,
- RSPAMD_REDIS_TERMINATED
- };
-
- struct redis_stat_runtime {
- struct redis_stat_ctx *ctx;
- struct rspamd_task *task;
- struct upstream *selected;
- struct event timeout_event;
- GArray *results;
- struct rspamd_statfile_config *stcf;
- gchar *redis_object_expanded;
- redisAsyncContext *redis;
- guint64 learned;
- gint id;
- gboolean has_event;
- GError *err;
- };
-
- /* Used to get statistics from redis */
- struct rspamd_redis_stat_cbdata;
-
- struct rspamd_redis_stat_elt {
- struct redis_stat_ctx *ctx;
- struct rspamd_stat_async_elt *async;
- struct event_base *ev_base;
- ucl_object_t *stat;
- struct rspamd_redis_stat_cbdata *cbdata;
- };
-
- struct rspamd_redis_stat_cbdata {
- struct rspamd_redis_stat_elt *elt;
- redisAsyncContext *redis;
- ucl_object_t *cur;
- GPtrArray *cur_keys;
- struct upstream *selected;
- guint inflight;
- gboolean wanna_die;
- };
-
- #define GET_TASK_ELT(task, elt) (task == NULL ? NULL : (task)->elt)
-
- static const gchar *M = "redis statistics";
-
- static GQuark
- rspamd_redis_stat_quark (void)
- {
- return g_quark_from_static_string (M);
- }
-
- static inline struct upstream_list *
- rspamd_redis_get_servers (struct redis_stat_ctx *ctx,
- const gchar *what)
- {
- lua_State *L = ctx->L;
- struct upstream_list *res;
-
- lua_rawgeti (L, LUA_REGISTRYINDEX, ctx->conf_ref);
- lua_pushstring (L, what);
- lua_gettable (L, -2);
- res = *((struct upstream_list**)lua_touserdata (L, -1));
- lua_settop (L, 0);
-
- return res;
- }
-
- /*
- * Non-static for lua unit testing
- */
- gsize
- rspamd_redis_expand_object (const gchar *pattern,
- struct redis_stat_ctx *ctx,
- struct rspamd_task *task,
- gchar **target)
- {
- gsize tlen = 0;
- const gchar *p = pattern, *elt;
- gchar *d, *end;
- enum {
- just_char,
- percent_char,
- mod_char
- } state = just_char;
- struct rspamd_statfile_config *stcf;
- lua_State *L = NULL;
- struct rspamd_task **ptask;
- GString *tb;
- const gchar *rcpt = NULL;
- gint err_idx;
-
- g_assert (ctx != NULL);
- stcf = ctx->stcf;
-
- L = task->cfg->lua_state;
-
- if (ctx->enable_users) {
- if (ctx->cbref_user == -1) {
- rcpt = rspamd_task_get_principal_recipient (task);
- }
- else if (L) {
- /* Execute lua function to get userdata */
- lua_pushcfunction (L, &rspamd_lua_traceback);
- err_idx = lua_gettop (L);
-
- lua_rawgeti (L, LUA_REGISTRYINDEX, ctx->cbref_user);
- ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
- *ptask = task;
- rspamd_lua_setclass (L, "rspamd{task}", -1);
-
- if (lua_pcall (L, 1, 1, err_idx) != 0) {
- tb = lua_touserdata (L, -1);
- msg_err_task ("call to user extraction script failed: %v", tb);
- g_string_free (tb, TRUE);
- }
- else {
- rcpt = rspamd_mempool_strdup (task->task_pool, lua_tostring (L, -1));
- }
-
- /* Result + error function */
- lua_pop (L, 2);
- }
-
- if (rcpt) {
- rspamd_mempool_set_variable (task->task_pool, "stat_user",
- (gpointer)rcpt, NULL);
- }
- }
-
- /* Length calculation */
- while (*p) {
- switch (state) {
- case just_char:
- if (*p == '%') {
- state = percent_char;
- }
- else {
- tlen ++;
- }
- p ++;
- break;
- case percent_char:
- switch (*p) {
- case '%':
- tlen ++;
- state = just_char;
- break;
- case 'u':
- elt = GET_TASK_ELT (task, user);
- if (elt) {
- tlen += strlen (elt);
- }
- break;
- case 'r':
-
- if (rcpt == NULL) {
- elt = rspamd_task_get_principal_recipient (task);
- }
- else {
- elt = rcpt;
- }
-
- if (elt) {
- tlen += strlen (elt);
- }
- break;
- case 'l':
- if (stcf->label) {
- tlen += strlen (stcf->label);
- }
- /* Label miss is OK */
- break;
- case 's':
- if (ctx->new_schema) {
- tlen += sizeof ("RS") - 1;
- }
- else {
- if (stcf->symbol) {
- tlen += strlen (stcf->symbol);
- }
- }
- break;
- default:
- state = just_char;
- tlen ++;
- break;
- }
-
- if (state == percent_char) {
- state = mod_char;
- }
- p ++;
- break;
-
- case mod_char:
- switch (*p) {
- case 'd':
- p ++;
- state = just_char;
- break;
- default:
- state = just_char;
- break;
- }
- break;
- }
- }
-
-
- if (target == NULL || task == NULL) {
- return -1;
- }
-
- *target = rspamd_mempool_alloc (task->task_pool, tlen + 1);
- d = *target;
- end = d + tlen + 1;
- d[tlen] = '\0';
- p = pattern;
- state = just_char;
-
- /* Expand string */
- while (*p && d < end) {
- switch (state) {
- case just_char:
- if (*p == '%') {
- state = percent_char;
- }
- else {
- *d++ = *p;
- }
- p ++;
- break;
- case percent_char:
- switch (*p) {
- case '%':
- *d++ = *p;
- state = just_char;
- break;
- case 'u':
- elt = GET_TASK_ELT (task, user);
- if (elt) {
- d += rspamd_strlcpy (d, elt, end - d);
- }
- break;
- case 'r':
- if (rcpt == NULL) {
- elt = rspamd_task_get_principal_recipient (task);
- }
- else {
- elt = rcpt;
- }
-
- if (elt) {
- d += rspamd_strlcpy (d, elt, end - d);
- }
- break;
- case 'l':
- if (stcf->label) {
- d += rspamd_strlcpy (d, stcf->label, end - d);
- }
- break;
- case 's':
- if (ctx->new_schema) {
- d += rspamd_strlcpy (d, "RS", end - d);
- }
- else {
- if (stcf->symbol) {
- d += rspamd_strlcpy (d, stcf->symbol, end - d);
- }
- }
- break;
- default:
- state = just_char;
- *d++ = *p;
- break;
- }
-
- if (state == percent_char) {
- state = mod_char;
- }
- p ++;
- break;
-
- case mod_char:
- switch (*p) {
- case 'd':
- /* TODO: not supported yet */
- p ++;
- state = just_char;
- break;
- default:
- state = just_char;
- break;
- }
- break;
- }
- }
-
- return tlen;
- }
-
- static void
- rspamd_redis_maybe_auth (struct redis_stat_ctx *ctx, redisAsyncContext *redis)
- {
- if (ctx->password) {
- redisAsyncCommand (redis, NULL, NULL, "AUTH %s", ctx->password);
- }
- if (ctx->dbname) {
- redisAsyncCommand (redis, NULL, NULL, "SELECT %s", ctx->dbname);
- }
- }
-
- static rspamd_fstring_t *
- rspamd_redis_tokens_to_query (struct rspamd_task *task,
- struct redis_stat_runtime *rt,
- GPtrArray *tokens,
- const gchar *command,
- const gchar *prefix,
- gboolean learn,
- gint idx,
- gboolean intvals)
- {
- rspamd_fstring_t *out;
- rspamd_token_t *tok;
- gchar n0[512], n1[64];
- guint i, l0, l1, cmd_len, prefix_len;
- gint ret;
-
- g_assert (tokens != NULL);
-
- cmd_len = strlen (command);
- prefix_len = strlen (prefix);
- out = rspamd_fstring_sized_new (1024);
-
- if (learn) {
- rspamd_printf_fstring (&out, "*1\r\n$5\r\nMULTI\r\n");
-
- ret = redisAsyncFormattedCommand (rt->redis, NULL, NULL,
- out->str, out->len);
-
- if (ret != REDIS_OK) {
- msg_err_task ("call to redis failed: %s", rt->redis->errstr);
- rspamd_fstring_free (out);
-
- return NULL;
- }
-
- out->len = 0;
- }
- else {
- if (rt->ctx->new_schema) {
- /* Multi + HGET */
- rspamd_printf_fstring (&out, "*1\r\n$5\r\nMULTI\r\n");
-
- ret = redisAsyncFormattedCommand (rt->redis, NULL, NULL,
- out->str, out->len);
-
- if (ret != REDIS_OK) {
- msg_err_task ("call to redis failed: %s", rt->redis->errstr);
- rspamd_fstring_free (out);
-
- return NULL;
- }
-
- out->len = 0;
- }
- else {
- rspamd_printf_fstring (&out, ""
- "*%d\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n",
- (tokens->len + 2),
- cmd_len, command,
- prefix_len, prefix);
- }
- }
-
- for (i = 0; i < tokens->len; i ++) {
- tok = g_ptr_array_index (tokens, i);
-
- if (learn) {
- if (intvals) {
- l1 = rspamd_snprintf (n1, sizeof (n1), "%L",
- (gint64) tok->values[idx]);
- } else {
- l1 = rspamd_snprintf (n1, sizeof (n1), "%f",
- tok->values[idx]);
- }
-
- if (rt->ctx->new_schema) {
- /*
- * HINCRBY <prefix_token> <0|1> <value>
- */
- l0 = rspamd_snprintf (n0, sizeof (n0), "%*s_%uL",
- prefix_len, prefix,
- tok->data);
-
- rspamd_printf_fstring (&out, ""
- "*4\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n",
- cmd_len, command,
- l0, n0,
- 1, rt->stcf->is_spam ? "S" : "H",
- l1, n1);
- }
- else {
- l0 = rspamd_snprintf (n0, sizeof (n0), "%uL", tok->data);
-
- /*
- * HINCRBY <prefix> <token> <value>
- */
- rspamd_printf_fstring (&out, ""
- "*4\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n",
- cmd_len, command,
- prefix_len, prefix,
- l0, n0,
- l1, n1);
- }
-
- ret = redisAsyncFormattedCommand (rt->redis, NULL, NULL,
- out->str, out->len);
-
- if (ret != REDIS_OK) {
- msg_err_task ("call to redis failed: %s", rt->redis->errstr);
- rspamd_fstring_free (out);
-
- return NULL;
- }
-
- if (rt->ctx->store_tokens) {
-
- if (!rt->ctx->new_schema) {
- /*
- * We store tokens in form
- * HSET prefix_tokens <token_id> "token_string"
- * ZINCRBY prefix_z 1.0 <token_id>
- */
- if (tok->t1 && tok->t2) {
- redisAsyncCommand (rt->redis, NULL, NULL,
- "HSET %b_tokens %b %b:%b",
- prefix, (size_t) prefix_len,
- n0, (size_t) l0,
- tok->t1->stemmed.begin, tok->t1->stemmed.len,
- tok->t2->stemmed.begin, tok->t2->stemmed.len);
- } else if (tok->t1) {
- redisAsyncCommand (rt->redis, NULL, NULL,
- "HSET %b_tokens %b %b",
- prefix, (size_t) prefix_len,
- n0, (size_t) l0,
- tok->t1->stemmed.begin, tok->t1->stemmed.len);
- }
- }
- else {
- /*
- * We store tokens in form
- * HSET <token_id> "tokens" "token_string"
- * ZINCRBY prefix_z 1.0 <token_id>
- */
- if (tok->t1 && tok->t2) {
- redisAsyncCommand (rt->redis, NULL, NULL,
- "HSET %b %s %b:%b",
- n0, (size_t) l0,
- "tokens",
- tok->t1->stemmed.begin, tok->t1->stemmed.len,
- tok->t2->stemmed.begin, tok->t2->stemmed.len);
- } else if (tok->t1) {
- redisAsyncCommand (rt->redis, NULL, NULL,
- "HSET %b %s %b",
- n0, (size_t) l0,
- "tokens",
- tok->t1->stemmed.begin, tok->t1->stemmed.len);
- }
- }
-
- redisAsyncCommand (rt->redis, NULL, NULL,
- "ZINCRBY %b_z %b %b",
- prefix, (size_t)prefix_len,
- n1, (size_t)l1,
- n0, (size_t)l0);
- }
-
- if (rt->ctx->new_schema && rt->ctx->expiry > 0) {
- out->len = 0;
- l1 = rspamd_snprintf (n1, sizeof (n1), "%d",
- rt->ctx->expiry);
-
- rspamd_printf_fstring (&out, ""
- "*3\r\n"
- "$6\r\n"
- "EXPIRE\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n",
- l0, n0,
- l1, n1);
- redisAsyncFormattedCommand (rt->redis, NULL, NULL,
- out->str, out->len);
- }
-
- out->len = 0;
- }
- else {
- if (rt->ctx->new_schema) {
- l0 = rspamd_snprintf (n0, sizeof (n0), "%*s_%uL",
- prefix_len, prefix,
- tok->data);
-
- rspamd_printf_fstring (&out, ""
- "*3\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n",
- cmd_len, command,
- l0, n0,
- 1, rt->stcf->is_spam ? "S" : "H");
-
- ret = redisAsyncFormattedCommand (rt->redis, NULL, NULL,
- out->str, out->len);
-
- if (ret != REDIS_OK) {
- msg_err_task ("call to redis failed: %s", rt->redis->errstr);
- rspamd_fstring_free (out);
-
- return NULL;
- }
-
- out->len = 0;
- }
- else {
- l0 = rspamd_snprintf (n0, sizeof (n0), "%uL", tok->data);
- rspamd_printf_fstring (&out, ""
- "$%d\r\n"
- "%s\r\n", l0, n0);
- }
- }
- }
-
- if (!learn && rt->ctx->new_schema) {
- rspamd_printf_fstring (&out, "*1\r\n$4\r\nEXEC\r\n");
- }
-
- return out;
- }
-
- static void
- rspamd_redis_store_stat_signature (struct rspamd_task *task,
- struct redis_stat_runtime *rt,
- GPtrArray *tokens,
- const gchar *prefix)
- {
- gchar *sig, keybuf[512], nbuf[64];
- rspamd_token_t *tok;
- guint i, blen, klen;
- rspamd_fstring_t *out;
-
- out = rspamd_fstring_sized_new (1024);
- sig = rspamd_mempool_get_variable (task->task_pool,
- RSPAMD_MEMPOOL_STAT_SIGNATURE);
-
- if (sig == NULL) {
- msg_err_task ("cannot get bayes signature");
- return;
- }
-
- klen = rspamd_snprintf (keybuf, sizeof (keybuf), "%s_%s_%s",
- prefix, sig, rt->stcf->is_spam ? "S" : "H");
-
- out->len = 0;
-
- /* Cleanup key */
- rspamd_printf_fstring (&out, ""
- "*2\r\n"
- "$3\r\n"
- "DEL\r\n"
- "$%d\r\n"
- "%s\r\n",
- klen, keybuf);
- redisAsyncFormattedCommand (rt->redis, NULL, NULL,
- out->str, out->len);
- out->len = 0;
-
- rspamd_printf_fstring (&out, ""
- "*%d\r\n"
- "$5\r\n"
- "LPUSH\r\n"
- "$%d\r\n"
- "%s\r\n",
- tokens->len + 2,
- klen, keybuf);
-
- PTR_ARRAY_FOREACH (tokens, i, tok) {
- blen = rspamd_snprintf (nbuf, sizeof (nbuf), "%uL", tok->data);
- rspamd_printf_fstring (&out, ""
- "$%d\r\n"
- "%s\r\n", blen, nbuf);
- }
-
- redisAsyncFormattedCommand (rt->redis, NULL, NULL,
- out->str, out->len);
- out->len = 0;
-
- if (rt->ctx->expiry > 0) {
- out->len = 0;
- blen = rspamd_snprintf (nbuf, sizeof (nbuf), "%d",
- rt->ctx->expiry);
-
- rspamd_printf_fstring (&out, ""
- "*3\r\n"
- "$6\r\n"
- "EXPIRE\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n",
- klen, keybuf,
- blen, nbuf);
- redisAsyncFormattedCommand (rt->redis, NULL, NULL,
- out->str, out->len);
- }
-
- rspamd_fstring_free (out);
- }
-
- static void
- rspamd_redis_async_cbdata_cleanup (struct rspamd_redis_stat_cbdata *cbdata)
- {
- guint i;
- gchar *k;
-
- if (cbdata && !cbdata->wanna_die) {
- /* Avoid double frees */
- cbdata->wanna_die = TRUE;
- redisAsyncFree (cbdata->redis);
-
- for (i = 0; i < cbdata->cur_keys->len; i ++) {
- k = g_ptr_array_index (cbdata->cur_keys, i);
- g_free (k);
- }
-
- g_ptr_array_free (cbdata->cur_keys, TRUE);
-
- if (cbdata->elt) {
- cbdata->elt->cbdata = NULL;
- /* Re-enable parent event */
- cbdata->elt->async->enabled = TRUE;
-
- /* Replace ucl object */
- if (cbdata->cur) {
- if (cbdata->elt->stat) {
- ucl_object_unref (cbdata->elt->stat);
- }
-
- cbdata->elt->stat = cbdata->cur;
- cbdata->cur = NULL;
- }
- }
-
- if (cbdata->cur) {
- ucl_object_unref (cbdata->cur);
- }
-
- g_free (cbdata);
- }
- }
-
- /* Called when we get number of learns for a specific key */
- static void
- rspamd_redis_stat_learns (redisAsyncContext *c, gpointer r, gpointer priv)
- {
- struct rspamd_redis_stat_cbdata *cbdata = priv;
- redisReply *reply = r;
- ucl_object_t *obj;
- gulong num = 0;
-
- if (cbdata->wanna_die) {
- return;
- }
-
- cbdata->inflight --;
-
- if (c->err == 0 && r != NULL) {
- if (G_LIKELY (reply->type == REDIS_REPLY_INTEGER)) {
- num = reply->integer;
- }
- else if (reply->type == REDIS_REPLY_STRING) {
- rspamd_strtoul (reply->str, reply->len, &num);
- }
-
- obj = (ucl_object_t *) ucl_object_lookup (cbdata->cur, "revision");
- if (obj) {
- obj->value.iv += num;
- }
- }
-
- if (cbdata->inflight == 0) {
- rspamd_redis_async_cbdata_cleanup (cbdata);
- }
- }
-
- /* Called when we get number of elements for a specific key */
- static void
- rspamd_redis_stat_key (redisAsyncContext *c, gpointer r, gpointer priv)
- {
- struct rspamd_redis_stat_cbdata *cbdata = priv;
- redisReply *reply = r;
- ucl_object_t *obj;
- glong num = 0;
-
- if (cbdata->wanna_die) {
- return;
- }
-
- cbdata->inflight --;
-
- if (c->err == 0 && r != NULL) {
- if (G_LIKELY (reply->type == REDIS_REPLY_INTEGER)) {
- num = reply->integer;
- }
- else if (reply->type == REDIS_REPLY_STRING) {
- rspamd_strtol (reply->str, reply->len, &num);
- }
-
- if (num < 0) {
- msg_err ("bad learns count: %L", (gint64)num);
- num = 0;
- }
-
- obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "used");
- if (obj) {
- obj->value.iv += num;
- }
-
- obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "total");
- if (obj) {
- obj->value.iv += num;
- }
-
- obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "size");
- if (obj) {
- /* Size of key + size of int64_t */
- obj->value.iv += num * (sizeof (G_STRINGIFY (G_MAXINT64)) +
- sizeof (guint64) + sizeof (gpointer));
- }
- }
-
- if (cbdata->inflight == 0) {
- rspamd_redis_async_cbdata_cleanup (cbdata);
- }
- }
-
- /* Called when we have connected to the redis server and got keys to check */
- static void
- rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv)
- {
- struct rspamd_redis_stat_cbdata *cbdata = priv;
- redisReply *reply = r, *elt;
- gchar **pk, *k;
- guint i, processed = 0;
-
-
- if (cbdata->wanna_die) {
- return;
- }
-
- cbdata->inflight --;
-
- if (c->err == 0 && r != NULL) {
- if (reply->type == REDIS_REPLY_ARRAY) {
- g_ptr_array_set_size (cbdata->cur_keys, reply->elements);
-
- for (i = 0; i < reply->elements; i ++) {
- elt = reply->element[i];
-
- if (elt->type == REDIS_REPLY_STRING) {
- pk = (gchar **)&g_ptr_array_index (cbdata->cur_keys, i);
- *pk = g_malloc (elt->len + 1);
- rspamd_strlcpy (*pk, elt->str, elt->len + 1);
- processed ++;
- }
- }
-
- if (processed) {
- for (i = 0; i < cbdata->cur_keys->len; i ++) {
- k = (gchar *)g_ptr_array_index (cbdata->cur_keys, i);
-
- if (k) {
- const gchar *learned_key = "learns";
-
- if (cbdata->elt->ctx->new_schema) {
- if (cbdata->elt->ctx->stcf->is_spam) {
- learned_key = "learns_spam";
- }
- else {
- learned_key = "learns_ham";
- }
- redisAsyncCommand (cbdata->redis,
- rspamd_redis_stat_learns,
- cbdata,
- "HGET %s %s",
- k, learned_key);
- cbdata->inflight += 1;
- }
- else {
- redisAsyncCommand (cbdata->redis,
- rspamd_redis_stat_key,
- cbdata,
- "HLEN %s",
- k);
- redisAsyncCommand (cbdata->redis,
- rspamd_redis_stat_learns,
- cbdata,
- "HGET %s %s",
- k, learned_key);
- cbdata->inflight += 2;
- }
- }
- }
- }
- }
-
- /* Set up the required keys */
- ucl_object_insert_key (cbdata->cur,
- ucl_object_typed_new (UCL_INT), "revision", 0, false);
- ucl_object_insert_key (cbdata->cur,
- ucl_object_typed_new (UCL_INT), "used", 0, false);
- ucl_object_insert_key (cbdata->cur,
- ucl_object_typed_new (UCL_INT), "total", 0, false);
- ucl_object_insert_key (cbdata->cur,
- ucl_object_typed_new (UCL_INT), "size", 0, false);
- ucl_object_insert_key (cbdata->cur,
- ucl_object_fromstring (cbdata->elt->ctx->stcf->symbol),
- "symbol", 0, false);
- ucl_object_insert_key (cbdata->cur, ucl_object_fromstring ("redis"),
- "type", 0, false);
- ucl_object_insert_key (cbdata->cur, ucl_object_fromint (0),
- "languages", 0, false);
- ucl_object_insert_key (cbdata->cur, ucl_object_fromint (processed),
- "users", 0, false);
-
- rspamd_upstream_ok (cbdata->selected);
-
- if (cbdata->inflight == 0) {
- rspamd_redis_async_cbdata_cleanup (cbdata);
- }
- }
- else {
- if (c->errstr) {
- msg_err ("cannot get keys to gather stat: %s", c->errstr);
- }
- else {
- msg_err ("cannot get keys to gather stat: unknown error");
- }
-
- rspamd_upstream_fail (cbdata->selected, FALSE);
- rspamd_redis_async_cbdata_cleanup (cbdata);
- }
- }
-
- static void
- rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d)
- {
- struct redis_stat_ctx *ctx;
- struct rspamd_redis_stat_elt *redis_elt = elt->ud;
- struct rspamd_redis_stat_cbdata *cbdata;
- rspamd_inet_addr_t *addr;
- struct upstream_list *ups;
-
- g_assert (redis_elt != NULL);
-
- ctx = redis_elt->ctx;
-
- if (redis_elt->cbdata) {
- /* We have some other process pending */
- rspamd_redis_async_cbdata_cleanup (redis_elt->cbdata);
- }
-
- /* Disable further events unless needed */
- elt->enabled = FALSE;
-
- ups = rspamd_redis_get_servers (ctx, "read_servers");
-
- if (!ups) {
- return;
- }
-
- cbdata = g_malloc0 (sizeof (*cbdata));
-
- cbdata->selected = rspamd_upstream_get (ups,
- RSPAMD_UPSTREAM_ROUND_ROBIN,
- NULL,
- 0);
-
- g_assert (cbdata->selected != NULL);
- addr = rspamd_upstream_addr (cbdata->selected);
- g_assert (addr != NULL);
-
- if (rspamd_inet_address_get_af (addr) == AF_UNIX) {
- cbdata->redis = redisAsyncConnectUnix (rspamd_inet_address_to_string (addr));
- }
- else {
- cbdata->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
- rspamd_inet_address_get_port (addr));
- }
-
- g_assert (cbdata->redis != NULL);
-
- redisLibeventAttach (cbdata->redis, redis_elt->ev_base);
-
- cbdata->inflight = 1;
- cbdata->cur = ucl_object_typed_new (UCL_OBJECT);
- cbdata->elt = redis_elt;
- cbdata->cur_keys = g_ptr_array_new ();
- redis_elt->cbdata = cbdata;
-
- /* XXX: deal with timeouts maybe */
- /* Get keys in redis that match our symbol */
- rspamd_redis_maybe_auth (ctx, cbdata->redis);
- redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, cbdata,
- "SMEMBERS %s_keys",
- ctx->stcf->symbol);
- }
-
- static void
- rspamd_redis_async_stat_fin (struct rspamd_stat_async_elt *elt, gpointer d)
- {
- struct rspamd_redis_stat_elt *redis_elt = elt->ud;
-
- rspamd_redis_async_cbdata_cleanup (redis_elt->cbdata);
- }
-
- /* Called on connection termination */
- static void
- rspamd_redis_fin (gpointer data)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (data);
- redisAsyncContext *redis;
-
- rt->has_event = FALSE;
- /* Stop timeout */
- if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
- event_del (&rt->timeout_event);
- }
-
- if (rt->redis) {
- redis = rt->redis;
- rt->redis = NULL;
- /* This calls for all callbacks pending */
- redisAsyncFree (redis);
- }
- }
-
- static void
- rspamd_redis_fin_learn (gpointer data)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (data);
- redisAsyncContext *redis;
-
- rt->has_event = FALSE;
- /* Stop timeout */
- if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
- event_del (&rt->timeout_event);
- }
-
- if (rt->redis) {
- redis = rt->redis;
- rt->redis = NULL;
- /* This calls for all callbacks pending */
- redisAsyncFree (redis);
- }
- }
-
- static void
- rspamd_redis_timeout (gint fd, short what, gpointer d)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (d);
- struct rspamd_task *task;
- redisAsyncContext *redis;
-
- task = rt->task;
-
- msg_err_task_check ("connection to redis server %s timed out",
- rspamd_upstream_name (rt->selected));
-
- rspamd_upstream_fail (rt->selected, FALSE);
-
- if (rt->redis) {
- redis = rt->redis;
- rt->redis = NULL;
- /* This calls for all callbacks pending */
- redisAsyncFree (redis);
- }
-
- if (!rt->err) {
- g_set_error (&rt->err, rspamd_redis_stat_quark (), ETIMEDOUT,
- "error getting reply from redis server %s: timeout",
- rspamd_upstream_name (rt->selected));
- }
- }
-
- /* Called when we have connected to the redis server and got stats */
- static void
- rspamd_redis_connected (redisAsyncContext *c, gpointer r, gpointer priv)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (priv);
- redisReply *reply = r;
- struct rspamd_task *task;
- glong val = 0;
-
- task = rt->task;
-
- if (c->err == 0) {
- if (r != NULL) {
- if (G_UNLIKELY (reply->type == REDIS_REPLY_INTEGER)) {
- val = reply->integer;
- }
- else if (reply->type == REDIS_REPLY_STRING) {
- rspamd_strtol (reply->str, reply->len, &val);
- }
- else {
- if (reply->type != REDIS_REPLY_NIL) {
- msg_err_task ("bad learned type for %s: %s, nil expected",
- rt->stcf->symbol,
- rspamd_redis_type_to_string (reply->type));
- }
-
- val = 0;
- }
-
- if (val < 0) {
- msg_warn_task ("invalid number of learns for %s: %L",
- rt->stcf->symbol, val);
- val = 0;
- }
-
- rt->learned = val;
- msg_debug_stat_redis ("connected to redis server, tokens learned for %s: %uL",
- rt->redis_object_expanded, rt->learned);
- rspamd_upstream_ok (rt->selected);
- }
- }
- else {
- msg_err_task ("error getting reply from redis server %s: %s",
- rspamd_upstream_name (rt->selected), c->errstr);
- rspamd_upstream_fail (rt->selected, FALSE);
-
- if (!rt->err) {
- g_set_error (&rt->err, rspamd_redis_stat_quark (), c->err,
- "error getting reply from redis server %s: %s",
- rspamd_upstream_name (rt->selected), c->errstr);
- }
- }
-
- }
-
- /* Called when we have received tokens values from redis */
- static void
- rspamd_redis_processed (redisAsyncContext *c, gpointer r, gpointer priv)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (priv);
- redisReply *reply = r, *elt;
- struct rspamd_task *task;
- rspamd_token_t *tok;
- guint i, processed = 0, found = 0;
- gulong val;
- gdouble float_val;
-
- task = rt->task;
-
- if (c->err == 0) {
- if (r != NULL) {
- if (reply->type == REDIS_REPLY_ARRAY) {
-
- if (reply->elements == task->tokens->len) {
- for (i = 0; i < reply->elements; i ++) {
- tok = g_ptr_array_index (task->tokens, i);
- elt = reply->element[i];
-
- if (G_UNLIKELY (elt->type == REDIS_REPLY_INTEGER)) {
- tok->values[rt->id] = elt->integer;
- found ++;
- }
- else if (elt->type == REDIS_REPLY_STRING) {
- if (rt->stcf->clcf->flags &
- RSPAMD_FLAG_CLASSIFIER_INTEGER) {
- rspamd_strtoul (elt->str, elt->len, &val);
- tok->values[rt->id] = val;
- }
- else {
- float_val = strtod (elt->str, NULL);
- tok->values[rt->id] = float_val;
- }
-
- found ++;
- }
- else {
- tok->values[rt->id] = 0;
- }
-
- processed ++;
- }
-
- if (rt->stcf->is_spam) {
- task->flags |= RSPAMD_TASK_FLAG_HAS_SPAM_TOKENS;
- }
- else {
- task->flags |= RSPAMD_TASK_FLAG_HAS_HAM_TOKENS;
- }
- }
- else {
- msg_err_task_check ("got invalid length of reply vector from redis: "
- "%d, expected: %d",
- (gint)reply->elements,
- (gint)task->tokens->len);
- }
- }
- else {
- msg_err_task_check ("got invalid reply from redis: %s, array expected",
- rspamd_redis_type_to_string (reply->type));
- }
-
- msg_debug_stat_redis ("received tokens for %s: %d processed, %d found",
- rt->redis_object_expanded, processed, found);
- rspamd_upstream_ok (rt->selected);
- }
- }
- else {
- msg_err_task ("error getting reply from redis server %s: %s",
- rspamd_upstream_name (rt->selected), c->errstr);
-
- if (rt->redis) {
- rspamd_upstream_fail (rt->selected, FALSE);
- }
-
- if (!rt->err) {
- g_set_error (&rt->err, rspamd_redis_stat_quark (), c->err,
- "cannot get values: error getting reply from redis server %s: %s",
- rspamd_upstream_name (rt->selected), c->errstr);
- }
- }
-
- if (rt->has_event) {
- rspamd_session_remove_event (task->s, rspamd_redis_fin, rt);
- }
- }
-
- /* Called when we have set tokens during learning */
- static void
- rspamd_redis_learned (redisAsyncContext *c, gpointer r, gpointer priv)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (priv);
- struct rspamd_task *task;
-
- task = rt->task;
-
- if (c->err == 0) {
- rspamd_upstream_ok (rt->selected);
- }
- else {
- msg_err_task_check ("error getting reply from redis server %s: %s",
- rspamd_upstream_name (rt->selected), c->errstr);
-
- if (rt->redis) {
- rspamd_upstream_fail (rt->selected, FALSE);
- }
-
- if (!rt->err) {
- g_set_error (&rt->err, rspamd_redis_stat_quark (), c->err,
- "cannot get learned: error getting reply from redis server %s: %s",
- rspamd_upstream_name (rt->selected), c->errstr);
- }
- }
-
- if (rt->has_event) {
- rspamd_session_remove_event (task->s, rspamd_redis_fin_learn, rt);
- }
- }
- static void
- rspamd_redis_parse_classifier_opts (struct redis_stat_ctx *backend,
- const ucl_object_t *obj,
- struct rspamd_config *cfg)
- {
- const gchar *lua_script;
- const ucl_object_t *elt, *users_enabled;
-
- users_enabled = ucl_object_lookup_any (obj, "per_user",
- "users_enabled", NULL);
-
- if (users_enabled != NULL) {
- if (ucl_object_type (users_enabled) == UCL_BOOLEAN) {
- backend->enable_users = ucl_object_toboolean (users_enabled);
- backend->cbref_user = -1;
- }
- else if (ucl_object_type (users_enabled) == UCL_STRING) {
- lua_script = ucl_object_tostring (users_enabled);
-
- if (luaL_dostring (cfg->lua_state, lua_script) != 0) {
- msg_err_config ("cannot execute lua script for users "
- "extraction: %s", lua_tostring (cfg->lua_state, -1));
- }
- else {
- if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) {
- backend->enable_users = TRUE;
- backend->cbref_user = luaL_ref (cfg->lua_state,
- LUA_REGISTRYINDEX);
- }
- else {
- msg_err_config ("lua script must return "
- "function(task) and not %s",
- lua_typename (cfg->lua_state, lua_type (
- cfg->lua_state, -1)));
- }
- }
- }
- }
- else {
- backend->enable_users = FALSE;
- backend->cbref_user = -1;
- }
-
- elt = ucl_object_lookup (obj, "prefix");
- if (elt == NULL || ucl_object_type (elt) != UCL_STRING) {
- /* Default non-users statistics */
- if (backend->enable_users || backend->cbref_user != -1) {
- backend->redis_object = REDIS_DEFAULT_USERS_OBJECT;
- }
- else {
- backend->redis_object = REDIS_DEFAULT_OBJECT;
- }
- }
- else {
- /* XXX: sanity check */
- backend->redis_object = ucl_object_tostring (elt);
- }
-
- elt = ucl_object_lookup (obj, "store_tokens");
- if (elt) {
- backend->store_tokens = ucl_object_toboolean (elt);
- }
- else {
- backend->store_tokens = FALSE;
- }
-
- elt = ucl_object_lookup (obj, "new_schema");
- if (elt) {
- backend->new_schema = ucl_object_toboolean (elt);
- }
- else {
- backend->new_schema = FALSE;
-
- msg_warn_config ("you are using old bayes schema for redis statistics, "
- "please consider converting it to a new one "
- "by using 'rspamadm configwizard statistics'");
- }
-
- elt = ucl_object_lookup (obj, "signatures");
- if (elt) {
- backend->enable_signatures = ucl_object_toboolean (elt);
- }
- else {
- backend->enable_signatures = FALSE;
- }
-
- elt = ucl_object_lookup_any (obj, "expiry", "expire", NULL);
- if (elt) {
- backend->expiry = ucl_object_toint (elt);
- }
- else {
- backend->expiry = 0;
- }
- }
-
- gpointer
- rspamd_redis_init (struct rspamd_stat_ctx *ctx,
- struct rspamd_config *cfg, struct rspamd_statfile *st)
- {
- struct redis_stat_ctx *backend;
- struct rspamd_statfile_config *stf = st->stcf;
- struct rspamd_redis_stat_elt *st_elt;
- const ucl_object_t *obj;
- gboolean ret = FALSE;
- gint conf_ref = -1;
- lua_State *L = (lua_State *)cfg->lua_state;
-
- backend = g_malloc0 (sizeof (*backend));
- backend->L = L;
- backend->timeout = REDIS_DEFAULT_TIMEOUT;
-
- /* First search in backend configuration */
- obj = ucl_object_lookup (st->classifier->cfg->opts, "backend");
- if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) {
- ret = rspamd_lua_try_load_redis (L, obj, cfg, &conf_ref);
- }
-
- /* Now try statfiles config */
- if (!ret && stf->opts) {
- ret = rspamd_lua_try_load_redis (L, stf->opts, cfg, &conf_ref);
- }
-
- /* Now try classifier config */
- if (!ret && st->classifier->cfg->opts) {
- ret = rspamd_lua_try_load_redis (L, st->classifier->cfg->opts, cfg, &conf_ref);
- }
-
- /* Now try global redis settings */
- if (!ret) {
- obj = ucl_object_lookup (cfg->rcl_obj, "redis");
-
- if (obj) {
- const ucl_object_t *specific_obj;
-
- specific_obj = ucl_object_lookup (obj, "statistics");
-
- if (specific_obj) {
- ret = rspamd_lua_try_load_redis (L,
- specific_obj, cfg, &conf_ref);
- }
- else {
- ret = rspamd_lua_try_load_redis (L,
- obj, cfg, &conf_ref);
- }
- }
- }
-
- if (!ret) {
- msg_err_config ("cannot init redis backend for %s", stf->symbol);
- g_free (backend);
- return NULL;
- }
-
- backend->conf_ref = conf_ref;
-
- /* Check some common table values */
- lua_rawgeti (L, LUA_REGISTRYINDEX, conf_ref);
-
- lua_pushstring (L, "timeout");
- lua_gettable (L, -2);
- if (lua_type (L, -1) == LUA_TNUMBER) {
- backend->timeout = lua_tonumber (L, -1);
- }
- lua_pop (L, 1);
-
- lua_pushstring (L, "db");
- lua_gettable (L, -2);
- if (lua_type (L, -1) == LUA_TSTRING) {
- backend->dbname = rspamd_mempool_strdup (cfg->cfg_pool,
- lua_tostring (L, -1));
- }
- lua_pop (L, 1);
-
- lua_pushstring (L, "password");
- lua_gettable (L, -2);
- if (lua_type (L, -1) == LUA_TSTRING) {
- backend->password = rspamd_mempool_strdup (cfg->cfg_pool,
- lua_tostring (L, -1));
- }
- lua_pop (L, 1);
-
- lua_settop (L, 0);
-
- rspamd_redis_parse_classifier_opts (backend, st->classifier->cfg->opts, cfg);
- stf->clcf->flags |= RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND;
- backend->stcf = stf;
-
- st_elt = g_malloc0 (sizeof (*st_elt));
- st_elt->ev_base = ctx->ev_base;
- st_elt->ctx = backend;
- backend->stat_elt = rspamd_stat_ctx_register_async (
- rspamd_redis_async_stat_cb,
- rspamd_redis_async_stat_fin,
- st_elt,
- REDIS_STAT_TIMEOUT);
- st_elt->async = backend->stat_elt;
-
- return (gpointer)backend;
- }
-
- gpointer
- rspamd_redis_runtime (struct rspamd_task *task,
- struct rspamd_statfile_config *stcf,
- gboolean learn, gpointer c)
- {
- struct redis_stat_ctx *ctx = REDIS_CTX (c);
- struct redis_stat_runtime *rt;
- struct upstream *up;
- struct upstream_list *ups;
- char *object_expanded = NULL;
- rspamd_inet_addr_t *addr;
-
- g_assert (ctx != NULL);
- g_assert (stcf != NULL);
-
- if (learn) {
- ups = rspamd_redis_get_servers (ctx, "write_servers");
-
- if (!ups) {
- msg_err_task ("no write servers defined for %s, cannot learn",
- stcf->symbol);
- return NULL;
- }
- up = rspamd_upstream_get (ups,
- RSPAMD_UPSTREAM_MASTER_SLAVE,
- NULL,
- 0);
- }
- else {
- ups = rspamd_redis_get_servers (ctx, "read_servers");
-
- if (!ups) {
- msg_err_task ("no read servers defined for %s, cannot stat",
- stcf->symbol);
- return NULL;
- }
- up = rspamd_upstream_get (ups,
- RSPAMD_UPSTREAM_ROUND_ROBIN,
- NULL,
- 0);
- }
-
- if (up == NULL) {
- msg_err_task ("no upstreams reachable");
- return NULL;
- }
-
- if (rspamd_redis_expand_object (ctx->redis_object, ctx, task,
- &object_expanded) == 0) {
- msg_err_task ("expansion for learning failed for symbol %s "
- "(maybe learning per user classifier with no user or recipient)",
- stcf->symbol);
- return NULL;
- }
-
- rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt));
- rspamd_mempool_add_destructor (task->task_pool,
- rspamd_gerror_free_maybe, &rt->err);
- rt->selected = up;
- rt->task = task;
- rt->ctx = ctx;
- rt->stcf = stcf;
- rt->redis_object_expanded = object_expanded;
-
- addr = rspamd_upstream_addr (up);
- g_assert (addr != NULL);
-
- if (rspamd_inet_address_get_af (addr) == AF_UNIX) {
- rt->redis = redisAsyncConnectUnix (rspamd_inet_address_to_string (addr));
- }
- else {
- rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
- rspamd_inet_address_get_port (addr));
- }
-
- if (rt->redis == NULL) {
- msg_err_task ("cannot connect redis");
- return NULL;
- }
-
- redisLibeventAttach (rt->redis, task->ev_base);
- rspamd_redis_maybe_auth (ctx, rt->redis);
-
- return rt;
- }
-
- void
- rspamd_redis_close (gpointer p)
- {
- struct redis_stat_ctx *ctx = REDIS_CTX (p);
- lua_State *L = ctx->L;
-
- if (ctx->conf_ref) {
- luaL_unref (L, LUA_REGISTRYINDEX, ctx->conf_ref);
- }
-
- g_free (ctx);
- }
-
- gboolean
- rspamd_redis_process_tokens (struct rspamd_task *task,
- GPtrArray *tokens,
- gint id, gpointer p)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
- rspamd_fstring_t *query;
- struct timeval tv;
- gint ret;
- const gchar *learned_key = "learns";
-
- if (rspamd_session_blocked (task->s)) {
- return FALSE;
- }
-
- if (tokens == NULL || tokens->len == 0 || rt->redis == NULL) {
- return FALSE;
- }
-
- rt->id = id;
-
- if (rt->ctx->new_schema) {
- if (rt->ctx->stcf->is_spam) {
- learned_key = "learns_spam";
- }
- else {
- learned_key = "learns_ham";
- }
- }
-
- if (redisAsyncCommand (rt->redis, rspamd_redis_connected, rt, "HGET %s %s",
- rt->redis_object_expanded, learned_key) == REDIS_OK) {
-
- rspamd_session_add_event (task->s, rspamd_redis_fin, rt, M);
- rt->has_event = TRUE;
-
- if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
- event_del (&rt->timeout_event);
- }
- event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
- event_base_set (task->ev_base, &rt->timeout_event);
- double_to_tv (rt->ctx->timeout, &tv);
- event_add (&rt->timeout_event, &tv);
-
- query = rspamd_redis_tokens_to_query (task, rt, tokens,
- rt->ctx->new_schema ? "HGET" : "HMGET",
- rt->redis_object_expanded, FALSE, -1,
- rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
- g_assert (query != NULL);
- rspamd_mempool_add_destructor (task->task_pool,
- (rspamd_mempool_destruct_t)rspamd_fstring_free, query);
-
- ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_processed, rt,
- query->str, query->len);
-
- if (ret == REDIS_OK) {
- return TRUE;
- }
- else {
- msg_err_task ("call to redis failed: %s", rt->redis->errstr);
- }
- }
-
- return FALSE;
- }
-
- gboolean
- rspamd_redis_finalize_process (struct rspamd_task *task, gpointer runtime,
- gpointer ctx)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime);
- redisAsyncContext *redis;
-
- if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
- event_del (&rt->timeout_event);
- }
-
- if (rt->redis) {
- redis = rt->redis;
- rt->redis = NULL;
- redisAsyncFree (redis);
- }
-
- if (rt->err) {
- return FALSE;
- }
-
- return TRUE;
- }
-
- gboolean
- rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens,
- gint id, gpointer p)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
- struct upstream *up;
- struct upstream_list *ups;
- rspamd_inet_addr_t *addr;
- struct timeval tv;
- rspamd_fstring_t *query;
- const gchar *redis_cmd;
- rspamd_token_t *tok;
- gint ret;
- goffset off;
- const gchar *learned_key = "learns";
-
- if (rspamd_session_blocked (task->s)) {
- return FALSE;
- }
-
- ups = rspamd_redis_get_servers (rt->ctx, "write_servers");
-
- if (!ups) {
- return FALSE;
- }
- up = rspamd_upstream_get (ups,
- RSPAMD_UPSTREAM_MASTER_SLAVE,
- NULL,
- 0);
-
- if (up == NULL) {
- msg_err_task ("no upstreams reachable");
- return FALSE;
- }
-
- rt->selected = up;
-
- if (rt->ctx->new_schema) {
- if (rt->ctx->stcf->is_spam) {
- learned_key = "learns_spam";
- }
- else {
- learned_key = "learns_ham";
- }
- }
-
- addr = rspamd_upstream_addr (up);
- g_assert (addr != NULL);
-
- if (rspamd_inet_address_get_af (addr) == AF_UNIX) {
- rt->redis = redisAsyncConnectUnix (rspamd_inet_address_to_string (addr));
- }
- else {
- rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
- rspamd_inet_address_get_port (addr));
- }
-
- g_assert (rt->redis != NULL);
-
- redisLibeventAttach (rt->redis, task->ev_base);
- rspamd_redis_maybe_auth (rt->ctx, rt->redis);
-
- /*
- * Add the current key to the set of learned keys
- */
- redisAsyncCommand (rt->redis, NULL, NULL, "SADD %s_keys %s",
- rt->stcf->symbol, rt->redis_object_expanded);
-
- if (rt->ctx->new_schema) {
- redisAsyncCommand (rt->redis, NULL, NULL, "HSET %s version 2",
- rt->redis_object_expanded);
- }
-
- if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) {
- redis_cmd = "HINCRBY";
- }
- else {
- redis_cmd = "HINCRBYFLOAT";
- }
-
- rt->id = id;
- query = rspamd_redis_tokens_to_query (task, rt, tokens,
- redis_cmd, rt->redis_object_expanded, TRUE, id,
- rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
- g_assert (query != NULL);
- query->len = 0;
-
- /*
- * XXX:
- * Dirty hack: we get a token and check if it's value is -1 or 1, so
- * we could understand that we are learning or unlearning
- */
-
- tok = g_ptr_array_index (task->tokens, 0);
-
- if (tok->values[id] > 0) {
- rspamd_printf_fstring (&query, ""
- "*4\r\n"
- "$7\r\n"
- "HINCRBY\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n" /* Learned key */
- "$1\r\n"
- "1\r\n",
- (gint)strlen (rt->redis_object_expanded),
- rt->redis_object_expanded,
- (gint)strlen (learned_key),
- learned_key);
- }
- else {
- rspamd_printf_fstring (&query, ""
- "*4\r\n"
- "$7\r\n"
- "HINCRBY\r\n"
- "$%d\r\n"
- "%s\r\n"
- "$%d\r\n"
- "%s\r\n" /* Learned key */
- "$2\r\n"
- "-1\r\n",
- (gint)strlen (rt->redis_object_expanded),
- rt->redis_object_expanded,
- (gint)strlen (learned_key),
- learned_key);
- }
-
- ret = redisAsyncFormattedCommand (rt->redis, NULL, NULL,
- query->str, query->len);
-
- if (ret != REDIS_OK) {
- msg_err_task ("call to redis failed: %s", rt->redis->errstr);
- rspamd_fstring_free (query);
-
- return FALSE;
- }
-
- off = query->len;
- ret = rspamd_printf_fstring (&query, "*1\r\n$4\r\nEXEC\r\n");
- ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_learned, rt,
- query->str + off, ret);
- rspamd_mempool_add_destructor (task->task_pool,
- (rspamd_mempool_destruct_t)rspamd_fstring_free, query);
-
- if (ret == REDIS_OK) {
-
- /* Add signature if needed */
- if (rt->ctx->enable_signatures) {
- rspamd_redis_store_stat_signature (task, rt, tokens,
- "RSIG");
- }
-
- rspamd_session_add_event (task->s, rspamd_redis_fin_learn, rt, M);
- rt->has_event = TRUE;
-
- /* Set timeout */
- if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
- event_del (&rt->timeout_event);
- }
- event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
- event_base_set (task->ev_base, &rt->timeout_event);
- double_to_tv (rt->ctx->timeout, &tv);
- event_add (&rt->timeout_event, &tv);
-
- return TRUE;
- }
- else {
- msg_err_task ("call to redis failed: %s", rt->redis->errstr);
- }
-
- return FALSE;
- }
-
-
- gboolean
- rspamd_redis_finalize_learn (struct rspamd_task *task, gpointer runtime,
- gpointer ctx, GError **err)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime);
- redisAsyncContext *redis;
-
- if (rspamd_event_pending (&rt->timeout_event, EV_TIMEOUT)) {
- event_del (&rt->timeout_event);
- }
-
- if (rt->redis) {
- redis = rt->redis;
- rt->redis = NULL;
- redisAsyncFree (redis);
- }
-
- if (rt->err) {
- g_propagate_error (err, rt->err);
- rt->err = NULL;
-
- return FALSE;
- }
-
- return TRUE;
- }
-
- gulong
- rspamd_redis_total_learns (struct rspamd_task *task, gpointer runtime,
- gpointer ctx)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime);
-
- return rt->learned;
- }
-
- gulong
- rspamd_redis_inc_learns (struct rspamd_task *task, gpointer runtime,
- gpointer ctx)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime);
-
- /* XXX: may cause races */
- return rt->learned + 1;
- }
-
- gulong
- rspamd_redis_dec_learns (struct rspamd_task *task, gpointer runtime,
- gpointer ctx)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime);
-
- /* XXX: may cause races */
- return rt->learned + 1;
- }
-
- gulong
- rspamd_redis_learns (struct rspamd_task *task, gpointer runtime,
- gpointer ctx)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime);
-
- return rt->learned;
- }
-
- ucl_object_t *
- rspamd_redis_get_stat (gpointer runtime,
- gpointer ctx)
- {
- struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime);
- struct rspamd_redis_stat_elt *st;
- redisAsyncContext *redis;
-
- if (rt->ctx->stat_elt) {
- st = rt->ctx->stat_elt->ud;
-
- if (rt->redis) {
- redis = rt->redis;
- rt->redis = NULL;
- redisAsyncFree (redis);
- }
-
- if (st->stat) {
- return ucl_object_ref (st->stat);
- }
- }
-
- return NULL;
- }
-
- gpointer
- rspamd_redis_load_tokenizer_config (gpointer runtime,
- gsize *len)
- {
- return NULL;
- }
-
- #endif
|