From 4854a5a4053a50743ce976392b458496a1e3fd60 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Sat, 25 Nov 2017 14:48:42 +0000 Subject: [PATCH] [Rework] Implement new version of fuzzy replies --- src/fuzzy_storage.c | 66 +++++++++++++++++++--------- src/libserver/fuzzy_backend_redis.c | 66 +++++++++++++++++++++++----- src/libserver/fuzzy_backend_sqlite.c | 29 +++++++----- src/libserver/fuzzy_wire.h | 12 ++++- 4 files changed, 128 insertions(+), 45 deletions(-) diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index 2ca300025..625fcbad3 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -591,11 +591,23 @@ rspamd_fuzzy_write_reply (struct fuzzy_session *session) session->cmd_type == CMD_ENCRYPTED_SHINGLE) { /* Encrypted reply */ data = &session->reply; - len = sizeof (session->reply); + + if (session->epoch > RSPAMD_FUZZY_EPOCH10) { + len = sizeof (session->reply); + } + else { + len = sizeof (session->reply.hdr + session->reply.rep.v1); + } } else { data = &session->reply.rep; - len = sizeof (session->reply.rep); + + if (session->epoch > RSPAMD_FUZZY_EPOCH10) { + len = sizeof (session->reply.rep); + } + else { + len = sizeof (session->reply.rep.v1); + } } r = rspamd_inet_address_sendto (session->fd, data, len, 0, @@ -700,18 +712,18 @@ rspamd_fuzzy_make_reply (struct rspamd_fuzzy_cmd *cmd, gboolean encrypted, gboolean is_shingle) { if (cmd) { - result->tag = cmd->tag; + result->v1.tag = cmd->tag; memcpy (&session->reply.rep, result, sizeof (*result)); rspamd_fuzzy_update_stats (session->ctx, session->epoch, - result->prob > 0.5, + result->v1.prob > 0.5, is_shingle, session->key_stat, session->ip_stat, cmd->cmd, - result->value); + result->v1.value); if (encrypted) { /* We need also to encrypt reply */ @@ -825,13 +837,13 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) session->ip_stat = ip_stat; } - result.flag = cmd->flag; + result.v1.flag = cmd->flag; if (cmd->cmd == FUZZY_CHECK) { if (G_UNLIKELY (session->ctx->collection_mode)) { - result.prob = 0; - result.value = 500; - result.flag = 0; + result.v1.prob = 0; + result.v1.value = 500; + result.v1.flag = 0; rspamd_fuzzy_make_reply (cmd, &result, session, encrypted, is_shingle); } else { @@ -842,15 +854,15 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) } else if (cmd->cmd == FUZZY_STAT) { if (G_UNLIKELY (session->ctx->collection_mode)) { - result.prob = 0; - result.value = 500; - result.flag = 0; + result.v1.prob = 0; + result.v1.value = 500; + result.v1.flag = 0; rspamd_fuzzy_make_reply (cmd, &result, session, encrypted, is_shingle); } else { - result.prob = 1.0; - result.value = 0; - result.flag = session->ctx->stat.fuzzy_hashes; + result.v1.prob = 1.0; + result.v1.value = 0; + result.v1.flag = session->ctx->stat.fuzzy_hashes; rspamd_fuzzy_make_reply (cmd, &result, session, encrypted, is_shingle); } } @@ -863,8 +875,8 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) hexbuf[sizeof (hexbuf) - 1] = '\0'; if (g_hash_table_lookup (session->ctx->skip_hashes, hexbuf)) { - result.value = 401; - result.prob = 0.0; + result.v1.value = 401; + result.v1.prob = 0.0; goto reply; } @@ -893,12 +905,12 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) event_add (&up_req->io_ev, NULL); } - result.value = 0; - result.prob = 1.0; + result.v1.value = 0; + result.v1.prob = 1.0; } else { - result.value = 403; - result.prob = 0.0; + result.v1.value = 403; + result.v1.prob = 0.0; } reply: rspamd_fuzzy_make_reply (cmd, &result, session, encrypted, is_shingle); @@ -912,6 +924,18 @@ rspamd_fuzzy_command_valid (struct rspamd_fuzzy_cmd *cmd, gint r) enum rspamd_fuzzy_epoch ret = RSPAMD_FUZZY_EPOCH_MAX; switch (cmd->version) { + case 4: + if (cmd->shingles_count > 0) { + if (r == sizeof (struct rspamd_fuzzy_shingle_cmd)) { + ret = RSPAMD_FUZZY_EPOCH11; + } + } + else { + if (r == sizeof (*cmd)) { + ret = RSPAMD_FUZZY_EPOCH11; + } + } + break; case 3: if (cmd->shingles_count > 0) { if (r == sizeof (struct rspamd_fuzzy_shingle_cmd)) { diff --git a/src/libserver/fuzzy_backend_redis.c b/src/libserver/fuzzy_backend_redis.c index a44f116a8..b77addea4 100644 --- a/src/libserver/fuzzy_backend_redis.c +++ b/src/libserver/fuzzy_backend_redis.c @@ -396,13 +396,13 @@ rspamd_fuzzy_redis_shingles_callback (redisAsyncContext *c, gpointer r, if (max_found > RSPAMD_SHINGLE_SIZE / 2) { session->prob = ((float)max_found) / RSPAMD_SHINGLE_SIZE; - rep.prob = session->prob; + rep.v1.prob = session->prob; g_assert (sel != NULL); /* Prepare new check command */ rspamd_fuzzy_redis_session_free_args (session); - session->nargs = 4; + session->nargs = 5; session->argv = g_malloc (sizeof (gchar *) * session->nargs); session->argv_lens = g_malloc (sizeof (gsize) * session->nargs); @@ -416,7 +416,10 @@ rspamd_fuzzy_redis_shingles_callback (redisAsyncContext *c, gpointer r, session->argv_lens[2] = 1; session->argv[3] = g_strdup ("F"); session->argv_lens[3] = 1; + session->argv[3] = g_strdup ("C"); + session->argv_lens[3] = 1; g_string_free (key, FALSE); /* Do not free underlying array */ + memcpy (rep.digest, sel->digest, sizeof (rep.digest)); g_assert (session->ctx != NULL); if (redisAsyncCommandArgv (session->ctx, @@ -535,12 +538,12 @@ rspamd_fuzzy_redis_check_callback (redisAsyncContext *c, gpointer r, if (c->err == 0) { rspamd_upstream_ok (session->up); - if (reply->type == REDIS_REPLY_ARRAY && reply->elements == 2) { + if (reply->type == REDIS_REPLY_ARRAY && reply->elements >= 2) { cur = reply->element[0]; if (cur->type == REDIS_REPLY_STRING) { value = strtoul (cur->str, NULL, 10); - rep.value = value; + rep.v1.value = value; found_elts ++; } @@ -548,12 +551,23 @@ rspamd_fuzzy_redis_check_callback (redisAsyncContext *c, gpointer r, if (cur->type == REDIS_REPLY_STRING) { value = strtoul (cur->str, NULL, 10); - rep.flag = value; + rep.v1.flag = value; found_elts ++; } - if (found_elts == 2) { - rep.prob = session->prob; + if (found_elts >= 2) { + rep.v1.prob = session->prob; + memcpy (rep.digest, session->cmd->digest, sizeof (rep.digest)); + } + + rep.ts = 0; + + if (reply->elements > 2) { + cur = reply->element[2]; + + if (cur->type == REDIS_REPLY_STRING) { + rep.ts = strtoul (cur->str, NULL, 10); + } } } @@ -619,7 +633,7 @@ rspamd_fuzzy_backend_check_redis (struct rspamd_fuzzy_backend *bk, session->ev_base = rspamd_fuzzy_backend_event_base (bk); /* First of all check digest */ - session->nargs = 4; + session->nargs = 5; session->argv = g_malloc (sizeof (gchar *) * session->nargs); session->argv_lens = g_malloc (sizeof (gsize) * session->nargs); @@ -633,6 +647,8 @@ rspamd_fuzzy_backend_check_redis (struct rspamd_fuzzy_backend *bk, session->argv_lens[2] = 1; session->argv[3] = g_strdup ("F"); session->argv_lens[3] = 1; + session->argv[4] = g_strdup ("C"); + session->argv_lens[4] = 1; g_string_free (key, FALSE); /* Do not free underlying array */ up = rspamd_upstream_get (backend->read_servers, @@ -963,8 +979,9 @@ rspamd_fuzzy_update_append_command (struct rspamd_fuzzy_backend *bk, if (cmd->cmd == FUZZY_WRITE) { /* - * For each normal hash addition we do 3 redis commands: + * For each normal hash addition we do 5 redis commands: * HSET F + * HSETNX C