From ffab92f4ff7f4ba5d34adf51ad9b9125ec7e1d27 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 6 Jul 2018 12:24:54 +0100 Subject: [PATCH] [Feature] Add more extended statistics about fuzzy updates --- src/fuzzy_storage.c | 13 ++++++-- src/libserver/fuzzy_backend.c | 15 +++++++--- src/libserver/fuzzy_backend.h | 7 ++++- src/libserver/fuzzy_backend_redis.c | 46 ++++++++++++++++++++--------- 4 files changed, 59 insertions(+), 22 deletions(-) diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index c6947f1cf..732e60035 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -506,7 +506,12 @@ rspamd_fuzzy_stat_callback (gint fd, gshort what, gpointer ud) } static void -rspamd_fuzzy_updates_cb (gboolean success, void *ud) +rspamd_fuzzy_updates_cb (gboolean success, + guint nadded, + guint ndeleted, + guint nextended, + guint nignored, + void *ud) { struct rspamd_updates_cbdata *cbdata = ud; struct rspamd_fuzzy_mirror *m; @@ -528,8 +533,10 @@ rspamd_fuzzy_updates_cb (gboolean success, void *ud) } } - msg_info ("successfully updated fuzzy storage: %d updates processed", - ctx->updates_pending->len); + msg_info ("successfully updated fuzzy storage: %d updates in queue; " + "%d added, %d deleted, %d extended, %d duplicates", + ctx->updates_pending->len, + nadded, ndeleted, nextended, nignored); /* Clear updates */ ctx->updates_pending->len = 0; rspamd_fuzzy_backend_version (ctx->backend, source, diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c index 13aef8ba8..6de977ff6 100644 --- a/src/libserver/fuzzy_backend.c +++ b/src/libserver/fuzzy_backend.c @@ -166,7 +166,7 @@ rspamd_fuzzy_backend_update_sqlite (struct rspamd_fuzzy_backend *bk, struct fuzzy_peer_cmd *io_cmd; struct rspamd_fuzzy_cmd *cmd; gpointer ptr; - guint nupdates = 0; + guint nupdates = 0, nadded = 0, ndeleted = 0, nextended = 0, nignored = 0; if (rspamd_fuzzy_backend_sqlite_prepare_update (sq, src)) { for (i = 0; i < updates->len; i ++) { @@ -183,14 +183,21 @@ rspamd_fuzzy_backend_update_sqlite (struct rspamd_fuzzy_backend *bk, if (cmd->cmd == FUZZY_WRITE) { rspamd_fuzzy_backend_sqlite_add (sq, ptr); + nadded ++; nupdates ++; } else if (cmd->cmd == FUZZY_DEL) { rspamd_fuzzy_backend_sqlite_del (sq, ptr); + ndeleted ++; nupdates ++; } else { - /* Do nothing for expire, this backend should no longer be used */ + if (cmd->cmd == FUZZY_REFRESH) { + nextended ++; + } + else { + nignored ++; + } } } @@ -201,7 +208,7 @@ rspamd_fuzzy_backend_update_sqlite (struct rspamd_fuzzy_backend *bk, } if (cb) { - cb (success, ud); + cb (success, nadded, ndeleted, nextended, nignored, ud); } } @@ -437,7 +444,7 @@ rspamd_fuzzy_backend_process_updates (struct rspamd_fuzzy_backend *bk, bk->subr->update (bk, updates, src, cb, ud, bk->subr_ud); } else if (cb) { - cb (TRUE, ud); + cb (TRUE, 0, 0, 0, 0, ud); } } diff --git a/src/libserver/fuzzy_backend.h b/src/libserver/fuzzy_backend.h index 032784465..f26f3a582 100644 --- a/src/libserver/fuzzy_backend.h +++ b/src/libserver/fuzzy_backend.h @@ -27,7 +27,12 @@ struct rspamd_config; * Callbacks for fuzzy methods */ typedef void (*rspamd_fuzzy_check_cb) (struct rspamd_fuzzy_reply *rep, void *ud); -typedef void (*rspamd_fuzzy_update_cb) (gboolean success, void *ud); +typedef void (*rspamd_fuzzy_update_cb) (gboolean success, + guint nadded, + guint ndeleted, + guint nextended, + guint nignored, + void *ud); typedef void (*rspamd_fuzzy_version_cb) (guint64 rev, void *ud); typedef void (*rspamd_fuzzy_count_cb) (guint64 count, void *ud); typedef gboolean (*rspamd_fuzzy_periodic_cb) (void *ud); diff --git a/src/libserver/fuzzy_backend_redis.c b/src/libserver/fuzzy_backend_redis.c index 89bc6728f..22857cca5 100644 --- a/src/libserver/fuzzy_backend_redis.c +++ b/src/libserver/fuzzy_backend_redis.c @@ -60,6 +60,13 @@ struct rspamd_fuzzy_backend_redis { ref_entry_t ref; }; +enum rspamd_fuzzy_redis_command { + RSPAMD_FUZZY_REDIS_COMMAND_COUNT, + RSPAMD_FUZZY_REDIS_COMMAND_VERSION, + RSPAMD_FUZZY_REDIS_COMMAND_UPDATES, + RSPAMD_FUZZY_REDIS_COMMAND_CHECK +}; + struct rspamd_fuzzy_redis_session { struct rspamd_fuzzy_backend_redis *backend; redisAsyncContext *ctx; @@ -69,14 +76,14 @@ struct rspamd_fuzzy_redis_session { float prob; gboolean shingles_checked; - enum { - RSPAMD_FUZZY_REDIS_COMMAND_COUNT, - RSPAMD_FUZZY_REDIS_COMMAND_VERSION, - RSPAMD_FUZZY_REDIS_COMMAND_UPDATES, - RSPAMD_FUZZY_REDIS_COMMAND_CHECK - } command; + enum rspamd_fuzzy_redis_command command; guint nargs; + guint nadded; + guint ndeleted; + guint nextended; + guint nignored; + union { rspamd_fuzzy_check_cb cb_check; rspamd_fuzzy_update_cb cb_update; @@ -1335,18 +1342,23 @@ rspamd_fuzzy_redis_update_callback (redisAsyncContext *c, gpointer r, if (reply->type == REDIS_REPLY_ARRAY) { /* TODO: check all replies somehow */ if (session->callback.cb_update) { - session->callback.cb_update (TRUE, session->cbdata); + session->callback.cb_update (TRUE, + session->nadded, + session->ndeleted, + session->nextended, + session->nignored, + session->cbdata); } } else { if (session->callback.cb_update) { - session->callback.cb_update (FALSE, session->cbdata); + session->callback.cb_update (FALSE, 0, 0, 0, 0, session->cbdata); } } } else { if (session->callback.cb_update) { - session->callback.cb_update (FALSE, session->cbdata); + session->callback.cb_update (FALSE, 0, 0, 0, 0, session->cbdata); } if (c->errstr) { @@ -1418,6 +1430,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, if (cmd->cmd == FUZZY_WRITE) { ncommands += 5; nargs += 17; + session->nadded ++; if (io_cmd->is_shingle) { ncommands += RSPAMD_SHINGLE_SIZE; @@ -1428,6 +1441,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, else if (cmd->cmd == FUZZY_DEL) { ncommands += 2; nargs += 4; + session->ndeleted ++; if (io_cmd->is_shingle) { ncommands += RSPAMD_SHINGLE_SIZE; @@ -1437,12 +1451,16 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, else if (cmd->cmd == FUZZY_REFRESH) { ncommands += 1; nargs += 3; + session->nextended ++; if (io_cmd->is_shingle) { ncommands += RSPAMD_SHINGLE_SIZE; nargs += RSPAMD_SHINGLE_SIZE * 3; } } + else { + session->nignored ++; + } } /* Now we need to create a new request */ @@ -1476,7 +1494,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, rspamd_fuzzy_redis_session_dtor (session, TRUE); if (cb) { - cb (FALSE, ud); + cb (FALSE, 0, 0, 0, 0, ud); } } else { @@ -1490,7 +1508,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, session->argv_lens) != REDIS_OK) { if (cb) { - cb (FALSE, ud); + cb (FALSE, 0, 0, 0, 0, ud); } rspamd_fuzzy_redis_session_dtor (session, TRUE); @@ -1506,7 +1524,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, if (!rspamd_fuzzy_update_append_command (bk, session, io_cmd, &cur_shift)) { if (cb) { - cb (FALSE, ud); + cb (FALSE, 0, 0, 0, 0, ud); } rspamd_fuzzy_redis_session_dtor (session, TRUE); @@ -1529,7 +1547,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, &session->argv_lens[cur_shift - 2]) != REDIS_OK) { if (cb) { - cb (FALSE, ud); + cb (FALSE, 0, 0, 0, 0, ud); } rspamd_fuzzy_redis_session_dtor (session, TRUE); @@ -1547,7 +1565,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk, &session->argv_lens[cur_shift]) != REDIS_OK) { if (cb) { - cb (FALSE, ud); + cb (FALSE, 0, 0, 0, 0, ud); } rspamd_fuzzy_redis_session_dtor (session, TRUE); -- 2.39.5