[Feature] Add more extended statistics about fuzzy updates

diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index c6947f1cf47241259535923796419a3dc06a26ad..732e600358331bd0bdeccec6dba5642c1722cd0b 100644 (file)
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -506,7 +506,12 @@ rspamd_fuzzy_stat_callback (gint fd, gshort what, gpointer ud)
 }
 
 static void
-rspamd_fuzzy_updates_cb (gboolean success, void *ud)
+rspamd_fuzzy_updates_cb (gboolean success,
+                                                guint nadded,
+                                                guint ndeleted,
+                                                guint nextended,
+                                                guint nignored,
+                                                void *ud)
 {
        struct rspamd_updates_cbdata *cbdata = ud;
        struct rspamd_fuzzy_mirror *m;
@@ -528,8 +533,10 @@ rspamd_fuzzy_updates_cb (gboolean success, void *ud)
                        }
                }
 
-               msg_info ("successfully updated fuzzy storage: %d updates processed",
-                               ctx->updates_pending->len);
+               msg_info ("successfully updated fuzzy storage: %d updates in queue; "
+                       "%d added, %d deleted, %d extended, %d duplicates",
+                               ctx->updates_pending->len,
+                               nadded, ndeleted, nextended, nignored);
                /* Clear updates */
                ctx->updates_pending->len = 0;
                rspamd_fuzzy_backend_version (ctx->backend, source,

diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c
index 13aef8ba8052bba5539bbec92f69c970585cb068..6de977ff6165f96398c92236d8415e2a88e8dd17 100644 (file)
--- a/src/libserver/fuzzy_backend.c
+++ b/src/libserver/fuzzy_backend.c
@@ -166,7 +166,7 @@ rspamd_fuzzy_backend_update_sqlite (struct rspamd_fuzzy_backend *bk,
        struct fuzzy_peer_cmd *io_cmd;
        struct rspamd_fuzzy_cmd *cmd;
        gpointer ptr;
-       guint nupdates = 0;
+       guint nupdates = 0, nadded = 0, ndeleted = 0, nextended = 0, nignored = 0;
 
        if (rspamd_fuzzy_backend_sqlite_prepare_update (sq, src)) {
                for (i = 0; i < updates->len; i ++) {
@@ -183,14 +183,21 @@ rspamd_fuzzy_backend_update_sqlite (struct rspamd_fuzzy_backend *bk,
 
                        if (cmd->cmd == FUZZY_WRITE) {
                                rspamd_fuzzy_backend_sqlite_add (sq, ptr);
+                               nadded ++;
                                nupdates ++;
                        }
                        else if (cmd->cmd == FUZZY_DEL) {
                                rspamd_fuzzy_backend_sqlite_del (sq, ptr);
+                               ndeleted ++;
                                nupdates ++;
                        }
                        else {
-                               /* Do nothing for expire, this backend should no longer be used */
+                               if (cmd->cmd == FUZZY_REFRESH) {
+                                       nextended ++;
+                               }
+                               else {
+                                       nignored ++;
+                               }
                        }
                }
 
@@ -201,7 +208,7 @@ rspamd_fuzzy_backend_update_sqlite (struct rspamd_fuzzy_backend *bk,
        }
 
        if (cb) {
-               cb (success, ud);
+               cb (success, nadded, ndeleted, nextended, nignored, ud);
        }
 }
 
@@ -437,7 +444,7 @@ rspamd_fuzzy_backend_process_updates (struct rspamd_fuzzy_backend *bk,
                bk->subr->update (bk, updates, src, cb, ud, bk->subr_ud);
        }
        else if (cb) {
-               cb (TRUE, ud);
+               cb (TRUE, 0, 0, 0, 0, ud);
        }
 }
 

diff --git a/src/libserver/fuzzy_backend.h b/src/libserver/fuzzy_backend.h
index 0327844650114992f4c5c7d777fe0b45f149935b..f26f3a582de3f4a5ef4e10c26cb5f69e29df2b93 100644 (file)
--- a/src/libserver/fuzzy_backend.h
+++ b/src/libserver/fuzzy_backend.h
@@ -27,7 +27,12 @@ struct rspamd_config;
  * Callbacks for fuzzy methods
  */
 typedef void (*rspamd_fuzzy_check_cb) (struct rspamd_fuzzy_reply *rep, void *ud);
-typedef void (*rspamd_fuzzy_update_cb) (gboolean success, void *ud);
+typedef void (*rspamd_fuzzy_update_cb) (gboolean success,
+                                                                               guint nadded,
+                                                                               guint ndeleted,
+                                                                               guint nextended,
+                                                                               guint nignored,
+                                                                               void *ud);
 typedef void (*rspamd_fuzzy_version_cb) (guint64 rev, void *ud);
 typedef void (*rspamd_fuzzy_count_cb) (guint64 count, void *ud);
 typedef gboolean (*rspamd_fuzzy_periodic_cb) (void *ud);

diff --git a/src/libserver/fuzzy_backend_redis.c b/src/libserver/fuzzy_backend_redis.c
index 89bc6728f4b910d4183834d2a04dc630d7632a2e..22857cca533ba8bbd2951a7e369cc9482d6139a8 100644 (file)
--- a/src/libserver/fuzzy_backend_redis.c
+++ b/src/libserver/fuzzy_backend_redis.c
@@ -60,6 +60,13 @@ struct rspamd_fuzzy_backend_redis {
        ref_entry_t ref;
 };
 
+enum rspamd_fuzzy_redis_command {
+       RSPAMD_FUZZY_REDIS_COMMAND_COUNT,
+       RSPAMD_FUZZY_REDIS_COMMAND_VERSION,
+       RSPAMD_FUZZY_REDIS_COMMAND_UPDATES,
+       RSPAMD_FUZZY_REDIS_COMMAND_CHECK
+};
+
 struct rspamd_fuzzy_redis_session {
        struct rspamd_fuzzy_backend_redis *backend;
        redisAsyncContext *ctx;
@@ -69,14 +76,14 @@ struct rspamd_fuzzy_redis_session {
        float prob;
        gboolean shingles_checked;
 
-       enum {
-               RSPAMD_FUZZY_REDIS_COMMAND_COUNT,
-               RSPAMD_FUZZY_REDIS_COMMAND_VERSION,
-               RSPAMD_FUZZY_REDIS_COMMAND_UPDATES,
-               RSPAMD_FUZZY_REDIS_COMMAND_CHECK
-       } command;
+       enum rspamd_fuzzy_redis_command command;
        guint nargs;
 
+       guint nadded;
+       guint ndeleted;
+       guint nextended;
+       guint nignored;
+
        union {
                rspamd_fuzzy_check_cb cb_check;
                rspamd_fuzzy_update_cb cb_update;
@@ -1335,18 +1342,23 @@ rspamd_fuzzy_redis_update_callback (redisAsyncContext *c, gpointer r,
                if (reply->type == REDIS_REPLY_ARRAY) {
                        /* TODO: check all replies somehow */
                        if (session->callback.cb_update) {
-                               session->callback.cb_update (TRUE, session->cbdata);
+                               session->callback.cb_update (TRUE,
+                                               session->nadded,
+                                               session->ndeleted,
+                                               session->nextended,
+                                               session->nignored,
+                                               session->cbdata);
                        }
                }
                else {
                        if (session->callback.cb_update) {
-                               session->callback.cb_update (FALSE, session->cbdata);
+                               session->callback.cb_update (FALSE, 0, 0, 0, 0, session->cbdata);
                        }
                }
        }
        else {
                if (session->callback.cb_update) {
-                       session->callback.cb_update (FALSE, session->cbdata);
+                       session->callback.cb_update (FALSE, 0, 0, 0, 0, session->cbdata);
                }
 
                if (c->errstr) {
@@ -1418,6 +1430,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
                if (cmd->cmd == FUZZY_WRITE) {
                        ncommands += 5;
                        nargs += 17;
+                       session->nadded ++;
 
                        if (io_cmd->is_shingle) {
                                ncommands += RSPAMD_SHINGLE_SIZE;
@@ -1428,6 +1441,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
                else if (cmd->cmd == FUZZY_DEL) {
                        ncommands += 2;
                        nargs += 4;
+                       session->ndeleted ++;
 
                        if (io_cmd->is_shingle) {
                                ncommands += RSPAMD_SHINGLE_SIZE;
@@ -1437,12 +1451,16 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
                else if (cmd->cmd == FUZZY_REFRESH) {
                        ncommands += 1;
                        nargs += 3;
+                       session->nextended ++;
 
                        if (io_cmd->is_shingle) {
                                ncommands += RSPAMD_SHINGLE_SIZE;
                                nargs += RSPAMD_SHINGLE_SIZE * 3;
                        }
                }
+               else {
+                       session->nignored ++;
+               }
        }
 
        /* Now we need to create a new request */
@@ -1476,7 +1494,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
                rspamd_fuzzy_redis_session_dtor (session, TRUE);
 
                if (cb) {
-                       cb (FALSE, ud);
+                       cb (FALSE, 0, 0, 0, 0, ud);
                }
        }
        else {
@@ -1490,7 +1508,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
                                session->argv_lens) != REDIS_OK) {
 
                        if (cb) {
-                               cb (FALSE, ud);
+                               cb (FALSE, 0, 0, 0, 0, ud);
                        }
                        rspamd_fuzzy_redis_session_dtor (session, TRUE);
 
@@ -1506,7 +1524,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
                        if (!rspamd_fuzzy_update_append_command (bk, session, io_cmd,
                                        &cur_shift)) {
                                if (cb) {
-                                       cb (FALSE, ud);
+                                       cb (FALSE, 0, 0, 0, 0, ud);
                                }
                                rspamd_fuzzy_redis_session_dtor (session, TRUE);
 
@@ -1529,7 +1547,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
                                &session->argv_lens[cur_shift - 2]) != REDIS_OK) {
 
                        if (cb) {
-                               cb (FALSE, ud);
+                               cb (FALSE, 0, 0, 0, 0, ud);
                        }
                        rspamd_fuzzy_redis_session_dtor (session, TRUE);
 
@@ -1547,7 +1565,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
                                &session->argv_lens[cur_shift]) != REDIS_OK) {
 
                        if (cb) {
-                               cb (FALSE, ud);
+                               cb (FALSE, 0, 0, 0, 0, ud);
                        }
                        rspamd_fuzzy_redis_session_dtor (session, TRUE);