[Feature] Add more extended statistics about fuzzy updates

src/fuzzy_storage.c

@@ -506,7 +506,12 @@ rspamd_fuzzy_stat_callback (gint fd, gshort what, gpointer ud)
 }
 
 static void
-rspamd_fuzzy_updates_cb (gboolean success, void *ud)
+rspamd_fuzzy_updates_cb (gboolean success,
+						 guint nadded,
+						 guint ndeleted,
+						 guint nextended,
+						 guint nignored,
+						 void *ud)
 {
 	struct rspamd_updates_cbdata *cbdata = ud;
 	struct rspamd_fuzzy_mirror *m;
@@ -528,8 +533,10 @@ rspamd_fuzzy_updates_cb (gboolean success, void *ud)
 			}
 		}
 
-		msg_info ("successfully updated fuzzy storage: %d updates processed",
-				ctx->updates_pending->len);
+		msg_info ("successfully updated fuzzy storage: %d updates in queue; "
+			"%d added, %d deleted, %d extended, %d duplicates",
+				ctx->updates_pending->len,
+				nadded, ndeleted, nextended, nignored);
 		/* Clear updates */
 		ctx->updates_pending->len = 0;
 		rspamd_fuzzy_backend_version (ctx->backend, source,

src/libserver/fuzzy_backend.c

@@ -166,7 +166,7 @@ rspamd_fuzzy_backend_update_sqlite (struct rspamd_fuzzy_backend *bk,
 	struct fuzzy_peer_cmd *io_cmd;
 	struct rspamd_fuzzy_cmd *cmd;
 	gpointer ptr;
-	guint nupdates = 0;
+	guint nupdates = 0, nadded = 0, ndeleted = 0, nextended = 0, nignored = 0;
 
 	if (rspamd_fuzzy_backend_sqlite_prepare_update (sq, src)) {
 		for (i = 0; i < updates->len; i ++) {
@@ -183,14 +183,21 @@ rspamd_fuzzy_backend_update_sqlite (struct rspamd_fuzzy_backend *bk,
 
 			if (cmd->cmd == FUZZY_WRITE) {
 				rspamd_fuzzy_backend_sqlite_add (sq, ptr);
+				nadded ++;
 				nupdates ++;
 			}
 			else if (cmd->cmd == FUZZY_DEL) {
 				rspamd_fuzzy_backend_sqlite_del (sq, ptr);
+				ndeleted ++;
 				nupdates ++;
 			}
 			else {
-				/* Do nothing for expire, this backend should no longer be used */
+				if (cmd->cmd == FUZZY_REFRESH) {
+					nextended ++;
+				}
+				else {
+					nignored ++;
+				}
 			}
 		}
 
@@ -201,7 +208,7 @@ rspamd_fuzzy_backend_update_sqlite (struct rspamd_fuzzy_backend *bk,
 	}
 
 	if (cb) {
-		cb (success, ud);
+		cb (success, nadded, ndeleted, nextended, nignored, ud);
 	}
 }
 
@@ -437,7 +444,7 @@ rspamd_fuzzy_backend_process_updates (struct rspamd_fuzzy_backend *bk,
 		bk->subr->update (bk, updates, src, cb, ud, bk->subr_ud);
 	}
 	else if (cb) {
-		cb (TRUE, ud);
+		cb (TRUE, 0, 0, 0, 0, ud);
 	}
 }
 

src/libserver/fuzzy_backend.h

@@ -27,7 +27,12 @@ struct rspamd_config;
  * Callbacks for fuzzy methods
  */
 typedef void (*rspamd_fuzzy_check_cb) (struct rspamd_fuzzy_reply *rep, void *ud);
-typedef void (*rspamd_fuzzy_update_cb) (gboolean success, void *ud);
+typedef void (*rspamd_fuzzy_update_cb) (gboolean success,
+										guint nadded,
+										guint ndeleted,
+										guint nextended,
+										guint nignored,
+										void *ud);
 typedef void (*rspamd_fuzzy_version_cb) (guint64 rev, void *ud);
 typedef void (*rspamd_fuzzy_count_cb) (guint64 count, void *ud);
 typedef gboolean (*rspamd_fuzzy_periodic_cb) (void *ud);

src/libserver/fuzzy_backend_redis.c

@@ -60,6 +60,13 @@ struct rspamd_fuzzy_backend_redis {
 	ref_entry_t ref;
 };
 
+enum rspamd_fuzzy_redis_command {
+	RSPAMD_FUZZY_REDIS_COMMAND_COUNT,
+	RSPAMD_FUZZY_REDIS_COMMAND_VERSION,
+	RSPAMD_FUZZY_REDIS_COMMAND_UPDATES,
+	RSPAMD_FUZZY_REDIS_COMMAND_CHECK
+};
+
 struct rspamd_fuzzy_redis_session {
 	struct rspamd_fuzzy_backend_redis *backend;
 	redisAsyncContext *ctx;
@@ -69,14 +76,14 @@ struct rspamd_fuzzy_redis_session {
 	float prob;
 	gboolean shingles_checked;
 
-	enum {
-		RSPAMD_FUZZY_REDIS_COMMAND_COUNT,
-		RSPAMD_FUZZY_REDIS_COMMAND_VERSION,
-		RSPAMD_FUZZY_REDIS_COMMAND_UPDATES,
-		RSPAMD_FUZZY_REDIS_COMMAND_CHECK
-	} command;
+	enum rspamd_fuzzy_redis_command command;
 	guint nargs;
 
+	guint nadded;
+	guint ndeleted;
+	guint nextended;
+	guint nignored;
+
 	union {
 		rspamd_fuzzy_check_cb cb_check;
 		rspamd_fuzzy_update_cb cb_update;
@@ -1335,18 +1342,23 @@ rspamd_fuzzy_redis_update_callback (redisAsyncContext *c, gpointer r,
 		if (reply->type == REDIS_REPLY_ARRAY) {
 			/* TODO: check all replies somehow */
 			if (session->callback.cb_update) {
-				session->callback.cb_update (TRUE, session->cbdata);
+				session->callback.cb_update (TRUE,
+						session->nadded,
+						session->ndeleted,
+						session->nextended,
+						session->nignored,
+						session->cbdata);
 			}
 		}
 		else {
 			if (session->callback.cb_update) {
-				session->callback.cb_update (FALSE, session->cbdata);
+				session->callback.cb_update (FALSE, 0, 0, 0, 0, session->cbdata);
 			}
 		}
 	}
 	else {
 		if (session->callback.cb_update) {
-			session->callback.cb_update (FALSE, session->cbdata);
+			session->callback.cb_update (FALSE, 0, 0, 0, 0, session->cbdata);
 		}
 
 		if (c->errstr) {
@@ -1418,6 +1430,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
 		if (cmd->cmd == FUZZY_WRITE) {
 			ncommands += 5;
 			nargs += 17;
+			session->nadded ++;
 
 			if (io_cmd->is_shingle) {
 				ncommands += RSPAMD_SHINGLE_SIZE;
@@ -1428,6 +1441,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
 		else if (cmd->cmd == FUZZY_DEL) {
 			ncommands += 2;
 			nargs += 4;
+			session->ndeleted ++;
 
 			if (io_cmd->is_shingle) {
 				ncommands += RSPAMD_SHINGLE_SIZE;
@@ -1437,12 +1451,16 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
 		else if (cmd->cmd == FUZZY_REFRESH) {
 			ncommands += 1;
 			nargs += 3;
+			session->nextended ++;
 
 			if (io_cmd->is_shingle) {
 				ncommands += RSPAMD_SHINGLE_SIZE;
 				nargs += RSPAMD_SHINGLE_SIZE * 3;
 			}
 		}
+		else {
+			session->nignored ++;
+		}
 	}
 
 	/* Now we need to create a new request */
@@ -1476,7 +1494,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
 		rspamd_fuzzy_redis_session_dtor (session, TRUE);
 
 		if (cb) {
-			cb (FALSE, ud);
+			cb (FALSE, 0, 0, 0, 0, ud);
 		}
 	}
 	else {
@@ -1490,7 +1508,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
 				session->argv_lens) != REDIS_OK) {
 
 			if (cb) {
-				cb (FALSE, ud);
+				cb (FALSE, 0, 0, 0, 0, ud);
 			}
 			rspamd_fuzzy_redis_session_dtor (session, TRUE);
 
@@ -1506,7 +1524,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
 			if (!rspamd_fuzzy_update_append_command (bk, session, io_cmd,
 					&cur_shift)) {
 				if (cb) {
-					cb (FALSE, ud);
+					cb (FALSE, 0, 0, 0, 0, ud);
 				}
 				rspamd_fuzzy_redis_session_dtor (session, TRUE);
 
@@ -1529,7 +1547,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
 				&session->argv_lens[cur_shift - 2]) != REDIS_OK) {
 
 			if (cb) {
-				cb (FALSE, ud);
+				cb (FALSE, 0, 0, 0, 0, ud);
 			}
 			rspamd_fuzzy_redis_session_dtor (session, TRUE);
 
@@ -1547,7 +1565,7 @@ rspamd_fuzzy_backend_update_redis (struct rspamd_fuzzy_backend *bk,
 				&session->argv_lens[cur_shift]) != REDIS_OK) {
 
 			if (cb) {
-				cb (FALSE, ud);
+				cb (FALSE, 0, 0, 0, 0, ud);
 			}
 			rspamd_fuzzy_redis_session_dtor (session, TRUE);