Remove fuzzy stat from generic server stat

diff --git a/src/controller.c b/src/controller.c
index d04aba5c760233bb7f44b83cd1d793d3f1e16e5c..f16bf42a072bfbe3b2f759ecf0a5de56fc9db23a 100644 (file)
--- a/src/controller.c
+++ b/src/controller.c
@@ -1871,37 +1871,12 @@ rspamd_controller_handle_stat_common (
        ucl_object_insert_key (top,
                ucl_object_fromint (
                        mem_st.oversized_chunks), "chunks_oversized", 0, false);
-       ucl_object_insert_key (top,
-               ucl_object_fromint (stat->fuzzy_hashes), "fuzzy_stored", 0, false);
-       ucl_object_insert_key (top,
-               ucl_object_fromint (
-                       stat->fuzzy_hashes_expired), "fuzzy_expired", 0, false);
-
-       /* Fuzzy epoch statistics */
-       sub = ucl_object_typed_new (UCL_ARRAY);
-
-       for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) {
-               ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_checked[i]));
-       }
-
-       ucl_object_insert_key (top, sub, "fuzzy_checked", 0, false);
-       sub = ucl_object_typed_new (UCL_ARRAY);
-
-       for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) {
-               ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_found[i]));
-       }
-
-       ucl_object_insert_key (top, sub, "fuzzy_found", 0, false);
 
        if (do_reset) {
                session->ctx->srv->stat->messages_scanned = 0;
                session->ctx->srv->stat->messages_learned = 0;
                session->ctx->srv->stat->connections_count = 0;
                session->ctx->srv->stat->control_connections_count = 0;
-               memset (stat->fuzzy_hashes_checked, 0,
-                               sizeof (stat->fuzzy_hashes_checked));
-               memset (stat->fuzzy_hashes_found, 0,
-                               sizeof (stat->fuzzy_hashes_found));
                rspamd_mempool_stat_reset ();
        }
 
@@ -2192,42 +2167,6 @@ rspamd_controller_load_saved_stats (struct rspamd_controller_worker_ctx *ctx)
                stat_copy.control_connections_count = ucl_object_toint (elt);
        }
 
-       elt = ucl_object_find_key (obj, "fuzzy_stored");
-
-       if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
-               stat_copy.fuzzy_hashes = ucl_object_toint (elt);
-       }
-
-       elt = ucl_object_find_key (obj, "fuzzy_expired");
-
-       if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
-               stat_copy.fuzzy_hashes_expired = ucl_object_toint (elt);
-       }
-
-       elt = ucl_object_find_key (obj, "fuzzy_checked");
-
-       if (elt && ucl_object_type (elt) == UCL_ARRAY) {
-               for (i = 0; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
-                       subelt = ucl_array_find_index (elt, i);
-
-                       if (subelt && ucl_object_type (subelt) == UCL_INT) {
-                               stat_copy.fuzzy_hashes_checked[i] = ucl_object_toint (subelt);
-                       }
-               }
-       }
-
-       elt = ucl_object_find_key (obj, "fuzzy_found");
-
-       if (elt && ucl_object_type (elt) == UCL_ARRAY) {
-               for (i = 0; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
-                       subelt = ucl_array_find_index (elt, i);
-
-                       if (subelt && ucl_object_type (subelt) == UCL_INT) {
-                               stat_copy.fuzzy_hashes_found[i] = ucl_object_toint (subelt);
-                       }
-               }
-       }
-
        ucl_object_unref (obj);
        memcpy (stat, &stat_copy, sizeof (stat_copy));
 }
@@ -2281,27 +2220,6 @@ rspamd_controller_store_saved_stats (struct rspamd_controller_worker_ctx *ctx)
                        ucl_object_fromint (stat->control_connections_count),
                        "control_connections", 0, false);
 
-       ucl_object_insert_key (top,
-                       ucl_object_fromint (stat->fuzzy_hashes), "fuzzy_stored", 0, false);
-       ucl_object_insert_key (top,
-                       ucl_object_fromint (
-                                       stat->fuzzy_hashes_expired), "fuzzy_expired", 0, false);
-
-       /* Fuzzy epoch statistics */
-       sub = ucl_object_typed_new (UCL_ARRAY);
-
-       for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) {
-               ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_checked[i]));
-       }
-
-       ucl_object_insert_key (top, sub, "fuzzy_checked", 0, false);
-       sub = ucl_object_typed_new (UCL_ARRAY);
-
-       for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) {
-               ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_found[i]));
-       }
-
-       ucl_object_insert_key (top, sub, "fuzzy_found", 0, false);
 
        ucl_object_emit_full (top, UCL_EMIT_JSON_COMPACT,
                        ucl_object_emit_fd_funcs (fd));

diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index 72e3bff0386f17d7b14123b1da84f6237d04ee93..bd9337a5419cac6e5eeaa5d78dfa8b7cd03f099a 100644 (file)
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -70,9 +70,32 @@ worker_t fuzzy_worker = {
 /* For evtimer */
 static struct timeval tmv;
 static struct event tev;
-static struct rspamd_stat *server_stat;
+
+struct fuzzy_global_stat {
+       guint64 fuzzy_hashes;
+       /**< number of fuzzy hashes stored                                      */
+       guint64 fuzzy_hashes_expired;
+       /**< number of fuzzy hashes expired                                     */
+       guint64 fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH_MAX];
+       /**< ammount of check requests for each epoch           */
+       guint64 fuzzy_shingles_checked[RSPAMD_FUZZY_EPOCH_MAX];
+       /**< ammount of shingle check requests for each epoch   */
+       guint64 fuzzy_hashes_found[RSPAMD_FUZZY_EPOCH_MAX];
+       /**< amount of hashes found by epoch                            */
+       guint64 invalid_requests;
+};
+
+struct fuzzy_key_stat {
+       guint64 checked;
+       guint64 matched;
+       guint64 added;
+       guint64 deleted;
+       guint64 errors;
+       rspamd_lru_hash_t *last_ips;
+};
 
 struct rspamd_fuzzy_storage_ctx {
+       struct fuzzy_global_stat stat;
        char *hashfile;
        gdouble expire;
        gdouble sync_timeout;
@@ -135,15 +158,6 @@ struct fuzzy_peer_request {
        struct fuzzy_peer_cmd cmd;
 };
 
-struct fuzzy_key_stat {
-       guint64 checked;
-       guint64 matched;
-       guint64 added;
-       guint64 deleted;
-       guint64 errors;
-       rspamd_lru_hash_t *last_ips;
-};
-
 struct fuzzy_key {
        struct rspamd_http_keypair *key;
        struct fuzzy_key_stat *stat;
@@ -214,7 +228,7 @@ rspamd_fuzzy_process_updates_queue (struct rspamd_fuzzy_storage_ctx *ctx)
                }
 
                if (rspamd_fuzzy_backend_finish_update (ctx->backend)) {
-                       server_stat->fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend);
+                       ctx->stat.fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend);
                        cur = ctx->updates_pending->head;
 
                        while (cur) {
@@ -301,25 +315,21 @@ fuzzy_peer_send_io (gint fd, gshort what, gpointer d)
 }
 
 static void
-rspamd_fuzzy_update_stats (enum rspamd_fuzzy_epoch epoch, gboolean matched,
-               struct fuzzy_key_stat *key_stat, struct fuzzy_key_stat *ip_stat,
+rspamd_fuzzy_update_stats (struct rspamd_fuzzy_storage_ctx *ctx,
+               enum rspamd_fuzzy_epoch epoch,
+               gboolean matched,
+               gboolean is_shingle,
+               struct fuzzy_key_stat *key_stat,
+               struct fuzzy_key_stat *ip_stat,
                guint cmd, guint reply)
 {
-#ifndef HAVE_ATOMIC_BUILTINS
-       server_stat->fuzzy_hashes_checked[epoch] ++;
-
+       ctx->stat.fuzzy_hashes_checked[epoch] ++;
        if (matched) {
-               server_stat->fuzzy_hashes_found[epoch] ++;
+               ctx->stat.fuzzy_hashes_found[epoch]++;
        }
-#else
-       __atomic_add_fetch (&server_stat->fuzzy_hashes_checked[epoch],
-                       1, __ATOMIC_RELEASE);
-
-       if (matched) {
-               __atomic_add_fetch (&server_stat->fuzzy_hashes_found[epoch],
-                               1, __ATOMIC_RELEASE);
+       if (is_shingle) {
+               ctx->stat.fuzzy_shingles_checked[epoch]++;
        }
-#endif
 
        if (key_stat) {
                if (!matched && reply != 0) {
@@ -367,7 +377,7 @@ rspamd_fuzzy_update_stats (enum rspamd_fuzzy_epoch epoch, gboolean matched,
 static void
 rspamd_fuzzy_process_command (struct fuzzy_session *session)
 {
-       gboolean encrypted = FALSE;
+       gboolean encrypted = FALSE, is_shingle = FALSE;
        struct rspamd_fuzzy_cmd *cmd;
        struct rspamd_fuzzy_reply result;
        struct fuzzy_peer_cmd *up_cmd;
@@ -384,6 +394,7 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session)
        case CMD_SHINGLE:
                cmd = &session->cmd.shingle.basic;
                up_len = sizeof (session->cmd.shingle);
+               is_shingle = TRUE;
                break;
        case CMD_ENCRYPTED_NORMAL:
                cmd = &session->cmd.enc_normal.cmd;
@@ -394,6 +405,7 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session)
                cmd = &session->cmd.enc_shingle.cmd.basic;
                up_len = sizeof (session->cmd.shingle);
                encrypted = TRUE;
+               is_shingle = TRUE;
                break;
        }
 
@@ -453,8 +465,13 @@ reply:
        result.tag = cmd->tag;
        memcpy (&session->reply.rep, &result, sizeof (result));
 
-       rspamd_fuzzy_update_stats (session->epoch, result.prob > 0.5,
-                       session->key_stat, ip_stat, cmd->cmd, result.value);
+       rspamd_fuzzy_update_stats (session->ctx,
+                       session->epoch,
+                       result.prob > 0.5,
+                       is_shingle,
+                       session->key_stat,
+                       ip_stat, cmd->cmd,
+                       result.value);
 
        if (encrypted) {
                /* We need also to encrypt reply */
@@ -700,7 +717,7 @@ accept_fuzzy_socket (gint fd, short what, void *arg)
                        }
                        else {
                                /* Discard input */
-                               server_stat->fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH6]++;
+                               session->ctx->stat.invalid_requests ++;
                                msg_debug ("invalid fuzzy command of size %z received", r);
                        }
 
@@ -727,7 +744,7 @@ sync_callback (gint fd, short what, void *arg)
                new_expired = rspamd_fuzzy_backend_expired (ctx->backend);
 
                if (old_expired < new_expired) {
-                       server_stat->fuzzy_hashes_expired += new_expired - old_expired;
+                       ctx->stat.fuzzy_hashes_expired += new_expired - old_expired;
                }
        }
 
@@ -1141,7 +1158,6 @@ start_fuzzy (struct rspamd_worker *worker)
                        "fuzzy",
                        NULL);
        ctx->peer_fd = -1;
-       server_stat = worker->srv->stat;
 
        /*
         * Open DB and perform VACUUM
@@ -1152,7 +1168,7 @@ start_fuzzy (struct rspamd_worker *worker)
                exit (EXIT_SUCCESS);
        }
 
-       server_stat->fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend);
+       ctx->stat.fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend);
 
        if (ctx->default_key && ctx->keypair_cache_size > 0) {
                /* Create keypairs cache */

diff --git a/src/fuzzy_storage.h b/src/fuzzy_storage.h
index f3a6f0f9c3a640d9b34231b1fcf336c06402faf4..ff3438b38609718f771ce047f0122ceab7c63bb0 100644 (file)
--- a/src/fuzzy_storage.h
+++ b/src/fuzzy_storage.h
@@ -14,6 +14,18 @@
 #define FUZZY_WRITE 1
 #define FUZZY_DEL 2
 
+
+/**
+ * The epoch of the fuzzy client
+ */
+enum rspamd_fuzzy_epoch {
+       RSPAMD_FUZZY_EPOCH6 = 0, /**< pre 0.6.x */
+       RSPAMD_FUZZY_EPOCH8, /**< 0.8 till 0.9 */
+       RSPAMD_FUZZY_EPOCH9, /**< 0.9 + */
+       RSPAMD_FUZZY_EPOCH10, /**< 1.0 + encryption */
+       RSPAMD_FUZZY_EPOCH_MAX
+};
+
 RSPAMD_PACKED(rspamd_fuzzy_cmd) {
        guint8 version;
        guint8 cmd;

diff --git a/src/rspamd.h b/src/rspamd.h
index 0095cc093090b9751185ba896607b7a2328c35da..a2a442a1f2fcdf49e500b96025683426778c68ba 100644 (file)
--- a/src/rspamd.h
+++ b/src/rspamd.h
@@ -128,17 +128,6 @@ struct mime_part;
 struct rspamd_dns_resolver;
 struct rspamd_task;
 
-/**
- * The epoch of the fuzzy client
- */
-enum rspamd_fuzzy_epoch {
-       RSPAMD_FUZZY_EPOCH6 = 0, /**< pre 0.6.x */
-       RSPAMD_FUZZY_EPOCH8,     /**< 0.8 till 0.9 */
-       RSPAMD_FUZZY_EPOCH9,     /**< 0.9 + */
-       RSPAMD_FUZZY_EPOCH10,    /**< 1.0 + encryption */
-       RSPAMD_FUZZY_EPOCH_MAX
-};
-
 /**
  * Server statistics
  */
@@ -148,10 +137,6 @@ struct rspamd_stat {
        guint connections_count;                            /**< total connections count                                                */
        guint control_connections_count;                    /**< connections count to control interface                 */
        guint messages_learned;                             /**< messages learned                                                               */
-       guint fuzzy_hashes;                                 /**< number of fuzzy hashes stored                                  */
-       guint fuzzy_hashes_expired;                         /**< number of fuzzy hashes expired                                 */
-       guint64 fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH_MAX]; /**< ammount of check requests for each epoch             */
-       guint64 fuzzy_hashes_found[RSPAMD_FUZZY_EPOCH_MAX]; /**< amount of hashes found by epoch                                */
 };
 
 /**