From e383fd8e4795ea6a24d28cd92be4bac79a91110d Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 16 Dec 2015 15:24:39 +0000 Subject: [PATCH] Remove fuzzy stat from generic server stat --- src/controller.c | 82 --------------------------------------------- src/fuzzy_storage.c | 80 +++++++++++++++++++++++++------------------ src/fuzzy_storage.h | 12 +++++++ src/rspamd.h | 15 --------- 4 files changed, 60 insertions(+), 129 deletions(-) diff --git a/src/controller.c b/src/controller.c index d04aba5c7..f16bf42a0 100644 --- a/src/controller.c +++ b/src/controller.c @@ -1871,37 +1871,12 @@ rspamd_controller_handle_stat_common ( ucl_object_insert_key (top, ucl_object_fromint ( mem_st.oversized_chunks), "chunks_oversized", 0, false); - ucl_object_insert_key (top, - ucl_object_fromint (stat->fuzzy_hashes), "fuzzy_stored", 0, false); - ucl_object_insert_key (top, - ucl_object_fromint ( - stat->fuzzy_hashes_expired), "fuzzy_expired", 0, false); - - /* Fuzzy epoch statistics */ - sub = ucl_object_typed_new (UCL_ARRAY); - - for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) { - ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_checked[i])); - } - - ucl_object_insert_key (top, sub, "fuzzy_checked", 0, false); - sub = ucl_object_typed_new (UCL_ARRAY); - - for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) { - ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_found[i])); - } - - ucl_object_insert_key (top, sub, "fuzzy_found", 0, false); if (do_reset) { session->ctx->srv->stat->messages_scanned = 0; session->ctx->srv->stat->messages_learned = 0; session->ctx->srv->stat->connections_count = 0; session->ctx->srv->stat->control_connections_count = 0; - memset (stat->fuzzy_hashes_checked, 0, - sizeof (stat->fuzzy_hashes_checked)); - memset (stat->fuzzy_hashes_found, 0, - sizeof (stat->fuzzy_hashes_found)); rspamd_mempool_stat_reset (); } @@ -2192,42 +2167,6 @@ rspamd_controller_load_saved_stats (struct rspamd_controller_worker_ctx *ctx) stat_copy.control_connections_count = ucl_object_toint (elt); } - elt = ucl_object_find_key (obj, "fuzzy_stored"); - - if (elt != NULL && ucl_object_type (elt) == UCL_INT) { - stat_copy.fuzzy_hashes = ucl_object_toint (elt); - } - - elt = ucl_object_find_key (obj, "fuzzy_expired"); - - if (elt != NULL && ucl_object_type (elt) == UCL_INT) { - stat_copy.fuzzy_hashes_expired = ucl_object_toint (elt); - } - - elt = ucl_object_find_key (obj, "fuzzy_checked"); - - if (elt && ucl_object_type (elt) == UCL_ARRAY) { - for (i = 0; i < RSPAMD_FUZZY_EPOCH_MAX; i++) { - subelt = ucl_array_find_index (elt, i); - - if (subelt && ucl_object_type (subelt) == UCL_INT) { - stat_copy.fuzzy_hashes_checked[i] = ucl_object_toint (subelt); - } - } - } - - elt = ucl_object_find_key (obj, "fuzzy_found"); - - if (elt && ucl_object_type (elt) == UCL_ARRAY) { - for (i = 0; i < RSPAMD_FUZZY_EPOCH_MAX; i++) { - subelt = ucl_array_find_index (elt, i); - - if (subelt && ucl_object_type (subelt) == UCL_INT) { - stat_copy.fuzzy_hashes_found[i] = ucl_object_toint (subelt); - } - } - } - ucl_object_unref (obj); memcpy (stat, &stat_copy, sizeof (stat_copy)); } @@ -2281,27 +2220,6 @@ rspamd_controller_store_saved_stats (struct rspamd_controller_worker_ctx *ctx) ucl_object_fromint (stat->control_connections_count), "control_connections", 0, false); - ucl_object_insert_key (top, - ucl_object_fromint (stat->fuzzy_hashes), "fuzzy_stored", 0, false); - ucl_object_insert_key (top, - ucl_object_fromint ( - stat->fuzzy_hashes_expired), "fuzzy_expired", 0, false); - - /* Fuzzy epoch statistics */ - sub = ucl_object_typed_new (UCL_ARRAY); - - for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) { - ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_checked[i])); - } - - ucl_object_insert_key (top, sub, "fuzzy_checked", 0, false); - sub = ucl_object_typed_new (UCL_ARRAY); - - for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) { - ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_found[i])); - } - - ucl_object_insert_key (top, sub, "fuzzy_found", 0, false); ucl_object_emit_full (top, UCL_EMIT_JSON_COMPACT, ucl_object_emit_fd_funcs (fd)); diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index 72e3bff03..bd9337a54 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -70,9 +70,32 @@ worker_t fuzzy_worker = { /* For evtimer */ static struct timeval tmv; static struct event tev; -static struct rspamd_stat *server_stat; + +struct fuzzy_global_stat { + guint64 fuzzy_hashes; + /**< number of fuzzy hashes stored */ + guint64 fuzzy_hashes_expired; + /**< number of fuzzy hashes expired */ + guint64 fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH_MAX]; + /**< ammount of check requests for each epoch */ + guint64 fuzzy_shingles_checked[RSPAMD_FUZZY_EPOCH_MAX]; + /**< ammount of shingle check requests for each epoch */ + guint64 fuzzy_hashes_found[RSPAMD_FUZZY_EPOCH_MAX]; + /**< amount of hashes found by epoch */ + guint64 invalid_requests; +}; + +struct fuzzy_key_stat { + guint64 checked; + guint64 matched; + guint64 added; + guint64 deleted; + guint64 errors; + rspamd_lru_hash_t *last_ips; +}; struct rspamd_fuzzy_storage_ctx { + struct fuzzy_global_stat stat; char *hashfile; gdouble expire; gdouble sync_timeout; @@ -135,15 +158,6 @@ struct fuzzy_peer_request { struct fuzzy_peer_cmd cmd; }; -struct fuzzy_key_stat { - guint64 checked; - guint64 matched; - guint64 added; - guint64 deleted; - guint64 errors; - rspamd_lru_hash_t *last_ips; -}; - struct fuzzy_key { struct rspamd_http_keypair *key; struct fuzzy_key_stat *stat; @@ -214,7 +228,7 @@ rspamd_fuzzy_process_updates_queue (struct rspamd_fuzzy_storage_ctx *ctx) } if (rspamd_fuzzy_backend_finish_update (ctx->backend)) { - server_stat->fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend); + ctx->stat.fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend); cur = ctx->updates_pending->head; while (cur) { @@ -301,25 +315,21 @@ fuzzy_peer_send_io (gint fd, gshort what, gpointer d) } static void -rspamd_fuzzy_update_stats (enum rspamd_fuzzy_epoch epoch, gboolean matched, - struct fuzzy_key_stat *key_stat, struct fuzzy_key_stat *ip_stat, +rspamd_fuzzy_update_stats (struct rspamd_fuzzy_storage_ctx *ctx, + enum rspamd_fuzzy_epoch epoch, + gboolean matched, + gboolean is_shingle, + struct fuzzy_key_stat *key_stat, + struct fuzzy_key_stat *ip_stat, guint cmd, guint reply) { -#ifndef HAVE_ATOMIC_BUILTINS - server_stat->fuzzy_hashes_checked[epoch] ++; - + ctx->stat.fuzzy_hashes_checked[epoch] ++; if (matched) { - server_stat->fuzzy_hashes_found[epoch] ++; + ctx->stat.fuzzy_hashes_found[epoch]++; } -#else - __atomic_add_fetch (&server_stat->fuzzy_hashes_checked[epoch], - 1, __ATOMIC_RELEASE); - - if (matched) { - __atomic_add_fetch (&server_stat->fuzzy_hashes_found[epoch], - 1, __ATOMIC_RELEASE); + if (is_shingle) { + ctx->stat.fuzzy_shingles_checked[epoch]++; } -#endif if (key_stat) { if (!matched && reply != 0) { @@ -367,7 +377,7 @@ rspamd_fuzzy_update_stats (enum rspamd_fuzzy_epoch epoch, gboolean matched, static void rspamd_fuzzy_process_command (struct fuzzy_session *session) { - gboolean encrypted = FALSE; + gboolean encrypted = FALSE, is_shingle = FALSE; struct rspamd_fuzzy_cmd *cmd; struct rspamd_fuzzy_reply result; struct fuzzy_peer_cmd *up_cmd; @@ -384,6 +394,7 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) case CMD_SHINGLE: cmd = &session->cmd.shingle.basic; up_len = sizeof (session->cmd.shingle); + is_shingle = TRUE; break; case CMD_ENCRYPTED_NORMAL: cmd = &session->cmd.enc_normal.cmd; @@ -394,6 +405,7 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) cmd = &session->cmd.enc_shingle.cmd.basic; up_len = sizeof (session->cmd.shingle); encrypted = TRUE; + is_shingle = TRUE; break; } @@ -453,8 +465,13 @@ reply: result.tag = cmd->tag; memcpy (&session->reply.rep, &result, sizeof (result)); - rspamd_fuzzy_update_stats (session->epoch, result.prob > 0.5, - session->key_stat, ip_stat, cmd->cmd, result.value); + rspamd_fuzzy_update_stats (session->ctx, + session->epoch, + result.prob > 0.5, + is_shingle, + session->key_stat, + ip_stat, cmd->cmd, + result.value); if (encrypted) { /* We need also to encrypt reply */ @@ -700,7 +717,7 @@ accept_fuzzy_socket (gint fd, short what, void *arg) } else { /* Discard input */ - server_stat->fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH6]++; + session->ctx->stat.invalid_requests ++; msg_debug ("invalid fuzzy command of size %z received", r); } @@ -727,7 +744,7 @@ sync_callback (gint fd, short what, void *arg) new_expired = rspamd_fuzzy_backend_expired (ctx->backend); if (old_expired < new_expired) { - server_stat->fuzzy_hashes_expired += new_expired - old_expired; + ctx->stat.fuzzy_hashes_expired += new_expired - old_expired; } } @@ -1141,7 +1158,6 @@ start_fuzzy (struct rspamd_worker *worker) "fuzzy", NULL); ctx->peer_fd = -1; - server_stat = worker->srv->stat; /* * Open DB and perform VACUUM @@ -1152,7 +1168,7 @@ start_fuzzy (struct rspamd_worker *worker) exit (EXIT_SUCCESS); } - server_stat->fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend); + ctx->stat.fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend); if (ctx->default_key && ctx->keypair_cache_size > 0) { /* Create keypairs cache */ diff --git a/src/fuzzy_storage.h b/src/fuzzy_storage.h index f3a6f0f9c..ff3438b38 100644 --- a/src/fuzzy_storage.h +++ b/src/fuzzy_storage.h @@ -14,6 +14,18 @@ #define FUZZY_WRITE 1 #define FUZZY_DEL 2 + +/** + * The epoch of the fuzzy client + */ +enum rspamd_fuzzy_epoch { + RSPAMD_FUZZY_EPOCH6 = 0, /**< pre 0.6.x */ + RSPAMD_FUZZY_EPOCH8, /**< 0.8 till 0.9 */ + RSPAMD_FUZZY_EPOCH9, /**< 0.9 + */ + RSPAMD_FUZZY_EPOCH10, /**< 1.0 + encryption */ + RSPAMD_FUZZY_EPOCH_MAX +}; + RSPAMD_PACKED(rspamd_fuzzy_cmd) { guint8 version; guint8 cmd; diff --git a/src/rspamd.h b/src/rspamd.h index 0095cc093..a2a442a1f 100644 --- a/src/rspamd.h +++ b/src/rspamd.h @@ -128,17 +128,6 @@ struct mime_part; struct rspamd_dns_resolver; struct rspamd_task; -/** - * The epoch of the fuzzy client - */ -enum rspamd_fuzzy_epoch { - RSPAMD_FUZZY_EPOCH6 = 0, /**< pre 0.6.x */ - RSPAMD_FUZZY_EPOCH8, /**< 0.8 till 0.9 */ - RSPAMD_FUZZY_EPOCH9, /**< 0.9 + */ - RSPAMD_FUZZY_EPOCH10, /**< 1.0 + encryption */ - RSPAMD_FUZZY_EPOCH_MAX -}; - /** * Server statistics */ @@ -148,10 +137,6 @@ struct rspamd_stat { guint connections_count; /**< total connections count */ guint control_connections_count; /**< connections count to control interface */ guint messages_learned; /**< messages learned */ - guint fuzzy_hashes; /**< number of fuzzy hashes stored */ - guint fuzzy_hashes_expired; /**< number of fuzzy hashes expired */ - guint64 fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH_MAX]; /**< ammount of check requests for each epoch */ - guint64 fuzzy_hashes_found[RSPAMD_FUZZY_EPOCH_MAX]; /**< amount of hashes found by epoch */ }; /** -- 2.39.5