aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/controller.c82
-rw-r--r--src/fuzzy_storage.c80
-rw-r--r--src/fuzzy_storage.h12
-rw-r--r--src/rspamd.h15
4 files changed, 60 insertions, 129 deletions
diff --git a/src/controller.c b/src/controller.c
index d04aba5c7..f16bf42a0 100644
--- a/src/controller.c
+++ b/src/controller.c
@@ -1871,37 +1871,12 @@ rspamd_controller_handle_stat_common (
ucl_object_insert_key (top,
ucl_object_fromint (
mem_st.oversized_chunks), "chunks_oversized", 0, false);
- ucl_object_insert_key (top,
- ucl_object_fromint (stat->fuzzy_hashes), "fuzzy_stored", 0, false);
- ucl_object_insert_key (top,
- ucl_object_fromint (
- stat->fuzzy_hashes_expired), "fuzzy_expired", 0, false);
-
- /* Fuzzy epoch statistics */
- sub = ucl_object_typed_new (UCL_ARRAY);
-
- for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) {
- ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_checked[i]));
- }
-
- ucl_object_insert_key (top, sub, "fuzzy_checked", 0, false);
- sub = ucl_object_typed_new (UCL_ARRAY);
-
- for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) {
- ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_found[i]));
- }
-
- ucl_object_insert_key (top, sub, "fuzzy_found", 0, false);
if (do_reset) {
session->ctx->srv->stat->messages_scanned = 0;
session->ctx->srv->stat->messages_learned = 0;
session->ctx->srv->stat->connections_count = 0;
session->ctx->srv->stat->control_connections_count = 0;
- memset (stat->fuzzy_hashes_checked, 0,
- sizeof (stat->fuzzy_hashes_checked));
- memset (stat->fuzzy_hashes_found, 0,
- sizeof (stat->fuzzy_hashes_found));
rspamd_mempool_stat_reset ();
}
@@ -2192,42 +2167,6 @@ rspamd_controller_load_saved_stats (struct rspamd_controller_worker_ctx *ctx)
stat_copy.control_connections_count = ucl_object_toint (elt);
}
- elt = ucl_object_find_key (obj, "fuzzy_stored");
-
- if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
- stat_copy.fuzzy_hashes = ucl_object_toint (elt);
- }
-
- elt = ucl_object_find_key (obj, "fuzzy_expired");
-
- if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
- stat_copy.fuzzy_hashes_expired = ucl_object_toint (elt);
- }
-
- elt = ucl_object_find_key (obj, "fuzzy_checked");
-
- if (elt && ucl_object_type (elt) == UCL_ARRAY) {
- for (i = 0; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
- subelt = ucl_array_find_index (elt, i);
-
- if (subelt && ucl_object_type (subelt) == UCL_INT) {
- stat_copy.fuzzy_hashes_checked[i] = ucl_object_toint (subelt);
- }
- }
- }
-
- elt = ucl_object_find_key (obj, "fuzzy_found");
-
- if (elt && ucl_object_type (elt) == UCL_ARRAY) {
- for (i = 0; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
- subelt = ucl_array_find_index (elt, i);
-
- if (subelt && ucl_object_type (subelt) == UCL_INT) {
- stat_copy.fuzzy_hashes_found[i] = ucl_object_toint (subelt);
- }
- }
- }
-
ucl_object_unref (obj);
memcpy (stat, &stat_copy, sizeof (stat_copy));
}
@@ -2281,27 +2220,6 @@ rspamd_controller_store_saved_stats (struct rspamd_controller_worker_ctx *ctx)
ucl_object_fromint (stat->control_connections_count),
"control_connections", 0, false);
- ucl_object_insert_key (top,
- ucl_object_fromint (stat->fuzzy_hashes), "fuzzy_stored", 0, false);
- ucl_object_insert_key (top,
- ucl_object_fromint (
- stat->fuzzy_hashes_expired), "fuzzy_expired", 0, false);
-
- /* Fuzzy epoch statistics */
- sub = ucl_object_typed_new (UCL_ARRAY);
-
- for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) {
- ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_checked[i]));
- }
-
- ucl_object_insert_key (top, sub, "fuzzy_checked", 0, false);
- sub = ucl_object_typed_new (UCL_ARRAY);
-
- for (i = RSPAMD_FUZZY_EPOCH6; i < RSPAMD_FUZZY_EPOCH_MAX; i ++) {
- ucl_array_append (sub, ucl_object_fromint (stat->fuzzy_hashes_found[i]));
- }
-
- ucl_object_insert_key (top, sub, "fuzzy_found", 0, false);
ucl_object_emit_full (top, UCL_EMIT_JSON_COMPACT,
ucl_object_emit_fd_funcs (fd));
diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index 72e3bff03..bd9337a54 100644
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -70,9 +70,32 @@ worker_t fuzzy_worker = {
/* For evtimer */
static struct timeval tmv;
static struct event tev;
-static struct rspamd_stat *server_stat;
+
+struct fuzzy_global_stat {
+ guint64 fuzzy_hashes;
+ /**< number of fuzzy hashes stored */
+ guint64 fuzzy_hashes_expired;
+ /**< number of fuzzy hashes expired */
+ guint64 fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH_MAX];
+ /**< ammount of check requests for each epoch */
+ guint64 fuzzy_shingles_checked[RSPAMD_FUZZY_EPOCH_MAX];
+ /**< ammount of shingle check requests for each epoch */
+ guint64 fuzzy_hashes_found[RSPAMD_FUZZY_EPOCH_MAX];
+ /**< amount of hashes found by epoch */
+ guint64 invalid_requests;
+};
+
+struct fuzzy_key_stat {
+ guint64 checked;
+ guint64 matched;
+ guint64 added;
+ guint64 deleted;
+ guint64 errors;
+ rspamd_lru_hash_t *last_ips;
+};
struct rspamd_fuzzy_storage_ctx {
+ struct fuzzy_global_stat stat;
char *hashfile;
gdouble expire;
gdouble sync_timeout;
@@ -135,15 +158,6 @@ struct fuzzy_peer_request {
struct fuzzy_peer_cmd cmd;
};
-struct fuzzy_key_stat {
- guint64 checked;
- guint64 matched;
- guint64 added;
- guint64 deleted;
- guint64 errors;
- rspamd_lru_hash_t *last_ips;
-};
-
struct fuzzy_key {
struct rspamd_http_keypair *key;
struct fuzzy_key_stat *stat;
@@ -214,7 +228,7 @@ rspamd_fuzzy_process_updates_queue (struct rspamd_fuzzy_storage_ctx *ctx)
}
if (rspamd_fuzzy_backend_finish_update (ctx->backend)) {
- server_stat->fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend);
+ ctx->stat.fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend);
cur = ctx->updates_pending->head;
while (cur) {
@@ -301,25 +315,21 @@ fuzzy_peer_send_io (gint fd, gshort what, gpointer d)
}
static void
-rspamd_fuzzy_update_stats (enum rspamd_fuzzy_epoch epoch, gboolean matched,
- struct fuzzy_key_stat *key_stat, struct fuzzy_key_stat *ip_stat,
+rspamd_fuzzy_update_stats (struct rspamd_fuzzy_storage_ctx *ctx,
+ enum rspamd_fuzzy_epoch epoch,
+ gboolean matched,
+ gboolean is_shingle,
+ struct fuzzy_key_stat *key_stat,
+ struct fuzzy_key_stat *ip_stat,
guint cmd, guint reply)
{
-#ifndef HAVE_ATOMIC_BUILTINS
- server_stat->fuzzy_hashes_checked[epoch] ++;
-
+ ctx->stat.fuzzy_hashes_checked[epoch] ++;
if (matched) {
- server_stat->fuzzy_hashes_found[epoch] ++;
+ ctx->stat.fuzzy_hashes_found[epoch]++;
}
-#else
- __atomic_add_fetch (&server_stat->fuzzy_hashes_checked[epoch],
- 1, __ATOMIC_RELEASE);
-
- if (matched) {
- __atomic_add_fetch (&server_stat->fuzzy_hashes_found[epoch],
- 1, __ATOMIC_RELEASE);
+ if (is_shingle) {
+ ctx->stat.fuzzy_shingles_checked[epoch]++;
}
-#endif
if (key_stat) {
if (!matched && reply != 0) {
@@ -367,7 +377,7 @@ rspamd_fuzzy_update_stats (enum rspamd_fuzzy_epoch epoch, gboolean matched,
static void
rspamd_fuzzy_process_command (struct fuzzy_session *session)
{
- gboolean encrypted = FALSE;
+ gboolean encrypted = FALSE, is_shingle = FALSE;
struct rspamd_fuzzy_cmd *cmd;
struct rspamd_fuzzy_reply result;
struct fuzzy_peer_cmd *up_cmd;
@@ -384,6 +394,7 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session)
case CMD_SHINGLE:
cmd = &session->cmd.shingle.basic;
up_len = sizeof (session->cmd.shingle);
+ is_shingle = TRUE;
break;
case CMD_ENCRYPTED_NORMAL:
cmd = &session->cmd.enc_normal.cmd;
@@ -394,6 +405,7 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session)
cmd = &session->cmd.enc_shingle.cmd.basic;
up_len = sizeof (session->cmd.shingle);
encrypted = TRUE;
+ is_shingle = TRUE;
break;
}
@@ -453,8 +465,13 @@ reply:
result.tag = cmd->tag;
memcpy (&session->reply.rep, &result, sizeof (result));
- rspamd_fuzzy_update_stats (session->epoch, result.prob > 0.5,
- session->key_stat, ip_stat, cmd->cmd, result.value);
+ rspamd_fuzzy_update_stats (session->ctx,
+ session->epoch,
+ result.prob > 0.5,
+ is_shingle,
+ session->key_stat,
+ ip_stat, cmd->cmd,
+ result.value);
if (encrypted) {
/* We need also to encrypt reply */
@@ -700,7 +717,7 @@ accept_fuzzy_socket (gint fd, short what, void *arg)
}
else {
/* Discard input */
- server_stat->fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH6]++;
+ session->ctx->stat.invalid_requests ++;
msg_debug ("invalid fuzzy command of size %z received", r);
}
@@ -727,7 +744,7 @@ sync_callback (gint fd, short what, void *arg)
new_expired = rspamd_fuzzy_backend_expired (ctx->backend);
if (old_expired < new_expired) {
- server_stat->fuzzy_hashes_expired += new_expired - old_expired;
+ ctx->stat.fuzzy_hashes_expired += new_expired - old_expired;
}
}
@@ -1141,7 +1158,6 @@ start_fuzzy (struct rspamd_worker *worker)
"fuzzy",
NULL);
ctx->peer_fd = -1;
- server_stat = worker->srv->stat;
/*
* Open DB and perform VACUUM
@@ -1152,7 +1168,7 @@ start_fuzzy (struct rspamd_worker *worker)
exit (EXIT_SUCCESS);
}
- server_stat->fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend);
+ ctx->stat.fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend);
if (ctx->default_key && ctx->keypair_cache_size > 0) {
/* Create keypairs cache */
diff --git a/src/fuzzy_storage.h b/src/fuzzy_storage.h
index f3a6f0f9c..ff3438b38 100644
--- a/src/fuzzy_storage.h
+++ b/src/fuzzy_storage.h
@@ -14,6 +14,18 @@
#define FUZZY_WRITE 1
#define FUZZY_DEL 2
+
+/**
+ * The epoch of the fuzzy client
+ */
+enum rspamd_fuzzy_epoch {
+ RSPAMD_FUZZY_EPOCH6 = 0, /**< pre 0.6.x */
+ RSPAMD_FUZZY_EPOCH8, /**< 0.8 till 0.9 */
+ RSPAMD_FUZZY_EPOCH9, /**< 0.9 + */
+ RSPAMD_FUZZY_EPOCH10, /**< 1.0 + encryption */
+ RSPAMD_FUZZY_EPOCH_MAX
+};
+
RSPAMD_PACKED(rspamd_fuzzy_cmd) {
guint8 version;
guint8 cmd;
diff --git a/src/rspamd.h b/src/rspamd.h
index 0095cc093..a2a442a1f 100644
--- a/src/rspamd.h
+++ b/src/rspamd.h
@@ -129,17 +129,6 @@ struct rspamd_dns_resolver;
struct rspamd_task;
/**
- * The epoch of the fuzzy client
- */
-enum rspamd_fuzzy_epoch {
- RSPAMD_FUZZY_EPOCH6 = 0, /**< pre 0.6.x */
- RSPAMD_FUZZY_EPOCH8, /**< 0.8 till 0.9 */
- RSPAMD_FUZZY_EPOCH9, /**< 0.9 + */
- RSPAMD_FUZZY_EPOCH10, /**< 1.0 + encryption */
- RSPAMD_FUZZY_EPOCH_MAX
-};
-
-/**
* Server statistics
*/
struct rspamd_stat {
@@ -148,10 +137,6 @@ struct rspamd_stat {
guint connections_count; /**< total connections count */
guint control_connections_count; /**< connections count to control interface */
guint messages_learned; /**< messages learned */
- guint fuzzy_hashes; /**< number of fuzzy hashes stored */
- guint fuzzy_hashes_expired; /**< number of fuzzy hashes expired */
- guint64 fuzzy_hashes_checked[RSPAMD_FUZZY_EPOCH_MAX]; /**< ammount of check requests for each epoch */
- guint64 fuzzy_hashes_found[RSPAMD_FUZZY_EPOCH_MAX]; /**< amount of hashes found by epoch */
};
/**
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-24 13:40:39 +0000