Rework statistics runtime structures.

diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c
index b3feda154632f45d8674cc3f99b3333b24458363..44bfe5a4ea7217ba5bb9a34f3b8dc1fd9df51486 100644 (file)
--- a/src/libserver/protocol.c
+++ b/src/libserver/protocol.c
@@ -773,6 +773,8 @@ rspamd_ucl_tolegacy_output (struct rspamd_task *task,
                                ucl_object_tostring (elt));
                }
 
+               g_assert (ucl_object_todouble (score) < 1000.0);
+
                iter = NULL;
                while ((elt = ucl_iterate_object (metric, &iter, true)) != NULL) {
                        if (elt->type == UCL_OBJECT) {

diff --git a/src/libstat/classifiers/bayes.c b/src/libstat/classifiers/bayes.c
index 06c54929265d9374f7882ebaca54ca92bc798252..54db73d9e8bb4c0069277d8f55c070db12701ec7 100644 (file)
--- a/src/libstat/classifiers/bayes.c
+++ b/src/libstat/classifiers/bayes.c
@@ -64,7 +64,7 @@ struct bayes_callback_data {
 static gboolean
 bayes_learn_callback (gpointer key, gpointer value, gpointer data)
 {
-       token_node_t *node = key;
+       rspamd_token_t *node = key;
        struct bayes_callback_data *cd = data;
        gint c;
        guint64 v;
@@ -144,7 +144,7 @@ static gboolean
 bayes_classify_callback (gpointer key, gpointer value, gpointer data)
 {
 
-       token_node_t *node = key;
+       rspamd_token_t *node = key;
        struct bayes_callback_data *cd = data;
        guint i;
        struct bayes_statfile_data *cur;
@@ -222,9 +222,7 @@ bayes_classify (struct classifier_ctx * ctx,
                (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
                minnodes = strtol (value, NULL, 10);
                nodes = g_tree_nnodes (input);
-               if (nodes > FEATURE_WINDOW_SIZE) {
-                       nodes = nodes / FEATURE_WINDOW_SIZE + FEATURE_WINDOW_SIZE;
-               }
+
                if (nodes < minnodes) {
                        return FALSE;
                }
@@ -331,9 +329,7 @@ bayes_learn_spam (struct classifier_ctx * ctx,
                (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
                minnodes = strtol (value, NULL, 10);
                nodes = g_tree_nnodes (input);
-               if (nodes > FEATURE_WINDOW_SIZE) {
-                       nodes = nodes / FEATURE_WINDOW_SIZE + FEATURE_WINDOW_SIZE;
-               }
+
                if (nodes < minnodes) {
                        g_set_error (err,
                                bayes_error_quark (),           /* error domain */

diff --git a/src/libstat/stat_api.h b/src/libstat/stat_api.h
index 0e2bf86b820c37b2adc4524e0f7cd6a7e2c06ebe..64b3f0b92c823e146e4d66fd652a3078e482711b 100644 (file)
--- a/src/libstat/stat_api.h
+++ b/src/libstat/stat_api.h
@@ -31,6 +31,35 @@
  * High level statistics API
  */
 
+struct rspamd_statfile_runtime {
+       struct rspamd_statfile_config *st;
+       guint64 hits;
+       guint64 total_hits;
+};
+
+struct rspamd_classifier_runtime {
+       double ham_prob;
+       double spam_prob;
+       guint64 total_spam;
+       guint64 total_ham;
+       guint64 processed_tokens;
+       gsize max_tokens;
+};
+
+struct rspamd_token_result {
+       double value;
+       struct rspamd_statfile_runtime *st_runtime;
+
+       struct rspamd_classifier_runtime *cl_runtime;
+};
+
+#define RSPAMD_MAX_TOKEN_LEN 64
+typedef struct token_node_s {
+       guchar data[RSPAMD_MAX_TOKEN_LEN];
+       guint datalen;
+       GArray *results;
+} rspamd_token_t;
+
 /**
  * Initialise statistics modules
  * @param cfg

diff --git a/src/libstat/tokenizers.h b/src/libstat/tokenizers.h
index 48f9b6e569d4cb3751bcbcfdf39a592faab8e2ea..73d07a5c43449f9c2fd3ffe7e8d84e938a1928bb 100644 (file)
--- a/src/libstat/tokenizers.h
+++ b/src/libstat/tokenizers.h
@@ -5,24 +5,7 @@
 #include "mem_pool.h"
 #include "fstring.h"
 #include "main.h"
-
-/* Size for features pipe */
-#define FEATURE_WINDOW_SIZE 5
-#define MAX_DATA_LEN 64
-#define MAX_VALUES 32
-
-struct token_result {
-       double value;
-       struct rspamd_statfile_config *st;
-       double *consolidated_value;
-};
-
-typedef struct token_node_s {
-       guchar data[MAX_DATA_LEN];
-       guint datalen;
-       struct token_result *results;
-       guint results_len;
-} token_node_t;
+#include "stat_api.h"
 
 /* Common tokenizer structure */
 struct tokenizer {

diff --git a/src/libstat/tokenizers/osb.c b/src/libstat/tokenizers/osb.c
index 4016842b6af52ae0d0e5366f72d6ee73f533f6ea..abf547f4339e16adb71da618502822e3191ef9e5 100644 (file)
--- a/src/libstat/tokenizers/osb.c
+++ b/src/libstat/tokenizers/osb.c
@@ -29,6 +29,9 @@
 #include <sys/types.h>
 #include "tokenizers.h"
 
+/* Size for features pipe */
+#define FEATURE_WINDOW_SIZE 5
+
 /* Minimum length of token */
 #define MIN_LEN 4
 
@@ -43,7 +46,7 @@ osb_tokenize_text (struct tokenizer *tokenizer,
        gboolean is_utf,
        GList *exceptions)
 {
-       token_node_t *new = NULL;
+       rspamd_token_t *new = NULL;
        rspamd_fstring_t *token;
        guint32 hashpipe[FEATURE_WINDOW_SIZE], h1, h2;
        gint i, processed = 0;
@@ -82,7 +85,7 @@ osb_tokenize_text (struct tokenizer *tokenizer,
                                h1 = hashpipe[0] * primes[0] + hashpipe[i] * primes[i << 1];
                                h2 = hashpipe[0] * primes[1] + hashpipe[i] *
                                        primes[(i << 1) - 1];
-                               new = rspamd_mempool_alloc0 (pool, sizeof (token_node_t));
+                               new = rspamd_mempool_alloc0 (pool, sizeof (rspamd_token_t));
                                new->datalen = sizeof(gint32) * 2;
                                memcpy(new->data, &h1, sizeof(h1));
                                memcpy(new->data + sizeof(h1), &h2, sizeof(h2));
@@ -98,7 +101,7 @@ osb_tokenize_text (struct tokenizer *tokenizer,
                for (i = 1; i < processed; i++) {
                        h1 = hashpipe[0] * primes[0] + hashpipe[i] * primes[i << 1];
                        h2 = hashpipe[0] * primes[1] + hashpipe[i] * primes[(i << 1) - 1];
-                       new = rspamd_mempool_alloc0 (pool, sizeof (token_node_t));
+                       new = rspamd_mempool_alloc0 (pool, sizeof (rspamd_token_t));
                        new->datalen = sizeof(gint32) * 2;
                        memcpy(new->data, &h1, sizeof(h1));
                        memcpy(new->data + sizeof(h1), &h2, sizeof(h2));

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 7d00f693aba0bf60a2ec9aec231d130ce5377b0a..10e4b92d5a9894939449943d3d392cf8ac2cd586 100644 (file)
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -92,7 +92,7 @@ rspamd_stat_get_tokenizer (const char *name)
 int
 token_node_compare_func (gconstpointer a, gconstpointer b)
 {
-       const token_node_t *aa = a, *bb = b;
+       const rspamd_token_t *aa = a, *bb = b;
 
        if (aa->datalen != bb->datalen) {
                return aa->datalen - bb->datalen;