aboutsummaryrefslogtreecommitdiffstats
path: root/src/classifiers/bayes.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/classifiers/bayes.c')
-rw-r--r--src/classifiers/bayes.c322
1 files changed, 192 insertions, 130 deletions
diff --git a/src/classifiers/bayes.c b/src/classifiers/bayes.c
index a8a18f5ff..2d3fca084 100644
--- a/src/classifiers/bayes.c
+++ b/src/classifiers/bayes.c
@@ -25,13 +25,13 @@
/*
* Bayesian classifier
*/
+#include "binlog.h"
+#include "cfg_file.h"
#include "classifiers.h"
-#include "tokenizers/tokenizers.h"
-#include "main.h"
#include "filter.h"
-#include "cfg_file.h"
-#include "binlog.h"
#include "lua/lua_common.h"
+#include "main.h"
+#include "tokenizers/tokenizers.h"
#define LOCAL_PROB_DENOM 16.0
@@ -42,56 +42,68 @@ bayes_error_quark (void)
}
struct bayes_statfile_data {
- guint64 hits;
- guint64 total_hits;
- double value;
- struct rspamd_statfile_config *st;
- stat_file_t *file;
+ guint64 hits;
+ guint64 total_hits;
+ double value;
+ struct rspamd_statfile_config *st;
+ stat_file_t *file;
};
struct bayes_callback_data {
- statfile_pool_t *pool;
- struct classifier_ctx *ctx;
- gboolean in_class;
- time_t now;
- stat_file_t *file;
- struct bayes_statfile_data *statfiles;
- guint32 statfiles_num;
- guint64 total_spam;
- guint64 total_ham;
- guint64 processed_tokens;
- gsize max_tokens;
- double spam_probability;
- double ham_probability;
+ statfile_pool_t *pool;
+ struct classifier_ctx *ctx;
+ gboolean in_class;
+ time_t now;
+ stat_file_t *file;
+ struct bayes_statfile_data *statfiles;
+ guint32 statfiles_num;
+ guint64 total_spam;
+ guint64 total_ham;
+ guint64 processed_tokens;
+ gsize max_tokens;
+ double spam_probability;
+ double ham_probability;
};
-static gboolean
+static gboolean
bayes_learn_callback (gpointer key, gpointer value, gpointer data)
{
- token_node_t *node = key;
- struct bayes_callback_data *cd = data;
- gint c;
- guint64 v;
+ token_node_t *node = key;
+ struct bayes_callback_data *cd = data;
+ gint c;
+ guint64 v;
c = (cd->in_class) ? 1 : -1;
/* Consider that not found blocks have value 1 */
- v = statfile_pool_get_block (cd->pool, cd->file, node->h1, node->h2, cd->now);
+ v =
+ statfile_pool_get_block (cd->pool, cd->file, node->h1, node->h2,
+ cd->now);
if (v == 0 && c > 0) {
- statfile_pool_set_block (cd->pool, cd->file, node->h1, node->h2, cd->now, c);
- cd->processed_tokens ++;
+ statfile_pool_set_block (cd->pool,
+ cd->file,
+ node->h1,
+ node->h2,
+ cd->now,
+ c);
+ cd->processed_tokens++;
}
else if (v != 0) {
if (G_LIKELY (c > 0)) {
- v ++;
+ v++;
}
- else if (c < 0){
+ else if (c < 0) {
if (v != 0) {
- v --;
+ v--;
}
}
- statfile_pool_set_block (cd->pool, cd->file, node->h1, node->h2, cd->now, v);
- cd->processed_tokens ++;
+ statfile_pool_set_block (cd->pool,
+ cd->file,
+ node->h1,
+ node->h2,
+ cd->now,
+ v);
+ cd->processed_tokens++;
}
if (cd->max_tokens != 0 && cd->processed_tokens > cd->max_tokens) {
@@ -133,7 +145,7 @@ inv_chi_square (gdouble value, gint freedom_deg)
return 0;
}
sum = prob;
- for (i = 1; i < freedom_deg / 2; i ++) {
+ for (i = 1; i < freedom_deg / 2; i++) {
prob *= value / (gdouble)i;
sum += prob;
}
@@ -148,16 +160,20 @@ static gboolean
bayes_classify_callback (gpointer key, gpointer value, gpointer data)
{
- token_node_t *node = key;
- struct bayes_callback_data *cd = data;
- guint i;
- struct bayes_statfile_data *cur;
- guint64 spam_count = 0, ham_count = 0, total_count = 0;
- double spam_prob, spam_freq, ham_freq, bayes_spam_prob;
+ token_node_t *node = key;
+ struct bayes_callback_data *cd = data;
+ guint i;
+ struct bayes_statfile_data *cur;
+ guint64 spam_count = 0, ham_count = 0, total_count = 0;
+ double spam_prob, spam_freq, ham_freq, bayes_spam_prob;
- for (i = 0; i < cd->statfiles_num; i ++) {
+ for (i = 0; i < cd->statfiles_num; i++) {
cur = &cd->statfiles[i];
- cur->value = statfile_pool_get_block (cd->pool, cur->file, node->h1, node->h2, cd->now);
+ cur->value = statfile_pool_get_block (cd->pool,
+ cur->file,
+ node->h1,
+ node->h2,
+ cd->now);
if (cur->value > 0) {
cur->total_hits += cur->value;
if (cur->st->is_spam) {
@@ -178,7 +194,7 @@ bayes_classify_callback (gpointer key, gpointer value, gpointer data)
bayes_spam_prob = (0.5 + spam_prob * total_count) / (1. + total_count);
cd->spam_probability += log (bayes_spam_prob);
cd->ham_probability += log (1. - bayes_spam_prob);
- cd->processed_tokens ++;
+ cd->processed_tokens++;
}
if (cd->max_tokens != 0 && cd->processed_tokens > cd->max_tokens) {
@@ -189,10 +205,11 @@ bayes_classify_callback (gpointer key, gpointer value, gpointer data)
return FALSE;
}
-struct classifier_ctx*
+struct classifier_ctx *
bayes_init (rspamd_mempool_t *pool, struct rspamd_classifier_config *cfg)
{
- struct classifier_ctx *ctx = rspamd_mempool_alloc (pool, sizeof (struct classifier_ctx));
+ struct classifier_ctx *ctx =
+ rspamd_mempool_alloc (pool, sizeof (struct classifier_ctx));
ctx->pool = pool;
ctx->cfg = cfg;
@@ -202,23 +219,28 @@ bayes_init (rspamd_mempool_t *pool, struct rspamd_classifier_config *cfg)
}
gboolean
-bayes_classify (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input, struct rspamd_task *task, lua_State *L)
+bayes_classify (struct classifier_ctx * ctx,
+ statfile_pool_t *pool,
+ GTree *input,
+ struct rspamd_task *task,
+ lua_State *L)
{
- struct bayes_callback_data data;
- gchar *value;
- gint nodes, i = 0, selected_st = -1, cnt;
- gint minnodes;
- guint64 maxhits = 0, rev;
- double final_prob, h, s;
- struct rspamd_statfile_config *st;
- stat_file_t *file;
- GList *cur;
- char *sumbuf;
+ struct bayes_callback_data data;
+ gchar *value;
+ gint nodes, i = 0, selected_st = -1, cnt;
+ gint minnodes;
+ guint64 maxhits = 0, rev;
+ double final_prob, h, s;
+ struct rspamd_statfile_config *st;
+ stat_file_t *file;
+ GList *cur;
+ char *sumbuf;
g_assert (pool != NULL);
g_assert (ctx != NULL);
- if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
+ if (ctx->cfg->opts &&
+ (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
minnodes = strtol (value, NULL, 10);
nodes = g_tree_nnodes (input);
if (nodes > FEATURE_WINDOW_SIZE) {
@@ -231,7 +253,8 @@ bayes_classify (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input,
cur = call_classifier_pre_callbacks (ctx->cfg, task, FALSE, FALSE, L);
if (cur) {
- rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)g_list_free, cur);
+ rspamd_mempool_add_destructor (task->task_pool,
+ (rspamd_mempool_destruct_t)g_list_free, cur);
}
else {
cur = ctx->cfg->statfiles;
@@ -248,7 +271,8 @@ bayes_classify (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input,
data.ham_probability = 0;
data.total_ham = 0;
data.total_spam = 0;
- if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "max_tokens")) != NULL) {
+ if (ctx->cfg->opts &&
+ (value = g_hash_table_lookup (ctx->cfg->opts, "max_tokens")) != NULL) {
minnodes = rspamd_config_parse_limit (value, -1);
data.max_tokens = minnodes;
}
@@ -260,10 +284,11 @@ bayes_classify (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input,
/* Select statfile to classify */
st = cur->data;
if ((file = statfile_pool_is_open (pool, st->path)) == NULL) {
- if ((file = statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) {
+ if ((file =
+ statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) {
msg_warn ("cannot open %s", st->path);
cur = g_list_next (cur);
- data.statfiles_num --;
+ data.statfiles_num--;
continue;
}
}
@@ -278,7 +303,7 @@ bayes_classify (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input,
}
cur = g_list_next (cur);
- i ++;
+ i++;
}
cnt = i;
@@ -289,17 +314,19 @@ bayes_classify (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input,
final_prob = 0;
}
else {
- h = 1 - inv_chi_square (-2. * data.spam_probability, 2 * data.processed_tokens);
- s = 1 - inv_chi_square (-2. * data.ham_probability, 2 * data.processed_tokens);
+ h = 1 - inv_chi_square (-2. * data.spam_probability,
+ 2 * data.processed_tokens);
+ s = 1 - inv_chi_square (-2. * data.ham_probability,
+ 2 * data.processed_tokens);
final_prob = (s + 1 - h) / 2.;
}
if (data.processed_tokens > 0 && fabs (final_prob - 0.5) > 0.05) {
sumbuf = rspamd_mempool_alloc (task->task_pool, 32);
- for (i = 0; i < cnt; i ++) {
+ for (i = 0; i < cnt; i++) {
if ((final_prob > 0.5 && !data.statfiles[i].st->is_spam) ||
- (final_prob < 0.5 && data.statfiles[i].st->is_spam)) {
+ (final_prob < 0.5 && data.statfiles[i].st->is_spam)) {
continue;
}
if (data.statfiles[i].total_hits > maxhits) {
@@ -308,7 +335,8 @@ bayes_classify (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input,
}
}
if (selected_st == -1) {
- msg_err ("unexpected classifier error: cannot select desired statfile");
+ msg_err (
+ "unexpected classifier error: cannot select desired statfile");
}
else {
/* Calculate ham probability correctly */
@@ -317,7 +345,10 @@ bayes_classify (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input,
}
rspamd_snprintf (sumbuf, 32, "%.2f%%", final_prob * 100.);
cur = g_list_prepend (NULL, sumbuf);
- insert_result (task, data.statfiles[selected_st].st->symbol, final_prob, cur);
+ insert_result (task,
+ data.statfiles[selected_st].st->symbol,
+ final_prob,
+ cur);
}
}
@@ -327,34 +358,44 @@ bayes_classify (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input,
}
gboolean
-bayes_learn (struct classifier_ctx* ctx, statfile_pool_t *pool, const char *symbol, GTree *input,
- gboolean in_class, double *sum, double multiplier, GError **err)
+bayes_learn (struct classifier_ctx * ctx,
+ statfile_pool_t *pool,
+ const char *symbol,
+ GTree *input,
+ gboolean in_class,
+ double *sum,
+ double multiplier,
+ GError **err)
{
- struct bayes_callback_data data;
- gchar *value;
- gint nodes;
- gint minnodes;
- struct rspamd_statfile_config *st, *sel_st = NULL;
- stat_file_t *to_learn;
- GList *cur;
+ struct bayes_callback_data data;
+ gchar *value;
+ gint nodes;
+ gint minnodes;
+ struct rspamd_statfile_config *st, *sel_st = NULL;
+ stat_file_t *to_learn;
+ GList *cur;
g_assert (pool != NULL);
g_assert (ctx != NULL);
- if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
+ if (ctx->cfg->opts &&
+ (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
minnodes = strtol (value, NULL, 10);
nodes = g_tree_nnodes (input);
if (nodes > FEATURE_WINDOW_SIZE) {
nodes = nodes / FEATURE_WINDOW_SIZE + FEATURE_WINDOW_SIZE;
}
if (nodes < minnodes) {
- msg_info ("do not learn message as it has too few tokens: %d, while %d min", nodes, minnodes);
+ msg_info (
+ "do not learn message as it has too few tokens: %d, while %d min",
+ nodes,
+ minnodes);
*sum = 0;
g_set_error (err,
- bayes_error_quark(), /* error domain */
- 1, /* error code */
- "message contains too few tokens: %d, while min is %d",
- nodes, (int)minnodes);
+ bayes_error_quark (), /* error domain */
+ 1, /* error code */
+ "message contains too few tokens: %d, while min is %d",
+ nodes, (int)minnodes);
return FALSE;
}
}
@@ -365,7 +406,8 @@ bayes_learn (struct classifier_ctx* ctx, statfile_pool_t *pool, const char *symb
data.ctx = ctx;
data.processed_tokens = 0;
data.processed_tokens = 0;
- if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "max_tokens")) != NULL) {
+ if (ctx->cfg->opts &&
+ (value = g_hash_table_lookup (ctx->cfg->opts, "max_tokens")) != NULL) {
minnodes = rspamd_config_parse_limit (value, -1);
data.max_tokens = minnodes;
}
@@ -384,31 +426,36 @@ bayes_learn (struct classifier_ctx* ctx, statfile_pool_t *pool, const char *symb
}
if (sel_st == NULL) {
g_set_error (err,
- bayes_error_quark(), /* error domain */
- 1, /* error code */
- "cannot find statfile for symbol: %s",
- symbol);
+ bayes_error_quark (), /* error domain */
+ 1, /* error code */
+ "cannot find statfile for symbol: %s",
+ symbol);
return FALSE;
}
if ((to_learn = statfile_pool_is_open (pool, sel_st->path)) == NULL) {
- if ((to_learn = statfile_pool_open (pool, sel_st->path, sel_st->size, FALSE)) == NULL) {
+ if ((to_learn =
+ statfile_pool_open (pool, sel_st->path, sel_st->size,
+ FALSE)) == NULL) {
msg_warn ("cannot open %s", sel_st->path);
if (statfile_pool_create (pool, sel_st->path, sel_st->size) == -1) {
msg_err ("cannot create statfile %s", sel_st->path);
g_set_error (err,
- bayes_error_quark(), /* error domain */
- 1, /* error code */
- "cannot create statfile: %s",
- sel_st->path);
+ bayes_error_quark (), /* error domain */
+ 1, /* error code */
+ "cannot create statfile: %s",
+ sel_st->path);
return FALSE;
}
- if ((to_learn = statfile_pool_open (pool, sel_st->path, sel_st->size, FALSE)) == NULL) {
+ if ((to_learn =
+ statfile_pool_open (pool, sel_st->path, sel_st->size,
+ FALSE)) == NULL) {
g_set_error (err,
- bayes_error_quark(), /* error domain */
- 1, /* error code */
- "cannot open statfile %s after creation",
- sel_st->path);
- msg_err ("cannot open statfile %s after creation", sel_st->path);
+ bayes_error_quark (), /* error domain */
+ 1, /* error code */
+ "cannot open statfile %s after creation",
+ sel_st->path);
+ msg_err ("cannot open statfile %s after creation",
+ sel_st->path);
return FALSE;
}
}
@@ -427,22 +474,28 @@ bayes_learn (struct classifier_ctx* ctx, statfile_pool_t *pool, const char *symb
}
gboolean
-bayes_learn_spam (struct classifier_ctx* ctx, statfile_pool_t *pool,
- GTree *input, struct rspamd_task *task, gboolean is_spam, lua_State *L, GError **err)
+bayes_learn_spam (struct classifier_ctx * ctx,
+ statfile_pool_t *pool,
+ GTree *input,
+ struct rspamd_task *task,
+ gboolean is_spam,
+ lua_State *L,
+ GError **err)
{
- struct bayes_callback_data data;
- gchar *value;
- gint nodes;
- gint minnodes;
- struct rspamd_statfile_config *st;
- stat_file_t *file;
- GList *cur;
- gboolean skip_labels;
+ struct bayes_callback_data data;
+ gchar *value;
+ gint nodes;
+ gint minnodes;
+ struct rspamd_statfile_config *st;
+ stat_file_t *file;
+ GList *cur;
+ gboolean skip_labels;
g_assert (pool != NULL);
g_assert (ctx != NULL);
- if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
+ if (ctx->cfg->opts &&
+ (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
minnodes = strtol (value, NULL, 10);
nodes = g_tree_nnodes (input);
if (nodes > FEATURE_WINDOW_SIZE) {
@@ -450,10 +503,10 @@ bayes_learn_spam (struct classifier_ctx* ctx, statfile_pool_t *pool,
}
if (nodes < minnodes) {
g_set_error (err,
- bayes_error_quark(), /* error domain */
- 1, /* error code */
- "message contains too few tokens: %d, while min is %d",
- nodes, (int)minnodes);
+ bayes_error_quark (), /* error domain */
+ 1, /* error code */
+ "message contains too few tokens: %d, while min is %d",
+ nodes, (int)minnodes);
return FALSE;
}
}
@@ -461,7 +514,8 @@ bayes_learn_spam (struct classifier_ctx* ctx, statfile_pool_t *pool,
cur = call_classifier_pre_callbacks (ctx->cfg, task, TRUE, is_spam, L);
if (cur) {
skip_labels = FALSE;
- rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)g_list_free, cur);
+ rspamd_mempool_add_destructor (task->task_pool,
+ (rspamd_mempool_destruct_t)g_list_free, cur);
}
else {
/* Do not try to learn specific statfiles if pre callback returned nil */
@@ -475,7 +529,8 @@ bayes_learn_spam (struct classifier_ctx* ctx, statfile_pool_t *pool,
data.in_class = TRUE;
data.processed_tokens = 0;
- if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "max_tokens")) != NULL) {
+ if (ctx->cfg->opts &&
+ (value = g_hash_table_lookup (ctx->cfg->opts, "max_tokens")) != NULL) {
minnodes = rspamd_config_parse_limit (value, -1);
data.max_tokens = minnodes;
}
@@ -491,24 +546,28 @@ bayes_learn_spam (struct classifier_ctx* ctx, statfile_pool_t *pool,
continue;
}
if ((file = statfile_pool_is_open (pool, st->path)) == NULL) {
- if ((file = statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) {
+ if ((file =
+ statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) {
msg_warn ("cannot open %s", st->path);
if (statfile_pool_create (pool, st->path, st->size) == -1) {
msg_err ("cannot create statfile %s", st->path);
g_set_error (err,
- bayes_error_quark(), /* error domain */
- 1, /* error code */
- "cannot create statfile: %s",
- st->path);
+ bayes_error_quark (), /* error domain */
+ 1, /* error code */
+ "cannot create statfile: %s",
+ st->path);
return FALSE;
}
- if ((file = statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) {
+ if ((file =
+ statfile_pool_open (pool, st->path, st->size,
+ FALSE)) == NULL) {
g_set_error (err,
- bayes_error_quark(), /* error domain */
- 1, /* error code */
- "cannot open statfile %s after creation",
- st->path);
- msg_err ("cannot open statfile %s after creation", st->path);
+ bayes_error_quark (), /* error domain */
+ 1, /* error code */
+ "cannot open statfile %s after creation",
+ st->path);
+ msg_err ("cannot open statfile %s after creation",
+ st->path);
return FALSE;
}
}
@@ -528,7 +587,10 @@ bayes_learn_spam (struct classifier_ctx* ctx, statfile_pool_t *pool,
}
GList *
-bayes_weights (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input, struct rspamd_task *task)
+bayes_weights (struct classifier_ctx * ctx,
+ statfile_pool_t *pool,
+ GTree *input,
+ struct rspamd_task *task)
{
/* This function is unimplemented with new normalizer */
return NULL;
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-11 15:48:03 +0000