diff options
author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2014-07-23 12:53:08 +0100 |
---|---|---|
committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2014-07-23 12:53:08 +0100 |
commit | fe79d8c5a39f2b717f78cc3f3ef21b3cfc46500b (patch) | |
tree | c84e6a5d4c5cd78a7a2cc3c7adbc7af5d0541682 /src/classifiers/winnow.c | |
parent | e0483657ff6cf1adc828ccce457814d61fe90a0d (diff) | |
download | rspamd-fe79d8c5a39f2b717f78cc3f3ef21b3cfc46500b.tar.gz rspamd-fe79d8c5a39f2b717f78cc3f3ef21b3cfc46500b.zip |
Revert "Unify code style."
This reverts commit e0483657ff6cf1adc828ccce457814d61fe90a0d.
Diffstat (limited to 'src/classifiers/winnow.c')
-rw-r--r-- | src/classifiers/winnow.c | 388 |
1 files changed, 149 insertions, 239 deletions
diff --git a/src/classifiers/winnow.c b/src/classifiers/winnow.c index 85d8cfa20..1fe8f16d2 100644 --- a/src/classifiers/winnow.c +++ b/src/classifiers/winnow.c @@ -26,12 +26,12 @@ * Winnow classifier */ -#include "cfg_file.h" #include "classifiers.h" +#include "tokenizers/tokenizers.h" +#include "main.h" #include "filter.h" +#include "cfg_file.h" #include "lua/lua_common.h" -#include "main.h" -#include "tokenizers/tokenizers.h" #define WINNOW_PROMOTION 1.23 #define WINNOW_DEMOTION 0.83 @@ -51,42 +51,40 @@ winnow_error_quark (void) } struct winnow_callback_data { - statfile_pool_t *pool; - struct classifier_ctx *ctx; - stat_file_t *file; - stat_file_t *learn_file; - long double sum; - long double start; - double multiplier; - guint32 count; - guint32 new_blocks; - gboolean in_class; - gboolean do_demote; - gboolean fresh_run; - time_t now; + statfile_pool_t *pool; + struct classifier_ctx *ctx; + stat_file_t *file; + stat_file_t *learn_file; + long double sum; + long double start; + double multiplier; + guint32 count; + guint32 new_blocks; + gboolean in_class; + gboolean do_demote; + gboolean fresh_run; + time_t now; }; static const double max_common_weight = MAX_WEIGHT * WINNOW_DEMOTION; -static gboolean +static gboolean winnow_classify_callback (gpointer key, gpointer value, gpointer data) { - token_node_t *node = key; - struct winnow_callback_data *cd = data; - double v; + token_node_t *node = key; + struct winnow_callback_data *cd = data; + double v; /* Consider that not found blocks have value 1 */ - v = - statfile_pool_get_block (cd->pool, cd->file, node->h1, node->h2, - cd->now); + v = statfile_pool_get_block (cd->pool, cd->file, node->h1, node->h2, cd->now); if (fabs (v) > ALPHA) { cd->sum += v; } else { cd->sum += 1.0; - cd->new_blocks++; + cd->new_blocks ++; } cd->count++; @@ -94,32 +92,24 @@ winnow_classify_callback (gpointer key, gpointer value, gpointer data) return FALSE; } -static gboolean +static gboolean winnow_learn_callback (gpointer key, gpointer value, gpointer data) { - token_node_t *node = key; - struct winnow_callback_data *cd = data; - double v, c; - - c = (cd->in_class) ? WINNOW_PROMOTION * cd->multiplier : WINNOW_DEMOTION / - cd->multiplier; + token_node_t *node = key; + struct winnow_callback_data *cd = data; + double v, c; + + c = (cd->in_class) ? WINNOW_PROMOTION * cd->multiplier : WINNOW_DEMOTION / cd->multiplier; /* Consider that not found blocks have value 1 */ - v = - statfile_pool_get_block (cd->pool, cd->file, node->h1, node->h2, - cd->now); + v = statfile_pool_get_block (cd->pool, cd->file, node->h1, node->h2, cd->now); if (fabs (v) < ALPHA) { /* Block not found, insert new */ cd->start += 1; if (cd->file == cd->learn_file) { - statfile_pool_set_block (cd->pool, - cd->file, - node->h1, - node->h2, - cd->now, - c); + statfile_pool_set_block (cd->pool, cd->file, node->h1, node->h2, cd->now, c); node->value = c; - cd->new_blocks++; + cd->new_blocks ++; } } else { @@ -129,23 +119,18 @@ winnow_learn_callback (gpointer key, gpointer value, gpointer data) node->extra = 0; } else { - node->extra++; + node->extra ++; } node->value = v; - + if (node->extra > 1) { - /* + /* * Assume that this node is common for several statfiles, so * decrease its weight proportianally */ if (node->value > max_common_weight) { /* Static fluctuation */ - statfile_pool_set_block (cd->pool, - cd->file, - node->h1, - node->h2, - cd->now, - 0.); + statfile_pool_set_block (cd->pool, cd->file, node->h1, node->h2, cd->now, 0.); node->value = 0.; } else if (node->value > WINNOW_PROMOTION * cd->multiplier) { @@ -156,7 +141,7 @@ winnow_learn_callback (gpointer key, gpointer value, gpointer data) node->value *= c; } else { - /* + /* * Too high token value that exists also in other * statfiles, may be statistic error, so decrease it * slightly @@ -167,13 +152,8 @@ winnow_learn_callback (gpointer key, gpointer value, gpointer data) else { node->value = WINNOW_DEMOTION / cd->multiplier; } - statfile_pool_set_block (cd->pool, - cd->file, - node->h1, - node->h2, - cd->now, - node->value); - } + statfile_pool_set_block (cd->pool, cd->file, node->h1, node->h2, cd->now, node->value); + } } else if (cd->file == cd->learn_file) { /* New block or block that is in only one statfile */ @@ -184,22 +164,12 @@ winnow_learn_callback (gpointer key, gpointer value, gpointer data) else { node->value *= c; } - statfile_pool_set_block (cd->pool, - cd->file, - node->h1, - node->h2, - cd->now, - node->value); + statfile_pool_set_block (cd->pool, cd->file, node->h1, node->h2, cd->now, node->value); } else if (cd->do_demote) { /* Demote blocks in file */ node->value *= WINNOW_DEMOTION / cd->multiplier; - statfile_pool_set_block (cd->pool, - cd->file, - node->h1, - node->h2, - cd->now, - node->value); + statfile_pool_set_block (cd->pool, cd->file, node->h1, node->h2, cd->now, node->value); } } @@ -211,11 +181,10 @@ winnow_learn_callback (gpointer key, gpointer value, gpointer data) return FALSE; } -struct classifier_ctx * +struct classifier_ctx * winnow_init (rspamd_mempool_t * pool, struct rspamd_classifier_config *cfg) { - struct classifier_ctx *ctx = - rspamd_mempool_alloc (pool, sizeof (struct classifier_ctx)); + struct classifier_ctx *ctx = rspamd_mempool_alloc (pool, sizeof (struct classifier_ctx)); ctx->pool = pool; ctx->cfg = cfg; @@ -224,18 +193,14 @@ winnow_init (rspamd_mempool_t * pool, struct rspamd_classifier_config *cfg) } gboolean -winnow_classify (struct classifier_ctx *ctx, - statfile_pool_t * pool, - GTree * input, - struct rspamd_task *task, - lua_State *L) +winnow_classify (struct classifier_ctx *ctx, statfile_pool_t * pool, GTree * input, struct rspamd_task *task, lua_State *L) { - struct winnow_callback_data data; - char *sumbuf, *value; - long double res = 0., max = 0.; - GList *cur; - struct rspamd_statfile_config *st, *sel = NULL; - int nodes, minnodes; + struct winnow_callback_data data; + char *sumbuf, *value; + long double res = 0., max = 0.; + GList *cur; + struct rspamd_statfile_config *st, *sel = NULL; + int nodes, minnodes; g_assert (pool != NULL); g_assert (ctx != NULL); @@ -243,27 +208,22 @@ winnow_classify (struct classifier_ctx *ctx, data.pool = pool; data.now = time (NULL); data.ctx = ctx; - - if (ctx->cfg->opts && - (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) { + + if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) { minnodes = strtol (value, NULL, 10); nodes = g_tree_nnodes (input); if (nodes > FEATURE_WINDOW_SIZE) { nodes = nodes / FEATURE_WINDOW_SIZE + FEATURE_WINDOW_SIZE; } if (nodes < minnodes) { - msg_info ( - "do not classify message as it has too few tokens: %d, while %d min", - nodes, - minnodes); + msg_info ("do not classify message as it has too few tokens: %d, while %d min", nodes, minnodes); return FALSE; } } cur = call_classifier_pre_callbacks (ctx->cfg, task, FALSE, FALSE, L); if (cur) { - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)g_list_free, cur); + rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)g_list_free, cur); } else { cur = ctx->cfg->statfiles; @@ -275,8 +235,7 @@ winnow_classify (struct classifier_ctx *ctx, data.count = 0; data.new_blocks = 0; if ((data.file = statfile_pool_is_open (pool, st->path)) == NULL) { - if ((data.file = - statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) { + if ((data.file = statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) { msg_warn ("cannot open %s, skip it", st->path); cur = g_list_next (cur); continue; @@ -302,16 +261,16 @@ winnow_classify (struct classifier_ctx *ctx, if (sel != NULL) { #ifdef WITH_LUA - max = call_classifier_post_callbacks (ctx->cfg, task, max, L); + max = call_classifier_post_callbacks (ctx->cfg, task, max, L); #endif #ifdef HAVE_TANHL - max = tanhl (max); + max = tanhl (max); #else - /* - * As some implementations of libm does not support tanhl, try to use - * tanh - */ - max = tanh ((double) max); + /* + * As some implementations of libm does not support tanhl, try to use + * tanh + */ + max = tanh ((double) max); #endif sumbuf = rspamd_mempool_alloc (task->task_pool, 32); rspamd_snprintf (sumbuf, 32, "%.2F", max); @@ -323,18 +282,15 @@ winnow_classify (struct classifier_ctx *ctx, } GList * -winnow_weights (struct classifier_ctx *ctx, - statfile_pool_t * pool, - GTree * input, - struct rspamd_task *task) +winnow_weights (struct classifier_ctx *ctx, statfile_pool_t * pool, GTree * input, struct rspamd_task *task) { - struct winnow_callback_data data; - long double res = 0.; - GList *cur, *resl = NULL; - struct rspamd_statfile_config *st; - struct classify_weight *w; - char *value; - int nodes, minnodes; + struct winnow_callback_data data; + long double res = 0.; + GList *cur, *resl = NULL; + struct rspamd_statfile_config *st; + struct classify_weight *w; + char *value; + int nodes, minnodes; g_assert (pool != NULL); g_assert (ctx != NULL); @@ -343,30 +299,25 @@ winnow_weights (struct classifier_ctx *ctx, data.now = time (NULL); data.ctx = ctx; - if (ctx->cfg->opts && - (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) { + if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) { minnodes = strtol (value, NULL, 10); nodes = g_tree_nnodes (input); if (nodes > FEATURE_WINDOW_SIZE) { nodes = nodes / FEATURE_WINDOW_SIZE + FEATURE_WINDOW_SIZE; } if (nodes < minnodes) { - msg_info ( - "do not classify message as it has too few tokens: %d, while %d min", - nodes, - minnodes); + msg_info ("do not classify message as it has too few tokens: %d, while %d min", nodes, minnodes); return NULL; } } - + cur = ctx->cfg->statfiles; while (cur) { st = cur->data; data.sum = 0; data.count = 0; if ((data.file = statfile_pool_is_open (pool, st->path)) == NULL) { - if ((data.file = - statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) { + if ((data.file = statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) { msg_warn ("cannot open %s, skip it", st->path); cur = g_list_next (cur); continue; @@ -377,9 +328,7 @@ winnow_weights (struct classifier_ctx *ctx, g_tree_foreach (input, winnow_classify_callback, &data); } - w = - rspamd_mempool_alloc0 (task->task_pool, - sizeof (struct classify_weight)); + w = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct classify_weight)); if (data.count != 0) { res = data.sum / (double)data.count; } @@ -391,10 +340,9 @@ winnow_weights (struct classifier_ctx *ctx, resl = g_list_prepend (resl, w); cur = g_list_next (cur); } - + if (resl != NULL) { - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)g_list_free, resl); + rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)g_list_free, resl); } return resl; @@ -403,27 +351,21 @@ winnow_weights (struct classifier_ctx *ctx, gboolean -winnow_learn (struct classifier_ctx *ctx, - statfile_pool_t *pool, - const char *symbol, - GTree * input, - int in_class, - double *sum, - double multiplier, - GError **err) +winnow_learn (struct classifier_ctx *ctx, statfile_pool_t *pool, const char *symbol, + GTree * input, int in_class, double *sum, double multiplier, GError **err) { - struct winnow_callback_data data = { + struct winnow_callback_data data = { .file = NULL, .multiplier = multiplier }; - char *value; - int nodes, minnodes, iterations = 0; - struct rspamd_statfile_config *st, *sel_st = NULL; - stat_file_t *sel = NULL, *to_learn; - long double res = 0., max = 0., start_value = 0., end_value = 0.; - double learn_threshold = 0.0; - GList *cur, *to_demote = NULL; - gboolean force_learn = FALSE; + char *value; + int nodes, minnodes, iterations = 0; + struct rspamd_statfile_config *st, *sel_st = NULL; + stat_file_t *sel = NULL, *to_learn; + long double res = 0., max = 0., start_value = 0., end_value = 0.; + double learn_threshold = 0.0; + GList *cur, *to_demote = NULL; + gboolean force_learn = FALSE; g_assert (pool != NULL); g_assert (ctx != NULL); @@ -434,35 +376,29 @@ winnow_learn (struct classifier_ctx *ctx, data.ctx = ctx; - if (ctx->cfg->opts && - (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) { + if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) { minnodes = strtol (value, NULL, 10); nodes = g_tree_nnodes (input); if (nodes > FEATURE_WINDOW_SIZE) { nodes = nodes / FEATURE_WINDOW_SIZE + FEATURE_WINDOW_SIZE; } if (nodes < minnodes) { - msg_info ( - "do not learn message as it has too few tokens: %d, while %d min", - nodes, - minnodes); + msg_info ("do not learn message as it has too few tokens: %d, while %d min", nodes, minnodes); if (sum != NULL) { *sum = 0; } g_set_error (err, - winnow_error_quark (), /* error domain */ - 1, /* error code */ - "message contains too few tokens: %d, while min is %d", - nodes, minnodes); + winnow_error_quark(), /* error domain */ + 1, /* error code */ + "message contains too few tokens: %d, while min is %d", + nodes, minnodes); return FALSE; } } - if (ctx->cfg->opts && - (value = - g_hash_table_lookup (ctx->cfg->opts, "learn_threshold")) != NULL) { + if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "learn_threshold")) != NULL) { learn_threshold = strtod (value, NULL); } - + if (learn_threshold <= 1.0 && learn_threshold >= 0) { /* Classify message and check target statfile score */ cur = ctx->cfg->statfiles; @@ -470,27 +406,24 @@ winnow_learn (struct classifier_ctx *ctx, /* Open or create all statfiles inside classifier */ st = cur->data; if (statfile_pool_is_open (pool, st->path) == NULL) { - if (statfile_pool_open (pool, st->path, st->size, - FALSE) == NULL) { + if (statfile_pool_open (pool, st->path, st->size, FALSE) == NULL) { msg_warn ("cannot open %s", st->path); if (statfile_pool_create (pool, st->path, st->size) == -1) { msg_err ("cannot create statfile %s", st->path); g_set_error (err, - winnow_error_quark (), /* error domain */ - 1, /* error code */ - "cannot create statfile: %s", - st->path); + winnow_error_quark(), /* error domain */ + 1, /* error code */ + "cannot create statfile: %s", + st->path); return FALSE; } - if (statfile_pool_open (pool, st->path, st->size, - FALSE) == NULL) { + if (statfile_pool_open (pool, st->path, st->size, FALSE) == NULL) { g_set_error (err, - winnow_error_quark (), /* error domain */ - 1, /* error code */ - "open statfile %s after creation", - st->path); - msg_err ("cannot open statfile %s after creation", - st->path); + winnow_error_quark(), /* error domain */ + 1, /* error code */ + "open statfile %s after creation", + st->path); + msg_err ("cannot open statfile %s after creation", st->path); return FALSE; } } @@ -504,10 +437,10 @@ winnow_learn (struct classifier_ctx *ctx, if (sel_st == NULL) { g_set_error (err, - winnow_error_quark (), /* error domain */ - 1, /* error code */ - "cannot find statfile for symbol %s", - symbol); + winnow_error_quark(), /* error domain */ + 1, /* error code */ + "cannot find statfile for symbol %s", + symbol); msg_err ("cannot find statfile for symbol %s", symbol); return FALSE; } @@ -515,10 +448,10 @@ winnow_learn (struct classifier_ctx *ctx, to_learn = statfile_pool_is_open (pool, sel_st->path); if (to_learn == NULL) { g_set_error (err, - winnow_error_quark (), /* error domain */ - 1, /* error code */ - "statfile %s is not opened this maybe if your statfile pool is too small to handle all statfiles", - sel_st->path); + winnow_error_quark(), /* error domain */ + 1, /* error code */ + "statfile %s is not opened this maybe if your statfile pool is too small to handle all statfiles", + sel_st->path); return FALSE; } /* Check target statfile */ @@ -544,10 +477,10 @@ winnow_learn (struct classifier_ctx *ctx, data.count = 0; if ((data.file = statfile_pool_is_open (pool, st->path)) == NULL) { g_set_error (err, - winnow_error_quark (), /* error domain */ - 1, /* error code */ - "statfile %s is not opened this maybe if your statfile pool is too small to handle all statfiles", - st->path); + winnow_error_quark(), /* error domain */ + 1, /* error code */ + "statfile %s is not opened this maybe if your statfile pool is too small to handle all statfiles", + st->path); return FALSE; } g_tree_foreach (input, winnow_classify_callback, &data); @@ -565,22 +498,18 @@ winnow_learn (struct classifier_ctx *ctx, } } else { - msg_err ( - "learn threshold is more than 1 or less than 0, so cannot do learn, please check your configuration"); + msg_err ("learn threshold is more than 1 or less than 0, so cannot do learn, please check your configuration"); g_set_error (err, - winnow_error_quark (), /* error domain */ - 1, /* error code */ - "bad learn_threshold setting: %.2f", - learn_threshold); + winnow_error_quark(), /* error domain */ + 1, /* error code */ + "bad learn_threshold setting: %.2f", + learn_threshold); return FALSE; } /* If to_demote list is empty this message is already classified correctly */ if (max > WINNOW_PROMOTION && to_demote == NULL && !force_learn) { - msg_info ( - "this message is already of class %s with threshold %.2f and weight %.2F", - sel_st->symbol, - learn_threshold, - max); + msg_info ("this message is already of class %s with threshold %.2f and weight %.2F", + sel_st->symbol, learn_threshold, max); goto end; } data.learn_file = to_learn; @@ -597,8 +526,7 @@ winnow_learn (struct classifier_ctx *ctx, if ((data.file = statfile_pool_is_open (pool, st->path)) == NULL) { return FALSE; } - if (to_demote != NULL && - g_list_find (to_demote, data.file) != NULL) { + if (to_demote != NULL && g_list_find (to_demote, data.file) != NULL) { data.do_demote = TRUE; } else { @@ -629,66 +557,48 @@ winnow_learn (struct classifier_ctx *ctx, } data.multiplier *= WINNOW_PROMOTION; - msg_info ( - "learn iteration %d for statfile %s: %G -> %G, multiplier: %.2f", - iterations + 1, - symbol, - start_value, - end_value, - data.multiplier); - } while ((in_class ? sel != to_learn : sel == - to_learn) && iterations++ < MAX_LEARN_ITERATIONS); - + msg_info ("learn iteration %d for statfile %s: %G -> %G, multiplier: %.2f", iterations + 1, symbol, + start_value, end_value, data.multiplier); + } while ((in_class ? sel != to_learn : sel == to_learn) && iterations ++ < MAX_LEARN_ITERATIONS); + if (iterations >= MAX_LEARN_ITERATIONS) { - msg_warn ( - "learning statfile %s was not fully successfull: iterations count is limited to %d, final sum is %G", - sel_st->symbol, - MAX_LEARN_ITERATIONS, - max); + msg_warn ("learning statfile %s was not fully successfull: iterations count is limited to %d, final sum is %G", + sel_st->symbol, MAX_LEARN_ITERATIONS, max); g_set_error (err, - winnow_error_quark (), /* error domain */ - 1, /* error code */ - "learning statfile %s was not fully successfull: iterations count is limited to %d", - sel_st->symbol, MAX_LEARN_ITERATIONS); + winnow_error_quark(), /* error domain */ + 1, /* error code */ + "learning statfile %s was not fully successfull: iterations count is limited to %d", + sel_st->symbol, MAX_LEARN_ITERATIONS); return FALSE; } else { - msg_info ( - "learned statfile %s successfully with %d iterations and sum %G", - sel_st->symbol, - iterations + 1, - max); + msg_info ("learned statfile %s successfully with %d iterations and sum %G", sel_st->symbol, iterations + 1, max); } end: if (sum) { #ifdef HAVE_TANHL - *sum = (double)tanhl (max); + *sum = (double)tanhl (max); #else - /* - * As some implementations of libm does not support tanhl, try to use - * tanh - */ - *sum = tanh ((double) max); + /* + * As some implementations of libm does not support tanhl, try to use + * tanh + */ + *sum = tanh ((double) max); #endif } return TRUE; } gboolean -winnow_learn_spam (struct classifier_ctx * ctx, - statfile_pool_t *pool, - GTree *input, - struct rspamd_task *task, - gboolean is_spam, - lua_State *L, - GError **err) +winnow_learn_spam (struct classifier_ctx* ctx, statfile_pool_t *pool, + GTree *input, struct rspamd_task *task, gboolean is_spam, lua_State *L, GError **err) { g_set_error (err, - winnow_error_quark (), /* error domain */ - 1, /* error code */ - "learn spam is not supported for winnow" - ); + winnow_error_quark(), /* error domain */ + 1, /* error code */ + "learn spam is not supported for winnow" + ); return FALSE; } |