+++ /dev/null
-/*
- * Copyright (c) 2009-2012, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/*
- * Winnow classifier
- */
-
-#include "classifiers.h"
-#include "tokenizers.h"
-#include "main.h"
-#include "filter.h"
-#include "cfg_file.h"
-#include "lua/lua_common.h"
-
-#define WINNOW_PROMOTION 1.23
-#define WINNOW_DEMOTION 0.83
-
-#define MEDIAN_WINDOW_SIZE 5
-
-#define MAX_WEIGHT G_MAXDOUBLE / 2.
-
-
-
-#define MAX_LEARN_ITERATIONS 100
-
-static inline GQuark
-winnow_error_quark (void)
-{
- return g_quark_from_static_string ("winnow-error");
-}
-
-struct winnow_callback_data {
- statfile_pool_t *pool;
- struct classifier_ctx *ctx;
- stat_file_t *file;
- stat_file_t *learn_file;
- long double sum;
- long double start;
- double multiplier;
- guint32 count;
- guint32 new_blocks;
- gboolean in_class;
- gboolean do_demote;
- gboolean fresh_run;
- time_t now;
-};
-
-static const double max_common_weight = MAX_WEIGHT * WINNOW_DEMOTION;
-
-
-
-static gboolean
-winnow_classify_callback (gpointer key, gpointer value, gpointer data)
-{
- token_node_t *node = key;
- struct winnow_callback_data *cd = data;
- double v;
-
- /* Consider that not found blocks have value 1 */
- v =
- statfile_pool_get_block (cd->pool, cd->file, node->h1, node->h2,
- cd->now);
- if (fabs (v) > ALPHA) {
- cd->sum += v;
- }
- else {
- cd->sum += 1.0;
- cd->new_blocks++;
- }
-
- cd->count++;
-
- return FALSE;
-}
-
-static gboolean
-winnow_learn_callback (gpointer key, gpointer value, gpointer data)
-{
- token_node_t *node = key;
- struct winnow_callback_data *cd = data;
- double v, c;
-
- c = (cd->in_class) ? WINNOW_PROMOTION * cd->multiplier : WINNOW_DEMOTION /
- cd->multiplier;
-
- /* Consider that not found blocks have value 1 */
- v =
- statfile_pool_get_block (cd->pool, cd->file, node->h1, node->h2,
- cd->now);
- if (fabs (v) < ALPHA) {
- /* Block not found, insert new */
- cd->start += 1;
- if (cd->file == cd->learn_file) {
- statfile_pool_set_block (cd->pool,
- cd->file,
- node->h1,
- node->h2,
- cd->now,
- c);
- node->value = c;
- cd->new_blocks++;
- }
- }
- else {
- cd->start += v;
- /* Here we just increase the extra value of block */
- if (cd->fresh_run) {
- node->extra = 0;
- }
- else {
- node->extra++;
- }
- node->value = v;
-
- if (node->extra > 1) {
- /*
- * Assume that this node is common for several statfiles, so
- * decrease its weight proportianally
- */
- if (node->value > max_common_weight) {
- /* Static fluctuation */
- statfile_pool_set_block (cd->pool,
- cd->file,
- node->h1,
- node->h2,
- cd->now,
- 0.);
- node->value = 0.;
- }
- else if (node->value > WINNOW_PROMOTION * cd->multiplier) {
- /* Try to decrease its value */
- /* XXX: it is more intelligent to add some adaptive filter here */
- if (cd->file == cd->learn_file) {
- if (node->value > max_common_weight / 2.) {
- node->value *= c;
- }
- else {
- /*
- * Too high token value that exists also in other
- * statfiles, may be statistic error, so decrease it
- * slightly
- */
- node->value *= WINNOW_DEMOTION;
- }
- }
- else {
- node->value = WINNOW_DEMOTION / cd->multiplier;
- }
- statfile_pool_set_block (cd->pool,
- cd->file,
- node->h1,
- node->h2,
- cd->now,
- node->value);
- }
- }
- else if (cd->file == cd->learn_file) {
- /* New block or block that is in only one statfile */
- /* Set some limit on growing */
- if (v > MAX_WEIGHT) {
- node->value = v;
- }
- else {
- node->value *= c;
- }
- statfile_pool_set_block (cd->pool,
- cd->file,
- node->h1,
- node->h2,
- cd->now,
- node->value);
- }
- else if (cd->do_demote) {
- /* Demote blocks in file */
- node->value *= WINNOW_DEMOTION / cd->multiplier;
- statfile_pool_set_block (cd->pool,
- cd->file,
- node->h1,
- node->h2,
- cd->now,
- node->value);
- }
- }
-
-
- cd->sum += node->value;
-
- cd->count++;
-
- return FALSE;
-}
-
-struct classifier_ctx *
-winnow_init (rspamd_mempool_t * pool, struct rspamd_classifier_config *cfg)
-{
- struct classifier_ctx *ctx =
- rspamd_mempool_alloc (pool, sizeof (struct classifier_ctx));
-
- ctx->pool = pool;
- ctx->cfg = cfg;
-
- return ctx;
-}
-
-gboolean
-winnow_classify (struct classifier_ctx *ctx,
- statfile_pool_t * pool,
- GTree * input,
- struct rspamd_task *task,
- lua_State *L)
-{
- struct winnow_callback_data data;
- char *sumbuf, *value;
- long double res = 0., max = 0.;
- GList *cur;
- struct rspamd_statfile_config *st, *sel = NULL;
- int nodes, minnodes;
-
- g_assert (pool != NULL);
- g_assert (ctx != NULL);
-
- data.pool = pool;
- data.now = time (NULL);
- data.ctx = ctx;
-
- if (ctx->cfg->opts &&
- (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
- minnodes = strtol (value, NULL, 10);
- nodes = g_tree_nnodes (input);
- if (nodes > FEATURE_WINDOW_SIZE) {
- nodes = nodes / FEATURE_WINDOW_SIZE + FEATURE_WINDOW_SIZE;
- }
- if (nodes < minnodes) {
- msg_info (
- "do not classify message as it has too few tokens: %d, while %d min",
- nodes,
- minnodes);
- return FALSE;
- }
- }
-
- cur = rspamd_lua_call_cls_pre_callbacks (ctx->cfg, task, FALSE, FALSE, L);
- if (cur) {
- rspamd_mempool_add_destructor (task->task_pool,
- (rspamd_mempool_destruct_t)g_list_free, cur);
- }
- else {
- cur = ctx->cfg->statfiles;
- }
-
- while (cur) {
- st = cur->data;
- data.sum = 0;
- data.count = 0;
- data.new_blocks = 0;
- if ((data.file = statfile_pool_is_open (pool, st->path)) == NULL) {
- if ((data.file =
- statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) {
- msg_warn ("cannot open %s, skip it", st->path);
- cur = g_list_next (cur);
- continue;
- }
- }
-
- if (data.file != NULL) {
- g_tree_foreach (input, winnow_classify_callback, &data);
- }
-
- if (data.count != 0) {
- res = data.sum / (double)data.count;
- }
- else {
- res = 0;
- }
- if (res > max) {
- max = res;
- sel = st;
- }
- cur = g_list_next (cur);
- }
-
- if (sel != NULL) {
-#ifdef WITH_LUA
- max = rspamd_lua_call_cls_post_callbacks (ctx->cfg, task, max, L);
-#endif
-#ifdef HAVE_TANHL
- max = tanhl (max);
-#else
- /*
- * As some implementations of libm does not support tanhl, try to use
- * tanh
- */
- max = tanh ((double) max);
-#endif
- sumbuf = rspamd_mempool_alloc (task->task_pool, 32);
- rspamd_snprintf (sumbuf, 32, "%.2F", max);
- cur = g_list_prepend (NULL, sumbuf);
- rspamd_task_insert_result (task, sel->symbol, max, cur);
- }
-
- return TRUE;
-}
-
-GList *
-winnow_weights (struct classifier_ctx *ctx,
- statfile_pool_t * pool,
- GTree * input,
- struct rspamd_task *task)
-{
- struct winnow_callback_data data;
- long double res = 0.;
- GList *cur, *resl = NULL;
- struct rspamd_statfile_config *st;
- struct classify_weight *w;
- char *value;
- int nodes, minnodes;
-
- g_assert (pool != NULL);
- g_assert (ctx != NULL);
-
- data.pool = pool;
- data.now = time (NULL);
- data.ctx = ctx;
-
- if (ctx->cfg->opts &&
- (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
- minnodes = strtol (value, NULL, 10);
- nodes = g_tree_nnodes (input);
- if (nodes > FEATURE_WINDOW_SIZE) {
- nodes = nodes / FEATURE_WINDOW_SIZE + FEATURE_WINDOW_SIZE;
- }
- if (nodes < minnodes) {
- msg_info (
- "do not classify message as it has too few tokens: %d, while %d min",
- nodes,
- minnodes);
- return NULL;
- }
- }
-
- cur = ctx->cfg->statfiles;
- while (cur) {
- st = cur->data;
- data.sum = 0;
- data.count = 0;
- if ((data.file = statfile_pool_is_open (pool, st->path)) == NULL) {
- if ((data.file =
- statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) {
- msg_warn ("cannot open %s, skip it", st->path);
- cur = g_list_next (cur);
- continue;
- }
- }
-
- if (data.file != NULL) {
- g_tree_foreach (input, winnow_classify_callback, &data);
- }
-
- w =
- rspamd_mempool_alloc0 (task->task_pool,
- sizeof (struct classify_weight));
- if (data.count != 0) {
- res = data.sum / (double)data.count;
- }
- else {
- res = 0;
- }
- w->name = st->symbol;
- w->weight = res;
- resl = g_list_prepend (resl, w);
- cur = g_list_next (cur);
- }
-
- if (resl != NULL) {
- rspamd_mempool_add_destructor (task->task_pool,
- (rspamd_mempool_destruct_t)g_list_free, resl);
- }
-
- return resl;
-
-}
-
-
-gboolean
-winnow_learn (struct classifier_ctx *ctx,
- statfile_pool_t *pool,
- const char *symbol,
- GTree * input,
- int in_class,
- double *sum,
- double multiplier,
- GError **err)
-{
- struct winnow_callback_data data = {
- .file = NULL,
- .multiplier = multiplier
- };
- char *value;
- int nodes, minnodes, iterations = 0;
- struct rspamd_statfile_config *st, *sel_st = NULL;
- stat_file_t *sel = NULL, *to_learn;
- long double res = 0., max = 0., start_value = 0., end_value = 0.;
- double learn_threshold = 0.0;
- GList *cur, *to_demote = NULL;
- gboolean force_learn = FALSE;
-
- g_assert (pool != NULL);
- g_assert (ctx != NULL);
-
- data.pool = pool;
- data.in_class = in_class;
- data.now = time (NULL);
- data.ctx = ctx;
-
-
- if (ctx->cfg->opts &&
- (value = g_hash_table_lookup (ctx->cfg->opts, "min_tokens")) != NULL) {
- minnodes = strtol (value, NULL, 10);
- nodes = g_tree_nnodes (input);
- if (nodes > FEATURE_WINDOW_SIZE) {
- nodes = nodes / FEATURE_WINDOW_SIZE + FEATURE_WINDOW_SIZE;
- }
- if (nodes < minnodes) {
- msg_info (
- "do not learn message as it has too few tokens: %d, while %d min",
- nodes,
- minnodes);
- if (sum != NULL) {
- *sum = 0;
- }
- g_set_error (err,
- winnow_error_quark (), /* error domain */
- 1, /* error code */
- "message contains too few tokens: %d, while min is %d",
- nodes, minnodes);
- return FALSE;
- }
- }
- if (ctx->cfg->opts &&
- (value =
- g_hash_table_lookup (ctx->cfg->opts, "learn_threshold")) != NULL) {
- learn_threshold = strtod (value, NULL);
- }
-
- if (learn_threshold <= 1.0 && learn_threshold >= 0) {
- /* Classify message and check target statfile score */
- cur = ctx->cfg->statfiles;
- while (cur) {
- /* Open or create all statfiles inside classifier */
- st = cur->data;
- if (statfile_pool_is_open (pool, st->path) == NULL) {
- if (statfile_pool_open (pool, st->path, st->size,
- FALSE) == NULL) {
- msg_warn ("cannot open %s", st->path);
- if (statfile_pool_create (pool, st->path, st->size) == -1) {
- msg_err ("cannot create statfile %s", st->path);
- g_set_error (err,
- winnow_error_quark (), /* error domain */
- 1, /* error code */
- "cannot create statfile: %s",
- st->path);
- return FALSE;
- }
- if (statfile_pool_open (pool, st->path, st->size,
- FALSE) == NULL) {
- g_set_error (err,
- winnow_error_quark (), /* error domain */
- 1, /* error code */
- "open statfile %s after creation",
- st->path);
- msg_err ("cannot open statfile %s after creation",
- st->path);
- return FALSE;
- }
- }
- }
- if (strcmp (st->symbol, symbol) == 0) {
- sel_st = st;
-
- }
- cur = g_list_next (cur);
- }
-
- if (sel_st == NULL) {
- g_set_error (err,
- winnow_error_quark (), /* error domain */
- 1, /* error code */
- "cannot find statfile for symbol %s",
- symbol);
- msg_err ("cannot find statfile for symbol %s", symbol);
- return FALSE;
- }
-
- to_learn = statfile_pool_is_open (pool, sel_st->path);
- if (to_learn == NULL) {
- g_set_error (err,
- winnow_error_quark (), /* error domain */
- 1, /* error code */
- "statfile %s is not opened this maybe if your statfile pool is too small to handle all statfiles",
- sel_st->path);
- return FALSE;
- }
- /* Check target statfile */
- data.file = to_learn;
- data.sum = 0;
- data.count = 0;
- data.new_blocks = 0;
- g_tree_foreach (input, winnow_classify_callback, &data);
- if (data.count > 0) {
- max = data.sum / (double)data.count;
- }
- else {
- max = 0;
- }
- /* If most of blocks are not presented in targeted statfile do forced learn */
- if (max < 1 + learn_threshold) {
- force_learn = TRUE;
- }
- /* Check other statfiles */
- while (cur) {
- st = cur->data;
- data.sum = 0;
- data.count = 0;
- if ((data.file = statfile_pool_is_open (pool, st->path)) == NULL) {
- g_set_error (err,
- winnow_error_quark (), /* error domain */
- 1, /* error code */
- "statfile %s is not opened this maybe if your statfile pool is too small to handle all statfiles",
- st->path);
- return FALSE;
- }
- g_tree_foreach (input, winnow_classify_callback, &data);
- if (data.count != 0) {
- res = data.sum / data.count;
- }
- else {
- res = 0;
- }
- if (to_learn != data.file && res - max > 1 - learn_threshold) {
- /* Demote tokens in this statfile */
- to_demote = g_list_prepend (to_demote, data.file);
- }
- cur = g_list_next (cur);
- }
- }
- else {
- msg_err (
- "learn threshold is more than 1 or less than 0, so cannot do learn, please check your configuration");
- g_set_error (err,
- winnow_error_quark (), /* error domain */
- 1, /* error code */
- "bad learn_threshold setting: %.2f",
- learn_threshold);
- return FALSE;
- }
- /* If to_demote list is empty this message is already classified correctly */
- if (max > WINNOW_PROMOTION && to_demote == NULL && !force_learn) {
- msg_info (
- "this message is already of class %s with threshold %.2f and weight %.2F",
- sel_st->symbol,
- learn_threshold,
- max);
- goto end;
- }
- data.learn_file = to_learn;
- end_value = max;
- do {
- cur = ctx->cfg->statfiles;
- data.fresh_run = TRUE;
- while (cur) {
- st = cur->data;
- data.sum = 0;
- data.count = 0;
- data.new_blocks = 0;
- data.start = 0;
- if ((data.file = statfile_pool_is_open (pool, st->path)) == NULL) {
- return FALSE;
- }
- if (to_demote != NULL &&
- g_list_find (to_demote, data.file) != NULL) {
- data.do_demote = TRUE;
- }
- else {
- data.do_demote = FALSE;
- }
-
- statfile_pool_lock_file (pool, data.file);
- g_tree_foreach (input, winnow_learn_callback, &data);
- statfile_pool_unlock_file (pool, data.file);
- if (data.count != 0) {
- res = data.sum / data.count;
- }
- else {
- res = 0;
- }
- if (res > max) {
- max = res;
- sel = data.file;
- }
- if (data.file == to_learn) {
- if (data.count > 0) {
- start_value = data.start / data.count;
- }
- end_value = res;
- }
- cur = g_list_next (cur);
- data.fresh_run = FALSE;
- }
-
- data.multiplier *= WINNOW_PROMOTION;
- msg_info (
- "learn iteration %d for statfile %s: %G -> %G, multiplier: %.2f",
- iterations + 1,
- symbol,
- start_value,
- end_value,
- data.multiplier);
- } while ((in_class ? sel != to_learn : sel ==
- to_learn) && iterations++ < MAX_LEARN_ITERATIONS);
-
- if (iterations >= MAX_LEARN_ITERATIONS) {
- msg_warn (
- "learning statfile %s was not fully successfull: iterations count is limited to %d, final sum is %G",
- sel_st->symbol,
- MAX_LEARN_ITERATIONS,
- max);
- g_set_error (err,
- winnow_error_quark (), /* error domain */
- 1, /* error code */
- "learning statfile %s was not fully successfull: iterations count is limited to %d",
- sel_st->symbol, MAX_LEARN_ITERATIONS);
- return FALSE;
- }
- else {
- msg_info (
- "learned statfile %s successfully with %d iterations and sum %G",
- sel_st->symbol,
- iterations + 1,
- max);
- }
-
-
-end:
- if (sum) {
-#ifdef HAVE_TANHL
- *sum = (double)tanhl (max);
-#else
- /*
- * As some implementations of libm does not support tanhl, try to use
- * tanh
- */
- *sum = tanh ((double) max);
-#endif
- }
- return TRUE;
-}
-
-gboolean
-winnow_learn_spam (struct classifier_ctx * ctx,
- statfile_pool_t *pool,
- GTree *input,
- struct rspamd_task *task,
- gboolean is_spam,
- lua_State *L,
- GError **err)
-{
- g_set_error (err,
- winnow_error_quark (), /* error domain */
- 1, /* error code */
- "learn spam is not supported for winnow"
- );
- return FALSE;
-}
projects / rspamd.git / commitdiff