From aef5bd31e18f108c51e4e3fc78b889a2e7c51b33 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Tue, 5 Jan 2016 17:24:40 +0000 Subject: [PATCH] Simplify learn and process functions --- src/libstat/backends/backends.h | 20 ++-- src/libstat/backends/mmaped_file.c | 43 +++---- src/libstat/backends/sqlite3_backend.c | 155 +++++++++++++------------ 3 files changed, 114 insertions(+), 104 deletions(-) diff --git a/src/libstat/backends/backends.h b/src/libstat/backends/backends.h index 42eba8d61..800030414 100644 --- a/src/libstat/backends/backends.h +++ b/src/libstat/backends/backends.h @@ -46,12 +46,14 @@ struct rspamd_stat_backend { struct rspamd_statfile *st); gpointer (*runtime)(struct rspamd_task *task, struct rspamd_statfile_config *stcf, gboolean learn, gpointer ctx); - gboolean (*process_token)(struct rspamd_task *task, struct token_node_s *tok, - struct rspamd_token_result *res, gpointer ctx); + gboolean (*process_tokens)(struct rspamd_task *task, GPtrArray *tokens, + gint id, + gpointer ctx); void (*finalize_process)(struct rspamd_task *task, gpointer runtime, gpointer ctx); - gboolean (*learn_token)(struct rspamd_task *task, struct token_node_s *tok, - struct rspamd_token_result *res, gpointer ctx); + gboolean (*learn_tokens)(struct rspamd_task *task, GPtrArray *tokens, + gint id, + gpointer ctx); gulong (*total_learns)(struct rspamd_task *task, gpointer runtime, gpointer ctx); void (*finalize_learn)(struct rspamd_task *task, @@ -73,16 +75,14 @@ struct rspamd_stat_backend { gpointer rspamd_##name##_runtime (struct rspamd_task *task, \ struct rspamd_statfile_config *stcf, \ gboolean learn, gpointer ctx); \ - gboolean rspamd_##name##_process_token (struct rspamd_task *task, \ - struct token_node_s *tok, \ - struct rspamd_token_result *res, \ + gboolean rspamd_##name##_process_tokens (struct rspamd_task *task, \ + GPtrArray *tokens, gint id, \ gpointer ctx); \ void rspamd_##name##_finalize_process (struct rspamd_task *task, \ gpointer runtime, \ gpointer ctx); \ - gboolean rspamd_##name##_learn_token (struct rspamd_task *task, \ - struct token_node_s *tok, \ - struct rspamd_token_result *res, \ + gboolean rspamd_##name##_learn_tokens (struct rspamd_task *task, \ + GPtrArray *tokens, gint id, \ gpointer ctx); \ void rspamd_##name##_finalize_learn (struct rspamd_task *task, \ gpointer runtime, \ diff --git a/src/libstat/backends/mmaped_file.c b/src/libstat/backends/mmaped_file.c index 651e39f38..5e0b176ef 100644 --- a/src/libstat/backends/mmaped_file.c +++ b/src/libstat/backends/mmaped_file.c @@ -844,45 +844,48 @@ rspamd_mmaped_file_runtime (struct rspamd_task *task, } gboolean -rspamd_mmaped_file_process_token (struct rspamd_task *task, rspamd_token_t *tok, - struct rspamd_token_result *res, +rspamd_mmaped_file_process_tokens (struct rspamd_task *task, GPtrArray *tokens, + gint id, gpointer p) { rspamd_mmaped_file_t *mf = p; guint32 h1, h2; + rspamd_token_t *tok; + guint i; - g_assert (res != NULL); + g_assert (tokens != NULL); g_assert (p != NULL); - g_assert (tok != NULL); - g_assert (tok->datalen >= sizeof (guint32) * 2); - memcpy (&h1, tok->data, sizeof (h1)); - memcpy (&h2, tok->data + sizeof (h1), sizeof (h2)); - res->value = rspamd_mmaped_file_get_block (ctx, mf, h1, h2); - - if (res->value > 0.0) { - return TRUE; + for (i = 0; i < tokens->len; i++) { + tok = g_ptr_array_index (tokens, i); + memcpy (&h1, tok->data, sizeof (h1)); + memcpy (&h2, tok->data + sizeof (h1), sizeof (h2)); + tok->values[id] = rspamd_mmaped_file_get_block (mf, h1, h2); } - return FALSE; + return TRUE; } gboolean -rspamd_mmaped_file_learn_token (struct rspamd_task *task, rspamd_token_t *tok, - struct rspamd_token_result *res, +rspamd_mmaped_file_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, + gint id, gpointer p) { rspamd_mmaped_file_t *mf = p; guint32 h1, h2; + rspamd_token_t *tok; + guint i; - g_assert (res != NULL); + g_assert (tokens != NULL); g_assert (p != NULL); - g_assert (tok != NULL); - g_assert (tok->datalen >= sizeof (guint32) * 2); - memcpy (&h1, tok->data, sizeof (h1)); - memcpy (&h2, tok->data + sizeof (h1), sizeof (h2)); - rspamd_mmaped_file_set_block (task->task_pool, ctx, mf, h1, h2, res->value); + for (i = 0; i < tokens->len; i++) { + tok = g_ptr_array_index (tokens, i); + memcpy (&h1, tok->data, sizeof (h1)); + memcpy (&h2, tok->data + sizeof (h1), sizeof (h2)); + rspamd_mmaped_file_set_block (task->task_pool, mf, h1, h2, + tok->values[id]); + } return TRUE; } diff --git a/src/libstat/backends/sqlite3_backend.c b/src/libstat/backends/sqlite3_backend.c index 591ab7669..3e05bee80 100644 --- a/src/libstat/backends/sqlite3_backend.c +++ b/src/libstat/backends/sqlite3_backend.c @@ -691,64 +691,64 @@ rspamd_sqlite3_runtime (struct rspamd_task *task, } gboolean -rspamd_sqlite3_process_token (struct rspamd_task *task, struct token_node_s *tok, - struct rspamd_token_result *res, gpointer p) +rspamd_sqlite3_process_tokens (struct rspamd_task *task, + GPtrArray *tokens, + gint id, gpointer p) { struct rspamd_stat_sqlite3_db *bk; struct rspamd_stat_sqlite3_rt *rt = p; gint64 iv = 0, idx; + guint i; + rspamd_token_t *tok; - g_assert (res != NULL); g_assert (p != NULL); - g_assert (tok != NULL); - g_assert (tok->datalen >= sizeof (guint32) * 2); + g_assert (tokens != NULL); bk = rt->db; - if (bk == NULL) { - /* Statfile is does not exist, so all values are zero */ - res->value = 0.0; - return FALSE; - } - - if (!bk->in_transaction) { - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF); - bk->in_transaction = TRUE; - } + for (i = 0; i < tokens->len; i ++) { + tok = g_ptr_array_index (tokens, i); - if (rt->user_id == -1) { - if (bk->enable_users) { - rt->user_id = rspamd_sqlite3_get_user (bk, task, FALSE); - } - else { - rt->user_id = 0; + if (bk == NULL) { + /* Statfile is does not exist, so all values are zero */ + tok->values[id] = 0.0; + continue; } - } - if (rt->lang_id == -1) { - if (bk->enable_languages) { - rt->lang_id = rspamd_sqlite3_get_language (bk, task, FALSE); + if (!bk->in_transaction) { + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF); + bk->in_transaction = TRUE; } - else { - rt->lang_id = 0; + + if (rt->user_id == -1) { + if (bk->enable_users) { + rt->user_id = rspamd_sqlite3_get_user (bk, task, FALSE); + } + else { + rt->user_id = 0; + } } - } - memcpy (&idx, tok->data, sizeof (idx)); + if (rt->lang_id == -1) { + if (bk->enable_languages) { + rt->lang_id = rspamd_sqlite3_get_language (bk, task, FALSE); + } + else { + rt->lang_id = 0; + } + } - if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_GET_TOKEN, - idx, rt->user_id, rt->lang_id, &iv) == SQLITE_OK) { - res->value = iv; + memcpy (&idx, tok->data, sizeof (idx)); - if (iv == 0) { - return FALSE; + if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_GET_TOKEN, + idx, rt->user_id, rt->lang_id, &iv) == SQLITE_OK) { + tok->values[id] = iv; + } + else { + tok->values[id] = 0.0; } - } - else { - res->value = 0.0; - return FALSE; } @@ -778,56 +778,63 @@ rspamd_sqlite3_finalize_process (struct rspamd_task *task, gpointer runtime, } gboolean -rspamd_sqlite3_learn_token (struct rspamd_task *task, struct token_node_s *tok, - struct rspamd_token_result *res, gpointer p) +rspamd_sqlite3_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, + gint id, gpointer p) { struct rspamd_stat_sqlite3_db *bk; struct rspamd_stat_sqlite3_rt *rt = p; gint64 iv = 0, idx; + guint i; + rspamd_token_t *tok; - g_assert (res != NULL); + g_assert (tokens != NULL); g_assert (p != NULL); - g_assert (tok != NULL); - g_assert (tok->datalen >= sizeof (guint32) * 2); bk = rt->db; - if (bk == NULL) { - /* Statfile is does not exist, so all values are zero */ - return FALSE; - } - - if (!bk->in_transaction) { - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); - bk->in_transaction = TRUE; - } - - if (rt->user_id == -1) { - if (bk->enable_users) { - rt->user_id = rspamd_sqlite3_get_user (bk, task, TRUE); + for (i = 0; i < tokens->len; i++) { + tok = g_ptr_array_index (tokens, i); + if (bk == NULL) { + /* Statfile is does not exist, so all values are zero */ + return FALSE; } - else { - rt->user_id = 0; + + if (!bk->in_transaction) { + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); + bk->in_transaction = TRUE; } - } - if (rt->lang_id == -1) { - if (bk->enable_languages) { - rt->lang_id = rspamd_sqlite3_get_language (bk, task, TRUE); + if (rt->user_id == -1) { + if (bk->enable_users) { + rt->user_id = rspamd_sqlite3_get_user (bk, task, TRUE); + } + else { + rt->user_id = 0; + } } - else { - rt->lang_id = 0; + + if (rt->lang_id == -1) { + if (bk->enable_languages) { + rt->lang_id = rspamd_sqlite3_get_language (bk, task, TRUE); + } + else { + rt->lang_id = 0; + } } - } - iv = res->value; - memcpy (&idx, tok->data, sizeof (idx)); + iv = tok->values[id]; + memcpy (&idx, tok->data, sizeof (idx)); + + if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_SET_TOKEN, + idx, rt->user_id, rt->lang_id, iv) != SQLITE_OK) { + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK); + bk->in_transaction = FALSE; - if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_SET_TOKEN, - idx, rt->user_id, rt->lang_id, iv) != SQLITE_OK) { - return FALSE; + return FALSE; + } } return TRUE; -- 2.39.5