From 5f0f8b6fd6eba17c11c6bf8e16b33323d31ae8c1 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Tue, 1 Sep 2015 16:58:57 +0100 Subject: [PATCH] Fix logging in sqlite3 modules. --- src/libserver/fuzzy_backend.c | 76 ++++++++++++++++--------- src/libstat/backends/sqlite3_backend.c | 70 ++++++++++++----------- src/libstat/learn_cache/sqlite3_cache.c | 27 +++++---- src/libutil/sqlite_utils.c | 30 +++++----- src/libutil/sqlite_utils.h | 5 +- 5 files changed, 120 insertions(+), 88 deletions(-) diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c index aebd9fd48..3d1749aa7 100644 --- a/src/libserver/fuzzy_backend.c +++ b/src/libserver/fuzzy_backend.c @@ -32,8 +32,26 @@ struct rspamd_fuzzy_backend { char *path; gsize count; gsize expired; + rspamd_mempool_t *pool; }; +#define msg_err_fuzzy_backend(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \ + backend->pool->tag.tagname, backend->pool->tag.uid, \ + G_STRFUNC, \ + __VA_ARGS__) +#define msg_warn_fuzzy_backend(...) rspamd_default_log_function (G_LOG_LEVEL_WARNING, \ + backend->pool->tag.tagname, backend->pool->tag.uid, \ + G_STRFUNC, \ + __VA_ARGS__) +#define msg_info_fuzzy_backend(...) rspamd_default_log_function (G_LOG_LEVEL_INFO, \ + backend->pool->tag.tagname, backend->pool->tag.uid, \ + G_STRFUNC, \ + __VA_ARGS__) +#define msg_debug_fuzzy_backend(...) rspamd_default_log_function (G_LOG_LEVEL_DEBUG, \ + backend->pool->tag.tagname, backend->pool->tag.uid, \ + G_STRFUNC, \ + __VA_ARGS__) + static const char *create_tables_sql = "BEGIN;" "CREATE TABLE digests(" @@ -204,7 +222,7 @@ rspamd_fuzzy_backend_prepare_stmts (struct rspamd_fuzzy_backend *bk, GError **er } static int -rspamd_fuzzy_backend_run_stmt (struct rspamd_fuzzy_backend *bk, int idx, ...) +rspamd_fuzzy_backend_run_stmt (struct rspamd_fuzzy_backend *backend, int idx, ...) { int retcode; va_list ap; @@ -219,17 +237,17 @@ rspamd_fuzzy_backend_run_stmt (struct rspamd_fuzzy_backend *bk, int idx, ...) stmt = prepared_stmts[idx].stmt; if (stmt == NULL) { - if ((retcode = sqlite3_prepare_v2 (bk->db, prepared_stmts[idx].sql, -1, + if ((retcode = sqlite3_prepare_v2 (backend->db, prepared_stmts[idx].sql, -1, &prepared_stmts[idx].stmt, NULL)) != SQLITE_OK) { - msg_err ("Cannot initialize prepared sql `%s`: %s", - prepared_stmts[idx].sql, sqlite3_errmsg (bk->db)); + msg_err_fuzzy_backend ("Cannot initialize prepared sql `%s`: %s", + prepared_stmts[idx].sql, sqlite3_errmsg (backend->db)); return retcode; } stmt = prepared_stmts[idx].stmt; } - msg_debug ("executing `%s`", prepared_stmts[idx].sql); + msg_debug_fuzzy_backend ("executing `%s`", prepared_stmts[idx].sql); argtypes = prepared_stmts[idx].args; sqlite3_reset (stmt); va_start (ap, idx); @@ -261,8 +279,8 @@ rspamd_fuzzy_backend_run_stmt (struct rspamd_fuzzy_backend *bk, int idx, ...) return SQLITE_OK; } else if (retcode != SQLITE_DONE) { - msg_debug ("failed to execute query %s: %d, %s", prepared_stmts[idx].sql, - retcode, sqlite3_errmsg (bk->db)); + msg_debug_fuzzy_backend ("failed to execute query %s: %d, %s", prepared_stmts[idx].sql, + retcode, sqlite3_errmsg (backend->db)); } return retcode; @@ -335,6 +353,7 @@ rspamd_fuzzy_backend_create_db (const gchar *path, gboolean add_index, bk->db = sqlite; bk->expired = 0; bk->count = 0; + bk->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "fuzzy_backend"); /* * Here we need to run create prior to preparing other statements @@ -377,6 +396,7 @@ rspamd_fuzzy_backend_open_db (const gchar *path, GError **err) bk->path = g_strdup (path); bk->db = sqlite; bk->expired = 0; + bk->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "fuzzy_backend"); return bk; } @@ -386,7 +406,7 @@ rspamd_fuzzy_backend_open (const gchar *path, GError **err) { gchar *dir; gint fd; - struct rspamd_fuzzy_backend *res; + struct rspamd_fuzzy_backend *backend; static const char sqlite_wal[] = "PRAGMA journal_mode=\"wal\";", fallback_journal[] = "PRAGMA journal_mode=\"off\";"; int rc; @@ -429,10 +449,10 @@ rspamd_fuzzy_backend_open (const gchar *path, GError **err) close (fd); /* Open database */ - if ((res = rspamd_fuzzy_backend_open_db (path, err)) == NULL) { + if ((backend = rspamd_fuzzy_backend_open_db (path, err)) == NULL) { GError *tmp = NULL; - if ((res = rspamd_fuzzy_backend_create_db (path, TRUE, &tmp)) == NULL) { + if ((backend = rspamd_fuzzy_backend_create_db (path, TRUE, &tmp)) == NULL) { g_clear_error (err); g_propagate_error (err, tmp); return NULL; @@ -440,25 +460,25 @@ rspamd_fuzzy_backend_open (const gchar *path, GError **err) g_clear_error (err); } - if ((rc = sqlite3_exec (res->db, sqlite_wal, NULL, NULL, NULL)) != SQLITE_OK) { - msg_warn ("WAL mode is not supported (%d), locking issues might occur", + if ((rc = sqlite3_exec (backend->db, sqlite_wal, NULL, NULL, NULL)) != SQLITE_OK) { + msg_warn_fuzzy_backend ("WAL mode is not supported (%d), locking issues might occur", rc); - sqlite3_exec (res->db, fallback_journal, NULL, NULL, NULL); + sqlite3_exec (backend->db, fallback_journal, NULL, NULL, NULL); } /* Cleanup database */ - rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_VACUUM, res, NULL); + rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_VACUUM, backend, NULL); - if (rspamd_fuzzy_backend_run_stmt (res, RSPAMD_FUZZY_BACKEND_COUNT) + if (rspamd_fuzzy_backend_run_stmt (backend, RSPAMD_FUZZY_BACKEND_COUNT) == SQLITE_OK) { - res->count = sqlite3_column_int64 ( + backend->count = sqlite3_column_int64 ( prepared_stmts[RSPAMD_FUZZY_BACKEND_COUNT].stmt, 0); } rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_TRANSACTION_START, - res, NULL); + backend, NULL); - return res; + return backend; } static gint @@ -490,7 +510,7 @@ rspamd_fuzzy_backend_check (struct rspamd_fuzzy_backend *backend, prepared_stmts[RSPAMD_FUZZY_BACKEND_CHECK].stmt, 1); if (time (NULL) - timestamp > expire) { /* Expire element */ - msg_debug ("requested hash has been expired"); + msg_debug_fuzzy_backend ("requested hash has been expired"); rspamd_fuzzy_backend_run_stmt (backend, RSPAMD_FUZZY_BACKEND_DELETE, cmd->digest); backend->expired ++; @@ -518,7 +538,7 @@ rspamd_fuzzy_backend_check (struct rspamd_fuzzy_backend *backend, else { shingle_values[i] = -1; } - msg_debug ("looking for shingle %d -> %L: %d", i, shcmd->sgl.hashes[i], rc); + msg_debug_fuzzy_backend ("looking for shingle %d -> %L: %d", i, shcmd->sgl.hashes[i], rc); } qsort (shingle_values, RSPAMD_SHINGLE_SIZE, sizeof (gint64), rspamd_fuzzy_backend_int64_cmp); @@ -553,7 +573,7 @@ rspamd_fuzzy_backend_check (struct rspamd_fuzzy_backend *backend, if (sel_id != -1) { /* We have some id selected here */ rep.prob = (gdouble)max_cnt / (gdouble)RSPAMD_SHINGLE_SIZE; - msg_debug ("found fuzzy hash with probability %.2f", rep.prob); + msg_debug_fuzzy_backend ("found fuzzy hash with probability %.2f", rep.prob); rc = rspamd_fuzzy_backend_run_stmt (backend, RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID, sel_id); if (rc == SQLITE_OK) { @@ -563,7 +583,7 @@ rspamd_fuzzy_backend_check (struct rspamd_fuzzy_backend *backend, prepared_stmts[RSPAMD_FUZZY_BACKEND_GET_DIGEST_BY_ID].stmt, 2); if (time (NULL) - timestamp > expire) { /* Expire element */ - msg_debug ("requested hash has been expired"); + msg_debug_fuzzy_backend ("requested hash has been expired"); backend->expired ++; rspamd_fuzzy_backend_run_stmt (backend, RSPAMD_FUZZY_BACKEND_DELETE, digest); @@ -613,7 +633,7 @@ rspamd_fuzzy_backend_add (struct rspamd_fuzzy_backend *backend, rspamd_fuzzy_backend_run_stmt (backend, RSPAMD_FUZZY_BACKEND_INSERT_SHINGLE, shcmd->sgl.hashes[i], i, id); - msg_debug ("add shingle %d -> %L: %d", i, shcmd->sgl.hashes[i], id); + msg_debug_fuzzy_backend ("add shingle %d -> %L: %d", i, shcmd->sgl.hashes[i], id); } } } @@ -658,11 +678,11 @@ rspamd_fuzzy_backend_sync (struct rspamd_fuzzy_backend *backend, gint64 expire) if (expired > 0) { backend->expired += expired; - msg_info ("expired %L hashes", expired); + msg_info_fuzzy_backend ("expired %L hashes", expired); } } else { - msg_warn ("cannot execute expired statement: %s", + msg_warn_fuzzy_backend ("cannot execute expired statement: %s", sqlite3_errmsg (backend->db)); } } @@ -676,7 +696,7 @@ rspamd_fuzzy_backend_sync (struct rspamd_fuzzy_backend *backend, gint64 expire) backend, NULL); } else { - msg_warn ("cannot synchronise fuzzy backend: %e", err); + msg_warn_fuzzy_backend ("cannot synchronise fuzzy backend: %e", err); g_error_free (err); } @@ -697,6 +717,10 @@ rspamd_fuzzy_backend_close (struct rspamd_fuzzy_backend *backend) g_free (backend->path); } + if (backend->pool) { + rspamd_mempool_delete (backend->pool); + } + g_slice_free1 (sizeof (*backend), backend); } } diff --git a/src/libstat/backends/sqlite3_backend.c b/src/libstat/backends/sqlite3_backend.c index 836fe18d5..50c9886df 100644 --- a/src/libstat/backends/sqlite3_backend.c +++ b/src/libstat/backends/sqlite3_backend.c @@ -44,6 +44,7 @@ struct rspamd_stat_sqlite3_db { struct rspamd_stat_sqlite3_ctx { GHashTable *files; + rspamd_mempool_t *pool; }; struct rspamd_stat_sqlite3_rt { @@ -307,18 +308,18 @@ rspamd_sqlite3_get_user (struct rspamd_stat_sqlite3_db *db, /* XXX: We ignore now mime recipients as they could be easily forged */ if (user != NULL) { - rc = rspamd_sqlite3_run_prstmt (db->sqlite, db->prstmt, + rc = rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, RSPAMD_STAT_BACKEND_GET_USER, user, &id); if (rc != SQLITE_OK && learn) { /* We need to insert a new user */ if (!db->in_transaction) { - rspamd_sqlite3_run_prstmt (db->sqlite, db->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); db->in_transaction = TRUE; } - rc = rspamd_sqlite3_run_prstmt (db->sqlite, db->prstmt, + rc = rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, RSPAMD_STAT_BACKEND_INSERT_USER, user, &id); } } @@ -348,18 +349,18 @@ rspamd_sqlite3_get_language (struct rspamd_stat_sqlite3_db *db, /* XXX: We ignore multiple languages but default + extra */ if (language != NULL) { - rc = rspamd_sqlite3_run_prstmt (db->sqlite, db->prstmt, + rc = rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, RSPAMD_STAT_BACKEND_GET_LANGUAGE, language, &id); if (rc != SQLITE_OK && learn) { /* We need to insert a new language */ if (!db->in_transaction) { - rspamd_sqlite3_run_prstmt (db->sqlite, db->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); db->in_transaction = TRUE; } - rc = rspamd_sqlite3_run_prstmt (db->sqlite, db->prstmt, + rc = rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, RSPAMD_STAT_BACKEND_INSERT_LANGUAGE, language, &id); } } @@ -383,7 +384,7 @@ rspamd_sqlite3_opendb (rspamd_mempool_t *pool, }; bk = g_slice_alloc0 (sizeof (*bk)); - bk->sqlite = rspamd_sqlite3_open_or_create (path, create_tables_sql, err); + bk->sqlite = rspamd_sqlite3_open_or_create (pool, path, create_tables_sql, err); if (bk->sqlite == NULL) { g_slice_free1 (sizeof (*bk), bk); @@ -405,19 +406,19 @@ rspamd_sqlite3_opendb (rspamd_mempool_t *pool, /* Check tokenizer configuration */ - while (rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + while (rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL) != SQLITE_OK) { nanosleep (&sleep_ts, NULL); } - if (rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + if (rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, &sz, &tk_conf) != SQLITE_OK) { g_assert (stcf->clcf->tokenizer != NULL); tokenizer = rspamd_stat_get_tokenizer (stcf->clcf->tokenizer->name); g_assert (tokenizer != NULL); tk_conf = tokenizer->get_config (pool, stcf->clcf->tokenizer, (gsize *)&sz); - if (rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + if (rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_SAVE_TOKENIZER, sz, tk_conf) != SQLITE_OK) { sqlite3_close (bk->sqlite); g_slice_free1 (sizeof (*bk), bk); @@ -429,7 +430,7 @@ rspamd_sqlite3_opendb (rspamd_mempool_t *pool, g_free (tk_conf); } - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); return bk; @@ -450,6 +451,7 @@ rspamd_sqlite3_init (struct rspamd_stat_ctx *ctx, new = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*new)); new->files = g_hash_table_new (g_direct_hash, g_direct_equal); + new->pool = cfg->cfg_pool; /* Iterate over all classifiers and load matching statfiles */ cur = cfg->classifiers; @@ -468,7 +470,7 @@ rspamd_sqlite3_init (struct rspamd_stat_ctx *ctx, if (filenameo == NULL || ucl_object_type (filenameo) != UCL_STRING) { filenameo = ucl_object_find_key (stf->opts, "path"); if (filenameo == NULL || ucl_object_type (filenameo) != UCL_STRING) { - msg_err ("statfile %s has no filename defined", stf->symbol); + msg_err_config ("statfile %s has no filename defined", stf->symbol); curst = curst->next; continue; } @@ -478,7 +480,7 @@ rspamd_sqlite3_init (struct rspamd_stat_ctx *ctx, if ((bk = rspamd_sqlite3_opendb (cfg->cfg_pool, stf, filename, stf->opts, TRUE, &err)) == NULL) { - msg_err ("cannot open sqlite3 db: %e", err); + msg_err_config ("cannot open sqlite3 db: %e", err); } if (bk != NULL) { @@ -534,7 +536,7 @@ rspamd_sqlite3_close (gpointer p) if (bk->sqlite) { if (bk->in_transaction) { - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (ctx->pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); } @@ -595,7 +597,7 @@ rspamd_sqlite3_process_token (struct rspamd_task *task, struct token_node_s *tok } if (!bk->in_transaction) { - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF); bk->in_transaction = TRUE; } @@ -620,7 +622,7 @@ rspamd_sqlite3_process_token (struct rspamd_task *task, struct token_node_s *tok memcpy (&idx, tok->data, sizeof (idx)); - if (rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_GET_TOKEN, idx, rt->user_id, rt->lang_id, &iv) == SQLITE_OK) { res->value = iv; @@ -649,7 +651,7 @@ rspamd_sqlite3_finalize_process (struct rspamd_task *task, gpointer runtime, bk = rt->db; if (bk->in_transaction) { - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); bk->in_transaction = FALSE; } @@ -680,7 +682,7 @@ rspamd_sqlite3_learn_token (struct rspamd_task *task, struct token_node_s *tok, } if (!bk->in_transaction) { - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); bk->in_transaction = TRUE; } @@ -696,7 +698,7 @@ rspamd_sqlite3_learn_token (struct rspamd_task *task, struct token_node_s *tok, iv = res->value; memcpy (&idx, tok->data, sizeof (idx)); - if (rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_SET_TOKEN, idx, rt->user_id, rt->lang_id, iv) != SQLITE_OK) { return FALSE; @@ -716,7 +718,7 @@ rspamd_sqlite3_finalize_learn (struct rspamd_task *task, gpointer runtime, bk = rt->db; if (bk->in_transaction) { - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); bk->in_transaction = FALSE; } @@ -734,7 +736,7 @@ rspamd_sqlite3_total_learns (struct rspamd_task *task, gpointer runtime, g_assert (rt != NULL); bk = rt->db; - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_GET_LEARNS, &res); return res; @@ -750,17 +752,17 @@ rspamd_sqlite3_inc_learns (struct rspamd_task *task, gpointer runtime, g_assert (rt != NULL); bk = rt->db; - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_INC_LEARNS, rt->user_id, rt->lang_id); if (bk->in_transaction) { - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); bk->in_transaction = FALSE; } - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_GET_LEARNS, &res); return res; @@ -776,17 +778,17 @@ rspamd_sqlite3_dec_learns (struct rspamd_task *task, gpointer runtime, g_assert (rt != NULL); bk = rt->db; - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_DEC_LEARNS, rt->user_id, rt->lang_id); if (bk->in_transaction) { - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); bk->in_transaction = FALSE; } - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_GET_LEARNS, &res); return res; @@ -802,7 +804,7 @@ rspamd_sqlite3_learns (struct rspamd_task *task, gpointer runtime, g_assert (rt != NULL); bk = rt->db; - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_GET_LEARNS, &res); return res; @@ -815,14 +817,16 @@ rspamd_sqlite3_get_stat (gpointer runtime, ucl_object_t *res = NULL; struct rspamd_stat_sqlite3_rt *rt = runtime; struct rspamd_stat_sqlite3_db *bk; + rspamd_mempool_t *pool; struct stat st; gint64 rev; g_assert (rt != NULL); bk = rt->db; + pool = rt->ctx->pool; (void)stat (bk->fname, &st); - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_GET_LEARNS, &rev); res = ucl_object_typed_new (UCL_OBJECT); @@ -830,7 +834,7 @@ rspamd_sqlite3_get_stat (gpointer runtime, 0, false); ucl_object_insert_key (res, ucl_object_fromint (st.st_size), "size", 0, false); - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_NTOKENS, &rev); ucl_object_insert_key (res, ucl_object_fromint (rev), "total", 0, false); ucl_object_insert_key (res, ucl_object_fromint (rev), "used", 0, false); @@ -838,11 +842,11 @@ rspamd_sqlite3_get_stat (gpointer runtime, "symbol", 0, false); ucl_object_insert_key (res, ucl_object_fromstring ("sqlite3"), "type", 0, false); - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_NLANGUAGES, &rev); ucl_object_insert_key (res, ucl_object_fromint (rev), "languages", 0, false); - rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_NUSERS, &rev); ucl_object_insert_key (res, ucl_object_fromint (rev), "users", 0, false); @@ -867,7 +871,7 @@ rspamd_sqlite3_load_tokenizer_config (gpointer runtime, g_assert (rt != NULL); bk = rt->db; - g_assert (rspamd_sqlite3_run_prstmt (bk->sqlite, bk->prstmt, + g_assert (rspamd_sqlite3_run_prstmt (rt->ctx->pool, bk->sqlite, bk->prstmt, RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, &sz, &tk_conf) == SQLITE_OK); g_assert (sz > 0); copied_conf = rspamd_mempool_alloc (rt->task->task_pool, sz); diff --git a/src/libstat/learn_cache/sqlite3_cache.c b/src/libstat/learn_cache/sqlite3_cache.c index 0b2aa5ed8..745b8839c 100644 --- a/src/libstat/learn_cache/sqlite3_cache.c +++ b/src/libstat/learn_cache/sqlite3_cache.c @@ -164,7 +164,8 @@ rspamd_stat_cache_sqlite3_init(struct rspamd_stat_ctx *ctx, } if (has_sqlite_cache) { - sqlite = rspamd_sqlite3_open_or_create (dbpath, create_tables_sql, &err); + sqlite = rspamd_sqlite3_open_or_create (cfg->cfg_pool, + dbpath, create_tables_sql, &err); if (sqlite == NULL) { msg_err_config ("cannot open sqlite3 cache: %e", err); @@ -192,17 +193,18 @@ rspamd_stat_cache_sqlite3_init(struct rspamd_stat_ctx *ctx, } static rspamd_learn_t -rspamd_stat_cache_sqlite3_check (const guchar *h, gsize len, gboolean is_spam, +rspamd_stat_cache_sqlite3_check (rspamd_mempool_t *pool, + const guchar *h, gsize len, gboolean is_spam, struct rspamd_stat_sqlite3_ctx *ctx) { gint rc, ret = RSPAMD_LEARN_OK; gint64 flag; - rspamd_sqlite3_run_prstmt (ctx->db, ctx->prstmt, + rspamd_sqlite3_run_prstmt (pool, ctx->db, ctx->prstmt, RSPAMD_STAT_CACHE_TRANSACTION_START_DEF); - rc = rspamd_sqlite3_run_prstmt (ctx->db, ctx->prstmt, + rc = rspamd_sqlite3_run_prstmt (pool, ctx->db, ctx->prstmt, RSPAMD_STAT_CACHE_GET_LEARN, (gint64)len, h, &flag); - rspamd_sqlite3_run_prstmt (ctx->db, ctx->prstmt, + rspamd_sqlite3_run_prstmt (pool, ctx->db, ctx->prstmt, RSPAMD_STAT_CACHE_TRANSACTION_COMMIT); if (rc == SQLITE_OK) { @@ -214,11 +216,11 @@ rspamd_stat_cache_sqlite3_check (const guchar *h, gsize len, gboolean is_spam, else { /* Need to relearn */ flag = is_spam ? 1 : 0; - rspamd_sqlite3_run_prstmt (ctx->db, ctx->prstmt, + rspamd_sqlite3_run_prstmt (pool, ctx->db, ctx->prstmt, RSPAMD_STAT_CACHE_TRANSACTION_START_IM); - rspamd_sqlite3_run_prstmt (ctx->db, ctx->prstmt, + rspamd_sqlite3_run_prstmt (pool, ctx->db, ctx->prstmt, RSPAMD_STAT_CACHE_UPDATE_LEARN, (gint64)len, h, flag); - rspamd_sqlite3_run_prstmt (ctx->db, ctx->prstmt, + rspamd_sqlite3_run_prstmt (pool, ctx->db, ctx->prstmt, RSPAMD_STAT_CACHE_TRANSACTION_COMMIT); return RSPAMD_LEARN_UNLEARN; @@ -227,11 +229,11 @@ rspamd_stat_cache_sqlite3_check (const guchar *h, gsize len, gboolean is_spam, else { /* Insert result new id */ flag = is_spam ? 1 : 0; - rspamd_sqlite3_run_prstmt (ctx->db, ctx->prstmt, + rspamd_sqlite3_run_prstmt (pool, ctx->db, ctx->prstmt, RSPAMD_STAT_CACHE_TRANSACTION_START_IM); - rspamd_sqlite3_run_prstmt (ctx->db, ctx->prstmt, + rspamd_sqlite3_run_prstmt (pool, ctx->db, ctx->prstmt, RSPAMD_STAT_CACHE_ADD_LEARN, (gint64)len, h, flag); - rspamd_sqlite3_run_prstmt (ctx->db, ctx->prstmt, + rspamd_sqlite3_run_prstmt (pool, ctx->db, ctx->prstmt, RSPAMD_STAT_CACHE_TRANSACTION_COMMIT); } @@ -265,7 +267,8 @@ rspamd_stat_cache_sqlite3_process (struct rspamd_task *task, blake2b_final (&st, out, sizeof (out)); - return rspamd_stat_cache_sqlite3_check (out, sizeof (out), is_spam, ctx); + return rspamd_stat_cache_sqlite3_check (task->task_pool, + out, sizeof (out), is_spam, ctx); } return RSPAMD_LEARN_OK; diff --git a/src/libutil/sqlite_utils.c b/src/libutil/sqlite_utils.c index 8bc86d331..18e3710ba 100644 --- a/src/libutil/sqlite_utils.c +++ b/src/libutil/sqlite_utils.c @@ -66,7 +66,7 @@ rspamd_sqlite3_init_prstmt (sqlite3 *db, } int -rspamd_sqlite3_run_prstmt (sqlite3 *db, GArray *stmts, +rspamd_sqlite3_run_prstmt (rspamd_mempool_t *pool, sqlite3 *db, GArray *stmts, gint idx, ...) { gint retcode; @@ -88,7 +88,7 @@ rspamd_sqlite3_run_prstmt (sqlite3 *db, GArray *stmts, g_assert (nst != NULL); - msg_debug ("executing `%s`", nst->sql); + msg_debug_pool ("executing `%s`", nst->sql); argtypes = nst->args; sqlite3_reset (stmt); va_start (ap, idx); @@ -176,7 +176,7 @@ rspamd_sqlite3_run_prstmt (sqlite3 *db, GArray *stmts, return SQLITE_OK; } else if (retcode != SQLITE_DONE) { - msg_debug ("failed to execute query %s: %d, %s", nst->sql, + msg_debug_pool ("failed to execute query %s: %d, %s", nst->sql, retcode, sqlite3_errmsg (db)); } @@ -206,7 +206,7 @@ rspamd_sqlite3_close_prstmt (sqlite3 *db, GArray *stmts) } static gboolean -rspamd_sqlite3_wait (const gchar *lock) +rspamd_sqlite3_wait (rspamd_mempool_t *pool, const gchar *lock) { gint fd; struct timespec sleep_ts = { @@ -217,7 +217,7 @@ rspamd_sqlite3_wait (const gchar *lock) fd = open (lock, O_RDONLY); if (fd == -1) { - msg_err ("cannot open lock file %s: %s", lock, strerror (errno)); + msg_err_pool ("cannot open lock file %s: %s", lock, strerror (errno)); return FALSE; } @@ -225,7 +225,7 @@ rspamd_sqlite3_wait (const gchar *lock) while (!rspamd_file_lock (fd, TRUE)) { if (nanosleep (&sleep_ts, NULL) == -1 && errno != EINTR) { close (fd); - msg_err ("cannot sleep open lock file %s: %s", lock, strerror (errno)); + msg_err_pool ("cannot sleep open lock file %s: %s", lock, strerror (errno)); return FALSE; } @@ -241,7 +241,7 @@ rspamd_sqlite3_wait (const gchar *lock) sqlite3 * -rspamd_sqlite3_open_or_create (const gchar *path, const +rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const gchar *create_sql, GError **err) { sqlite3 *sqlite; @@ -282,9 +282,9 @@ rspamd_sqlite3_open_or_create (const gchar *path, const lock_fd = open (lock_path, O_WRONLY|O_CREAT|O_EXCL, 00600); if (lock_fd == -1 && (errno == EEXIST || errno == EBUSY)) { - msg_debug ("checking %s to wait for db being initialized", lock_path); + msg_debug_pool ("checking %s to wait for db being initialized", lock_path); - if (!rspamd_sqlite3_wait (lock_path)) { + if (!rspamd_sqlite3_wait (pool, lock_path)) { g_set_error (err, rspamd_sqlite3_quark (), errno, "cannot create sqlite file %s: %s", path, strerror (errno)); @@ -297,7 +297,7 @@ rspamd_sqlite3_open_or_create (const gchar *path, const has_lock = FALSE; } else { - msg_debug ("locking %s to block other processes", lock_path); + msg_debug_pool ("locking %s to block other processes", lock_path); g_assert (rspamd_file_lock (lock_fd, FALSE)); has_lock = TRUE; @@ -320,12 +320,12 @@ rspamd_sqlite3_open_or_create (const gchar *path, const if (create) { if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) { - msg_warn ("WAL mode is not supported (%s), locking issues might occur", + msg_warn_pool ("WAL mode is not supported (%s), locking issues might occur", sqlite3_errmsg (sqlite)); } if (sqlite3_exec (sqlite, exclusive_lock_sql, NULL, NULL, NULL) != SQLITE_OK) { - msg_warn ("cannot exclusively lock database to create schema: %s", + msg_warn_pool ("cannot exclusively lock database to create schema: %s", sqlite3_errmsg (sqlite)); } @@ -345,7 +345,7 @@ rspamd_sqlite3_open_or_create (const gchar *path, const /* Reopen in normal mode */ - msg_debug ("reopening %s in normal mode", path); + msg_debug_pool ("reopening %s in normal mode", path); flags &= ~SQLITE_OPEN_CREATE; if ((rc = sqlite3_open_v2 (path, &sqlite, @@ -367,12 +367,12 @@ rspamd_sqlite3_open_or_create (const gchar *path, const } if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) { - msg_warn ("WAL mode is not supported (%s), locking issues might occur", + msg_warn_pool ("WAL mode is not supported (%s), locking issues might occur", sqlite3_errmsg (sqlite)); } if (has_lock) { - msg_debug ("removing lock from %s", lock_path); + msg_debug_pool ("removing lock from %s", lock_path); rspamd_file_unlock (lock_fd, FALSE); unlink (lock_path); close (lock_fd); diff --git a/src/libutil/sqlite_utils.h b/src/libutil/sqlite_utils.h index 3a8f81b18..fba1b0091 100644 --- a/src/libutil/sqlite_utils.h +++ b/src/libutil/sqlite_utils.h @@ -27,6 +27,7 @@ #define SRC_LIBUTIL_SQLITE_UTILS_H_ #include "config.h" +#include "mem_pool.h" #include "sqlite3.h" #define RSPAMD_SQLITE3_STMT_MULTIPLE (1 << 0) @@ -61,7 +62,7 @@ GArray* rspamd_sqlite3_init_prstmt (sqlite3 *db, * @param idx * @return */ -gint rspamd_sqlite3_run_prstmt (sqlite3 *db, GArray *stmts, +gint rspamd_sqlite3_run_prstmt (rspamd_mempool_t *pool, sqlite3 *db, GArray *stmts, gint idx, ...); /** @@ -77,7 +78,7 @@ void rspamd_sqlite3_close_prstmt (sqlite3 *db, GArray *stmts); * @param create_sql * @return */ -sqlite3 * rspamd_sqlite3_open_or_create (const gchar *path, const +sqlite3 * rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const gchar *create_sql, GError **err); #endif /* SRC_LIBUTIL_SQLITE_UTILS_H_ */ -- 2.39.5