diff options
author | Vsevolod Stakhov <vsevolod@rspamd.com> | 2023-07-26 10:49:23 +0100 |
---|---|---|
committer | Vsevolod Stakhov <vsevolod@rspamd.com> | 2023-07-26 10:49:23 +0100 |
commit | 537a7180a0d5132c11636c4fd8b1450cd99d352c (patch) | |
tree | fb9f8c84955a411bdffbd6371ea32f2716fb3687 /src/libstat/backends | |
parent | 5fd7a90fdaa33f52c59bdb0ca84451e5c1e22365 (diff) | |
download | rspamd-537a7180a0d5132c11636c4fd8b1450cd99d352c.tar.gz rspamd-537a7180a0d5132c11636c4fd8b1450cd99d352c.zip |
[Rework] Use clang-format to unify formatting in all sources
No meaningful changes.
Diffstat (limited to 'src/libstat/backends')
-rw-r--r-- | src/libstat/backends/backends.h | 124 | ||||
-rw-r--r-- | src/libstat/backends/cdb_backend.cxx | 151 | ||||
-rw-r--r-- | src/libstat/backends/http_backend.cxx | 142 | ||||
-rw-r--r-- | src/libstat/backends/mmaped_file.c | 880 | ||||
-rw-r--r-- | src/libstat/backends/redis_backend.c | 1465 | ||||
-rw-r--r-- | src/libstat/backends/sqlite3_backend.c | 857 |
6 files changed, 1733 insertions, 1886 deletions
diff --git a/src/libstat/backends/backends.h b/src/libstat/backends/backends.h index 67d612383..4b16950bb 100644 --- a/src/libstat/backends/backends.h +++ b/src/libstat/backends/backends.h @@ -21,7 +21,7 @@ #define RSPAMD_DEFAULT_BACKEND "mmap" -#ifdef __cplusplus +#ifdef __cplusplus extern "C" { #endif @@ -38,81 +38,81 @@ struct rspamd_stat_backend { const char *name; bool read_only; - gpointer (*init) (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg, - struct rspamd_statfile *st); + gpointer (*init)(struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg, + struct rspamd_statfile *st); - gpointer (*runtime) (struct rspamd_task *task, - struct rspamd_statfile_config *stcf, - gboolean learn, gpointer ctx, - gint id); + gpointer (*runtime)(struct rspamd_task *task, + struct rspamd_statfile_config *stcf, + gboolean learn, gpointer ctx, + gint id); - gboolean (*process_tokens) (struct rspamd_task *task, GPtrArray *tokens, - gint id, - gpointer ctx); + gboolean (*process_tokens)(struct rspamd_task *task, GPtrArray *tokens, + gint id, + gpointer ctx); - gboolean (*finalize_process) (struct rspamd_task *task, - gpointer runtime, gpointer ctx); + gboolean (*finalize_process)(struct rspamd_task *task, + gpointer runtime, gpointer ctx); - gboolean (*learn_tokens) (struct rspamd_task *task, GPtrArray *tokens, - gint id, - gpointer ctx); + gboolean (*learn_tokens)(struct rspamd_task *task, GPtrArray *tokens, + gint id, + gpointer ctx); - gulong (*total_learns) (struct rspamd_task *task, - gpointer runtime, gpointer ctx); + gulong (*total_learns)(struct rspamd_task *task, + gpointer runtime, gpointer ctx); - gboolean (*finalize_learn) (struct rspamd_task *task, - gpointer runtime, gpointer ctx, GError **err); + gboolean (*finalize_learn)(struct rspamd_task *task, + gpointer runtime, gpointer ctx, GError **err); - gulong (*inc_learns) (struct rspamd_task *task, - gpointer runtime, gpointer ctx); + gulong (*inc_learns)(struct rspamd_task *task, + gpointer runtime, gpointer ctx); - gulong (*dec_learns) (struct rspamd_task *task, - gpointer runtime, gpointer ctx); + gulong (*dec_learns)(struct rspamd_task *task, + gpointer runtime, gpointer ctx); - ucl_object_t *(*get_stat) (gpointer runtime, gpointer ctx); + ucl_object_t *(*get_stat)(gpointer runtime, gpointer ctx); - void (*close) (gpointer ctx); + void (*close)(gpointer ctx); - gpointer (*load_tokenizer_config) (gpointer runtime, gsize *sz); + gpointer (*load_tokenizer_config)(gpointer runtime, gsize *sz); gpointer ctx; }; -#define RSPAMD_STAT_BACKEND_DEF(name) \ - gpointer rspamd_##name##_init (struct rspamd_stat_ctx *ctx, \ - struct rspamd_config *cfg, struct rspamd_statfile *st); \ - gpointer rspamd_##name##_runtime (struct rspamd_task *task, \ - struct rspamd_statfile_config *stcf, \ - gboolean learn, gpointer ctx, gint id); \ - gboolean rspamd_##name##_process_tokens (struct rspamd_task *task, \ - GPtrArray *tokens, gint id, \ - gpointer runtime); \ - gboolean rspamd_##name##_finalize_process (struct rspamd_task *task, \ - gpointer runtime, \ - gpointer ctx); \ - gboolean rspamd_##name##_learn_tokens (struct rspamd_task *task, \ - GPtrArray *tokens, gint id, \ - gpointer runtime); \ - gboolean rspamd_##name##_finalize_learn (struct rspamd_task *task, \ - gpointer runtime, \ - gpointer ctx, GError **err); \ - gulong rspamd_##name##_total_learns (struct rspamd_task *task, \ - gpointer runtime, \ - gpointer ctx); \ - gulong rspamd_##name##_inc_learns (struct rspamd_task *task, \ - gpointer runtime, \ - gpointer ctx); \ - gulong rspamd_##name##_dec_learns (struct rspamd_task *task, \ - gpointer runtime, \ - gpointer ctx); \ - gulong rspamd_##name##_learns (struct rspamd_task *task, \ - gpointer runtime, \ - gpointer ctx); \ - ucl_object_t * rspamd_##name##_get_stat (gpointer runtime, \ - gpointer ctx); \ - gpointer rspamd_##name##_load_tokenizer_config (gpointer runtime, \ - gsize *len); \ - void rspamd_##name##_close (gpointer ctx) +#define RSPAMD_STAT_BACKEND_DEF(name) \ + gpointer rspamd_##name##_init(struct rspamd_stat_ctx *ctx, \ + struct rspamd_config *cfg, struct rspamd_statfile *st); \ + gpointer rspamd_##name##_runtime(struct rspamd_task *task, \ + struct rspamd_statfile_config *stcf, \ + gboolean learn, gpointer ctx, gint id); \ + gboolean rspamd_##name##_process_tokens(struct rspamd_task *task, \ + GPtrArray *tokens, gint id, \ + gpointer runtime); \ + gboolean rspamd_##name##_finalize_process(struct rspamd_task *task, \ + gpointer runtime, \ + gpointer ctx); \ + gboolean rspamd_##name##_learn_tokens(struct rspamd_task *task, \ + GPtrArray *tokens, gint id, \ + gpointer runtime); \ + gboolean rspamd_##name##_finalize_learn(struct rspamd_task *task, \ + gpointer runtime, \ + gpointer ctx, GError **err); \ + gulong rspamd_##name##_total_learns(struct rspamd_task *task, \ + gpointer runtime, \ + gpointer ctx); \ + gulong rspamd_##name##_inc_learns(struct rspamd_task *task, \ + gpointer runtime, \ + gpointer ctx); \ + gulong rspamd_##name##_dec_learns(struct rspamd_task *task, \ + gpointer runtime, \ + gpointer ctx); \ + gulong rspamd_##name##_learns(struct rspamd_task *task, \ + gpointer runtime, \ + gpointer ctx); \ + ucl_object_t *rspamd_##name##_get_stat(gpointer runtime, \ + gpointer ctx); \ + gpointer rspamd_##name##_load_tokenizer_config(gpointer runtime, \ + gsize *len); \ + void rspamd_##name##_close(gpointer ctx) RSPAMD_STAT_BACKEND_DEF(mmaped_file); RSPAMD_STAT_BACKEND_DEF(sqlite3); @@ -120,7 +120,7 @@ RSPAMD_STAT_BACKEND_DEF(cdb); RSPAMD_STAT_BACKEND_DEF(redis); RSPAMD_STAT_BACKEND_DEF(http); -#ifdef __cplusplus +#ifdef __cplusplus } #endif diff --git a/src/libstat/backends/cdb_backend.cxx b/src/libstat/backends/cdb_backend.cxx index 22a6c3dab..81d87f319 100644 --- a/src/libstat/backends/cdb_backend.cxx +++ b/src/libstat/backends/cdb_backend.cxx @@ -41,7 +41,8 @@ public: using cdb_element_t = std::shared_ptr<struct cdb>; cdb_shared_storage() = default; - auto get_cdb(const char *path) const -> std::optional<cdb_element_t> { + auto get_cdb(const char *path) const -> std::optional<cdb_element_t> + { auto found = elts.find(path); if (found != elts.end()) { @@ -53,13 +54,15 @@ public: return std::nullopt; } /* Create a new smart pointer over POD cdb structure */ - static auto new_cdb() -> cdb_element_t { + static auto new_cdb() -> cdb_element_t + { auto ret = cdb_element_t(new struct cdb, cdb_deleter()); memset(ret.get(), 0, sizeof(struct cdb)); return ret; } /* Enclose cdb into storage */ - auto push_cdb(const char *path, cdb_element_t cdbp) -> cdb_element_t { + auto push_cdb(const char *path, cdb_element_t cdbp) -> cdb_element_t + { auto found = elts.find(path); if (found != elts.end()) { @@ -82,6 +85,7 @@ public: return cdbp; } } + private: /* * We store weak pointers here to allow owning cdb statfiles to free @@ -90,7 +94,8 @@ private: ankerl::unordered_dense::map<std::string, std::weak_ptr<struct cdb>> elts; struct cdb_deleter { - void operator()(struct cdb *c) const { + void operator()(struct cdb *c) const + { cdb_free(c); delete c; } @@ -102,13 +107,16 @@ static cdb_shared_storage cdb_shared_storage; class ro_backend final { public: explicit ro_backend(struct rspamd_statfile *_st, cdb_shared_storage::cdb_element_t _db) - : st(_st), db(std::move(_db)) {} + : st(_st), db(std::move(_db)) + { + } ro_backend() = delete; ro_backend(const ro_backend &) = delete; - ro_backend(ro_backend &&other) noexcept { + ro_backend(ro_backend &&other) noexcept + { *this = std::move(other); } - ro_backend& operator=(ro_backend &&other) noexcept + ro_backend &operator=(ro_backend &&other) noexcept { std::swap(st, other.st); std::swap(db, other.db); @@ -118,14 +126,18 @@ public: return *this; } - ~ro_backend() {} + ~ro_backend() + { + } auto load_cdb() -> tl::expected<bool, std::string>; auto process_token(const rspamd_token_t *tok) const -> std::optional<float>; - constexpr auto is_spam() const -> bool { + constexpr auto is_spam() const -> bool + { return st->stcf->is_spam; } - auto get_learns() const -> std::uint64_t { + auto get_learns() const -> std::uint64_t + { if (is_spam()) { return learns_spam; } @@ -133,9 +145,11 @@ public: return learns_ham; } } - auto get_total_learns() const -> std::uint64_t { + auto get_total_learns() const -> std::uint64_t + { return learns_spam + learns_ham; } + private: struct rspamd_statfile *st; cdb_shared_storage::cdb_element_t db; @@ -148,7 +162,7 @@ template<typename T> static inline auto cdb_get_key_as_int64(struct cdb *cdb, T key) -> std::optional<std::int64_t> { - auto pos = cdb_find(cdb, (void *)&key, sizeof(key)); + auto pos = cdb_find(cdb, (void *) &key, sizeof(key)); if (pos > 0) { auto vpos = cdb_datapos(cdb); @@ -156,7 +170,7 @@ cdb_get_key_as_int64(struct cdb *cdb, T key) -> std::optional<std::int64_t> if (vlen == sizeof(std::int64_t)) { std::int64_t ret; - cdb_read(cdb, (void *)&ret, vlen, vpos); + cdb_read(cdb, (void *) &ret, vlen, vpos); return ret; } @@ -169,7 +183,7 @@ template<typename T> static inline auto cdb_get_key_as_float_pair(struct cdb *cdb, T key) -> std::optional<std::pair<float, float>> { - auto pos = cdb_find(cdb, (void *)&key, sizeof(key)); + auto pos = cdb_find(cdb, (void *) &key, sizeof(key)); if (pos > 0) { auto vpos = cdb_datapos(cdb); @@ -183,7 +197,7 @@ cdb_get_key_as_float_pair(struct cdb *cdb, T key) -> std::optional<std::pair<flo } d; char c[sizeof(float) * 2]; } u; - cdb_read(cdb, (void *)u.c, vlen, vpos); + cdb_read(cdb, (void *) u.c, vlen, vpos); return std::make_pair(u.d.v1, u.d.v2); } @@ -193,8 +207,7 @@ cdb_get_key_as_float_pair(struct cdb *cdb, T key) -> std::optional<std::pair<flo } -auto -ro_backend::load_cdb() -> tl::expected<bool, std::string> +auto ro_backend::load_cdb() -> tl::expected<bool, std::string> { if (!db) { return tl::make_unexpected("no database loaded"); @@ -205,7 +218,7 @@ ro_backend::load_cdb() -> tl::expected<bool, std::string> static const char learn_spam_key[9] = "_lrnspam", learn_ham_key[9] = "_lrnham_"; auto check_key = [&](const char *key, std::uint64_t &target) -> tl::expected<bool, std::string> { - memcpy((void *)&cdb_key, key, sizeof(cdb_key)); + memcpy((void *) &cdb_key, key, sizeof(cdb_key)); auto maybe_value = cdb_get_key_as_int64(db.get(), cdb_key); @@ -213,7 +226,7 @@ ro_backend::load_cdb() -> tl::expected<bool, std::string> return tl::make_unexpected(fmt::format("missing {} key", key)); } - target = (std::uint64_t)maybe_value.value(); + target = (std::uint64_t) maybe_value.value(); return true; }; @@ -232,11 +245,10 @@ ro_backend::load_cdb() -> tl::expected<bool, std::string> loaded = true; - return true; // expected + return true;// expected } -auto -ro_backend::process_token(const rspamd_token_t *tok) const -> std::optional<float> +auto ro_backend::process_token(const rspamd_token_t *tok) const -> std::optional<float> { if (!loaded) { return std::nullopt; @@ -258,15 +270,14 @@ ro_backend::process_token(const rspamd_token_t *tok) const -> std::optional<floa return std::nullopt; } -auto -open_cdb(struct rspamd_statfile *st) -> tl::expected<ro_backend, std::string> +auto open_cdb(struct rspamd_statfile *st) -> tl::expected<ro_backend, std::string> { const char *path = nullptr; const auto *stf = st->stcf; auto get_filename = [](const ucl_object_t *obj) -> const char * { const auto *filename = ucl_object_lookup_any(obj, - "filename", "path", "cdb", nullptr); + "filename", "path", "cdb", nullptr); if (filename && ucl_object_type(filename) == UCL_STRING) { return ucl_object_tostring(filename); @@ -276,8 +287,8 @@ open_cdb(struct rspamd_statfile *st) -> tl::expected<ro_backend, std::string> }; /* First search in backend configuration */ - const auto *obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); - if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) { + const auto *obj = ucl_object_lookup(st->classifier->cfg->opts, "backend"); + if (obj != NULL && ucl_object_type(obj) == UCL_OBJECT) { path = get_filename(obj); } @@ -304,7 +315,7 @@ open_cdb(struct rspamd_statfile *st) -> tl::expected<ro_backend, std::string> if (fd == -1) { return tl::make_unexpected(fmt::format("cannot open {}: {}", - path, strerror(errno))); + path, strerror(errno))); } cdbp = cdb_shared_storage::new_cdb(); @@ -313,7 +324,7 @@ open_cdb(struct rspamd_statfile *st) -> tl::expected<ro_backend, std::string> close(fd); return tl::make_unexpected(fmt::format("cannot init cdb in {}: {}", - path, strerror(errno))); + path, strerror(errno))); } cdbp = cdb_shared_storage.push_cdb(path, cdbp); @@ -326,7 +337,7 @@ open_cdb(struct rspamd_statfile *st) -> tl::expected<ro_backend, std::string> if (!cdbp) { return tl::make_unexpected(fmt::format("cannot init cdb in {}: internal error", - path)); + path)); } ro_backend bk{st, std::move(cdbp)}; @@ -340,15 +351,15 @@ open_cdb(struct rspamd_statfile *st) -> tl::expected<ro_backend, std::string> return bk; } -} +}// namespace rspamd::stat::cdb #define CDB_FROM_RAW(p) (reinterpret_cast<rspamd::stat::cdb::ro_backend *>(p)) /* C exports */ gpointer -rspamd_cdb_init(struct rspamd_stat_ctx* ctx, - struct rspamd_config* cfg, - struct rspamd_statfile* st) +rspamd_cdb_init(struct rspamd_stat_ctx *ctx, + struct rspamd_config *cfg, + struct rspamd_statfile *st) { auto maybe_backend = rspamd::stat::cdb::open_cdb(st); @@ -365,21 +376,21 @@ rspamd_cdb_init(struct rspamd_stat_ctx* ctx, return nullptr; } gpointer -rspamd_cdb_runtime(struct rspamd_task* task, - struct rspamd_statfile_config* stcf, - gboolean learn, - gpointer ctx, - gint _id) +rspamd_cdb_runtime(struct rspamd_task *task, + struct rspamd_statfile_config *stcf, + gboolean learn, + gpointer ctx, + gint _id) { /* In CDB we don't have any dynamic stuff */ return ctx; } gboolean -rspamd_cdb_process_tokens(struct rspamd_task* task, - GPtrArray* tokens, - gint id, - gpointer runtime) +rspamd_cdb_process_tokens(struct rspamd_task *task, + GPtrArray *tokens, + gint id, + gpointer runtime) { auto *cdbp = CDB_FROM_RAW(runtime); bool seen_values = false; @@ -409,33 +420,32 @@ rspamd_cdb_process_tokens(struct rspamd_task* task, } return true; - } gboolean -rspamd_cdb_finalize_process(struct rspamd_task* task, - gpointer runtime, - gpointer ctx) +rspamd_cdb_finalize_process(struct rspamd_task *task, + gpointer runtime, + gpointer ctx) { return true; } gboolean -rspamd_cdb_learn_tokens(struct rspamd_task* task, - GPtrArray* tokens, - gint id, - gpointer ctx) +rspamd_cdb_learn_tokens(struct rspamd_task *task, + GPtrArray *tokens, + gint id, + gpointer ctx) { return false; } gboolean -rspamd_cdb_finalize_learn(struct rspamd_task* task, - gpointer runtime, - gpointer ctx, - GError** err) +rspamd_cdb_finalize_learn(struct rspamd_task *task, + gpointer runtime, + gpointer ctx, + GError **err) { return false; } -gulong rspamd_cdb_total_learns(struct rspamd_task* task, +gulong rspamd_cdb_total_learns(struct rspamd_task *task, gpointer runtime, gpointer ctx) { @@ -443,39 +453,38 @@ gulong rspamd_cdb_total_learns(struct rspamd_task* task, return cdbp->get_total_learns(); } gulong -rspamd_cdb_inc_learns(struct rspamd_task* task, - gpointer runtime, - gpointer ctx) +rspamd_cdb_inc_learns(struct rspamd_task *task, + gpointer runtime, + gpointer ctx) { - return (gulong)-1; + return (gulong) -1; } gulong -rspamd_cdb_dec_learns(struct rspamd_task* task, - gpointer runtime, - gpointer ctx) +rspamd_cdb_dec_learns(struct rspamd_task *task, + gpointer runtime, + gpointer ctx) { - return (gulong)-1; + return (gulong) -1; } gulong -rspamd_cdb_learns(struct rspamd_task* task, - gpointer runtime, - gpointer ctx) +rspamd_cdb_learns(struct rspamd_task *task, + gpointer runtime, + gpointer ctx) { auto *cdbp = CDB_FROM_RAW(ctx); return cdbp->get_learns(); } -ucl_object_t* +ucl_object_t * rspamd_cdb_get_stat(gpointer runtime, gpointer ctx) { return nullptr; } gpointer -rspamd_cdb_load_tokenizer_config(gpointer runtime, gsize* len) +rspamd_cdb_load_tokenizer_config(gpointer runtime, gsize *len) { return nullptr; } -void -rspamd_cdb_close(gpointer ctx) +void rspamd_cdb_close(gpointer ctx) { auto *cdbp = CDB_FROM_RAW(ctx); delete cdbp; diff --git a/src/libstat/backends/http_backend.cxx b/src/libstat/backends/http_backend.cxx index 69c735a09..3b2e3efe0 100644 --- a/src/libstat/backends/http_backend.cxx +++ b/src/libstat/backends/http_backend.cxx @@ -24,10 +24,10 @@ namespace rspamd::stat::http { -#define msg_debug_stat_http(...) rspamd_conditional_debug_fast (NULL, NULL, \ - rspamd_stat_http_log_id, "stat_http", task->task_pool->tag.uid, \ - RSPAMD_LOG_FUNC, \ - __VA_ARGS__) +#define msg_debug_stat_http(...) rspamd_conditional_debug_fast(NULL, NULL, \ + rspamd_stat_http_log_id, "stat_http", task->task_pool->tag.uid, \ + RSPAMD_LOG_FUNC, \ + __VA_ARGS__) INIT_LOG_MODULE(stat_http) @@ -37,8 +37,10 @@ class http_backends_collection { double timeout = 1.0; /* Default timeout */ struct upstream_list *read_servers = nullptr; struct upstream_list *write_servers = nullptr; + public: - static auto get() -> http_backends_collection& { + static auto get() -> http_backends_collection & + { static http_backends_collection *singleton = nullptr; if (singleton == nullptr) { @@ -81,26 +83,31 @@ class http_backend_runtime final { public: static auto create(struct rspamd_task *task, bool is_learn) -> http_backend_runtime *; /* Add a new statfile with a specific id to the list of statfiles */ - auto notice_statfile(int id, const struct rspamd_statfile_config *st) -> void { + auto notice_statfile(int id, const struct rspamd_statfile_config *st) -> void + { seen_statfiles[id] = st; } - auto process_tokens(struct rspamd_task* task, - GPtrArray* tokens, + auto process_tokens(struct rspamd_task *task, + GPtrArray *tokens, gint id, bool learn) -> bool; + private: http_backends_collection *all_backends; ankerl::unordered_dense::map<int, const struct rspamd_statfile_config *> seen_statfiles; struct upstream *selected; + private: - http_backend_runtime(struct rspamd_task *task, bool is_learn) : - all_backends(&http_backends_collection::get()) { + http_backend_runtime(struct rspamd_task *task, bool is_learn) + : all_backends(&http_backends_collection::get()) + { selected = all_backends->get_upstream(is_learn); } ~http_backend_runtime() = default; - static auto dtor(void *p) -> void { - ((http_backend_runtime *)p)->~http_backend_runtime(); + static auto dtor(void *p) -> void + { + ((http_backend_runtime *) p)->~http_backend_runtime(); } }; @@ -126,14 +133,15 @@ stat_tokens_to_msgpack(GPtrArray *tokens) -> std::vector<std::uint8_t> ret.resize(tokens->len * (sizeof(std::uint64_t) + 1) + 5); ret.push_back('\xdd'); std::uint32_t ulen = GUINT32_TO_BE(tokens->len); - std::copy((const std::uint8_t *)&ulen, - ((const std::uint8_t *)&ulen) + sizeof(ulen), std::back_inserter(ret)); + std::copy((const std::uint8_t *) &ulen, + ((const std::uint8_t *) &ulen) + sizeof(ulen), std::back_inserter(ret)); - PTR_ARRAY_FOREACH(tokens, i, cur) { + PTR_ARRAY_FOREACH(tokens, i, cur) + { ret.push_back('\xcf'); std::uint64_t val = GUINT64_TO_BE(cur->data); - std::copy((const std::uint8_t *)&val, - ((const std::uint8_t *)&val) + sizeof(val), std::back_inserter(ret)); + std::copy((const std::uint8_t *) &val, + ((const std::uint8_t *) &val) + sizeof(val), std::back_inserter(ret)); } return ret; @@ -149,8 +157,7 @@ auto http_backend_runtime::create(struct rspamd_task *task, bool is_learn) -> ht return new (allocated_runtime) http_backend_runtime{task, is_learn}; } -auto -http_backend_runtime::process_tokens(struct rspamd_task *task, GPtrArray *tokens, gint id, bool learn) -> bool +auto http_backend_runtime::process_tokens(struct rspamd_task *task, GPtrArray *tokens, gint id, bool learn) -> bool { if (!learn) { if (id == seen_statfiles.size() - 1) { @@ -171,10 +178,9 @@ http_backend_runtime::process_tokens(struct rspamd_task *task, GPtrArray *tokens return true; } -auto -http_backends_collection::add_backend(struct rspamd_stat_ctx *ctx, - struct rspamd_config *cfg, - struct rspamd_statfile *st) -> bool +auto http_backends_collection::add_backend(struct rspamd_stat_ctx *ctx, + struct rspamd_config *cfg, + struct rspamd_statfile *st) -> bool { /* On empty list of backends we know that we need to load backend data actually */ if (backends.empty()) { @@ -235,7 +241,7 @@ auto http_backends_collection::first_init(struct rspamd_stat_ctx *ctx, }; auto ret = false; - auto obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); + auto obj = ucl_object_lookup(st->classifier->cfg->opts, "backend"); if (obj != nullptr) { ret = try_load_backend_config(obj); } @@ -291,14 +297,14 @@ upstream *http_backends_collection::get_upstream(bool is_learn) return rspamd_upstream_get(ups_list, RSPAMD_UPSTREAM_ROUND_ROBIN, nullptr, 0); } -} +}// namespace rspamd::stat::http /* C API */ gpointer -rspamd_http_init(struct rspamd_stat_ctx* ctx, - struct rspamd_config* cfg, - struct rspamd_statfile* st) +rspamd_http_init(struct rspamd_stat_ctx *ctx, + struct rspamd_config *cfg, + struct rspamd_statfile *st) { auto &collections = rspamd::stat::http::http_backends_collection::get(); @@ -308,11 +314,11 @@ rspamd_http_init(struct rspamd_stat_ctx* ctx, return nullptr; } - return (void *)&collections; + return (void *) &collections; } gpointer -rspamd_http_runtime(struct rspamd_task* task, - struct rspamd_statfile_config* stcf, +rspamd_http_runtime(struct rspamd_task *task, + struct rspamd_statfile_config *stcf, gboolean learn, gpointer ctx, gint id) @@ -320,7 +326,7 @@ rspamd_http_runtime(struct rspamd_task* task, auto maybe_existing = rspamd_mempool_get_variable(task->task_pool, RSPAMD_MEMPOOL_HTTP_STAT_BACKEND_RUNTIME); if (maybe_existing != nullptr) { - auto real_runtime = (rspamd::stat::http::http_backend_runtime *)maybe_existing; + auto real_runtime = (rspamd::stat::http::http_backend_runtime *) maybe_existing; real_runtime->notice_statfile(id, stcf); return maybe_existing; @@ -331,19 +337,19 @@ rspamd_http_runtime(struct rspamd_task* task, if (runtime) { runtime->notice_statfile(id, stcf); rspamd_mempool_set_variable(task->task_pool, RSPAMD_MEMPOOL_HTTP_STAT_BACKEND_RUNTIME, - (void *)runtime, nullptr); + (void *) runtime, nullptr); } - return (void *)runtime; + return (void *) runtime; } gboolean -rspamd_http_process_tokens(struct rspamd_task* task, - GPtrArray* tokens, - gint id, - gpointer runtime) +rspamd_http_process_tokens(struct rspamd_task *task, + GPtrArray *tokens, + gint id, + gpointer runtime) { - auto real_runtime = (rspamd::stat::http::http_backend_runtime *)runtime; + auto real_runtime = (rspamd::stat::http::http_backend_runtime *) runtime; if (real_runtime) { return real_runtime->process_tokens(task, tokens, id, false); @@ -351,24 +357,23 @@ rspamd_http_process_tokens(struct rspamd_task* task, return false; - } gboolean -rspamd_http_finalize_process(struct rspamd_task* task, - gpointer runtime, - gpointer ctx) +rspamd_http_finalize_process(struct rspamd_task *task, + gpointer runtime, + gpointer ctx) { /* Not needed */ return true; } gboolean -rspamd_http_learn_tokens(struct rspamd_task* task, - GPtrArray* tokens, - gint id, - gpointer runtime) +rspamd_http_learn_tokens(struct rspamd_task *task, + GPtrArray *tokens, + gint id, + gpointer runtime) { - auto real_runtime = (rspamd::stat::http::http_backend_runtime *)runtime; + auto real_runtime = (rspamd::stat::http::http_backend_runtime *) runtime; if (real_runtime) { return real_runtime->process_tokens(task, tokens, id, true); @@ -378,58 +383,57 @@ rspamd_http_learn_tokens(struct rspamd_task* task, return false; } gboolean -rspamd_http_finalize_learn(struct rspamd_task* task, - gpointer runtime, - gpointer ctx, - GError** err) +rspamd_http_finalize_learn(struct rspamd_task *task, + gpointer runtime, + gpointer ctx, + GError **err) { return false; } -gulong rspamd_http_total_learns(struct rspamd_task* task, - gpointer runtime, - gpointer ctx) +gulong rspamd_http_total_learns(struct rspamd_task *task, + gpointer runtime, + gpointer ctx) { /* TODO */ return 0; } gulong -rspamd_http_inc_learns(struct rspamd_task* task, - gpointer runtime, - gpointer ctx) +rspamd_http_inc_learns(struct rspamd_task *task, + gpointer runtime, + gpointer ctx) { /* TODO */ return 0; } gulong -rspamd_http_dec_learns(struct rspamd_task* task, - gpointer runtime, - gpointer ctx) +rspamd_http_dec_learns(struct rspamd_task *task, + gpointer runtime, + gpointer ctx) { /* TODO */ - return (gulong)-1; + return (gulong) -1; } gulong -rspamd_http_learns(struct rspamd_task* task, - gpointer runtime, - gpointer ctx) +rspamd_http_learns(struct rspamd_task *task, + gpointer runtime, + gpointer ctx) { /* TODO */ return 0; } -ucl_object_t* +ucl_object_t * rspamd_http_get_stat(gpointer runtime, gpointer ctx) { /* TODO */ return nullptr; } gpointer -rspamd_http_load_tokenizer_config(gpointer runtime, gsize* len) +rspamd_http_load_tokenizer_config(gpointer runtime, gsize *len) { return nullptr; } -void -rspamd_http_close(gpointer ctx) +void rspamd_http_close(gpointer ctx) { /* TODO */ }
\ No newline at end of file diff --git a/src/libstat/backends/mmaped_file.c b/src/libstat/backends/mmaped_file.c index 046c024fb..5c2020787 100644 --- a/src/libstat/backends/mmaped_file.c +++ b/src/libstat/backends/mmaped_file.c @@ -26,42 +26,42 @@ * Common statfile header */ struct stat_file_header { - u_char magic[3]; /**< magic signature ('r' 's' 'd') */ - u_char version[2]; /**< version of statfile */ - u_char padding[3]; /**< padding */ - guint64 create_time; /**< create time (time_t->guint64) */ - guint64 revision; /**< revision number */ - guint64 rev_time; /**< revision time */ - guint64 used_blocks; /**< used blocks number */ - guint64 total_blocks; /**< total number of blocks */ - guint64 tokenizer_conf_len; /**< length of tokenizer configuration */ - u_char unused[231]; /**< some bytes that can be used in future */ + u_char magic[3]; /**< magic signature ('r' 's' 'd') */ + u_char version[2]; /**< version of statfile */ + u_char padding[3]; /**< padding */ + guint64 create_time; /**< create time (time_t->guint64) */ + guint64 revision; /**< revision number */ + guint64 rev_time; /**< revision time */ + guint64 used_blocks; /**< used blocks number */ + guint64 total_blocks; /**< total number of blocks */ + guint64 tokenizer_conf_len; /**< length of tokenizer configuration */ + u_char unused[231]; /**< some bytes that can be used in future */ }; /** * Section header */ struct stat_file_section { - guint64 code; /**< section's code */ - guint64 length; /**< section's length in blocks */ + guint64 code; /**< section's code */ + guint64 length; /**< section's length in blocks */ }; /** * Block of data in statfile */ struct stat_file_block { - guint32 hash1; /**< hash1 (also acts as index) */ - guint32 hash2; /**< hash2 */ - double value; /**< double value */ + guint32 hash1; /**< hash1 (also acts as index) */ + guint32 hash2; /**< hash2 */ + double value; /**< double value */ }; /** * Statistic file */ struct stat_file { - struct stat_file_header header; /**< header */ - struct stat_file_section section; /**< first section */ - struct stat_file_block blocks[1]; /**< first block of data */ + struct stat_file_header header; /**< header */ + struct stat_file_section section; /**< first section */ + struct stat_file_block blocks[1]; /**< first block of data */ }; /** @@ -69,40 +69,43 @@ struct stat_file { */ typedef struct { #ifdef HAVE_PATH_MAX - gchar filename[PATH_MAX]; /**< name of file */ + gchar filename[PATH_MAX]; /**< name of file */ #else - gchar filename[MAXPATHLEN]; /**< name of file */ + gchar filename[MAXPATHLEN]; /**< name of file */ #endif rspamd_mempool_t *pool; - gint fd; /**< descriptor */ - void *map; /**< mmaped area */ - off_t seek_pos; /**< current seek position */ - struct stat_file_section cur_section; /**< current section */ - size_t len; /**< length of file(in bytes) */ + gint fd; /**< descriptor */ + void *map; /**< mmaped area */ + off_t seek_pos; /**< current seek position */ + struct stat_file_section cur_section; /**< current section */ + size_t len; /**< length of file(in bytes) */ struct rspamd_statfile_config *cf; } rspamd_mmaped_file_t; -#define RSPAMD_STATFILE_VERSION {'1', '2'} +#define RSPAMD_STATFILE_VERSION \ + { \ + '1', '2' \ + } #define BACKUP_SUFFIX ".old" -static void rspamd_mmaped_file_set_block_common (rspamd_mempool_t *pool, - rspamd_mmaped_file_t *file, - guint32 h1, guint32 h2, double value); +static void rspamd_mmaped_file_set_block_common(rspamd_mempool_t *pool, + rspamd_mmaped_file_t *file, + guint32 h1, guint32 h2, double value); -rspamd_mmaped_file_t * rspamd_mmaped_file_open (rspamd_mempool_t *pool, - const gchar *filename, size_t size, - struct rspamd_statfile_config *stcf); -gint rspamd_mmaped_file_create (const gchar *filename, size_t size, - struct rspamd_statfile_config *stcf, - rspamd_mempool_t *pool); -gint rspamd_mmaped_file_close_file (rspamd_mempool_t *pool, - rspamd_mmaped_file_t * file); +rspamd_mmaped_file_t *rspamd_mmaped_file_open(rspamd_mempool_t *pool, + const gchar *filename, size_t size, + struct rspamd_statfile_config *stcf); +gint rspamd_mmaped_file_create(const gchar *filename, size_t size, + struct rspamd_statfile_config *stcf, + rspamd_mempool_t *pool); +gint rspamd_mmaped_file_close_file(rspamd_mempool_t *pool, + rspamd_mmaped_file_t *file); double -rspamd_mmaped_file_get_block (rspamd_mmaped_file_t * file, - guint32 h1, - guint32 h2) +rspamd_mmaped_file_get_block(rspamd_mmaped_file_t *file, + guint32 h1, + guint32 h2) { struct stat_file_block *block; guint i, blocknum; @@ -113,9 +116,8 @@ rspamd_mmaped_file_get_block (rspamd_mmaped_file_t * file, } blocknum = h1 % file->cur_section.length; - c = (u_char *) file->map + file->seek_pos + blocknum * - sizeof (struct stat_file_block); - block = (struct stat_file_block *)c; + c = (u_char *) file->map + file->seek_pos + blocknum * sizeof(struct stat_file_block); + block = (struct stat_file_block *) c; for (i = 0; i < CHAIN_LENGTH; i++) { if (i + blocknum >= file->cur_section.length) { @@ -124,8 +126,8 @@ rspamd_mmaped_file_get_block (rspamd_mmaped_file_t * file, if (block->hash1 == h1 && block->hash2 == h2) { return block->value; } - c += sizeof (struct stat_file_block); - block = (struct stat_file_block *)c; + c += sizeof(struct stat_file_block); + block = (struct stat_file_block *) c; } @@ -133,9 +135,9 @@ rspamd_mmaped_file_get_block (rspamd_mmaped_file_t * file, } static void -rspamd_mmaped_file_set_block_common (rspamd_mempool_t *pool, - rspamd_mmaped_file_t *file, - guint32 h1, guint32 h2, double value) +rspamd_mmaped_file_set_block_common(rspamd_mempool_t *pool, + rspamd_mmaped_file_t *file, + guint32 h1, guint32 h2, double value) { struct stat_file_block *block, *to_expire = NULL; struct stat_file_header *header; @@ -148,38 +150,37 @@ rspamd_mmaped_file_set_block_common (rspamd_mempool_t *pool, } blocknum = h1 % file->cur_section.length; - header = (struct stat_file_header *)file->map; - c = (u_char *) file->map + file->seek_pos + blocknum * - sizeof (struct stat_file_block); - block = (struct stat_file_block *)c; + header = (struct stat_file_header *) file->map; + c = (u_char *) file->map + file->seek_pos + blocknum * sizeof(struct stat_file_block); + block = (struct stat_file_block *) c; for (i = 0; i < CHAIN_LENGTH; i++) { if (i + blocknum >= file->cur_section.length) { /* Need to expire some block in chain */ - msg_info_pool ("chain %ud is full in statfile %s, starting expire", - blocknum, - file->filename); + msg_info_pool("chain %ud is full in statfile %s, starting expire", + blocknum, + file->filename); break; } /* First try to find block in chain */ if (block->hash1 == h1 && block->hash2 == h2) { - msg_debug_pool ("%s found existing block %ud in chain %ud, value %.2f", - file->filename, - i, - blocknum, - value); + msg_debug_pool("%s found existing block %ud in chain %ud, value %.2f", + file->filename, + i, + blocknum, + value); block->value = value; return; } /* Check whether we have a free block in chain */ if (block->hash1 == 0 && block->hash2 == 0) { /* Write new block here */ - msg_debug_pool ("%s found free block %ud in chain %ud, set h1=%ud, h2=%ud", - file->filename, - i, - blocknum, - h1, - h2); + msg_debug_pool("%s found free block %ud in chain %ud, set h1=%ud, h2=%ud", + file->filename, + i, + blocknum, + h1, + h2); block->hash1 = h1; block->hash2 = h2; block->value = value; @@ -193,8 +194,8 @@ rspamd_mmaped_file_set_block_common (rspamd_mempool_t *pool, to_expire = block; min = block->value; } - c += sizeof (struct stat_file_block); - block = (struct stat_file_block *)c; + c += sizeof(struct stat_file_block); + block = (struct stat_file_block *) c; } /* Try expire some block */ @@ -203,9 +204,8 @@ rspamd_mmaped_file_set_block_common (rspamd_mempool_t *pool, } else { /* Expire first block in chain */ - c = (u_char *) file->map + file->seek_pos + blocknum * - sizeof (struct stat_file_block); - block = (struct stat_file_block *)c; + c = (u_char *) file->map + file->seek_pos + blocknum * sizeof(struct stat_file_block); + block = (struct stat_file_block *) c; } block->hash1 = h1; @@ -213,18 +213,17 @@ rspamd_mmaped_file_set_block_common (rspamd_mempool_t *pool, block->value = value; } -void -rspamd_mmaped_file_set_block (rspamd_mempool_t *pool, - rspamd_mmaped_file_t * file, - guint32 h1, - guint32 h2, - double value) +void rspamd_mmaped_file_set_block(rspamd_mempool_t *pool, + rspamd_mmaped_file_t *file, + guint32 h1, + guint32 h2, + double value) { - rspamd_mmaped_file_set_block_common (pool, file, h1, h2, value); + rspamd_mmaped_file_set_block_common(pool, file, h1, h2, value); } gboolean -rspamd_mmaped_file_set_revision (rspamd_mmaped_file_t *file, guint64 rev, time_t time) +rspamd_mmaped_file_set_revision(rspamd_mmaped_file_t *file, guint64 rev, time_t time) { struct stat_file_header *header; @@ -232,7 +231,7 @@ rspamd_mmaped_file_set_revision (rspamd_mmaped_file_t *file, guint64 rev, time_t return FALSE; } - header = (struct stat_file_header *)file->map; + header = (struct stat_file_header *) file->map; header->revision = rev; header->rev_time = time; @@ -241,7 +240,7 @@ rspamd_mmaped_file_set_revision (rspamd_mmaped_file_t *file, guint64 rev, time_t } gboolean -rspamd_mmaped_file_inc_revision (rspamd_mmaped_file_t *file) +rspamd_mmaped_file_inc_revision(rspamd_mmaped_file_t *file) { struct stat_file_header *header; @@ -249,7 +248,7 @@ rspamd_mmaped_file_inc_revision (rspamd_mmaped_file_t *file) return FALSE; } - header = (struct stat_file_header *)file->map; + header = (struct stat_file_header *) file->map; header->revision++; @@ -257,7 +256,7 @@ rspamd_mmaped_file_inc_revision (rspamd_mmaped_file_t *file) } gboolean -rspamd_mmaped_file_dec_revision (rspamd_mmaped_file_t *file) +rspamd_mmaped_file_dec_revision(rspamd_mmaped_file_t *file) { struct stat_file_header *header; @@ -265,7 +264,7 @@ rspamd_mmaped_file_dec_revision (rspamd_mmaped_file_t *file) return FALSE; } - header = (struct stat_file_header *)file->map; + header = (struct stat_file_header *) file->map; header->revision--; @@ -274,7 +273,7 @@ rspamd_mmaped_file_dec_revision (rspamd_mmaped_file_t *file) gboolean -rspamd_mmaped_file_get_revision (rspamd_mmaped_file_t *file, guint64 *rev, time_t *time) +rspamd_mmaped_file_get_revision(rspamd_mmaped_file_t *file, guint64 *rev, time_t *time) { struct stat_file_header *header; @@ -282,7 +281,7 @@ rspamd_mmaped_file_get_revision (rspamd_mmaped_file_t *file, guint64 *rev, time_ return FALSE; } - header = (struct stat_file_header *)file->map; + header = (struct stat_file_header *) file->map; if (rev != NULL) { *rev = header->revision; @@ -295,29 +294,29 @@ rspamd_mmaped_file_get_revision (rspamd_mmaped_file_t *file, guint64 *rev, time_ } guint64 -rspamd_mmaped_file_get_used (rspamd_mmaped_file_t *file) +rspamd_mmaped_file_get_used(rspamd_mmaped_file_t *file) { struct stat_file_header *header; if (file == NULL || file->map == NULL) { - return (guint64) - 1; + return (guint64) -1; } - header = (struct stat_file_header *)file->map; + header = (struct stat_file_header *) file->map; return header->used_blocks; } guint64 -rspamd_mmaped_file_get_total (rspamd_mmaped_file_t *file) +rspamd_mmaped_file_get_total(rspamd_mmaped_file_t *file) { struct stat_file_header *header; if (file == NULL || file->map == NULL) { - return (guint64) - 1; + return (guint64) -1; } - header = (struct stat_file_header *)file->map; + header = (struct stat_file_header *) file->map; /* If total blocks is 0 we have old version of header, so set total blocks correctly */ if (header->total_blocks == 0) { @@ -329,7 +328,7 @@ rspamd_mmaped_file_get_total (rspamd_mmaped_file_t *file) /* Check whether specified file is statistic file and calculate its len in blocks */ static gint -rspamd_mmaped_file_check (rspamd_mempool_t *pool, rspamd_mmaped_file_t * file) +rspamd_mmaped_file_check(rspamd_mempool_t *pool, rspamd_mmaped_file_t *file) { struct stat_file *f; gchar *c; @@ -340,18 +339,18 @@ rspamd_mmaped_file_check (rspamd_mempool_t *pool, rspamd_mmaped_file_t * file) return -1; } - if (file->len < sizeof (struct stat_file)) { - msg_info_pool ("file %s is too short to be stat file: %z", - file->filename, - file->len); + if (file->len < sizeof(struct stat_file)) { + msg_info_pool("file %s is too short to be stat file: %z", + file->filename, + file->len); return -1; } - f = (struct stat_file *)file->map; + f = (struct stat_file *) file->map; c = &f->header.magic[0]; /* Check magic and version */ if (*c++ != 'r' || *c++ != 's' || *c++ != 'd') { - msg_info_pool ("file %s is invalid stat file", file->filename); + msg_info_pool("file %s is invalid stat file", file->filename); return -1; } @@ -360,39 +359,39 @@ rspamd_mmaped_file_check (rspamd_mempool_t *pool, rspamd_mmaped_file_t * file) if (*c == 1 && *(c + 1) == 0) { return -1; } - else if (memcmp (c, valid_version, sizeof (valid_version)) != 0) { + else if (memcmp(c, valid_version, sizeof(valid_version)) != 0) { /* Unknown version */ - msg_info_pool ("file %s has invalid version %c.%c", - file->filename, - '0' + *c, - '0' + *(c + 1)); + msg_info_pool("file %s has invalid version %c.%c", + file->filename, + '0' + *c, + '0' + *(c + 1)); return -1; } /* Check first section and set new offset */ file->cur_section.code = f->section.code; file->cur_section.length = f->section.length; - if (file->cur_section.length * sizeof (struct stat_file_block) > + if (file->cur_section.length * sizeof(struct stat_file_block) > file->len) { - msg_info_pool ("file %s is truncated: %z, must be %z", - file->filename, - file->len, - file->cur_section.length * sizeof (struct stat_file_block)); + msg_info_pool("file %s is truncated: %z, must be %z", + file->filename, + file->len, + file->cur_section.length * sizeof(struct stat_file_block)); return -1; } - file->seek_pos = sizeof (struct stat_file) - - sizeof (struct stat_file_block); + file->seek_pos = sizeof(struct stat_file) - + sizeof(struct stat_file_block); return 0; } static rspamd_mmaped_file_t * -rspamd_mmaped_file_reindex (rspamd_mempool_t *pool, - const gchar *filename, - size_t old_size, - size_t size, - struct rspamd_statfile_config *stcf) +rspamd_mmaped_file_reindex(rspamd_mempool_t *pool, + const gchar *filename, + size_t old_size, + size_t size, + struct rspamd_statfile_config *stcf) { gchar *backup, *lock; gint fd, lock_fd; @@ -401,285 +400,276 @@ rspamd_mmaped_file_reindex (rspamd_mempool_t *pool, struct stat_file_block *block; struct stat_file_header *header, *nh; struct timespec sleep_ts = { - .tv_sec = 0, - .tv_nsec = 1000000 - }; + .tv_sec = 0, + .tv_nsec = 1000000}; if (size < - sizeof (struct stat_file_header) + sizeof (struct stat_file_section) + - sizeof (block)) { - msg_err_pool ("file %s is too small to carry any statistic: %z", - filename, - size); + sizeof(struct stat_file_header) + sizeof(struct stat_file_section) + + sizeof(block)) { + msg_err_pool("file %s is too small to carry any statistic: %z", + filename, + size); return NULL; } - lock = g_strconcat (filename, ".lock", NULL); - lock_fd = open (lock, O_WRONLY|O_CREAT|O_EXCL, 00600); + lock = g_strconcat(filename, ".lock", NULL); + lock_fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600); while (lock_fd == -1) { /* Wait for lock */ - lock_fd = open (lock, O_WRONLY|O_CREAT|O_EXCL, 00600); + lock_fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600); if (lock_fd != -1) { - unlink (lock); - close (lock_fd); - g_free (lock); + unlink(lock); + close(lock_fd); + g_free(lock); - return rspamd_mmaped_file_open (pool, filename, size, stcf); + return rspamd_mmaped_file_open(pool, filename, size, stcf); } else { - nanosleep (&sleep_ts, NULL); + nanosleep(&sleep_ts, NULL); } } - backup = g_strconcat (filename, ".old", NULL); - if (rename (filename, backup) == -1) { - msg_err_pool ("cannot rename %s to %s: %s", filename, backup, strerror ( - errno)); - g_free (backup); - unlink (lock); - g_free (lock); - close (lock_fd); + backup = g_strconcat(filename, ".old", NULL); + if (rename(filename, backup) == -1) { + msg_err_pool("cannot rename %s to %s: %s", filename, backup, strerror(errno)); + g_free(backup); + unlink(lock); + g_free(lock); + close(lock_fd); return NULL; } - old = rspamd_mmaped_file_open (pool, backup, old_size, stcf); + old = rspamd_mmaped_file_open(pool, backup, old_size, stcf); if (old == NULL) { - msg_warn_pool ("old file %s is invalid mmapped file, just move it", - backup); + msg_warn_pool("old file %s is invalid mmapped file, just move it", + backup); } /* We need to release our lock here */ - unlink (lock); - close (lock_fd); - g_free (lock); + unlink(lock); + close(lock_fd); + g_free(lock); /* Now create new file with required size */ - if (rspamd_mmaped_file_create (filename, size, stcf, pool) != 0) { - msg_err_pool ("cannot create new file"); - rspamd_mmaped_file_close (old); - g_free (backup); + if (rspamd_mmaped_file_create(filename, size, stcf, pool) != 0) { + msg_err_pool("cannot create new file"); + rspamd_mmaped_file_close(old); + g_free(backup); return NULL; } - new = rspamd_mmaped_file_open (pool, filename, size, stcf); + new = rspamd_mmaped_file_open(pool, filename, size, stcf); if (old) { /* Now open new file and start copying */ - fd = open (backup, O_RDONLY); + fd = open(backup, O_RDONLY); if (fd == -1 || new == NULL) { if (fd != -1) { - close (fd); + close(fd); } - msg_err_pool ("cannot open file: %s", strerror (errno)); - rspamd_mmaped_file_close (old); - g_free (backup); + msg_err_pool("cannot open file: %s", strerror(errno)); + rspamd_mmaped_file_close(old); + g_free(backup); return NULL; } - /* Now start reading blocks from old statfile */ if ((map = - mmap (NULL, old_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { - msg_err_pool ("cannot mmap file: %s", strerror (errno)); - close (fd); - rspamd_mmaped_file_close (old); - g_free (backup); + mmap(NULL, old_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { + msg_err_pool("cannot mmap file: %s", strerror(errno)); + close(fd); + rspamd_mmaped_file_close(old); + g_free(backup); return NULL; } - pos = map + (sizeof (struct stat_file) - sizeof (struct stat_file_block)); + pos = map + (sizeof(struct stat_file) - sizeof(struct stat_file_block)); - if (pos - map < (gssize)old_size) { - while ((gssize)old_size - (pos - map) >= (gssize)sizeof (struct stat_file_block)) { - block = (struct stat_file_block *)pos; + if (pos - map < (gssize) old_size) { + while ((gssize) old_size - (pos - map) >= (gssize) sizeof(struct stat_file_block)) { + block = (struct stat_file_block *) pos; if (block->hash1 != 0 && block->value != 0) { - rspamd_mmaped_file_set_block_common (pool, - new, block->hash1, - block->hash2, block->value); + rspamd_mmaped_file_set_block_common(pool, + new, block->hash1, + block->hash2, block->value); } - pos += sizeof (block); + pos += sizeof(block); } } - header = (struct stat_file_header *)map; - rspamd_mmaped_file_set_revision (new, header->revision, header->rev_time); + header = (struct stat_file_header *) map; + rspamd_mmaped_file_set_revision(new, header->revision, header->rev_time); nh = new->map; /* Copy tokenizer configuration */ - memcpy (nh->unused, header->unused, sizeof (header->unused)); + memcpy(nh->unused, header->unused, sizeof(header->unused)); nh->tokenizer_conf_len = header->tokenizer_conf_len; - munmap (map, old_size); - close (fd); - rspamd_mmaped_file_close_file (pool, old); + munmap(map, old_size); + close(fd); + rspamd_mmaped_file_close_file(pool, old); } - unlink (backup); - g_free (backup); + unlink(backup); + g_free(backup); return new; - } /* * Pre-load mmaped file into memory */ static void -rspamd_mmaped_file_preload (rspamd_mmaped_file_t *file) +rspamd_mmaped_file_preload(rspamd_mmaped_file_t *file) { guint8 *pos, *end; volatile guint8 t; gsize size; - pos = (guint8 *)file->map; - end = (guint8 *)file->map + file->len; + pos = (guint8 *) file->map; + end = (guint8 *) file->map + file->len; - if (madvise (pos, end - pos, MADV_SEQUENTIAL) == -1) { - msg_info ("madvise failed: %s", strerror (errno)); + if (madvise(pos, end - pos, MADV_SEQUENTIAL) == -1) { + msg_info("madvise failed: %s", strerror(errno)); } else { /* Load pages of file */ #ifdef HAVE_GETPAGESIZE - size = getpagesize (); + size = getpagesize(); #else - size = sysconf (_SC_PAGESIZE); + size = sysconf(_SC_PAGESIZE); #endif while (pos < end) { t = *pos; - (void)t; + (void) t; pos += size; } } } rspamd_mmaped_file_t * -rspamd_mmaped_file_open (rspamd_mempool_t *pool, - const gchar *filename, size_t size, - struct rspamd_statfile_config *stcf) +rspamd_mmaped_file_open(rspamd_mempool_t *pool, + const gchar *filename, size_t size, + struct rspamd_statfile_config *stcf) { struct stat st; rspamd_mmaped_file_t *new_file; gchar *lock; gint lock_fd; - lock = g_strconcat (filename, ".lock", NULL); - lock_fd = open (lock, O_WRONLY|O_CREAT|O_EXCL, 00600); + lock = g_strconcat(filename, ".lock", NULL); + lock_fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600); if (lock_fd == -1) { - g_free (lock); - msg_info_pool ("cannot open file %s, it is locked by another process", - filename); + g_free(lock); + msg_info_pool("cannot open file %s, it is locked by another process", + filename); return NULL; } - close (lock_fd); - unlink (lock); - g_free (lock); + close(lock_fd); + unlink(lock); + g_free(lock); - if (stat (filename, &st) == -1) { - msg_info_pool ("cannot stat file %s, error %s, %d", filename, strerror ( - errno), errno); + if (stat(filename, &st) == -1) { + msg_info_pool("cannot stat file %s, error %s, %d", filename, strerror(errno), errno); return NULL; } - if (labs ((glong)size - st.st_size) > (long)sizeof (struct stat_file) * 2 - && size > sizeof (struct stat_file)) { - msg_warn_pool ("need to reindex statfile old size: %Hz, new size: %Hz", - (size_t)st.st_size, size); - return rspamd_mmaped_file_reindex (pool, filename, st.st_size, size, stcf); + if (labs((glong) size - st.st_size) > (long) sizeof(struct stat_file) * 2 && size > sizeof(struct stat_file)) { + msg_warn_pool("need to reindex statfile old size: %Hz, new size: %Hz", + (size_t) st.st_size, size); + return rspamd_mmaped_file_reindex(pool, filename, st.st_size, size, stcf); } - else if (size < sizeof (struct stat_file)) { - msg_err_pool ("requested to shrink statfile to %Hz but it is too small", - size); + else if (size < sizeof(struct stat_file)) { + msg_err_pool("requested to shrink statfile to %Hz but it is too small", + size); } - new_file = g_malloc0 (sizeof (rspamd_mmaped_file_t)); - if ((new_file->fd = open (filename, O_RDWR)) == -1) { - msg_info_pool ("cannot open file %s, error %d, %s", - filename, - errno, - strerror (errno)); - g_free (new_file); + new_file = g_malloc0(sizeof(rspamd_mmaped_file_t)); + if ((new_file->fd = open(filename, O_RDWR)) == -1) { + msg_info_pool("cannot open file %s, error %d, %s", + filename, + errno, + strerror(errno)); + g_free(new_file); return NULL; } if ((new_file->map = - mmap (NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, - new_file->fd, 0)) == MAP_FAILED) { - close (new_file->fd); - msg_info_pool ("cannot mmap file %s, error %d, %s", - filename, - errno, - strerror (errno)); - g_free (new_file); + mmap(NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, + new_file->fd, 0)) == MAP_FAILED) { + close(new_file->fd); + msg_info_pool("cannot mmap file %s, error %d, %s", + filename, + errno, + strerror(errno)); + g_free(new_file); return NULL; - } - rspamd_strlcpy (new_file->filename, filename, sizeof (new_file->filename)); + rspamd_strlcpy(new_file->filename, filename, sizeof(new_file->filename)); new_file->len = st.st_size; /* Try to lock pages in RAM */ /* Acquire lock for this operation */ - if (!rspamd_file_lock (new_file->fd, FALSE)) { - close (new_file->fd); - munmap (new_file->map, st.st_size); - msg_info_pool ("cannot lock file %s, error %d, %s", - filename, - errno, - strerror (errno)); - g_free (new_file); + if (!rspamd_file_lock(new_file->fd, FALSE)) { + close(new_file->fd); + munmap(new_file->map, st.st_size); + msg_info_pool("cannot lock file %s, error %d, %s", + filename, + errno, + strerror(errno)); + g_free(new_file); return NULL; } - if (rspamd_mmaped_file_check (pool, new_file) == -1) { - close (new_file->fd); - rspamd_file_unlock (new_file->fd, FALSE); - munmap (new_file->map, st.st_size); - g_free (new_file); + if (rspamd_mmaped_file_check(pool, new_file) == -1) { + close(new_file->fd); + rspamd_file_unlock(new_file->fd, FALSE); + munmap(new_file->map, st.st_size); + g_free(new_file); return NULL; } - rspamd_file_unlock (new_file->fd, FALSE); + rspamd_file_unlock(new_file->fd, FALSE); new_file->cf = stcf; new_file->pool = pool; - rspamd_mmaped_file_preload (new_file); + rspamd_mmaped_file_preload(new_file); - g_assert (stcf->clcf != NULL); + g_assert(stcf->clcf != NULL); - msg_debug_pool ("opened statfile %s of size %l", filename, (long)size); + msg_debug_pool("opened statfile %s of size %l", filename, (long) size); return new_file; } -gint -rspamd_mmaped_file_close_file (rspamd_mempool_t *pool, - rspamd_mmaped_file_t * file) +gint rspamd_mmaped_file_close_file(rspamd_mempool_t *pool, + rspamd_mmaped_file_t *file) { if (file->map) { - msg_info_pool ("syncing statfile %s", file->filename); - msync (file->map, file->len, MS_ASYNC); - munmap (file->map, file->len); + msg_info_pool("syncing statfile %s", file->filename); + msync(file->map, file->len, MS_ASYNC); + munmap(file->map, file->len); } if (file->fd != -1) { - close (file->fd); + close(file->fd); } - g_free (file); + g_free(file); return 0; } -gint -rspamd_mmaped_file_create (const gchar *filename, - size_t size, - struct rspamd_statfile_config *stcf, - rspamd_mempool_t *pool) +gint rspamd_mmaped_file_create(const gchar *filename, + size_t size, + struct rspamd_statfile_config *stcf, + rspamd_mempool_t *pool) { struct stat_file_header header = { .magic = {'r', 's', 'd'}, @@ -687,12 +677,11 @@ rspamd_mmaped_file_create (const gchar *filename, .padding = {0, 0, 0}, .revision = 0, .rev_time = 0, - .used_blocks = 0 - }; + .used_blocks = 0}; struct stat_file_section section = { .code = STATFILE_SECTION_COMMON, }; - struct stat_file_block block = { 0, 0, 0 }; + struct stat_file_block block = {0, 0, 0}; struct rspamd_stat_tokenizer *tokenizer; gint fd, lock_fd; guint buflen = 0, nblocks; @@ -701,31 +690,30 @@ rspamd_mmaped_file_create (const gchar *filename, gpointer tok_conf; gsize tok_conf_len; struct timespec sleep_ts = { - .tv_sec = 0, - .tv_nsec = 1000000 - }; + .tv_sec = 0, + .tv_nsec = 1000000}; if (size < - sizeof (struct stat_file_header) + sizeof (struct stat_file_section) + - sizeof (block)) { - msg_err_pool ("file %s is too small to carry any statistic: %z", - filename, - size); + sizeof(struct stat_file_header) + sizeof(struct stat_file_section) + + sizeof(block)) { + msg_err_pool("file %s is too small to carry any statistic: %z", + filename, + size); return -1; } - lock = g_strconcat (filename, ".lock", NULL); - lock_fd = open (lock, O_WRONLY|O_CREAT|O_EXCL, 00600); + lock = g_strconcat(filename, ".lock", NULL); + lock_fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600); while (lock_fd == -1) { /* Wait for lock */ - lock_fd = open (lock, O_WRONLY|O_CREAT|O_EXCL, 00600); + lock_fd = open(lock, O_WRONLY | O_CREAT | O_EXCL, 00600); if (lock_fd != -1) { - if (stat (filename, &sb) != -1) { + if (stat(filename, &sb) != -1) { /* File has been created by some other process */ - unlink (lock); - close (lock_fd); - g_free (lock); + unlink(lock); + close(lock_fd); + g_free(lock); return 0; } @@ -734,110 +722,111 @@ rspamd_mmaped_file_create (const gchar *filename, goto create; } else { - nanosleep (&sleep_ts, NULL); + nanosleep(&sleep_ts, NULL); } } create: - msg_debug_pool ("create statfile %s of size %l", filename, (long)size); + msg_debug_pool("create statfile %s of size %l", filename, (long) size); nblocks = - (size - sizeof (struct stat_file_header) - - sizeof (struct stat_file_section)) / sizeof (struct stat_file_block); + (size - sizeof(struct stat_file_header) - + sizeof(struct stat_file_section)) / + sizeof(struct stat_file_block); header.total_blocks = nblocks; if ((fd = - open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { - msg_info_pool ("cannot create file %s, error %d, %s", - filename, - errno, - strerror (errno)); - unlink (lock); - close (lock_fd); - g_free (lock); + open(filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { + msg_info_pool("cannot create file %s, error %d, %s", + filename, + errno, + strerror(errno)); + unlink(lock); + close(lock_fd); + g_free(lock); return -1; } - rspamd_fallocate (fd, - 0, - sizeof (header) + sizeof (section) + sizeof (block) * nblocks); + rspamd_fallocate(fd, + 0, + sizeof(header) + sizeof(section) + sizeof(block) * nblocks); - header.create_time = (guint64) time (NULL); - g_assert (stcf->clcf != NULL); - g_assert (stcf->clcf->tokenizer != NULL); - tokenizer = rspamd_stat_get_tokenizer (stcf->clcf->tokenizer->name); - g_assert (tokenizer != NULL); - tok_conf = tokenizer->get_config (pool, stcf->clcf->tokenizer, &tok_conf_len); + header.create_time = (guint64) time(NULL); + g_assert(stcf->clcf != NULL); + g_assert(stcf->clcf->tokenizer != NULL); + tokenizer = rspamd_stat_get_tokenizer(stcf->clcf->tokenizer->name); + g_assert(tokenizer != NULL); + tok_conf = tokenizer->get_config(pool, stcf->clcf->tokenizer, &tok_conf_len); header.tokenizer_conf_len = tok_conf_len; - g_assert (tok_conf_len < sizeof (header.unused) - sizeof (guint64)); - memcpy (header.unused, tok_conf, tok_conf_len); - - if (write (fd, &header, sizeof (header)) == -1) { - msg_info_pool ("cannot write header to file %s, error %d, %s", - filename, - errno, - strerror (errno)); - close (fd); - unlink (lock); - close (lock_fd); - g_free (lock); + g_assert(tok_conf_len < sizeof(header.unused) - sizeof(guint64)); + memcpy(header.unused, tok_conf, tok_conf_len); + + if (write(fd, &header, sizeof(header)) == -1) { + msg_info_pool("cannot write header to file %s, error %d, %s", + filename, + errno, + strerror(errno)); + close(fd); + unlink(lock); + close(lock_fd); + g_free(lock); return -1; } section.length = (guint64) nblocks; - if (write (fd, §ion, sizeof (section)) == -1) { - msg_info_pool ("cannot write section header to file %s, error %d, %s", - filename, - errno, - strerror (errno)); - close (fd); - unlink (lock); - close (lock_fd); - g_free (lock); + if (write(fd, §ion, sizeof(section)) == -1) { + msg_info_pool("cannot write section header to file %s, error %d, %s", + filename, + errno, + strerror(errno)); + close(fd); + unlink(lock); + close(lock_fd); + g_free(lock); return -1; } /* Buffer for write 256 blocks at once */ if (nblocks > 256) { - buflen = sizeof (block) * 256; - buf = g_malloc0 (buflen); + buflen = sizeof(block) * 256; + buf = g_malloc0(buflen); } while (nblocks) { if (nblocks > 256) { /* Just write buffer */ - if (write (fd, buf, buflen) == -1) { - msg_info_pool ("cannot write blocks buffer to file %s, error %d, %s", - filename, - errno, - strerror (errno)); - close (fd); - g_free (buf); - unlink (lock); - close (lock_fd); - g_free (lock); + if (write(fd, buf, buflen) == -1) { + msg_info_pool("cannot write blocks buffer to file %s, error %d, %s", + filename, + errno, + strerror(errno)); + close(fd); + g_free(buf); + unlink(lock); + close(lock_fd); + g_free(lock); return -1; } nblocks -= 256; } else { - if (write (fd, &block, sizeof (block)) == -1) { - msg_info_pool ("cannot write block to file %s, error %d, %s", - filename, - errno, - strerror (errno)); - close (fd); + if (write(fd, &block, sizeof(block)) == -1) { + msg_info_pool("cannot write block to file %s, error %d, %s", + filename, + errno, + strerror(errno)); + close(fd); if (buf) { - g_free (buf); + g_free(buf); } - unlink (lock); - close (lock_fd); - g_free (lock); + unlink(lock); + close(lock_fd); + g_free(lock); return -1; } @@ -845,23 +834,23 @@ create: } } - close (fd); + close(fd); if (buf) { - g_free (buf); + g_free(buf); } - unlink (lock); - close (lock_fd); - g_free (lock); - msg_debug_pool ("created statfile %s of size %l", filename, (long)size); + unlink(lock); + close(lock_fd); + g_free(lock); + msg_debug_pool("created statfile %s of size %l", filename, (long) size); return 0; } gpointer -rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, - struct rspamd_config *cfg, struct rspamd_statfile *st) +rspamd_mmaped_file_init(struct rspamd_stat_ctx *ctx, + struct rspamd_config *cfg, struct rspamd_statfile *st) { struct rspamd_statfile_config *stf = st->stcf; rspamd_mmaped_file_t *mf; @@ -869,105 +858,104 @@ rspamd_mmaped_file_init (struct rspamd_stat_ctx *ctx, const gchar *filename; gsize size; - filenameo = ucl_object_lookup (stf->opts, "filename"); + filenameo = ucl_object_lookup(stf->opts, "filename"); - if (filenameo == NULL || ucl_object_type (filenameo) != UCL_STRING) { - filenameo = ucl_object_lookup (stf->opts, "path"); + if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) { + filenameo = ucl_object_lookup(stf->opts, "path"); - if (filenameo == NULL || ucl_object_type (filenameo) != UCL_STRING) { - msg_err_config ("statfile %s has no filename defined", stf->symbol); + if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) { + msg_err_config("statfile %s has no filename defined", stf->symbol); return NULL; } } - filename = ucl_object_tostring (filenameo); + filename = ucl_object_tostring(filenameo); - sizeo = ucl_object_lookup (stf->opts, "size"); + sizeo = ucl_object_lookup(stf->opts, "size"); - if (sizeo == NULL || ucl_object_type (sizeo) != UCL_INT) { - msg_err_config ("statfile %s has no size defined", stf->symbol); + if (sizeo == NULL || ucl_object_type(sizeo) != UCL_INT) { + msg_err_config("statfile %s has no size defined", stf->symbol); return NULL; } - size = ucl_object_toint (sizeo); - mf = rspamd_mmaped_file_open (cfg->cfg_pool, filename, size, stf); + size = ucl_object_toint(sizeo); + mf = rspamd_mmaped_file_open(cfg->cfg_pool, filename, size, stf); if (mf != NULL) { mf->pool = cfg->cfg_pool; - } else { + } + else { /* Create file here */ - filenameo = ucl_object_find_key (stf->opts, "filename"); - if (filenameo == NULL || ucl_object_type (filenameo) != UCL_STRING) { - filenameo = ucl_object_find_key (stf->opts, "path"); - if (filenameo == NULL || ucl_object_type (filenameo) != UCL_STRING) { - msg_err_config ("statfile %s has no filename defined", stf->symbol); + filenameo = ucl_object_find_key(stf->opts, "filename"); + if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) { + filenameo = ucl_object_find_key(stf->opts, "path"); + if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) { + msg_err_config("statfile %s has no filename defined", stf->symbol); return NULL; } } - filename = ucl_object_tostring (filenameo); + filename = ucl_object_tostring(filenameo); - sizeo = ucl_object_find_key (stf->opts, "size"); - if (sizeo == NULL || ucl_object_type (sizeo) != UCL_INT) { - msg_err_config ("statfile %s has no size defined", stf->symbol); + sizeo = ucl_object_find_key(stf->opts, "size"); + if (sizeo == NULL || ucl_object_type(sizeo) != UCL_INT) { + msg_err_config("statfile %s has no size defined", stf->symbol); return NULL; } - size = ucl_object_toint (sizeo); + size = ucl_object_toint(sizeo); - if (rspamd_mmaped_file_create (filename, size, stf, cfg->cfg_pool) != 0) { - msg_err_config ("cannot create new file"); + if (rspamd_mmaped_file_create(filename, size, stf, cfg->cfg_pool) != 0) { + msg_err_config("cannot create new file"); } - mf = rspamd_mmaped_file_open (cfg->cfg_pool, filename, size, stf); + mf = rspamd_mmaped_file_open(cfg->cfg_pool, filename, size, stf); } - return (gpointer)mf; + return (gpointer) mf; } -void -rspamd_mmaped_file_close (gpointer p) +void rspamd_mmaped_file_close(gpointer p) { rspamd_mmaped_file_t *mf = p; if (mf) { - rspamd_mmaped_file_close_file (mf->pool, mf); + rspamd_mmaped_file_close_file(mf->pool, mf); } - } gpointer -rspamd_mmaped_file_runtime (struct rspamd_task *task, - struct rspamd_statfile_config *stcf, - gboolean learn, - gpointer p, - gint _id) +rspamd_mmaped_file_runtime(struct rspamd_task *task, + struct rspamd_statfile_config *stcf, + gboolean learn, + gpointer p, + gint _id) { rspamd_mmaped_file_t *mf = p; - return (gpointer)mf; + return (gpointer) mf; } gboolean -rspamd_mmaped_file_process_tokens (struct rspamd_task *task, GPtrArray *tokens, - gint id, - gpointer p) +rspamd_mmaped_file_process_tokens(struct rspamd_task *task, GPtrArray *tokens, + gint id, + gpointer p) { rspamd_mmaped_file_t *mf = p; guint32 h1, h2; rspamd_token_t *tok; guint i; - g_assert (tokens != NULL); - g_assert (p != NULL); + g_assert(tokens != NULL); + g_assert(p != NULL); for (i = 0; i < tokens->len; i++) { - tok = g_ptr_array_index (tokens, i); - memcpy (&h1, (guchar *)&tok->data, sizeof (h1)); - memcpy (&h2, ((guchar *)&tok->data) + sizeof (h1), sizeof (h2)); - tok->values[id] = rspamd_mmaped_file_get_block (mf, h1, h2); + tok = g_ptr_array_index(tokens, i); + memcpy(&h1, (guchar *) &tok->data, sizeof(h1)); + memcpy(&h2, ((guchar *) &tok->data) + sizeof(h1), sizeof(h2)); + tok->values[id] = rspamd_mmaped_file_get_block(mf, h1, h2); } if (mf->cf->is_spam) { @@ -981,71 +969,71 @@ rspamd_mmaped_file_process_tokens (struct rspamd_task *task, GPtrArray *tokens, } gboolean -rspamd_mmaped_file_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, - gint id, - gpointer p) +rspamd_mmaped_file_learn_tokens(struct rspamd_task *task, GPtrArray *tokens, + gint id, + gpointer p) { rspamd_mmaped_file_t *mf = p; guint32 h1, h2; rspamd_token_t *tok; guint i; - g_assert (tokens != NULL); - g_assert (p != NULL); + g_assert(tokens != NULL); + g_assert(p != NULL); for (i = 0; i < tokens->len; i++) { - tok = g_ptr_array_index (tokens, i); - memcpy (&h1, (guchar *)&tok->data, sizeof (h1)); - memcpy (&h2, ((guchar *)&tok->data) + sizeof (h1), sizeof (h2)); - rspamd_mmaped_file_set_block (task->task_pool, mf, h1, h2, - tok->values[id]); + tok = g_ptr_array_index(tokens, i); + memcpy(&h1, (guchar *) &tok->data, sizeof(h1)); + memcpy(&h2, ((guchar *) &tok->data) + sizeof(h1), sizeof(h2)); + rspamd_mmaped_file_set_block(task->task_pool, mf, h1, h2, + tok->values[id]); } return TRUE; } gulong -rspamd_mmaped_file_total_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_mmaped_file_total_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { - rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *)runtime; + rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *) runtime; guint64 rev = 0; time_t t; if (mf != NULL) { - rspamd_mmaped_file_get_revision (mf, &rev, &t); + rspamd_mmaped_file_get_revision(mf, &rev, &t); } return rev; } gulong -rspamd_mmaped_file_inc_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_mmaped_file_inc_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { - rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *)runtime; + rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *) runtime; guint64 rev = 0; time_t t; if (mf != NULL) { - rspamd_mmaped_file_inc_revision (mf); - rspamd_mmaped_file_get_revision (mf, &rev, &t); + rspamd_mmaped_file_inc_revision(mf); + rspamd_mmaped_file_get_revision(mf, &rev, &t); } return rev; } gulong -rspamd_mmaped_file_dec_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_mmaped_file_dec_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { - rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *)runtime; + rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *) runtime; guint64 rev = 0; time_t t; if (mf != NULL) { - rspamd_mmaped_file_dec_revision (mf); - rspamd_mmaped_file_get_revision (mf, &rev, &t); + rspamd_mmaped_file_dec_revision(mf); + rspamd_mmaped_file_get_revision(mf, &rev, &t); } return rev; @@ -1053,36 +1041,34 @@ rspamd_mmaped_file_dec_learns (struct rspamd_task *task, gpointer runtime, ucl_object_t * -rspamd_mmaped_file_get_stat (gpointer runtime, - gpointer ctx) +rspamd_mmaped_file_get_stat(gpointer runtime, + gpointer ctx) { ucl_object_t *res = NULL; guint64 rev; - rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *)runtime; + rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *) runtime; if (mf != NULL) { - res = ucl_object_typed_new (UCL_OBJECT); - rspamd_mmaped_file_get_revision (mf, &rev, NULL); - ucl_object_insert_key (res, ucl_object_fromint (rev), "revision", - 0, false); - ucl_object_insert_key (res, ucl_object_fromint (mf->len), "size", - 0, false); - ucl_object_insert_key (res, ucl_object_fromint ( - rspamd_mmaped_file_get_total (mf)), "total", 0, false); - ucl_object_insert_key (res, ucl_object_fromint ( - rspamd_mmaped_file_get_used (mf)), "used", 0, false); - ucl_object_insert_key (res, ucl_object_fromstring (mf->cf->symbol), - "symbol", 0, false); - ucl_object_insert_key (res, ucl_object_fromstring ("mmap"), - "type", 0, false); - ucl_object_insert_key (res, ucl_object_fromint (0), - "languages", 0, false); - ucl_object_insert_key (res, ucl_object_fromint (0), - "users", 0, false); + res = ucl_object_typed_new(UCL_OBJECT); + rspamd_mmaped_file_get_revision(mf, &rev, NULL); + ucl_object_insert_key(res, ucl_object_fromint(rev), "revision", + 0, false); + ucl_object_insert_key(res, ucl_object_fromint(mf->len), "size", + 0, false); + ucl_object_insert_key(res, ucl_object_fromint(rspamd_mmaped_file_get_total(mf)), "total", 0, false); + ucl_object_insert_key(res, ucl_object_fromint(rspamd_mmaped_file_get_used(mf)), "used", 0, false); + ucl_object_insert_key(res, ucl_object_fromstring(mf->cf->symbol), + "symbol", 0, false); + ucl_object_insert_key(res, ucl_object_fromstring("mmap"), + "type", 0, false); + ucl_object_insert_key(res, ucl_object_fromint(0), + "languages", 0, false); + ucl_object_insert_key(res, ucl_object_fromint(0), + "users", 0, false); if (mf->cf->label) { - ucl_object_insert_key (res, ucl_object_fromstring (mf->cf->label), - "label", 0, false); + ucl_object_insert_key(res, ucl_object_fromstring(mf->cf->label), + "label", 0, false); } } @@ -1090,33 +1076,33 @@ rspamd_mmaped_file_get_stat (gpointer runtime, } gboolean -rspamd_mmaped_file_finalize_learn (struct rspamd_task *task, gpointer runtime, - gpointer ctx, GError **err) +rspamd_mmaped_file_finalize_learn(struct rspamd_task *task, gpointer runtime, + gpointer ctx, GError **err) { - rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *)runtime; + rspamd_mmaped_file_t *mf = (rspamd_mmaped_file_t *) runtime; if (mf != NULL) { - msync (mf->map, mf->len, MS_INVALIDATE | MS_ASYNC); + msync(mf->map, mf->len, MS_INVALIDATE | MS_ASYNC); } return TRUE; } gboolean -rspamd_mmaped_file_finalize_process (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_mmaped_file_finalize_process(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { return TRUE; } gpointer -rspamd_mmaped_file_load_tokenizer_config (gpointer runtime, - gsize *len) +rspamd_mmaped_file_load_tokenizer_config(gpointer runtime, + gsize *len) { rspamd_mmaped_file_t *mf = runtime; struct stat_file_header *header; - g_assert (mf != NULL); + g_assert(mf != NULL); header = mf->map; if (len) { diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c index 883d3d00c..2e4711ae9 100644 --- a/src/libstat/backends/redis_backend.c +++ b/src/libstat/backends/redis_backend.c @@ -23,15 +23,15 @@ #include "adapters/libev.h" #include "ref.h" -#define msg_debug_stat_redis(...) rspamd_conditional_debug_fast (NULL, NULL, \ - rspamd_stat_redis_log_id, "stat_redis", task->task_pool->tag.uid, \ - RSPAMD_LOG_FUNC, \ - __VA_ARGS__) +#define msg_debug_stat_redis(...) rspamd_conditional_debug_fast(NULL, NULL, \ + rspamd_stat_redis_log_id, "stat_redis", task->task_pool->tag.uid, \ + RSPAMD_LOG_FUNC, \ + __VA_ARGS__) INIT_LOG_MODULE(stat_redis) -#define REDIS_CTX(p) (struct redis_stat_ctx *)(p) -#define REDIS_RUNTIME(p) (struct redis_stat_runtime *)(p) +#define REDIS_CTX(p) (struct redis_stat_ctx *) (p) +#define REDIS_RUNTIME(p) (struct redis_stat_runtime *) (p) #define REDIS_BACKEND_TYPE "redis" #define REDIS_DEFAULT_PORT 6379 #define REDIS_DEFAULT_OBJECT "%s%l" @@ -106,23 +106,23 @@ struct rspamd_redis_stat_cbdata { static const gchar *M = "redis statistics"; static GQuark -rspamd_redis_stat_quark (void) +rspamd_redis_stat_quark(void) { - return g_quark_from_static_string (M); + return g_quark_from_static_string(M); } static inline struct upstream_list * -rspamd_redis_get_servers (struct redis_stat_ctx *ctx, - const gchar *what) +rspamd_redis_get_servers(struct redis_stat_ctx *ctx, + const gchar *what) { lua_State *L = ctx->L; struct upstream_list *res; - lua_rawgeti (L, LUA_REGISTRYINDEX, ctx->conf_ref); - lua_pushstring (L, what); - lua_gettable (L, -2); - res = *((struct upstream_list**)lua_touserdata (L, -1)); - lua_settop (L, 0); + lua_rawgeti(L, LUA_REGISTRYINDEX, ctx->conf_ref); + lua_pushstring(L, what); + lua_gettable(L, -2); + res = *((struct upstream_list **) lua_touserdata(L, -1)); + lua_settop(L, 0); return res; } @@ -130,16 +130,15 @@ rspamd_redis_get_servers (struct redis_stat_ctx *ctx, /* * Non-static for lua unit testing */ -gsize -rspamd_redis_expand_object (const gchar *pattern, - struct redis_stat_ctx *ctx, - struct rspamd_task *task, - gchar **target) +gsize rspamd_redis_expand_object(const gchar *pattern, + struct redis_stat_ctx *ctx, + struct rspamd_task *task, + gchar **target) { gsize tlen = 0; const gchar *p = pattern, *elt; gchar *d, *end; - enum { + enum { just_char, percent_char, mod_char @@ -150,42 +149,42 @@ rspamd_redis_expand_object (const gchar *pattern, const gchar *rcpt = NULL; gint err_idx; - g_assert (ctx != NULL); - g_assert (task != NULL); + g_assert(ctx != NULL); + g_assert(task != NULL); stcf = ctx->stcf; L = task->cfg->lua_state; - g_assert (L != NULL); + g_assert(L != NULL); if (ctx->enable_users) { if (ctx->cbref_user == -1) { - rcpt = rspamd_task_get_principal_recipient (task); + rcpt = rspamd_task_get_principal_recipient(task); } else { /* Execute lua function to get userdata */ - lua_pushcfunction (L, &rspamd_lua_traceback); - err_idx = lua_gettop (L); + lua_pushcfunction(L, &rspamd_lua_traceback); + err_idx = lua_gettop(L); - lua_rawgeti (L, LUA_REGISTRYINDEX, ctx->cbref_user); - ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); + lua_rawgeti(L, LUA_REGISTRYINDEX, ctx->cbref_user); + ptask = lua_newuserdata(L, sizeof(struct rspamd_task *)); *ptask = task; - rspamd_lua_setclass (L, "rspamd{task}", -1); + rspamd_lua_setclass(L, "rspamd{task}", -1); - if (lua_pcall (L, 1, 1, err_idx) != 0) { - msg_err_task ("call to user extraction script failed: %s", - lua_tostring (L, -1)); + if (lua_pcall(L, 1, 1, err_idx) != 0) { + msg_err_task("call to user extraction script failed: %s", + lua_tostring(L, -1)); } else { - rcpt = rspamd_mempool_strdup (task->task_pool, lua_tostring (L, -1)); + rcpt = rspamd_mempool_strdup(task->task_pool, lua_tostring(L, -1)); } /* Result + error function */ - lua_settop (L, err_idx - 1); + lua_settop(L, err_idx - 1); } if (rcpt) { - rspamd_mempool_set_variable (task->task_pool, "stat_user", - (gpointer)rcpt, NULL); + rspamd_mempool_set_variable(task->task_pool, "stat_user", + (gpointer) rcpt, NULL); } } @@ -197,67 +196,67 @@ rspamd_redis_expand_object (const gchar *pattern, state = percent_char; } else { - tlen ++; + tlen++; } - p ++; + p++; break; case percent_char: switch (*p) { case '%': - tlen ++; + tlen++; state = just_char; break; case 'u': - elt = GET_TASK_ELT (task, auth_user); + elt = GET_TASK_ELT(task, auth_user); if (elt) { - tlen += strlen (elt); + tlen += strlen(elt); } break; case 'r': if (rcpt == NULL) { - elt = rspamd_task_get_principal_recipient (task); + elt = rspamd_task_get_principal_recipient(task); } else { elt = rcpt; } if (elt) { - tlen += strlen (elt); + tlen += strlen(elt); } break; case 'l': if (stcf->label) { - tlen += strlen (stcf->label); + tlen += strlen(stcf->label); } /* Label miss is OK */ break; case 's': if (ctx->new_schema) { - tlen += sizeof ("RS") - 1; + tlen += sizeof("RS") - 1; } else { if (stcf->symbol) { - tlen += strlen (stcf->symbol); + tlen += strlen(stcf->symbol); } } break; default: state = just_char; - tlen ++; + tlen++; break; } if (state == percent_char) { state = mod_char; } - p ++; + p++; break; case mod_char: switch (*p) { case 'd': - p ++; + p++; state = just_char; break; default: @@ -273,7 +272,7 @@ rspamd_redis_expand_object (const gchar *pattern, return -1; } - *target = rspamd_mempool_alloc (task->task_pool, tlen + 1); + *target = rspamd_mempool_alloc(task->task_pool, tlen + 1); d = *target; end = d + tlen + 1; d[tlen] = '\0'; @@ -290,7 +289,7 @@ rspamd_redis_expand_object (const gchar *pattern, else { *d++ = *p; } - p ++; + p++; break; case percent_char: switch (*p) { @@ -299,35 +298,35 @@ rspamd_redis_expand_object (const gchar *pattern, state = just_char; break; case 'u': - elt = GET_TASK_ELT (task, auth_user); + elt = GET_TASK_ELT(task, auth_user); if (elt) { - d += rspamd_strlcpy (d, elt, end - d); + d += rspamd_strlcpy(d, elt, end - d); } break; case 'r': if (rcpt == NULL) { - elt = rspamd_task_get_principal_recipient (task); + elt = rspamd_task_get_principal_recipient(task); } else { elt = rcpt; } if (elt) { - d += rspamd_strlcpy (d, elt, end - d); + d += rspamd_strlcpy(d, elt, end - d); } break; case 'l': if (stcf->label) { - d += rspamd_strlcpy (d, stcf->label, end - d); + d += rspamd_strlcpy(d, stcf->label, end - d); } break; case 's': if (ctx->new_schema) { - d += rspamd_strlcpy (d, "RS", end - d); + d += rspamd_strlcpy(d, "RS", end - d); } else { if (stcf->symbol) { - d += rspamd_strlcpy (d, stcf->symbol, end - d); + d += rspamd_strlcpy(d, stcf->symbol, end - d); } } break; @@ -340,14 +339,14 @@ rspamd_redis_expand_object (const gchar *pattern, if (state == percent_char) { state = mod_char; } - p ++; + p++; break; case mod_char: switch (*p) { case 'd': /* TODO: not supported yet */ - p ++; + p++; state = just_char; break; default: @@ -362,13 +361,13 @@ rspamd_redis_expand_object (const gchar *pattern, } static void -rspamd_redis_maybe_auth (struct redis_stat_ctx *ctx, redisAsyncContext *redis) +rspamd_redis_maybe_auth(struct redis_stat_ctx *ctx, redisAsyncContext *redis) { if (ctx->password) { - redisAsyncCommand (redis, NULL, NULL, "AUTH %s", ctx->password); + redisAsyncCommand(redis, NULL, NULL, "AUTH %s", ctx->password); } if (ctx->dbname) { - redisAsyncCommand (redis, NULL, NULL, "SELECT %s", ctx->dbname); + redisAsyncCommand(redis, NULL, NULL, "SELECT %s", ctx->dbname); } } @@ -379,14 +378,14 @@ rspamd_redis_maybe_auth (struct redis_stat_ctx *ctx, redisAsyncContext *redis) #pragma GCC diagnostic ignored "-Wformat-extra-args" #endif static rspamd_fstring_t * -rspamd_redis_tokens_to_query (struct rspamd_task *task, - struct redis_stat_runtime *rt, - GPtrArray *tokens, - const gchar *command, - const gchar *prefix, - gboolean learn, - gint idx, - gboolean intvals) +rspamd_redis_tokens_to_query(struct rspamd_task *task, + struct redis_stat_runtime *rt, + GPtrArray *tokens, + const gchar *command, + const gchar *prefix, + gboolean learn, + gint idx, + gboolean intvals) { rspamd_fstring_t *out; rspamd_token_t *tok; @@ -394,21 +393,21 @@ rspamd_redis_tokens_to_query (struct rspamd_task *task, guint i, l0, l1, cmd_len, prefix_len; gint ret; - g_assert (tokens != NULL); + g_assert(tokens != NULL); - cmd_len = strlen (command); - prefix_len = strlen (prefix); - out = rspamd_fstring_sized_new (1024); + cmd_len = strlen(command); + prefix_len = strlen(prefix); + out = rspamd_fstring_sized_new(1024); if (learn) { - rspamd_printf_fstring (&out, "*1\r\n$5\r\nMULTI\r\n"); + rspamd_printf_fstring(&out, "*1\r\n$5\r\nMULTI\r\n"); - ret = redisAsyncFormattedCommand (rt->redis, NULL, NULL, - out->str, out->len); + ret = redisAsyncFormattedCommand(rt->redis, NULL, NULL, + out->str, out->len); if (ret != REDIS_OK) { - msg_err_task ("call to redis failed: %s", rt->redis->errstr); - rspamd_fstring_free (out); + msg_err_task("call to redis failed: %s", rt->redis->errstr); + rspamd_fstring_free(out); return NULL; } @@ -418,14 +417,14 @@ rspamd_redis_tokens_to_query (struct rspamd_task *task, else { if (rt->ctx->new_schema) { /* Multi + HGET */ - rspamd_printf_fstring (&out, "*1\r\n$5\r\nMULTI\r\n"); + rspamd_printf_fstring(&out, "*1\r\n$5\r\nMULTI\r\n"); - ret = redisAsyncFormattedCommand (rt->redis, NULL, NULL, - out->str, out->len); + ret = redisAsyncFormattedCommand(rt->redis, NULL, NULL, + out->str, out->len); if (ret != REDIS_OK) { - msg_err_task ("call to redis failed: %s", rt->redis->errstr); - rspamd_fstring_free (out); + msg_err_task("call to redis failed: %s", rt->redis->errstr); + rspamd_fstring_free(out); return NULL; } @@ -433,81 +432,82 @@ rspamd_redis_tokens_to_query (struct rspamd_task *task, out->len = 0; } else { - rspamd_printf_fstring (&out, "" - "*%d\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n", - (tokens->len + 2), - cmd_len, command, - prefix_len, prefix); + rspamd_printf_fstring(&out, "" + "*%d\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n", + (tokens->len + 2), + cmd_len, command, + prefix_len, prefix); } } - for (i = 0; i < tokens->len; i ++) { - tok = g_ptr_array_index (tokens, i); + for (i = 0; i < tokens->len; i++) { + tok = g_ptr_array_index(tokens, i); if (learn) { if (intvals) { - l1 = rspamd_snprintf (n1, sizeof (n1), "%L", - (gint64) tok->values[idx]); - } else { - l1 = rspamd_snprintf (n1, sizeof (n1), "%f", - tok->values[idx]); + l1 = rspamd_snprintf(n1, sizeof(n1), "%L", + (gint64) tok->values[idx]); + } + else { + l1 = rspamd_snprintf(n1, sizeof(n1), "%f", + tok->values[idx]); } if (rt->ctx->new_schema) { /* * HINCRBY <prefix_token> <0|1> <value> */ - l0 = rspamd_snprintf (n0, sizeof (n0), "%*s_%uL", - prefix_len, prefix, - tok->data); - - rspamd_printf_fstring (&out, "" - "*4\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n", - cmd_len, command, - l0, n0, - 1, rt->stcf->is_spam ? "S" : "H", - l1, n1); + l0 = rspamd_snprintf(n0, sizeof(n0), "%*s_%uL", + prefix_len, prefix, + tok->data); + + rspamd_printf_fstring(&out, "" + "*4\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n", + cmd_len, command, + l0, n0, + 1, rt->stcf->is_spam ? "S" : "H", + l1, n1); } else { - l0 = rspamd_snprintf (n0, sizeof (n0), "%uL", tok->data); + l0 = rspamd_snprintf(n0, sizeof(n0), "%uL", tok->data); /* * HINCRBY <prefix> <token> <value> */ - rspamd_printf_fstring (&out, "" - "*4\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n", - cmd_len, command, - prefix_len, prefix, - l0, n0, - l1, n1); + rspamd_printf_fstring(&out, "" + "*4\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n", + cmd_len, command, + prefix_len, prefix, + l0, n0, + l1, n1); } - ret = redisAsyncFormattedCommand (rt->redis, NULL, NULL, - out->str, out->len); + ret = redisAsyncFormattedCommand(rt->redis, NULL, NULL, + out->str, out->len); if (ret != REDIS_OK) { - msg_err_task ("call to redis failed: %s", rt->redis->errstr); - rspamd_fstring_free (out); + msg_err_task("call to redis failed: %s", rt->redis->errstr); + rspamd_fstring_free(out); return NULL; } @@ -521,19 +521,20 @@ rspamd_redis_tokens_to_query (struct rspamd_task *task, * ZINCRBY prefix_z 1.0 <token_id> */ if (tok->t1 && tok->t2) { - redisAsyncCommand (rt->redis, NULL, NULL, - "HSET %b_tokens %b %b:%b", - prefix, (size_t) prefix_len, - n0, (size_t) l0, - tok->t1->stemmed.begin, tok->t1->stemmed.len, - tok->t2->stemmed.begin, tok->t2->stemmed.len); - } else if (tok->t1) { - redisAsyncCommand (rt->redis, NULL, NULL, - "HSET %b_tokens %b %b", - prefix, (size_t) prefix_len, - n0, (size_t) l0, - tok->t1->stemmed.begin, - tok->t1->stemmed.len); + redisAsyncCommand(rt->redis, NULL, NULL, + "HSET %b_tokens %b %b:%b", + prefix, (size_t) prefix_len, + n0, (size_t) l0, + tok->t1->stemmed.begin, tok->t1->stemmed.len, + tok->t2->stemmed.begin, tok->t2->stemmed.len); + } + else if (tok->t1) { + redisAsyncCommand(rt->redis, NULL, NULL, + "HSET %b_tokens %b %b", + prefix, (size_t) prefix_len, + n0, (size_t) l0, + tok->t1->stemmed.begin, + tok->t1->stemmed.len); } } else { @@ -543,73 +544,74 @@ rspamd_redis_tokens_to_query (struct rspamd_task *task, * ZINCRBY prefix_z 1.0 <token_id> */ if (tok->t1 && tok->t2) { - redisAsyncCommand (rt->redis, NULL, NULL, - "HSET %b %s %b:%b", - n0, (size_t) l0, - "tokens", - tok->t1->stemmed.begin, tok->t1->stemmed.len, - tok->t2->stemmed.begin, tok->t2->stemmed.len); - } else if (tok->t1) { - redisAsyncCommand (rt->redis, NULL, NULL, - "HSET %b %s %b", - n0, (size_t) l0, - "tokens", - tok->t1->stemmed.begin, tok->t1->stemmed.len); + redisAsyncCommand(rt->redis, NULL, NULL, + "HSET %b %s %b:%b", + n0, (size_t) l0, + "tokens", + tok->t1->stemmed.begin, tok->t1->stemmed.len, + tok->t2->stemmed.begin, tok->t2->stemmed.len); + } + else if (tok->t1) { + redisAsyncCommand(rt->redis, NULL, NULL, + "HSET %b %s %b", + n0, (size_t) l0, + "tokens", + tok->t1->stemmed.begin, tok->t1->stemmed.len); } } - redisAsyncCommand (rt->redis, NULL, NULL, - "ZINCRBY %b_z %b %b", - prefix, (size_t)prefix_len, - n1, (size_t)l1, - n0, (size_t)l0); + redisAsyncCommand(rt->redis, NULL, NULL, + "ZINCRBY %b_z %b %b", + prefix, (size_t) prefix_len, + n1, (size_t) l1, + n0, (size_t) l0); } if (rt->ctx->new_schema && rt->ctx->expiry > 0) { out->len = 0; - l1 = rspamd_snprintf (n1, sizeof (n1), "%d", - rt->ctx->expiry); - - rspamd_printf_fstring (&out, "" - "*3\r\n" - "$6\r\n" - "EXPIRE\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n", - l0, n0, - l1, n1); - redisAsyncFormattedCommand (rt->redis, NULL, NULL, - out->str, out->len); + l1 = rspamd_snprintf(n1, sizeof(n1), "%d", + rt->ctx->expiry); + + rspamd_printf_fstring(&out, "" + "*3\r\n" + "$6\r\n" + "EXPIRE\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n", + l0, n0, + l1, n1); + redisAsyncFormattedCommand(rt->redis, NULL, NULL, + out->str, out->len); } out->len = 0; } else { if (rt->ctx->new_schema) { - l0 = rspamd_snprintf (n0, sizeof (n0), "%*s_%uL", - prefix_len, prefix, - tok->data); - - rspamd_printf_fstring (&out, "" - "*3\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n", - cmd_len, command, - l0, n0, - 1, rt->stcf->is_spam ? "S" : "H"); - - ret = redisAsyncFormattedCommand (rt->redis, NULL, NULL, - out->str, out->len); + l0 = rspamd_snprintf(n0, sizeof(n0), "%*s_%uL", + prefix_len, prefix, + tok->data); + + rspamd_printf_fstring(&out, "" + "*3\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n", + cmd_len, command, + l0, n0, + 1, rt->stcf->is_spam ? "S" : "H"); + + ret = redisAsyncFormattedCommand(rt->redis, NULL, NULL, + out->str, out->len); if (ret != REDIS_OK) { - msg_err_task ("call to redis failed: %s", rt->redis->errstr); - rspamd_fstring_free (out); + msg_err_task("call to redis failed: %s", rt->redis->errstr); + rspamd_fstring_free(out); return NULL; } @@ -617,16 +619,17 @@ rspamd_redis_tokens_to_query (struct rspamd_task *task, out->len = 0; } else { - l0 = rspamd_snprintf (n0, sizeof (n0), "%uL", tok->data); - rspamd_printf_fstring (&out, "" - "$%d\r\n" - "%s\r\n", l0, n0); + l0 = rspamd_snprintf(n0, sizeof(n0), "%uL", tok->data); + rspamd_printf_fstring(&out, "" + "$%d\r\n" + "%s\r\n", + l0, n0); } } } if (!learn && rt->ctx->new_schema) { - rspamd_printf_fstring (&out, "*1\r\n$4\r\nEXEC\r\n"); + rspamd_printf_fstring(&out, "*1\r\n$4\r\nEXEC\r\n"); } return out; @@ -636,84 +639,86 @@ rspamd_redis_tokens_to_query (struct rspamd_task *task, #endif static void -rspamd_redis_store_stat_signature (struct rspamd_task *task, - struct redis_stat_runtime *rt, - GPtrArray *tokens, - const gchar *prefix) +rspamd_redis_store_stat_signature(struct rspamd_task *task, + struct redis_stat_runtime *rt, + GPtrArray *tokens, + const gchar *prefix) { gchar *sig, keybuf[512], nbuf[64]; rspamd_token_t *tok; guint i, blen, klen; rspamd_fstring_t *out; - sig = rspamd_mempool_get_variable (task->task_pool, - RSPAMD_MEMPOOL_STAT_SIGNATURE); + sig = rspamd_mempool_get_variable(task->task_pool, + RSPAMD_MEMPOOL_STAT_SIGNATURE); if (sig == NULL) { - msg_err_task ("cannot get bayes signature"); + msg_err_task("cannot get bayes signature"); return; } - out = rspamd_fstring_sized_new (1024); - klen = rspamd_snprintf (keybuf, sizeof (keybuf), "%s_%s_%s", - prefix, sig, rt->stcf->is_spam ? "S" : "H"); + out = rspamd_fstring_sized_new(1024); + klen = rspamd_snprintf(keybuf, sizeof(keybuf), "%s_%s_%s", + prefix, sig, rt->stcf->is_spam ? "S" : "H"); /* Cleanup key */ - rspamd_printf_fstring (&out, "" - "*2\r\n" - "$3\r\n" - "DEL\r\n" - "$%d\r\n" - "%s\r\n", - klen, keybuf); - redisAsyncFormattedCommand (rt->redis, NULL, NULL, - out->str, out->len); + rspamd_printf_fstring(&out, "" + "*2\r\n" + "$3\r\n" + "DEL\r\n" + "$%d\r\n" + "%s\r\n", + klen, keybuf); + redisAsyncFormattedCommand(rt->redis, NULL, NULL, + out->str, out->len); out->len = 0; - rspamd_printf_fstring (&out, "" - "*%d\r\n" - "$5\r\n" - "LPUSH\r\n" - "$%d\r\n" - "%s\r\n", - tokens->len + 2, - klen, keybuf); + rspamd_printf_fstring(&out, "" + "*%d\r\n" + "$5\r\n" + "LPUSH\r\n" + "$%d\r\n" + "%s\r\n", + tokens->len + 2, + klen, keybuf); - PTR_ARRAY_FOREACH (tokens, i, tok) { - blen = rspamd_snprintf (nbuf, sizeof (nbuf), "%uL", tok->data); - rspamd_printf_fstring (&out, "" - "$%d\r\n" - "%s\r\n", blen, nbuf); + PTR_ARRAY_FOREACH(tokens, i, tok) + { + blen = rspamd_snprintf(nbuf, sizeof(nbuf), "%uL", tok->data); + rspamd_printf_fstring(&out, "" + "$%d\r\n" + "%s\r\n", + blen, nbuf); } - redisAsyncFormattedCommand (rt->redis, NULL, NULL, - out->str, out->len); + redisAsyncFormattedCommand(rt->redis, NULL, NULL, + out->str, out->len); out->len = 0; if (rt->ctx->expiry > 0) { out->len = 0; - blen = rspamd_snprintf (nbuf, sizeof (nbuf), "%d", - rt->ctx->expiry); - - rspamd_printf_fstring (&out, "" - "*3\r\n" - "$6\r\n" - "EXPIRE\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n", - klen, keybuf, - blen, nbuf); - redisAsyncFormattedCommand (rt->redis, NULL, NULL, - out->str, out->len); - } - - rspamd_fstring_free (out); + blen = rspamd_snprintf(nbuf, sizeof(nbuf), "%d", + rt->ctx->expiry); + + rspamd_printf_fstring(&out, "" + "*3\r\n" + "$6\r\n" + "EXPIRE\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n", + klen, keybuf, + blen, nbuf); + redisAsyncFormattedCommand(rt->redis, NULL, NULL, + out->str, out->len); + } + + rspamd_fstring_free(out); } static void -rspamd_redis_async_cbdata_cleanup (struct rspamd_redis_stat_cbdata *cbdata) +rspamd_redis_async_cbdata_cleanup(struct rspamd_redis_stat_cbdata *cbdata) { guint i; gchar *k; @@ -721,14 +726,14 @@ rspamd_redis_async_cbdata_cleanup (struct rspamd_redis_stat_cbdata *cbdata) if (cbdata && !cbdata->wanna_die) { /* Avoid double frees */ cbdata->wanna_die = TRUE; - redisAsyncFree (cbdata->redis); + redisAsyncFree(cbdata->redis); - for (i = 0; i < cbdata->cur_keys->len; i ++) { - k = g_ptr_array_index (cbdata->cur_keys, i); - g_free (k); + for (i = 0; i < cbdata->cur_keys->len; i++) { + k = g_ptr_array_index(cbdata->cur_keys, i); + g_free(k); } - g_ptr_array_free (cbdata->cur_keys, TRUE); + g_ptr_array_free(cbdata->cur_keys, TRUE); if (cbdata->elt) { cbdata->elt->cbdata = NULL; @@ -738,7 +743,7 @@ rspamd_redis_async_cbdata_cleanup (struct rspamd_redis_stat_cbdata *cbdata) /* Replace ucl object */ if (cbdata->cur) { if (cbdata->elt->stat) { - ucl_object_unref (cbdata->elt->stat); + ucl_object_unref(cbdata->elt->stat); } cbdata->elt->stat = cbdata->cur; @@ -747,18 +752,18 @@ rspamd_redis_async_cbdata_cleanup (struct rspamd_redis_stat_cbdata *cbdata) } if (cbdata->cur) { - ucl_object_unref (cbdata->cur); + ucl_object_unref(cbdata->cur); } - g_free (cbdata); + g_free(cbdata); } } /* Called when we get number of learns for a specific key */ static void -rspamd_redis_stat_learns (redisAsyncContext *c, gpointer r, gpointer priv) +rspamd_redis_stat_learns(redisAsyncContext *c, gpointer r, gpointer priv) { - struct rspamd_redis_stat_elt *redis_elt = (struct rspamd_redis_stat_elt *)priv; + struct rspamd_redis_stat_elt *redis_elt = (struct rspamd_redis_stat_elt *) priv; struct rspamd_redis_stat_cbdata *cbdata; redisReply *reply = r; ucl_object_t *obj; @@ -770,33 +775,33 @@ rspamd_redis_stat_learns (redisAsyncContext *c, gpointer r, gpointer priv) return; } - cbdata->inflight --; + cbdata->inflight--; if (c->err == 0 && r != NULL) { - if (G_LIKELY (reply->type == REDIS_REPLY_INTEGER)) { + if (G_LIKELY(reply->type == REDIS_REPLY_INTEGER)) { num = reply->integer; } else if (reply->type == REDIS_REPLY_STRING) { - rspamd_strtoul (reply->str, reply->len, &num); + rspamd_strtoul(reply->str, reply->len, &num); } - obj = (ucl_object_t *) ucl_object_lookup (cbdata->cur, "revision"); + obj = (ucl_object_t *) ucl_object_lookup(cbdata->cur, "revision"); if (obj) { obj->value.iv += num; } } if (cbdata->inflight == 0) { - rspamd_redis_async_cbdata_cleanup (cbdata); + rspamd_redis_async_cbdata_cleanup(cbdata); redis_elt->cbdata = NULL; } } /* Called when we get number of elements for a specific key */ static void -rspamd_redis_stat_key (redisAsyncContext *c, gpointer r, gpointer priv) +rspamd_redis_stat_key(redisAsyncContext *c, gpointer r, gpointer priv) { - struct rspamd_redis_stat_elt *redis_elt = (struct rspamd_redis_stat_elt *)priv; + struct rspamd_redis_stat_elt *redis_elt = (struct rspamd_redis_stat_elt *) priv; struct rspamd_redis_stat_cbdata *cbdata; redisReply *reply = r; ucl_object_t *obj; @@ -808,50 +813,50 @@ rspamd_redis_stat_key (redisAsyncContext *c, gpointer r, gpointer priv) return; } - cbdata->inflight --; + cbdata->inflight--; if (c->err == 0 && r != NULL) { - if (G_LIKELY (reply->type == REDIS_REPLY_INTEGER)) { + if (G_LIKELY(reply->type == REDIS_REPLY_INTEGER)) { num = reply->integer; } else if (reply->type == REDIS_REPLY_STRING) { - rspamd_strtol (reply->str, reply->len, &num); + rspamd_strtol(reply->str, reply->len, &num); } if (num < 0) { - msg_err ("bad learns count: %L", (gint64)num); + msg_err("bad learns count: %L", (gint64) num); num = 0; } - obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "used"); + obj = (ucl_object_t *) ucl_object_lookup(cbdata->cur, "used"); if (obj) { obj->value.iv += num; } - obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "total"); + obj = (ucl_object_t *) ucl_object_lookup(cbdata->cur, "total"); if (obj) { obj->value.iv += num; } - obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "size"); + obj = (ucl_object_t *) ucl_object_lookup(cbdata->cur, "size"); if (obj) { /* Size of key + size of int64_t */ - obj->value.iv += num * (sizeof (G_STRINGIFY (G_MAXINT64)) + - sizeof (guint64) + sizeof (gpointer)); + obj->value.iv += num * (sizeof(G_STRINGIFY(G_MAXINT64)) + + sizeof(guint64) + sizeof(gpointer)); } } if (cbdata->inflight == 0) { - rspamd_redis_async_cbdata_cleanup (cbdata); + rspamd_redis_async_cbdata_cleanup(cbdata); redis_elt->cbdata = NULL; } } /* Called when we have connected to the redis server and got keys to check */ static void -rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) +rspamd_redis_stat_keys(redisAsyncContext *c, gpointer r, gpointer priv) { - struct rspamd_redis_stat_elt *redis_elt = (struct rspamd_redis_stat_elt *)priv; + struct rspamd_redis_stat_elt *redis_elt = (struct rspamd_redis_stat_elt *) priv; struct rspamd_redis_stat_cbdata *cbdata; redisReply *reply = r, *more_elt, *elts, *elt; gchar **pk, *k; @@ -864,43 +869,45 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) return; } - cbdata->inflight --; + cbdata->inflight--; if (c->err == 0 && r != NULL) { if (reply->type == REDIS_REPLY_ARRAY) { more_elt = reply->element[0]; elts = reply->element[1]; - if (more_elt != NULL && more_elt->str != NULL && strcmp (more_elt->str, "0") != 0) { + if (more_elt != NULL && more_elt->str != NULL && strcmp(more_elt->str, "0") != 0) { more = true; } /* Clear the existing stuff */ - PTR_ARRAY_FOREACH (cbdata->cur_keys, i, k) { + PTR_ARRAY_FOREACH(cbdata->cur_keys, i, k) + { if (k) { - g_free (k); + g_free(k); } } - g_ptr_array_set_size (cbdata->cur_keys, elts->elements); + g_ptr_array_set_size(cbdata->cur_keys, elts->elements); - for (i = 0; i < elts->elements; i ++) { + for (i = 0; i < elts->elements; i++) { elt = elts->element[i]; if (elt->type == REDIS_REPLY_STRING) { - pk = (gchar **)&g_ptr_array_index (cbdata->cur_keys, i); - *pk = g_malloc (elt->len + 1); - rspamd_strlcpy (*pk, elt->str, elt->len + 1); - processed ++; + pk = (gchar **) &g_ptr_array_index(cbdata->cur_keys, i); + *pk = g_malloc(elt->len + 1); + rspamd_strlcpy(*pk, elt->str, elt->len + 1); + processed++; } else { - pk = (gchar **)&g_ptr_array_index (cbdata->cur_keys, i); + pk = (gchar **) &g_ptr_array_index(cbdata->cur_keys, i); *pk = NULL; } } if (processed) { - PTR_ARRAY_FOREACH (cbdata->cur_keys, i, k) { + PTR_ARRAY_FOREACH(cbdata->cur_keys, i, k) + { if (k) { const gchar *learned_key = "learns"; @@ -911,24 +918,24 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) else { learned_key = "learns_ham"; } - redisAsyncCommand (cbdata->redis, - rspamd_redis_stat_learns, - redis_elt, - "HGET %s %s", - k, learned_key); + redisAsyncCommand(cbdata->redis, + rspamd_redis_stat_learns, + redis_elt, + "HGET %s %s", + k, learned_key); cbdata->inflight += 1; } else { - redisAsyncCommand (cbdata->redis, - rspamd_redis_stat_key, - redis_elt, - "HLEN %s", - k); - redisAsyncCommand (cbdata->redis, - rspamd_redis_stat_learns, - redis_elt, - "HGET %s %s", - k, learned_key); + redisAsyncCommand(cbdata->redis, + rspamd_redis_stat_key, + redis_elt, + "HLEN %s", + k); + redisAsyncCommand(cbdata->redis, + rspamd_redis_stat_learns, + redis_elt, + "HGET %s %s", + k, learned_key); cbdata->inflight += 2; } } @@ -938,56 +945,56 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) if (more) { /* Get more stat keys */ - redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, redis_elt, - "SSCAN %s_keys %s COUNT 1000", - cbdata->elt->ctx->stcf->symbol, more_elt->str); + redisAsyncCommand(cbdata->redis, rspamd_redis_stat_keys, redis_elt, + "SSCAN %s_keys %s COUNT 1000", + cbdata->elt->ctx->stcf->symbol, more_elt->str); cbdata->inflight += 1; } else { /* Set up the required keys */ - ucl_object_insert_key (cbdata->cur, - ucl_object_typed_new (UCL_INT), "revision", 0, false); - ucl_object_insert_key (cbdata->cur, - ucl_object_typed_new (UCL_INT), "used", 0, false); - ucl_object_insert_key (cbdata->cur, - ucl_object_typed_new (UCL_INT), "total", 0, false); - ucl_object_insert_key (cbdata->cur, - ucl_object_typed_new (UCL_INT), "size", 0, false); - ucl_object_insert_key (cbdata->cur, - ucl_object_fromstring (cbdata->elt->ctx->stcf->symbol), - "symbol", 0, false); - ucl_object_insert_key (cbdata->cur, ucl_object_fromstring ("redis"), - "type", 0, false); - ucl_object_insert_key (cbdata->cur, ucl_object_fromint (0), - "languages", 0, false); - ucl_object_insert_key (cbdata->cur, ucl_object_fromint (processed), - "users", 0, false); - - rspamd_upstream_ok (cbdata->selected); + ucl_object_insert_key(cbdata->cur, + ucl_object_typed_new(UCL_INT), "revision", 0, false); + ucl_object_insert_key(cbdata->cur, + ucl_object_typed_new(UCL_INT), "used", 0, false); + ucl_object_insert_key(cbdata->cur, + ucl_object_typed_new(UCL_INT), "total", 0, false); + ucl_object_insert_key(cbdata->cur, + ucl_object_typed_new(UCL_INT), "size", 0, false); + ucl_object_insert_key(cbdata->cur, + ucl_object_fromstring(cbdata->elt->ctx->stcf->symbol), + "symbol", 0, false); + ucl_object_insert_key(cbdata->cur, ucl_object_fromstring("redis"), + "type", 0, false); + ucl_object_insert_key(cbdata->cur, ucl_object_fromint(0), + "languages", 0, false); + ucl_object_insert_key(cbdata->cur, ucl_object_fromint(processed), + "users", 0, false); + + rspamd_upstream_ok(cbdata->selected); if (cbdata->inflight == 0) { - rspamd_redis_async_cbdata_cleanup (cbdata); + rspamd_redis_async_cbdata_cleanup(cbdata); redis_elt->cbdata = NULL; } } } else { if (c->errstr) { - msg_err ("cannot get keys to gather stat: %s", c->errstr); + msg_err("cannot get keys to gather stat: %s", c->errstr); } else { - msg_err ("cannot get keys to gather stat: unknown error"); + msg_err("cannot get keys to gather stat: unknown error"); } - rspamd_upstream_fail (cbdata->selected, FALSE, c->errstr); - rspamd_redis_async_cbdata_cleanup (cbdata); + rspamd_upstream_fail(cbdata->selected, FALSE, c->errstr); + rspamd_redis_async_cbdata_cleanup(cbdata); redis_elt->cbdata = NULL; } } static void -rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d) +rspamd_redis_async_stat_cb(struct rspamd_stat_async_elt *elt, gpointer d) { struct redis_stat_ctx *ctx; struct rspamd_redis_stat_elt *redis_elt = elt->ud; @@ -997,83 +1004,83 @@ rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d) redisAsyncContext *redis_ctx; struct upstream *selected; - g_assert (redis_elt != NULL); + g_assert(redis_elt != NULL); ctx = redis_elt->ctx; if (redis_elt->cbdata) { /* We have some other process pending */ - rspamd_redis_async_cbdata_cleanup (redis_elt->cbdata); + rspamd_redis_async_cbdata_cleanup(redis_elt->cbdata); redis_elt->cbdata = NULL; } /* Disable further events unless needed */ elt->enabled = FALSE; - ups = rspamd_redis_get_servers (ctx, "read_servers"); + ups = rspamd_redis_get_servers(ctx, "read_servers"); if (!ups) { return; } - selected = rspamd_upstream_get (ups, - RSPAMD_UPSTREAM_ROUND_ROBIN, - NULL, - 0); + selected = rspamd_upstream_get(ups, + RSPAMD_UPSTREAM_ROUND_ROBIN, + NULL, + 0); - g_assert (selected != NULL); - addr = rspamd_upstream_addr_next (selected); - g_assert (addr != NULL); + g_assert(selected != NULL); + addr = rspamd_upstream_addr_next(selected); + g_assert(addr != NULL); - if (rspamd_inet_address_get_af (addr) == AF_UNIX) { - redis_ctx = redisAsyncConnectUnix (rspamd_inet_address_to_string (addr)); + if (rspamd_inet_address_get_af(addr) == AF_UNIX) { + redis_ctx = redisAsyncConnectUnix(rspamd_inet_address_to_string(addr)); } else { - redis_ctx = redisAsyncConnect (rspamd_inet_address_to_string (addr), - rspamd_inet_address_get_port (addr)); + redis_ctx = redisAsyncConnect(rspamd_inet_address_to_string(addr), + rspamd_inet_address_get_port(addr)); } if (redis_ctx == NULL) { - msg_warn ("cannot connect to redis server %s: %s", - rspamd_inet_address_to_string_pretty (addr), - strerror (errno)); + msg_warn("cannot connect to redis server %s: %s", + rspamd_inet_address_to_string_pretty(addr), + strerror(errno)); return; } else if (redis_ctx->err != REDIS_OK) { - msg_warn ("cannot connect to redis server %s: %s", - rspamd_inet_address_to_string_pretty (addr), - redis_ctx->errstr); - redisAsyncFree (redis_ctx); + msg_warn("cannot connect to redis server %s: %s", + rspamd_inet_address_to_string_pretty(addr), + redis_ctx->errstr); + redisAsyncFree(redis_ctx); return; } - redisLibevAttach (redis_elt->event_loop, redis_ctx); - cbdata = g_malloc0 (sizeof (*cbdata)); + redisLibevAttach(redis_elt->event_loop, redis_ctx); + cbdata = g_malloc0(sizeof(*cbdata)); cbdata->redis = redis_ctx; cbdata->selected = selected; cbdata->inflight = 1; - cbdata->cur = ucl_object_typed_new (UCL_OBJECT); + cbdata->cur = ucl_object_typed_new(UCL_OBJECT); cbdata->elt = redis_elt; - cbdata->cur_keys = g_ptr_array_sized_new (1000); + cbdata->cur_keys = g_ptr_array_sized_new(1000); redis_elt->cbdata = cbdata; /* XXX: deal with timeouts maybe */ /* Get keys in redis that match our symbol */ - rspamd_redis_maybe_auth (ctx, cbdata->redis); - redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, redis_elt, - "SSCAN %s_keys 0 COUNT 1000", - ctx->stcf->symbol); + rspamd_redis_maybe_auth(ctx, cbdata->redis); + redisAsyncCommand(cbdata->redis, rspamd_redis_stat_keys, redis_elt, + "SSCAN %s_keys 0 COUNT 1000", + ctx->stcf->symbol); } static void -rspamd_redis_async_stat_fin (struct rspamd_stat_async_elt *elt, gpointer d) +rspamd_redis_async_stat_fin(struct rspamd_stat_async_elt *elt, gpointer d) { struct rspamd_redis_stat_elt *redis_elt = elt->ud; if (redis_elt->cbdata != NULL) { - rspamd_redis_async_cbdata_cleanup (redis_elt->cbdata); + rspamd_redis_async_cbdata_cleanup(redis_elt->cbdata); redis_elt->cbdata = NULL; } @@ -1083,29 +1090,29 @@ rspamd_redis_async_stat_fin (struct rspamd_stat_async_elt *elt, gpointer d) redis_elt->stat = NULL; } - g_free (redis_elt); + g_free(redis_elt); } /* Called on connection termination */ static void -rspamd_redis_fin (gpointer data) +rspamd_redis_fin(gpointer data) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (data); + struct redis_stat_runtime *rt = REDIS_RUNTIME(data); redisAsyncContext *redis; if (rt->has_event) { /* Should not happen ! */ - msg_err ("FIXME: this code path should not be reached!"); - rspamd_session_remove_event (rt->task->s, NULL, rt); + msg_err("FIXME: this code path should not be reached!"); + rspamd_session_remove_event(rt->task->s, NULL, rt); rt->has_event = FALSE; } /* Stop timeout */ - if (ev_can_stop (&rt->timeout_event)) { - ev_timer_stop (rt->task->event_loop, &rt->timeout_event); + if (ev_can_stop(&rt->timeout_event)) { + ev_timer_stop(rt->task->event_loop, &rt->timeout_event); } if (rt->tokens) { - g_ptr_array_unref (rt->tokens); + g_ptr_array_unref(rt->tokens); rt->tokens = NULL; } @@ -1113,56 +1120,56 @@ rspamd_redis_fin (gpointer data) redis = rt->redis; rt->redis = NULL; /* This calls for all callbacks pending */ - redisAsyncFree (redis); + redisAsyncFree(redis); } if (rt->err) { - g_error_free (rt->err); + g_error_free(rt->err); } } static void -rspamd_redis_timeout (EV_P_ ev_timer *w, int revents) +rspamd_redis_timeout(EV_P_ ev_timer *w, int revents) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (w->data); + struct redis_stat_runtime *rt = REDIS_RUNTIME(w->data); struct rspamd_task *task; redisAsyncContext *redis; task = rt->task; - msg_err_task_check ("connection to redis server %s timed out", - rspamd_upstream_name (rt->selected)); + msg_err_task_check("connection to redis server %s timed out", + rspamd_upstream_name(rt->selected)); - rspamd_upstream_fail (rt->selected, FALSE, "timeout"); + rspamd_upstream_fail(rt->selected, FALSE, "timeout"); if (rt->redis) { redis = rt->redis; rt->redis = NULL; /* This calls for all callbacks pending */ - redisAsyncFree (redis); + redisAsyncFree(redis); } if (rt->tokens) { - g_ptr_array_unref (rt->tokens); + g_ptr_array_unref(rt->tokens); rt->tokens = NULL; } if (!rt->err) { - g_set_error (&rt->err, rspamd_redis_stat_quark (), ETIMEDOUT, - "error getting reply from redis server %s: timeout", - rspamd_upstream_name (rt->selected)); + g_set_error(&rt->err, rspamd_redis_stat_quark(), ETIMEDOUT, + "error getting reply from redis server %s: timeout", + rspamd_upstream_name(rt->selected)); } if (rt->has_event) { rt->has_event = FALSE; - rspamd_session_remove_event (task->s, NULL, rt); + rspamd_session_remove_event(task->s, NULL, rt); } } /* Called when we have received tokens values from redis */ static void -rspamd_redis_processed (redisAsyncContext *c, gpointer r, gpointer priv) +rspamd_redis_processed(redisAsyncContext *c, gpointer r, gpointer priv) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (priv); + struct redis_stat_runtime *rt = REDIS_RUNTIME(priv); redisReply *reply = r, *elt; struct rspamd_task *task; rspamd_token_t *tok; @@ -1177,32 +1184,32 @@ rspamd_redis_processed (redisAsyncContext *c, gpointer r, gpointer priv) if (reply->type == REDIS_REPLY_ARRAY) { if (reply->elements == task->tokens->len) { - for (i = 0; i < reply->elements; i ++) { - tok = g_ptr_array_index (task->tokens, i); + for (i = 0; i < reply->elements; i++) { + tok = g_ptr_array_index(task->tokens, i); elt = reply->element[i]; - if (G_UNLIKELY (elt->type == REDIS_REPLY_INTEGER)) { + if (G_UNLIKELY(elt->type == REDIS_REPLY_INTEGER)) { tok->values[rt->id] = elt->integer; - found ++; + found++; } else if (elt->type == REDIS_REPLY_STRING) { if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) { - rspamd_strtoul (elt->str, elt->len, &val); + rspamd_strtoul(elt->str, elt->len, &val); tok->values[rt->id] = val; } else { - float_val = strtof (elt->str, NULL); + float_val = strtof(elt->str, NULL); tok->values[rt->id] = float_val; } - found ++; + found++; } else { tok->values[rt->id] = 0; } - processed ++; + processed++; } if (rt->stcf->is_spam) { @@ -1213,54 +1220,54 @@ rspamd_redis_processed (redisAsyncContext *c, gpointer r, gpointer priv) } } else { - msg_err_task_check ("got invalid length of reply vector from redis: " - "%d, expected: %d", - (gint)reply->elements, - (gint)task->tokens->len); + msg_err_task_check("got invalid length of reply vector from redis: " + "%d, expected: %d", + (gint) reply->elements, + (gint) task->tokens->len); } } else { if (reply->type == REDIS_REPLY_ERROR) { - msg_err_task_check ("cannot learn %s: redis error: \"%s\"", - rt->stcf->symbol, reply->str); + msg_err_task_check("cannot learn %s: redis error: \"%s\"", + rt->stcf->symbol, reply->str); } else { - msg_err_task_check ("got invalid reply from redis: %s, array expected", - rspamd_redis_type_to_string(reply->type)); + msg_err_task_check("got invalid reply from redis: %s, array expected", + rspamd_redis_type_to_string(reply->type)); } } - msg_debug_stat_redis ("received tokens for %s: %d processed, %d found", - rt->redis_object_expanded, processed, found); - rspamd_upstream_ok (rt->selected); + msg_debug_stat_redis("received tokens for %s: %d processed, %d found", + rt->redis_object_expanded, processed, found); + rspamd_upstream_ok(rt->selected); } } else { - msg_err_task ("error getting reply from redis server %s: %s", - rspamd_upstream_name (rt->selected), c->errstr); + msg_err_task("error getting reply from redis server %s: %s", + rspamd_upstream_name(rt->selected), c->errstr); if (rt->redis) { - rspamd_upstream_fail (rt->selected, FALSE, c->errstr); + rspamd_upstream_fail(rt->selected, FALSE, c->errstr); } if (!rt->err) { - g_set_error (&rt->err, rspamd_redis_stat_quark (), c->err, - "cannot get values: error getting reply from redis server %s: %s", - rspamd_upstream_name (rt->selected), c->errstr); + g_set_error(&rt->err, rspamd_redis_stat_quark(), c->err, + "cannot get values: error getting reply from redis server %s: %s", + rspamd_upstream_name(rt->selected), c->errstr); } } if (rt->has_event) { rt->has_event = FALSE; - rspamd_session_remove_event (task->s, NULL, rt); + rspamd_session_remove_event(task->s, NULL, rt); } } /* Called when we have connected to the redis server and got stats */ static void -rspamd_redis_connected (redisAsyncContext *c, gpointer r, gpointer priv) +rspamd_redis_connected(redisAsyncContext *c, gpointer r, gpointer priv) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (priv); + struct redis_stat_runtime *rt = REDIS_RUNTIME(priv); redisReply *reply = r; struct rspamd_task *task; glong val = 0; @@ -1270,22 +1277,22 @@ rspamd_redis_connected (redisAsyncContext *c, gpointer r, gpointer priv) if (c->err == 0 && rt->has_event) { if (r != NULL) { - if (G_UNLIKELY (reply->type == REDIS_REPLY_INTEGER)) { + if (G_UNLIKELY(reply->type == REDIS_REPLY_INTEGER)) { val = reply->integer; } else if (reply->type == REDIS_REPLY_STRING) { - rspamd_strtol (reply->str, reply->len, &val); + rspamd_strtol(reply->str, reply->len, &val); } else { if (reply->type != REDIS_REPLY_NIL) { if (reply->type == REDIS_REPLY_ERROR) { - msg_err_task ("cannot learn %s: redis error: \"%s\"", - rt->stcf->symbol, reply->str); + msg_err_task("cannot learn %s: redis error: \"%s\"", + rt->stcf->symbol, reply->str); } else { - msg_err_task ("bad learned type for %s: %s, nil expected", - rt->stcf->symbol, - rspamd_redis_type_to_string(reply->type)); + msg_err_task("bad learned type for %s: %s, nil expected", + rt->stcf->symbol, + rspamd_redis_type_to_string(reply->type)); } } @@ -1293,15 +1300,15 @@ rspamd_redis_connected (redisAsyncContext *c, gpointer r, gpointer priv) } if (val < 0) { - msg_warn_task ("invalid number of learns for %s: %L", - rt->stcf->symbol, val); + msg_warn_task("invalid number of learns for %s: %L", + rt->stcf->symbol, val); val = 0; } rt->learned = val; - msg_debug_stat_redis ("connected to redis server, tokens learned for %s: %uL", - rt->redis_object_expanded, rt->learned); - rspamd_upstream_ok (rt->selected); + msg_debug_stat_redis("connected to redis server, tokens learned for %s: %uL", + rt->redis_object_expanded, rt->learned); + rspamd_upstream_ok(rt->selected); /* Save learn count in mempool variable */ gint64 *learns_cnt; @@ -1314,148 +1321,149 @@ rspamd_redis_connected (redisAsyncContext *c, gpointer r, gpointer priv) var_name = RSPAMD_MEMPOOL_HAM_LEARNS; } - learns_cnt = rspamd_mempool_get_variable (task->task_pool, - var_name); + learns_cnt = rspamd_mempool_get_variable(task->task_pool, + var_name); if (learns_cnt) { (*learns_cnt) += rt->learned; } else { - learns_cnt = rspamd_mempool_alloc (task->task_pool, - sizeof (*learns_cnt)); + learns_cnt = rspamd_mempool_alloc(task->task_pool, + sizeof(*learns_cnt)); *learns_cnt = rt->learned; - rspamd_mempool_set_variable (task->task_pool, - var_name, - learns_cnt, NULL); + rspamd_mempool_set_variable(task->task_pool, + var_name, + learns_cnt, NULL); } if (rt->learned >= rt->stcf->clcf->min_learns && rt->learned > 0) { - rspamd_fstring_t *query = rspamd_redis_tokens_to_query ( - task, - rt, - rt->tokens, - rt->ctx->new_schema ? "HGET" : "HMGET", - rt->redis_object_expanded, FALSE, -1, - rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER); - g_assert (query != NULL); - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)rspamd_fstring_free, query); - - int ret = redisAsyncFormattedCommand (rt->redis, - rspamd_redis_processed, rt, - query->str, query->len); + rspamd_fstring_t *query = rspamd_redis_tokens_to_query( + task, + rt, + rt->tokens, + rt->ctx->new_schema ? "HGET" : "HMGET", + rt->redis_object_expanded, FALSE, -1, + rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER); + g_assert(query != NULL); + rspamd_mempool_add_destructor(task->task_pool, + (rspamd_mempool_destruct_t) rspamd_fstring_free, query); + + int ret = redisAsyncFormattedCommand(rt->redis, + rspamd_redis_processed, rt, + query->str, query->len); if (ret != REDIS_OK) { - msg_err_task ("call to redis failed: %s", rt->redis->errstr); + msg_err_task("call to redis failed: %s", rt->redis->errstr); } else { /* Further is handled by rspamd_redis_processed */ final = FALSE; /* Restart timeout */ - if (ev_can_stop (&rt->timeout_event)) { + if (ev_can_stop(&rt->timeout_event)) { rt->timeout_event.repeat = rt->ctx->timeout; - ev_timer_again (task->event_loop, &rt->timeout_event); + ev_timer_again(task->event_loop, &rt->timeout_event); } else { rt->timeout_event.data = rt; - ev_timer_init (&rt->timeout_event, rspamd_redis_timeout, - rt->ctx->timeout, 0.); - ev_timer_start (task->event_loop, &rt->timeout_event); + ev_timer_init(&rt->timeout_event, rspamd_redis_timeout, + rt->ctx->timeout, 0.); + ev_timer_start(task->event_loop, &rt->timeout_event); } } } else { - msg_warn_task ("skip obtaining bayes tokens for %s of classifier " - "%s: not enough learns %d; %d required", - rt->stcf->symbol, rt->stcf->clcf->name, - (int)rt->learned, rt->stcf->clcf->min_learns); + msg_warn_task("skip obtaining bayes tokens for %s of classifier " + "%s: not enough learns %d; %d required", + rt->stcf->symbol, rt->stcf->clcf->name, + (int) rt->learned, rt->stcf->clcf->min_learns); } } } else if (rt->has_event) { - msg_err_task ("error getting reply from redis server %s: %s", - rspamd_upstream_name (rt->selected), c->errstr); - rspamd_upstream_fail (rt->selected, FALSE, c->errstr); + msg_err_task("error getting reply from redis server %s: %s", + rspamd_upstream_name(rt->selected), c->errstr); + rspamd_upstream_fail(rt->selected, FALSE, c->errstr); if (!rt->err) { - g_set_error (&rt->err, rspamd_redis_stat_quark (), c->err, - "error getting reply from redis server %s: %s", - rspamd_upstream_name (rt->selected), c->errstr); + g_set_error(&rt->err, rspamd_redis_stat_quark(), c->err, + "error getting reply from redis server %s: %s", + rspamd_upstream_name(rt->selected), c->errstr); } } if (final && rt->has_event) { rt->has_event = FALSE; - rspamd_session_remove_event (task->s, NULL, rt); + rspamd_session_remove_event(task->s, NULL, rt); } } /* Called when we have set tokens during learning */ static void -rspamd_redis_learned (redisAsyncContext *c, gpointer r, gpointer priv) +rspamd_redis_learned(redisAsyncContext *c, gpointer r, gpointer priv) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (priv); + struct redis_stat_runtime *rt = REDIS_RUNTIME(priv); struct rspamd_task *task; task = rt->task; if (c->err == 0) { - rspamd_upstream_ok (rt->selected); + rspamd_upstream_ok(rt->selected); } else { - msg_err_task_check ("error getting reply from redis server %s: %s", - rspamd_upstream_name (rt->selected), c->errstr); + msg_err_task_check("error getting reply from redis server %s: %s", + rspamd_upstream_name(rt->selected), c->errstr); if (rt->redis) { - rspamd_upstream_fail (rt->selected, FALSE, c->errstr); + rspamd_upstream_fail(rt->selected, FALSE, c->errstr); } if (!rt->err) { - g_set_error (&rt->err, rspamd_redis_stat_quark (), c->err, - "cannot get learned: error getting reply from redis server %s: %s", - rspamd_upstream_name (rt->selected), c->errstr); + g_set_error(&rt->err, rspamd_redis_stat_quark(), c->err, + "cannot get learned: error getting reply from redis server %s: %s", + rspamd_upstream_name(rt->selected), c->errstr); } } if (rt->has_event) { rt->has_event = FALSE; - rspamd_session_remove_event (task->s, NULL, rt); + rspamd_session_remove_event(task->s, NULL, rt); } } static void -rspamd_redis_parse_classifier_opts (struct redis_stat_ctx *backend, - const ucl_object_t *obj, - struct rspamd_config *cfg) +rspamd_redis_parse_classifier_opts(struct redis_stat_ctx *backend, + const ucl_object_t *obj, + struct rspamd_config *cfg) { const gchar *lua_script; const ucl_object_t *elt, *users_enabled; - users_enabled = ucl_object_lookup_any (obj, "per_user", - "users_enabled", NULL); + users_enabled = ucl_object_lookup_any(obj, "per_user", + "users_enabled", NULL); if (users_enabled != NULL) { - if (ucl_object_type (users_enabled) == UCL_BOOLEAN) { - backend->enable_users = ucl_object_toboolean (users_enabled); + if (ucl_object_type(users_enabled) == UCL_BOOLEAN) { + backend->enable_users = ucl_object_toboolean(users_enabled); backend->cbref_user = -1; } - else if (ucl_object_type (users_enabled) == UCL_STRING) { - lua_script = ucl_object_tostring (users_enabled); + else if (ucl_object_type(users_enabled) == UCL_STRING) { + lua_script = ucl_object_tostring(users_enabled); - if (luaL_dostring (cfg->lua_state, lua_script) != 0) { - msg_err_config ("cannot execute lua script for users " - "extraction: %s", lua_tostring (cfg->lua_state, -1)); + if (luaL_dostring(cfg->lua_state, lua_script) != 0) { + msg_err_config("cannot execute lua script for users " + "extraction: %s", + lua_tostring(cfg->lua_state, -1)); } else { - if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) { + if (lua_type(cfg->lua_state, -1) == LUA_TFUNCTION) { backend->enable_users = TRUE; - backend->cbref_user = luaL_ref (cfg->lua_state, - LUA_REGISTRYINDEX); + backend->cbref_user = luaL_ref(cfg->lua_state, + LUA_REGISTRYINDEX); } else { - msg_err_config ("lua script must return " - "function(task) and not %s", - lua_typename (cfg->lua_state, lua_type ( - cfg->lua_state, -1))); + msg_err_config("lua script must return " + "function(task) and not %s", + lua_typename(cfg->lua_state, lua_type( + cfg->lua_state, -1))); } } } @@ -1465,8 +1473,8 @@ rspamd_redis_parse_classifier_opts (struct redis_stat_ctx *backend, backend->cbref_user = -1; } - elt = ucl_object_lookup (obj, "prefix"); - if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { + elt = ucl_object_lookup(obj, "prefix"); + if (elt == NULL || ucl_object_type(elt) != UCL_STRING) { /* Default non-users statistics */ if (backend->enable_users || backend->cbref_user != -1) { backend->redis_object = REDIS_DEFAULT_USERS_OBJECT; @@ -1477,40 +1485,40 @@ rspamd_redis_parse_classifier_opts (struct redis_stat_ctx *backend, } else { /* XXX: sanity check */ - backend->redis_object = ucl_object_tostring (elt); + backend->redis_object = ucl_object_tostring(elt); } - elt = ucl_object_lookup (obj, "store_tokens"); + elt = ucl_object_lookup(obj, "store_tokens"); if (elt) { - backend->store_tokens = ucl_object_toboolean (elt); + backend->store_tokens = ucl_object_toboolean(elt); } else { backend->store_tokens = FALSE; } - elt = ucl_object_lookup (obj, "new_schema"); + elt = ucl_object_lookup(obj, "new_schema"); if (elt) { - backend->new_schema = ucl_object_toboolean (elt); + backend->new_schema = ucl_object_toboolean(elt); } else { backend->new_schema = FALSE; - msg_warn_config ("you are using old bayes schema for redis statistics, " - "please consider converting it to a new one " - "by using 'rspamadm configwizard statistics'"); + msg_warn_config("you are using old bayes schema for redis statistics, " + "please consider converting it to a new one " + "by using 'rspamadm configwizard statistics'"); } - elt = ucl_object_lookup (obj, "signatures"); + elt = ucl_object_lookup(obj, "signatures"); if (elt) { - backend->enable_signatures = ucl_object_toboolean (elt); + backend->enable_signatures = ucl_object_toboolean(elt); } else { backend->enable_signatures = FALSE; } - elt = ucl_object_lookup_any (obj, "expiry", "expire", NULL); + elt = ucl_object_lookup_any(obj, "expiry", "expire", NULL); if (elt) { - backend->expiry = ucl_object_toint (elt); + backend->expiry = ucl_object_toint(elt); } else { backend->expiry = 0; @@ -1518,8 +1526,8 @@ rspamd_redis_parse_classifier_opts (struct redis_stat_ctx *backend, } gpointer -rspamd_redis_init (struct rspamd_stat_ctx *ctx, - struct rspamd_config *cfg, struct rspamd_statfile *st) +rspamd_redis_init(struct rspamd_stat_ctx *ctx, + struct rspamd_config *cfg, struct rspamd_statfile *st) { struct redis_stat_ctx *backend; struct rspamd_statfile_config *stf = st->stcf; @@ -1527,99 +1535,99 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, const ucl_object_t *obj; gboolean ret = FALSE; gint conf_ref = -1; - lua_State *L = (lua_State *)cfg->lua_state; + lua_State *L = (lua_State *) cfg->lua_state; - backend = g_malloc0 (sizeof (*backend)); + backend = g_malloc0(sizeof(*backend)); backend->L = L; backend->timeout = REDIS_DEFAULT_TIMEOUT; /* First search in backend configuration */ - obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); - if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) { - ret = rspamd_lua_try_load_redis (L, obj, cfg, &conf_ref); + obj = ucl_object_lookup(st->classifier->cfg->opts, "backend"); + if (obj != NULL && ucl_object_type(obj) == UCL_OBJECT) { + ret = rspamd_lua_try_load_redis(L, obj, cfg, &conf_ref); } /* Now try statfiles config */ if (!ret && stf->opts) { - ret = rspamd_lua_try_load_redis (L, stf->opts, cfg, &conf_ref); + ret = rspamd_lua_try_load_redis(L, stf->opts, cfg, &conf_ref); } /* Now try classifier config */ if (!ret && st->classifier->cfg->opts) { - ret = rspamd_lua_try_load_redis (L, st->classifier->cfg->opts, cfg, &conf_ref); + ret = rspamd_lua_try_load_redis(L, st->classifier->cfg->opts, cfg, &conf_ref); } /* Now try global redis settings */ if (!ret) { - obj = ucl_object_lookup (cfg->rcl_obj, "redis"); + obj = ucl_object_lookup(cfg->rcl_obj, "redis"); if (obj) { const ucl_object_t *specific_obj; - specific_obj = ucl_object_lookup (obj, "statistics"); + specific_obj = ucl_object_lookup(obj, "statistics"); if (specific_obj) { - ret = rspamd_lua_try_load_redis (L, - specific_obj, cfg, &conf_ref); + ret = rspamd_lua_try_load_redis(L, + specific_obj, cfg, &conf_ref); } else { - ret = rspamd_lua_try_load_redis (L, - obj, cfg, &conf_ref); + ret = rspamd_lua_try_load_redis(L, + obj, cfg, &conf_ref); } } } if (!ret) { - msg_err_config ("cannot init redis backend for %s", stf->symbol); - g_free (backend); + msg_err_config("cannot init redis backend for %s", stf->symbol); + g_free(backend); return NULL; } backend->conf_ref = conf_ref; /* Check some common table values */ - lua_rawgeti (L, LUA_REGISTRYINDEX, conf_ref); + lua_rawgeti(L, LUA_REGISTRYINDEX, conf_ref); - lua_pushstring (L, "timeout"); - lua_gettable (L, -2); - if (lua_type (L, -1) == LUA_TNUMBER) { - backend->timeout = lua_tonumber (L, -1); + lua_pushstring(L, "timeout"); + lua_gettable(L, -2); + if (lua_type(L, -1) == LUA_TNUMBER) { + backend->timeout = lua_tonumber(L, -1); } - lua_pop (L, 1); + lua_pop(L, 1); - lua_pushstring (L, "db"); - lua_gettable (L, -2); - if (lua_type (L, -1) == LUA_TSTRING) { - backend->dbname = rspamd_mempool_strdup (cfg->cfg_pool, - lua_tostring (L, -1)); + lua_pushstring(L, "db"); + lua_gettable(L, -2); + if (lua_type(L, -1) == LUA_TSTRING) { + backend->dbname = rspamd_mempool_strdup(cfg->cfg_pool, + lua_tostring(L, -1)); } - lua_pop (L, 1); + lua_pop(L, 1); - lua_pushstring (L, "password"); - lua_gettable (L, -2); - if (lua_type (L, -1) == LUA_TSTRING) { - backend->password = rspamd_mempool_strdup (cfg->cfg_pool, - lua_tostring (L, -1)); + lua_pushstring(L, "password"); + lua_gettable(L, -2); + if (lua_type(L, -1) == LUA_TSTRING) { + backend->password = rspamd_mempool_strdup(cfg->cfg_pool, + lua_tostring(L, -1)); } - lua_pop (L, 1); + lua_pop(L, 1); - lua_settop (L, 0); + lua_settop(L, 0); - rspamd_redis_parse_classifier_opts (backend, st->classifier->cfg->opts, cfg); + rspamd_redis_parse_classifier_opts(backend, st->classifier->cfg->opts, cfg); stf->clcf->flags |= RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND; backend->stcf = stf; - st_elt = g_malloc0 (sizeof (*st_elt)); + st_elt = g_malloc0(sizeof(*st_elt)); st_elt->event_loop = ctx->event_loop; st_elt->ctx = backend; - backend->stat_elt = rspamd_stat_ctx_register_async ( - rspamd_redis_async_stat_cb, - rspamd_redis_async_stat_fin, - st_elt, - REDIS_STAT_TIMEOUT); + backend->stat_elt = rspamd_stat_ctx_register_async( + rspamd_redis_async_stat_cb, + rspamd_redis_async_stat_fin, + st_elt, + REDIS_STAT_TIMEOUT); st_elt->async = backend->stat_elt; - return (gpointer)backend; + return (gpointer) backend; } /* @@ -1629,10 +1637,10 @@ rspamd_redis_init (struct rspamd_stat_ctx *ctx, static void rspamd_stat_redis_on_disconnect(const struct redisAsyncContext *ac, int status) { - struct redis_stat_runtime *rt = (struct redis_stat_runtime *)ac->data; + struct redis_stat_runtime *rt = (struct redis_stat_runtime *) ac->data; - if (ev_can_stop (&rt->timeout_event)) { - ev_timer_stop (rt->task->event_loop, &rt->timeout_event); + if (ev_can_stop(&rt->timeout_event)) { + ev_timer_stop(rt->task->event_loop, &rt->timeout_event); } rt->redis = NULL; } @@ -1640,7 +1648,7 @@ rspamd_stat_redis_on_disconnect(const struct redisAsyncContext *ac, int status) static void rspamd_stat_redis_on_connect(const struct redisAsyncContext *ac, int status) { - struct redis_stat_runtime *rt = (struct redis_stat_runtime *)ac->data; + struct redis_stat_runtime *rt = (struct redis_stat_runtime *) ac->data; if (status == REDIS_ERR) { @@ -1648,136 +1656,135 @@ rspamd_stat_redis_on_connect(const struct redisAsyncContext *ac, int status) * We also need to reset rt->redis as it will be subsequently freed without * calling for redis_on_disconnect callback... */ - if (ev_can_stop (&rt->timeout_event)) { - ev_timer_stop (rt->task->event_loop, &rt->timeout_event); + if (ev_can_stop(&rt->timeout_event)) { + ev_timer_stop(rt->task->event_loop, &rt->timeout_event); } rt->redis = NULL; } } gpointer -rspamd_redis_runtime (struct rspamd_task *task, - struct rspamd_statfile_config *stcf, - gboolean learn, gpointer c, gint _id) +rspamd_redis_runtime(struct rspamd_task *task, + struct rspamd_statfile_config *stcf, + gboolean learn, gpointer c, gint _id) { - struct redis_stat_ctx *ctx = REDIS_CTX (c); + struct redis_stat_ctx *ctx = REDIS_CTX(c); struct redis_stat_runtime *rt; struct upstream *up; struct upstream_list *ups; char *object_expanded = NULL; rspamd_inet_addr_t *addr; - g_assert (ctx != NULL); - g_assert (stcf != NULL); + g_assert(ctx != NULL); + g_assert(stcf != NULL); if (learn) { - ups = rspamd_redis_get_servers (ctx, "write_servers"); + ups = rspamd_redis_get_servers(ctx, "write_servers"); if (!ups) { - msg_err_task ("no write servers defined for %s, cannot learn", - stcf->symbol); + msg_err_task("no write servers defined for %s, cannot learn", + stcf->symbol); return NULL; } - up = rspamd_upstream_get (ups, - RSPAMD_UPSTREAM_MASTER_SLAVE, - NULL, - 0); + up = rspamd_upstream_get(ups, + RSPAMD_UPSTREAM_MASTER_SLAVE, + NULL, + 0); } else { - ups = rspamd_redis_get_servers (ctx, "read_servers"); + ups = rspamd_redis_get_servers(ctx, "read_servers"); if (!ups) { - msg_err_task ("no read servers defined for %s, cannot stat", - stcf->symbol); + msg_err_task("no read servers defined for %s, cannot stat", + stcf->symbol); return NULL; } - up = rspamd_upstream_get (ups, - RSPAMD_UPSTREAM_ROUND_ROBIN, - NULL, - 0); + up = rspamd_upstream_get(ups, + RSPAMD_UPSTREAM_ROUND_ROBIN, + NULL, + 0); } if (up == NULL) { - msg_err_task ("no upstreams reachable"); + msg_err_task("no upstreams reachable"); return NULL; } - if (rspamd_redis_expand_object (ctx->redis_object, ctx, task, - &object_expanded) == 0) { - msg_err_task ("expansion for %s failed for symbol %s " - "(maybe learning per user classifier with no user or recipient)", - learn ? "learning" : "classifying", - stcf->symbol); + if (rspamd_redis_expand_object(ctx->redis_object, ctx, task, + &object_expanded) == 0) { + msg_err_task("expansion for %s failed for symbol %s " + "(maybe learning per user classifier with no user or recipient)", + learn ? "learning" : "classifying", + stcf->symbol); return NULL; } - rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt)); + rt = rspamd_mempool_alloc0(task->task_pool, sizeof(*rt)); rt->selected = up; rt->task = task; rt->ctx = ctx; rt->stcf = stcf; rt->redis_object_expanded = object_expanded; - addr = rspamd_upstream_addr_next (up); - g_assert (addr != NULL); + addr = rspamd_upstream_addr_next(up); + g_assert(addr != NULL); - if (rspamd_inet_address_get_af (addr) == AF_UNIX) { - rt->redis = redisAsyncConnectUnix (rspamd_inet_address_to_string (addr)); + if (rspamd_inet_address_get_af(addr) == AF_UNIX) { + rt->redis = redisAsyncConnectUnix(rspamd_inet_address_to_string(addr)); } else { - rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), - rspamd_inet_address_get_port (addr)); + rt->redis = redisAsyncConnect(rspamd_inet_address_to_string(addr), + rspamd_inet_address_get_port(addr)); } if (rt->redis == NULL) { - msg_warn_task ("cannot connect to redis server %s: %s", - rspamd_inet_address_to_string_pretty (addr), - strerror (errno)); + msg_warn_task("cannot connect to redis server %s: %s", + rspamd_inet_address_to_string_pretty(addr), + strerror(errno)); return NULL; } else if (rt->redis->err != REDIS_OK) { - msg_warn_task ("cannot connect to redis server %s: %s", - rspamd_inet_address_to_string_pretty (addr), - rt->redis->errstr); - redisAsyncFree (rt->redis); + msg_warn_task("cannot connect to redis server %s: %s", + rspamd_inet_address_to_string_pretty(addr), + rt->redis->errstr); + redisAsyncFree(rt->redis); rt->redis = NULL; return NULL; } - redisLibevAttach (task->event_loop, rt->redis); - rspamd_redis_maybe_auth (ctx, rt->redis); + redisLibevAttach(task->event_loop, rt->redis); + rspamd_redis_maybe_auth(ctx, rt->redis); rt->redis->data = rt; - redisAsyncSetDisconnectCallback (rt->redis, rspamd_stat_redis_on_disconnect); - redisAsyncSetConnectCallback (rt->redis, rspamd_stat_redis_on_connect); + redisAsyncSetDisconnectCallback(rt->redis, rspamd_stat_redis_on_disconnect); + redisAsyncSetConnectCallback(rt->redis, rspamd_stat_redis_on_connect); - rspamd_mempool_add_destructor (task->task_pool, rspamd_redis_fin, rt); + rspamd_mempool_add_destructor(task->task_pool, rspamd_redis_fin, rt); return rt; } -void -rspamd_redis_close (gpointer p) +void rspamd_redis_close(gpointer p) { - struct redis_stat_ctx *ctx = REDIS_CTX (p); + struct redis_stat_ctx *ctx = REDIS_CTX(p); lua_State *L = ctx->L; if (ctx->conf_ref) { - luaL_unref (L, LUA_REGISTRYINDEX, ctx->conf_ref); + luaL_unref(L, LUA_REGISTRYINDEX, ctx->conf_ref); } - g_free (ctx); + g_free(ctx); } gboolean -rspamd_redis_process_tokens (struct rspamd_task *task, - GPtrArray *tokens, - gint id, gpointer p) +rspamd_redis_process_tokens(struct rspamd_task *task, + GPtrArray *tokens, + gint id, gpointer p) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (p); + struct redis_stat_runtime *rt = REDIS_RUNTIME(p); const gchar *learned_key = "learns"; - if (rspamd_session_blocked (task->s)) { + if (rspamd_session_blocked(task->s)) { return FALSE; } @@ -1796,22 +1803,22 @@ rspamd_redis_process_tokens (struct rspamd_task *task, } } - if (redisAsyncCommand (rt->redis, rspamd_redis_connected, rt, "HGET %s %s", - rt->redis_object_expanded, learned_key) == REDIS_OK) { + if (redisAsyncCommand(rt->redis, rspamd_redis_connected, rt, "HGET %s %s", + rt->redis_object_expanded, learned_key) == REDIS_OK) { - rspamd_session_add_event (task->s, NULL, rt, M); + rspamd_session_add_event(task->s, NULL, rt, M); rt->has_event = TRUE; - rt->tokens = g_ptr_array_ref (tokens); + rt->tokens = g_ptr_array_ref(tokens); - if (ev_can_stop (&rt->timeout_event)) { + if (ev_can_stop(&rt->timeout_event)) { rt->timeout_event.repeat = rt->ctx->timeout; - ev_timer_again (task->event_loop, &rt->timeout_event); + ev_timer_again(task->event_loop, &rt->timeout_event); } else { rt->timeout_event.data = rt; - ev_timer_init (&rt->timeout_event, rspamd_redis_timeout, - rt->ctx->timeout, 0.); - ev_timer_start (task->event_loop, &rt->timeout_event); + ev_timer_init(&rt->timeout_event, rspamd_redis_timeout, + rt->ctx->timeout, 0.); + ev_timer_start(task->event_loop, &rt->timeout_event); } } @@ -1819,30 +1826,30 @@ rspamd_redis_process_tokens (struct rspamd_task *task, } gboolean -rspamd_redis_finalize_process (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_redis_finalize_process(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime); + struct redis_stat_runtime *rt = REDIS_RUNTIME(runtime); if (rt->err) { - msg_info_task ("cannot retrieve stat tokens from Redis: %e", rt->err); - g_error_free (rt->err); + msg_info_task("cannot retrieve stat tokens from Redis: %e", rt->err); + g_error_free(rt->err); rt->err = NULL; - rspamd_redis_fin (rt); + rspamd_redis_fin(rt); return FALSE; } - rspamd_redis_fin (rt); + rspamd_redis_fin(rt); return TRUE; } gboolean -rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, - gint id, gpointer p) +rspamd_redis_learn_tokens(struct rspamd_task *task, GPtrArray *tokens, + gint id, gpointer p) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (p); + struct redis_stat_runtime *rt = REDIS_RUNTIME(p); rspamd_fstring_t *query; const gchar *redis_cmd; rspamd_token_t *tok; @@ -1850,7 +1857,7 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, goffset off; const gchar *learned_key = "learns"; - if (rspamd_session_blocked (task->s)) { + if (rspamd_session_blocked(task->s)) { return FALSE; } @@ -1866,12 +1873,12 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, /* * Add the current key to the set of learned keys */ - redisAsyncCommand (rt->redis, NULL, NULL, "SADD %s_keys %s", - rt->stcf->symbol, rt->redis_object_expanded); + redisAsyncCommand(rt->redis, NULL, NULL, "SADD %s_keys %s", + rt->stcf->symbol, rt->redis_object_expanded); if (rt->ctx->new_schema) { - redisAsyncCommand (rt->redis, NULL, NULL, "HSET %s version 2", - rt->redis_object_expanded); + redisAsyncCommand(rt->redis, NULL, NULL, "HSET %s version 2", + rt->redis_object_expanded); } if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) { @@ -1882,10 +1889,10 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, } rt->id = id; - query = rspamd_redis_tokens_to_query (task, rt, tokens, - redis_cmd, rt->redis_object_expanded, TRUE, id, - rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER); - g_assert (query != NULL); + query = rspamd_redis_tokens_to_query(task, rt, tokens, + redis_cmd, rt->redis_object_expanded, TRUE, id, + rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER); + g_assert(query != NULL); query->len = 0; /* @@ -1894,85 +1901,85 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, * we could understand that we are learning or unlearning */ - tok = g_ptr_array_index (task->tokens, 0); + tok = g_ptr_array_index(task->tokens, 0); if (tok->values[id] > 0) { - rspamd_printf_fstring (&query, "" - "*4\r\n" - "$7\r\n" - "HINCRBY\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n" /* Learned key */ - "$1\r\n" - "1\r\n", - (gint)strlen (rt->redis_object_expanded), - rt->redis_object_expanded, - (gint)strlen (learned_key), - learned_key); + rspamd_printf_fstring(&query, "" + "*4\r\n" + "$7\r\n" + "HINCRBY\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n" /* Learned key */ + "$1\r\n" + "1\r\n", + (gint) strlen(rt->redis_object_expanded), + rt->redis_object_expanded, + (gint) strlen(learned_key), + learned_key); } else { - rspamd_printf_fstring (&query, "" - "*4\r\n" - "$7\r\n" - "HINCRBY\r\n" - "$%d\r\n" - "%s\r\n" - "$%d\r\n" - "%s\r\n" /* Learned key */ - "$2\r\n" - "-1\r\n", - (gint)strlen (rt->redis_object_expanded), - rt->redis_object_expanded, - (gint)strlen (learned_key), - learned_key); - } - - ret = redisAsyncFormattedCommand (rt->redis, NULL, NULL, - query->str, query->len); + rspamd_printf_fstring(&query, "" + "*4\r\n" + "$7\r\n" + "HINCRBY\r\n" + "$%d\r\n" + "%s\r\n" + "$%d\r\n" + "%s\r\n" /* Learned key */ + "$2\r\n" + "-1\r\n", + (gint) strlen(rt->redis_object_expanded), + rt->redis_object_expanded, + (gint) strlen(learned_key), + learned_key); + } + + ret = redisAsyncFormattedCommand(rt->redis, NULL, NULL, + query->str, query->len); if (ret != REDIS_OK) { - msg_err_task ("call to redis failed: %s", rt->redis->errstr); - rspamd_fstring_free (query); + msg_err_task("call to redis failed: %s", rt->redis->errstr); + rspamd_fstring_free(query); return FALSE; } off = query->len; - ret = rspamd_printf_fstring (&query, "*1\r\n$4\r\nEXEC\r\n"); - ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_learned, rt, - query->str + off, ret); - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)rspamd_fstring_free, query); + ret = rspamd_printf_fstring(&query, "*1\r\n$4\r\nEXEC\r\n"); + ret = redisAsyncFormattedCommand(rt->redis, rspamd_redis_learned, rt, + query->str + off, ret); + rspamd_mempool_add_destructor(task->task_pool, + (rspamd_mempool_destruct_t) rspamd_fstring_free, query); if (ret == REDIS_OK) { /* Add signature if needed */ if (rt->ctx->enable_signatures) { - rspamd_redis_store_stat_signature (task, rt, tokens, - "RSIG"); + rspamd_redis_store_stat_signature(task, rt, tokens, + "RSIG"); } - rspamd_session_add_event (task->s, NULL, rt, M); + rspamd_session_add_event(task->s, NULL, rt, M); rt->has_event = TRUE; /* Set timeout */ - if (ev_can_stop (&rt->timeout_event)) { + if (ev_can_stop(&rt->timeout_event)) { rt->timeout_event.repeat = rt->ctx->timeout; - ev_timer_again (task->event_loop, &rt->timeout_event); + ev_timer_again(task->event_loop, &rt->timeout_event); } else { rt->timeout_event.data = rt; - ev_timer_init (&rt->timeout_event, rspamd_redis_timeout, - rt->ctx->timeout, 0.); - ev_timer_start (task->event_loop, &rt->timeout_event); + ev_timer_init(&rt->timeout_event, rspamd_redis_timeout, + rt->ctx->timeout, 0.); + ev_timer_start(task->event_loop, &rt->timeout_event); } return TRUE; } else { - msg_err_task ("call to redis failed: %s", rt->redis->errstr); + msg_err_task("call to redis failed: %s", rt->redis->errstr); } return FALSE; @@ -1980,67 +1987,67 @@ rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, gboolean -rspamd_redis_finalize_learn (struct rspamd_task *task, gpointer runtime, - gpointer ctx, GError **err) +rspamd_redis_finalize_learn(struct rspamd_task *task, gpointer runtime, + gpointer ctx, GError **err) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime); + struct redis_stat_runtime *rt = REDIS_RUNTIME(runtime); if (rt->err) { - g_propagate_error (err, rt->err); + g_propagate_error(err, rt->err); rt->err = NULL; - rspamd_redis_fin (rt); + rspamd_redis_fin(rt); return FALSE; } - rspamd_redis_fin (rt); + rspamd_redis_fin(rt); return TRUE; } gulong -rspamd_redis_total_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_redis_total_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime); + struct redis_stat_runtime *rt = REDIS_RUNTIME(runtime); return rt->learned; } gulong -rspamd_redis_inc_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_redis_inc_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime); + struct redis_stat_runtime *rt = REDIS_RUNTIME(runtime); /* XXX: may cause races */ return rt->learned + 1; } gulong -rspamd_redis_dec_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_redis_dec_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime); + struct redis_stat_runtime *rt = REDIS_RUNTIME(runtime); /* XXX: may cause races */ return rt->learned + 1; } gulong -rspamd_redis_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_redis_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime); + struct redis_stat_runtime *rt = REDIS_RUNTIME(runtime); return rt->learned; } ucl_object_t * -rspamd_redis_get_stat (gpointer runtime, - gpointer ctx) +rspamd_redis_get_stat(gpointer runtime, + gpointer ctx) { - struct redis_stat_runtime *rt = REDIS_RUNTIME (runtime); + struct redis_stat_runtime *rt = REDIS_RUNTIME(runtime); struct rspamd_redis_stat_elt *st; redisAsyncContext *redis; @@ -2050,11 +2057,11 @@ rspamd_redis_get_stat (gpointer runtime, if (rt->redis) { redis = rt->redis; rt->redis = NULL; - redisAsyncFree (redis); + redisAsyncFree(redis); } if (st->stat) { - return ucl_object_ref (st->stat); + return ucl_object_ref(st->stat); } } @@ -2062,8 +2069,8 @@ rspamd_redis_get_stat (gpointer runtime, } gpointer -rspamd_redis_load_tokenizer_config (gpointer runtime, - gsize *len) +rspamd_redis_load_tokenizer_config(gpointer runtime, + gsize *len) { return NULL; } diff --git a/src/libstat/backends/sqlite3_backend.c b/src/libstat/backends/sqlite3_backend.c index 45e51fa7b..2fd34d83f 100644 --- a/src/libstat/backends/sqlite3_backend.c +++ b/src/libstat/backends/sqlite3_backend.c @@ -48,33 +48,33 @@ struct rspamd_stat_sqlite3_rt { }; static const char *create_tables_sql = - "BEGIN IMMEDIATE;" - "CREATE TABLE tokenizer(data BLOB);" - "CREATE TABLE users(" - "id INTEGER PRIMARY KEY," - "name TEXT," - "learns INTEGER" - ");" - "CREATE TABLE languages(" - "id INTEGER PRIMARY KEY," - "name TEXT," - "learns INTEGER" - ");" - "CREATE TABLE tokens(" - "token INTEGER NOT NULL," - "user INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE," - "language INTEGER NOT NULL REFERENCES languages(id) ON DELETE CASCADE," - "value INTEGER," - "modified INTEGER," - "CONSTRAINT tid UNIQUE (token, user, language) ON CONFLICT REPLACE" - ");" - "CREATE UNIQUE INDEX IF NOT EXISTS un ON users(name);" - "CREATE INDEX IF NOT EXISTS tok ON tokens(token);" - "CREATE UNIQUE INDEX IF NOT EXISTS ln ON languages(name);" - "PRAGMA user_version=" SQLITE3_SCHEMA_VERSION ";" - "INSERT INTO users(id, name, learns) VALUES(0, '" SQLITE3_DEFAULT "',0);" - "INSERT INTO languages(id, name, learns) VALUES(0, '" SQLITE3_DEFAULT "',0);" - "COMMIT;"; + "BEGIN IMMEDIATE;" + "CREATE TABLE tokenizer(data BLOB);" + "CREATE TABLE users(" + "id INTEGER PRIMARY KEY," + "name TEXT," + "learns INTEGER" + ");" + "CREATE TABLE languages(" + "id INTEGER PRIMARY KEY," + "name TEXT," + "learns INTEGER" + ");" + "CREATE TABLE tokens(" + "token INTEGER NOT NULL," + "user INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE," + "language INTEGER NOT NULL REFERENCES languages(id) ON DELETE CASCADE," + "value INTEGER," + "modified INTEGER," + "CONSTRAINT tid UNIQUE (token, user, language) ON CONFLICT REPLACE" + ");" + "CREATE UNIQUE INDEX IF NOT EXISTS un ON users(name);" + "CREATE INDEX IF NOT EXISTS tok ON tokens(token);" + "CREATE UNIQUE INDEX IF NOT EXISTS ln ON languages(name);" + "PRAGMA user_version=" SQLITE3_SCHEMA_VERSION ";" + "INSERT INTO users(id, name, learns) VALUES(0, '" SQLITE3_DEFAULT "',0);" + "INSERT INTO languages(id, name, learns) VALUES(0, '" SQLITE3_DEFAULT "',0);" + "COMMIT;"; enum rspamd_stat_sqlite3_stmt_idx { RSPAMD_STAT_BACKEND_TRANSACTION_START_IM = 0, @@ -103,221 +103,62 @@ enum rspamd_stat_sqlite3_stmt_idx { }; static struct rspamd_sqlite3_prstmt prepared_stmts[RSPAMD_STAT_BACKEND_MAX] = -{ - [RSPAMD_STAT_BACKEND_TRANSACTION_START_IM] = { - .idx = RSPAMD_STAT_BACKEND_TRANSACTION_START_IM, - .sql = "BEGIN IMMEDIATE TRANSACTION;", - .args = "", - .stmt = NULL, - .result = SQLITE_DONE, - .flags = 0, - .ret = "", - }, - [RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF] = { - .idx = RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF, - .sql = "BEGIN DEFERRED TRANSACTION;", - .args = "", - .stmt = NULL, - .result = SQLITE_DONE, - .flags = 0, - .ret = "" - }, - [RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL] = { - .idx = RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL, - .sql = "BEGIN EXCLUSIVE TRANSACTION;", - .args = "", - .stmt = NULL, - .result = SQLITE_DONE, - .flags = 0, - .ret = "" - }, - [RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT] = { - .idx = RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT, - .sql = "COMMIT;", - .args = "", - .stmt = NULL, - .result = SQLITE_DONE, - .flags = 0, - .ret = "" - }, - [RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK] = { - .idx = RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK, - .sql = "ROLLBACK;", - .args = "", - .stmt = NULL, - .result = SQLITE_DONE, - .flags = 0, - .ret = "" - }, - [RSPAMD_STAT_BACKEND_GET_TOKEN_FULL] = { - .idx = RSPAMD_STAT_BACKEND_GET_TOKEN_FULL, - .sql = "SELECT value FROM tokens " - "LEFT JOIN languages ON tokens.language=languages.id " - "LEFT JOIN users ON tokens.user=users.id " - "WHERE token=?1 AND (users.id=?2) " - "AND (languages.id=?3 OR languages.id=0);", - .stmt = NULL, - .args = "III", - .result = SQLITE_ROW, - .flags = 0, - .ret = "I" - }, - [RSPAMD_STAT_BACKEND_GET_TOKEN_SIMPLE] = { - .idx = RSPAMD_STAT_BACKEND_GET_TOKEN_SIMPLE, - .sql = "SELECT value FROM tokens WHERE token=?1", + { + [RSPAMD_STAT_BACKEND_TRANSACTION_START_IM] = { + .idx = RSPAMD_STAT_BACKEND_TRANSACTION_START_IM, + .sql = "BEGIN IMMEDIATE TRANSACTION;", + .args = "", .stmt = NULL, - .args = "I", - .result = SQLITE_ROW, + .result = SQLITE_DONE, .flags = 0, - .ret = "I" - }, - [RSPAMD_STAT_BACKEND_SET_TOKEN] = { - .idx = RSPAMD_STAT_BACKEND_SET_TOKEN, - .sql = "INSERT OR REPLACE INTO tokens (token, user, language, value, modified) " - "VALUES (?1, ?2, ?3, ?4, strftime('%s','now'))", - .stmt = NULL, - .args = "IIII", - .result = SQLITE_DONE, - .flags = 0, - .ret = "" - }, - [RSPAMD_STAT_BACKEND_INC_LEARNS_LANG] = { - .idx = RSPAMD_STAT_BACKEND_INC_LEARNS_LANG, - .sql = "UPDATE languages SET learns=learns + 1 WHERE id=?1", - .stmt = NULL, - .args = "I", - .result = SQLITE_DONE, - .flags = 0, - .ret = "" - }, - [RSPAMD_STAT_BACKEND_INC_LEARNS_USER] = { - .idx = RSPAMD_STAT_BACKEND_INC_LEARNS_USER, - .sql = "UPDATE users SET learns=learns + 1 WHERE id=?1", - .stmt = NULL, - .args = "I", - .result = SQLITE_DONE, - .flags = 0, - .ret = "" - }, - [RSPAMD_STAT_BACKEND_DEC_LEARNS_LANG] = { - .idx = RSPAMD_STAT_BACKEND_DEC_LEARNS_LANG, - .sql = "UPDATE languages SET learns=MAX(0, learns - 1) WHERE id=?1", - .stmt = NULL, - .args = "I", - .result = SQLITE_DONE, - .flags = 0, - .ret = "" - }, - [RSPAMD_STAT_BACKEND_DEC_LEARNS_USER] = { - .idx = RSPAMD_STAT_BACKEND_DEC_LEARNS_USER, - .sql = "UPDATE users SET learns=MAX(0, learns - 1) WHERE id=?1", - .stmt = NULL, - .args = "I", - .result = SQLITE_DONE, - .flags = 0, - .ret = "" - }, - [RSPAMD_STAT_BACKEND_GET_LEARNS] = { - .idx = RSPAMD_STAT_BACKEND_GET_LEARNS, - .sql = "SELECT SUM(MAX(0, learns)) FROM languages", - .stmt = NULL, - .args = "", - .result = SQLITE_ROW, - .flags = 0, - .ret = "I" - }, - [RSPAMD_STAT_BACKEND_GET_LANGUAGE] = { - .idx = RSPAMD_STAT_BACKEND_GET_LANGUAGE, - .sql = "SELECT id FROM languages WHERE name=?1", - .stmt = NULL, - .args = "T", - .result = SQLITE_ROW, - .flags = 0, - .ret = "I" - }, - [RSPAMD_STAT_BACKEND_GET_USER] = { - .idx = RSPAMD_STAT_BACKEND_GET_USER, - .sql = "SELECT id FROM users WHERE name=?1", - .stmt = NULL, - .args = "T", - .result = SQLITE_ROW, - .flags = 0, - .ret = "I" - }, - [RSPAMD_STAT_BACKEND_INSERT_USER] = { - .idx = RSPAMD_STAT_BACKEND_INSERT_USER, - .sql = "INSERT INTO users (name, learns) VALUES (?1, 0)", - .stmt = NULL, - .args = "T", - .result = SQLITE_DONE, - .flags = 0, - .ret = "L" - }, - [RSPAMD_STAT_BACKEND_INSERT_LANGUAGE] = { - .idx = RSPAMD_STAT_BACKEND_INSERT_LANGUAGE, - .sql = "INSERT INTO languages (name, learns) VALUES (?1, 0)", - .stmt = NULL, - .args = "T", - .result = SQLITE_DONE, - .flags = 0, - .ret = "L" - }, - [RSPAMD_STAT_BACKEND_SAVE_TOKENIZER] = { - .idx = RSPAMD_STAT_BACKEND_SAVE_TOKENIZER, - .sql = "INSERT INTO tokenizer(data) VALUES (?1)", - .stmt = NULL, - .args = "B", - .result = SQLITE_DONE, - .flags = 0, - .ret = "" - }, - [RSPAMD_STAT_BACKEND_LOAD_TOKENIZER] = { - .idx = RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, - .sql = "SELECT data FROM tokenizer", - .stmt = NULL, - .args = "", - .result = SQLITE_ROW, - .flags = 0, - .ret = "B" - }, - [RSPAMD_STAT_BACKEND_NTOKENS] = { - .idx = RSPAMD_STAT_BACKEND_NTOKENS, - .sql = "SELECT COUNT(*) FROM tokens", - .stmt = NULL, - .args = "", - .result = SQLITE_ROW, - .flags = 0, - .ret = "I" - }, - [RSPAMD_STAT_BACKEND_NLANGUAGES] = { - .idx = RSPAMD_STAT_BACKEND_NLANGUAGES, - .sql = "SELECT COUNT(*) FROM languages", - .stmt = NULL, - .args = "", - .result = SQLITE_ROW, - .flags = 0, - .ret = "I" - }, - [RSPAMD_STAT_BACKEND_NUSERS] = { - .idx = RSPAMD_STAT_BACKEND_NUSERS, - .sql = "SELECT COUNT(*) FROM users", - .stmt = NULL, - .args = "", - .result = SQLITE_ROW, - .flags = 0, - .ret = "I" - } -}; + .ret = "", + }, + [RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF] = {.idx = RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF, .sql = "BEGIN DEFERRED TRANSACTION;", .args = "", .stmt = NULL, .result = SQLITE_DONE, .flags = 0, .ret = ""}, + [RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL] = {.idx = RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL, .sql = "BEGIN EXCLUSIVE TRANSACTION;", .args = "", .stmt = NULL, .result = SQLITE_DONE, .flags = 0, .ret = ""}, + [RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT] = {.idx = RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT, .sql = "COMMIT;", .args = "", .stmt = NULL, .result = SQLITE_DONE, .flags = 0, .ret = ""}, + [RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK] = {.idx = RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK, .sql = "ROLLBACK;", .args = "", .stmt = NULL, .result = SQLITE_DONE, .flags = 0, .ret = ""}, + [RSPAMD_STAT_BACKEND_GET_TOKEN_FULL] = {.idx = RSPAMD_STAT_BACKEND_GET_TOKEN_FULL, .sql = "SELECT value FROM tokens " + "LEFT JOIN languages ON tokens.language=languages.id " + "LEFT JOIN users ON tokens.user=users.id " + "WHERE token=?1 AND (users.id=?2) " + "AND (languages.id=?3 OR languages.id=0);", + .stmt = NULL, + .args = "III", + .result = SQLITE_ROW, + .flags = 0, + .ret = "I"}, + [RSPAMD_STAT_BACKEND_GET_TOKEN_SIMPLE] = {.idx = RSPAMD_STAT_BACKEND_GET_TOKEN_SIMPLE, .sql = "SELECT value FROM tokens WHERE token=?1", .stmt = NULL, .args = "I", .result = SQLITE_ROW, .flags = 0, .ret = "I"}, + [RSPAMD_STAT_BACKEND_SET_TOKEN] = {.idx = RSPAMD_STAT_BACKEND_SET_TOKEN, .sql = "INSERT OR REPLACE INTO tokens (token, user, language, value, modified) " + "VALUES (?1, ?2, ?3, ?4, strftime('%s','now'))", + .stmt = NULL, + .args = "IIII", + .result = SQLITE_DONE, + .flags = 0, + .ret = ""}, + [RSPAMD_STAT_BACKEND_INC_LEARNS_LANG] = {.idx = RSPAMD_STAT_BACKEND_INC_LEARNS_LANG, .sql = "UPDATE languages SET learns=learns + 1 WHERE id=?1", .stmt = NULL, .args = "I", .result = SQLITE_DONE, .flags = 0, .ret = ""}, + [RSPAMD_STAT_BACKEND_INC_LEARNS_USER] = {.idx = RSPAMD_STAT_BACKEND_INC_LEARNS_USER, .sql = "UPDATE users SET learns=learns + 1 WHERE id=?1", .stmt = NULL, .args = "I", .result = SQLITE_DONE, .flags = 0, .ret = ""}, + [RSPAMD_STAT_BACKEND_DEC_LEARNS_LANG] = {.idx = RSPAMD_STAT_BACKEND_DEC_LEARNS_LANG, .sql = "UPDATE languages SET learns=MAX(0, learns - 1) WHERE id=?1", .stmt = NULL, .args = "I", .result = SQLITE_DONE, .flags = 0, .ret = ""}, + [RSPAMD_STAT_BACKEND_DEC_LEARNS_USER] = {.idx = RSPAMD_STAT_BACKEND_DEC_LEARNS_USER, .sql = "UPDATE users SET learns=MAX(0, learns - 1) WHERE id=?1", .stmt = NULL, .args = "I", .result = SQLITE_DONE, .flags = 0, .ret = ""}, + [RSPAMD_STAT_BACKEND_GET_LEARNS] = {.idx = RSPAMD_STAT_BACKEND_GET_LEARNS, .sql = "SELECT SUM(MAX(0, learns)) FROM languages", .stmt = NULL, .args = "", .result = SQLITE_ROW, .flags = 0, .ret = "I"}, + [RSPAMD_STAT_BACKEND_GET_LANGUAGE] = {.idx = RSPAMD_STAT_BACKEND_GET_LANGUAGE, .sql = "SELECT id FROM languages WHERE name=?1", .stmt = NULL, .args = "T", .result = SQLITE_ROW, .flags = 0, .ret = "I"}, + [RSPAMD_STAT_BACKEND_GET_USER] = {.idx = RSPAMD_STAT_BACKEND_GET_USER, .sql = "SELECT id FROM users WHERE name=?1", .stmt = NULL, .args = "T", .result = SQLITE_ROW, .flags = 0, .ret = "I"}, + [RSPAMD_STAT_BACKEND_INSERT_USER] = {.idx = RSPAMD_STAT_BACKEND_INSERT_USER, .sql = "INSERT INTO users (name, learns) VALUES (?1, 0)", .stmt = NULL, .args = "T", .result = SQLITE_DONE, .flags = 0, .ret = "L"}, + [RSPAMD_STAT_BACKEND_INSERT_LANGUAGE] = {.idx = RSPAMD_STAT_BACKEND_INSERT_LANGUAGE, .sql = "INSERT INTO languages (name, learns) VALUES (?1, 0)", .stmt = NULL, .args = "T", .result = SQLITE_DONE, .flags = 0, .ret = "L"}, + [RSPAMD_STAT_BACKEND_SAVE_TOKENIZER] = {.idx = RSPAMD_STAT_BACKEND_SAVE_TOKENIZER, .sql = "INSERT INTO tokenizer(data) VALUES (?1)", .stmt = NULL, .args = "B", .result = SQLITE_DONE, .flags = 0, .ret = ""}, + [RSPAMD_STAT_BACKEND_LOAD_TOKENIZER] = {.idx = RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, .sql = "SELECT data FROM tokenizer", .stmt = NULL, .args = "", .result = SQLITE_ROW, .flags = 0, .ret = "B"}, + [RSPAMD_STAT_BACKEND_NTOKENS] = {.idx = RSPAMD_STAT_BACKEND_NTOKENS, .sql = "SELECT COUNT(*) FROM tokens", .stmt = NULL, .args = "", .result = SQLITE_ROW, .flags = 0, .ret = "I"}, + [RSPAMD_STAT_BACKEND_NLANGUAGES] = {.idx = RSPAMD_STAT_BACKEND_NLANGUAGES, .sql = "SELECT COUNT(*) FROM languages", .stmt = NULL, .args = "", .result = SQLITE_ROW, .flags = 0, .ret = "I"}, + [RSPAMD_STAT_BACKEND_NUSERS] = {.idx = RSPAMD_STAT_BACKEND_NUSERS, .sql = "SELECT COUNT(*) FROM users", .stmt = NULL, .args = "", .result = SQLITE_ROW, .flags = 0, .ret = "I"}}; static GQuark -rspamd_sqlite3_backend_quark (void) +rspamd_sqlite3_backend_quark(void) { - return g_quark_from_static_string ("sqlite3-stat-backend"); + return g_quark_from_static_string("sqlite3-stat-backend"); } static gint64 -rspamd_sqlite3_get_user (struct rspamd_stat_sqlite3_db *db, - struct rspamd_task *task, gboolean learn) +rspamd_sqlite3_get_user(struct rspamd_stat_sqlite3_db *db, + struct rspamd_task *task, gboolean learn) { gint64 id = 0; /* Default user is 0 */ gint rc, err_idx; @@ -326,48 +167,48 @@ rspamd_sqlite3_get_user (struct rspamd_stat_sqlite3_db *db, lua_State *L = db->L; if (db->cbref_user == -1) { - user = rspamd_task_get_principal_recipient (task); + user = rspamd_task_get_principal_recipient(task); } else { /* Execute lua function to get userdata */ - lua_pushcfunction (L, &rspamd_lua_traceback); - err_idx = lua_gettop (L); + lua_pushcfunction(L, &rspamd_lua_traceback); + err_idx = lua_gettop(L); - lua_rawgeti (L, LUA_REGISTRYINDEX, db->cbref_user); - ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); + lua_rawgeti(L, LUA_REGISTRYINDEX, db->cbref_user); + ptask = lua_newuserdata(L, sizeof(struct rspamd_task *)); *ptask = task; - rspamd_lua_setclass (L, "rspamd{task}", -1); + rspamd_lua_setclass(L, "rspamd{task}", -1); - if (lua_pcall (L, 1, 1, err_idx) != 0) { - msg_err_task ("call to user extraction script failed: %s", - lua_tostring (L, -1)); + if (lua_pcall(L, 1, 1, err_idx) != 0) { + msg_err_task("call to user extraction script failed: %s", + lua_tostring(L, -1)); } else { - user = rspamd_mempool_strdup (task->task_pool, lua_tostring (L, -1)); + user = rspamd_mempool_strdup(task->task_pool, lua_tostring(L, -1)); } /* Result + error function */ - lua_settop (L, err_idx - 1); + lua_settop(L, err_idx - 1); } if (user != NULL) { - rspamd_mempool_set_variable (task->task_pool, "stat_user", - (gpointer)user, NULL); + rspamd_mempool_set_variable(task->task_pool, "stat_user", + (gpointer) user, NULL); - rc = rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, - RSPAMD_STAT_BACKEND_GET_USER, user, &id); + rc = rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt, + RSPAMD_STAT_BACKEND_GET_USER, user, &id); if (rc != SQLITE_OK && learn) { /* We need to insert a new user */ if (!db->in_transaction) { - rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); + rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); db->in_transaction = TRUE; } - rc = rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, - RSPAMD_STAT_BACKEND_INSERT_USER, user, &id); + rc = rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt, + RSPAMD_STAT_BACKEND_INSERT_USER, user, &id); } } @@ -375,8 +216,8 @@ rspamd_sqlite3_get_user (struct rspamd_stat_sqlite3_db *db, } static gint64 -rspamd_sqlite3_get_language (struct rspamd_stat_sqlite3_db *db, - struct rspamd_task *task, gboolean learn) +rspamd_sqlite3_get_language(struct rspamd_stat_sqlite3_db *db, + struct rspamd_task *task, gboolean learn) { gint64 id = 0; /* Default language is 0 */ gint rc, err_idx; @@ -387,10 +228,11 @@ rspamd_sqlite3_get_language (struct rspamd_stat_sqlite3_db *db, lua_State *L = db->L; if (db->cbref_language == -1) { - PTR_ARRAY_FOREACH (MESSAGE_FIELD (task, text_parts), i, tp) { + PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, tp) + { if (tp->language != NULL && tp->language[0] != '\0' && - strcmp (tp->language, "en") != 0) { + strcmp(tp->language, "en") != 0) { language = tp->language; break; } @@ -398,43 +240,43 @@ rspamd_sqlite3_get_language (struct rspamd_stat_sqlite3_db *db, } else { /* Execute lua function to get userdata */ - lua_pushcfunction (L, &rspamd_lua_traceback); - err_idx = lua_gettop (L); + lua_pushcfunction(L, &rspamd_lua_traceback); + err_idx = lua_gettop(L); - lua_rawgeti (L, LUA_REGISTRYINDEX, db->cbref_language); - ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); + lua_rawgeti(L, LUA_REGISTRYINDEX, db->cbref_language); + ptask = lua_newuserdata(L, sizeof(struct rspamd_task *)); *ptask = task; - rspamd_lua_setclass (L, "rspamd{task}", -1); + rspamd_lua_setclass(L, "rspamd{task}", -1); - if (lua_pcall (L, 1, 1, err_idx) != 0) { - msg_err_task ("call to language extraction script failed: %s", - lua_tostring (L, -1)); + if (lua_pcall(L, 1, 1, err_idx) != 0) { + msg_err_task("call to language extraction script failed: %s", + lua_tostring(L, -1)); } else { - language = rspamd_mempool_strdup (task->task_pool, - lua_tostring (L, -1)); + language = rspamd_mempool_strdup(task->task_pool, + lua_tostring(L, -1)); } /* Result + error function */ - lua_settop (L, err_idx - 1); + lua_settop(L, err_idx - 1); } /* XXX: We ignore multiple languages but default + extra */ if (language != NULL) { - rc = rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, - RSPAMD_STAT_BACKEND_GET_LANGUAGE, language, &id); + rc = rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt, + RSPAMD_STAT_BACKEND_GET_LANGUAGE, language, &id); if (rc != SQLITE_OK && learn) { /* We need to insert a new language */ if (!db->in_transaction) { - rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); + rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); db->in_transaction = TRUE; } - rc = rspamd_sqlite3_run_prstmt (task->task_pool, db->sqlite, db->prstmt, - RSPAMD_STAT_BACKEND_INSERT_LANGUAGE, language, &id); + rc = rspamd_sqlite3_run_prstmt(task->task_pool, db->sqlite, db->prstmt, + RSPAMD_STAT_BACKEND_INSERT_LANGUAGE, language, &id); } } @@ -442,10 +284,10 @@ rspamd_sqlite3_get_language (struct rspamd_stat_sqlite3_db *db, } static struct rspamd_stat_sqlite3_db * -rspamd_sqlite3_opendb (rspamd_mempool_t *pool, - struct rspamd_statfile_config *stcf, - const gchar *path, const ucl_object_t *opts, - gboolean create, GError **err) +rspamd_sqlite3_opendb(rspamd_mempool_t *pool, + struct rspamd_statfile_config *stcf, + const gchar *path, const ucl_object_t *opts, + gboolean create, GError **err) { struct rspamd_stat_sqlite3_db *bk; struct rspamd_stat_tokenizer *tokenizer; @@ -456,80 +298,79 @@ rspamd_sqlite3_opendb (rspamd_mempool_t *pool, gint ret, ntries = 0; const gint max_tries = 100; struct timespec sleep_ts = { - .tv_sec = 0, - .tv_nsec = 1000000 - }; + .tv_sec = 0, + .tv_nsec = 1000000}; - bk = g_malloc0 (sizeof (*bk)); - bk->sqlite = rspamd_sqlite3_open_or_create (pool, path, create_tables_sql, - 0, err); + bk = g_malloc0(sizeof(*bk)); + bk->sqlite = rspamd_sqlite3_open_or_create(pool, path, create_tables_sql, + 0, err); bk->pool = pool; if (bk->sqlite == NULL) { - g_free (bk); + g_free(bk); return NULL; } - bk->fname = g_strdup (path); + bk->fname = g_strdup(path); - bk->prstmt = rspamd_sqlite3_init_prstmt (bk->sqlite, prepared_stmts, - RSPAMD_STAT_BACKEND_MAX, err); + bk->prstmt = rspamd_sqlite3_init_prstmt(bk->sqlite, prepared_stmts, + RSPAMD_STAT_BACKEND_MAX, err); if (bk->prstmt == NULL) { - sqlite3_close (bk->sqlite); - g_free (bk); + sqlite3_close(bk->sqlite); + g_free(bk); return NULL; } /* Check tokenizer configuration */ - if (rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, &sz64, &tk_conf) != SQLITE_OK || - sz64 == 0) { + if (rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, &sz64, &tk_conf) != SQLITE_OK || + sz64 == 0) { - while ((ret = rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL)) == SQLITE_BUSY && + while ((ret = rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_START_EXCL)) == SQLITE_BUSY && ++ntries <= max_tries) { - nanosleep (&sleep_ts, NULL); + nanosleep(&sleep_ts, NULL); } - msg_info_pool ("absent tokenizer conf in %s, creating a new one", - bk->fname); - g_assert (stcf->clcf->tokenizer != NULL); - tokenizer = rspamd_stat_get_tokenizer (stcf->clcf->tokenizer->name); - g_assert (tokenizer != NULL); - tk_conf = tokenizer->get_config (pool, stcf->clcf->tokenizer, &sz); + msg_info_pool("absent tokenizer conf in %s, creating a new one", + bk->fname); + g_assert(stcf->clcf->tokenizer != NULL); + tokenizer = rspamd_stat_get_tokenizer(stcf->clcf->tokenizer->name); + g_assert(tokenizer != NULL); + tk_conf = tokenizer->get_config(pool, stcf->clcf->tokenizer, &sz); /* Encode to base32 */ - tok_conf_encoded = rspamd_encode_base32 (tk_conf, sz, RSPAMD_BASE32_DEFAULT); + tok_conf_encoded = rspamd_encode_base32(tk_conf, sz, RSPAMD_BASE32_DEFAULT); - if (rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_SAVE_TOKENIZER, - (gint64)strlen (tok_conf_encoded), - tok_conf_encoded) != SQLITE_OK) { - sqlite3_close (bk->sqlite); - g_free (bk); - g_free (tok_conf_encoded); + if (rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_SAVE_TOKENIZER, + (gint64) strlen(tok_conf_encoded), + tok_conf_encoded) != SQLITE_OK) { + sqlite3_close(bk->sqlite); + g_free(bk); + g_free(tok_conf_encoded); return NULL; } - rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); - g_free (tok_conf_encoded); + rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); + g_free(tok_conf_encoded); } else { - g_free (tk_conf); + g_free(tk_conf); } return bk; } gpointer -rspamd_sqlite3_init (struct rspamd_stat_ctx *ctx, - struct rspamd_config *cfg, - struct rspamd_statfile *st) +rspamd_sqlite3_init(struct rspamd_stat_ctx *ctx, + struct rspamd_config *cfg, + struct rspamd_statfile *st) { struct rspamd_classifier_config *clf = st->classifier->cfg; struct rspamd_statfile_config *stf = st->stcf; @@ -538,51 +379,52 @@ rspamd_sqlite3_init (struct rspamd_stat_ctx *ctx, struct rspamd_stat_sqlite3_db *bk; GError *err = NULL; - filenameo = ucl_object_lookup (stf->opts, "filename"); - if (filenameo == NULL || ucl_object_type (filenameo) != UCL_STRING) { - filenameo = ucl_object_lookup (stf->opts, "path"); - if (filenameo == NULL || ucl_object_type (filenameo) != UCL_STRING) { - msg_err_config ("statfile %s has no filename defined", stf->symbol); + filenameo = ucl_object_lookup(stf->opts, "filename"); + if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) { + filenameo = ucl_object_lookup(stf->opts, "path"); + if (filenameo == NULL || ucl_object_type(filenameo) != UCL_STRING) { + msg_err_config("statfile %s has no filename defined", stf->symbol); return NULL; } } - filename = ucl_object_tostring (filenameo); + filename = ucl_object_tostring(filenameo); - if ((bk = rspamd_sqlite3_opendb (cfg->cfg_pool, stf, filename, - stf->opts, TRUE, &err)) == NULL) { - msg_err_config ("cannot open sqlite3 db %s: %e", filename, err); - g_error_free (err); + if ((bk = rspamd_sqlite3_opendb(cfg->cfg_pool, stf, filename, + stf->opts, TRUE, &err)) == NULL) { + msg_err_config("cannot open sqlite3 db %s: %e", filename, err); + g_error_free(err); return NULL; } bk->L = cfg->lua_state; - users_enabled = ucl_object_lookup_any (clf->opts, "per_user", - "users_enabled", NULL); + users_enabled = ucl_object_lookup_any(clf->opts, "per_user", + "users_enabled", NULL); if (users_enabled != NULL) { - if (ucl_object_type (users_enabled) == UCL_BOOLEAN) { - bk->enable_users = ucl_object_toboolean (users_enabled); + if (ucl_object_type(users_enabled) == UCL_BOOLEAN) { + bk->enable_users = ucl_object_toboolean(users_enabled); bk->cbref_user = -1; } - else if (ucl_object_type (users_enabled) == UCL_STRING) { - lua_script = ucl_object_tostring (users_enabled); + else if (ucl_object_type(users_enabled) == UCL_STRING) { + lua_script = ucl_object_tostring(users_enabled); - if (luaL_dostring (cfg->lua_state, lua_script) != 0) { - msg_err_config ("cannot execute lua script for users " - "extraction: %s", lua_tostring (cfg->lua_state, -1)); + if (luaL_dostring(cfg->lua_state, lua_script) != 0) { + msg_err_config("cannot execute lua script for users " + "extraction: %s", + lua_tostring(cfg->lua_state, -1)); } else { - if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) { + if (lua_type(cfg->lua_state, -1) == LUA_TFUNCTION) { bk->enable_users = TRUE; - bk->cbref_user = luaL_ref (cfg->lua_state, - LUA_REGISTRYINDEX); + bk->cbref_user = luaL_ref(cfg->lua_state, + LUA_REGISTRYINDEX); } else { - msg_err_config ("lua script must return " - "function(task) and not %s", - lua_typename (cfg->lua_state, lua_type ( - cfg->lua_state, -1))); + msg_err_config("lua script must return " + "function(task) and not %s", + lua_typename(cfg->lua_state, lua_type( + cfg->lua_state, -1))); } } } @@ -591,34 +433,34 @@ rspamd_sqlite3_init (struct rspamd_stat_ctx *ctx, bk->enable_users = FALSE; } - lang_enabled = ucl_object_lookup_any (clf->opts, - "per_language", "languages_enabled", NULL); + lang_enabled = ucl_object_lookup_any(clf->opts, + "per_language", "languages_enabled", NULL); if (lang_enabled != NULL) { - if (ucl_object_type (lang_enabled) == UCL_BOOLEAN) { - bk->enable_languages = ucl_object_toboolean (lang_enabled); + if (ucl_object_type(lang_enabled) == UCL_BOOLEAN) { + bk->enable_languages = ucl_object_toboolean(lang_enabled); bk->cbref_language = -1; } - else if (ucl_object_type (lang_enabled) == UCL_STRING) { - lua_script = ucl_object_tostring (lang_enabled); - - if (luaL_dostring (cfg->lua_state, lua_script) != 0) { - msg_err_config ( - "cannot execute lua script for languages " - "extraction: %s", - lua_tostring (cfg->lua_state, -1)); + else if (ucl_object_type(lang_enabled) == UCL_STRING) { + lua_script = ucl_object_tostring(lang_enabled); + + if (luaL_dostring(cfg->lua_state, lua_script) != 0) { + msg_err_config( + "cannot execute lua script for languages " + "extraction: %s", + lua_tostring(cfg->lua_state, -1)); } else { - if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) { + if (lua_type(cfg->lua_state, -1) == LUA_TFUNCTION) { bk->enable_languages = TRUE; - bk->cbref_language = luaL_ref (cfg->lua_state, - LUA_REGISTRYINDEX); + bk->cbref_language = luaL_ref(cfg->lua_state, + LUA_REGISTRYINDEX); } else { - msg_err_config ("lua script must return " - "function(task) and not %s", - lua_typename (cfg->lua_state, - lua_type (cfg->lua_state, -1))); + msg_err_config("lua script must return " + "function(task) and not %s", + lua_typename(cfg->lua_state, + lua_type(cfg->lua_state, -1))); } } } @@ -628,46 +470,45 @@ rspamd_sqlite3_init (struct rspamd_stat_ctx *ctx, } if (bk->enable_languages) { - msg_info_config ("enable per language statistics for %s", - stf->symbol); + msg_info_config("enable per language statistics for %s", + stf->symbol); } if (bk->enable_users) { - msg_info_config ("enable per users statistics for %s", - stf->symbol); + msg_info_config("enable per users statistics for %s", + stf->symbol); } return (gpointer) bk; } -void -rspamd_sqlite3_close (gpointer p) +void rspamd_sqlite3_close(gpointer p) { struct rspamd_stat_sqlite3_db *bk = p; if (bk->sqlite) { if (bk->in_transaction) { - rspamd_sqlite3_run_prstmt (bk->pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); + rspamd_sqlite3_run_prstmt(bk->pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); } - rspamd_sqlite3_close_prstmt (bk->sqlite, bk->prstmt); - sqlite3_close (bk->sqlite); - g_free (bk->fname); - g_free (bk); + rspamd_sqlite3_close_prstmt(bk->sqlite, bk->prstmt); + sqlite3_close(bk->sqlite); + g_free(bk->fname); + g_free(bk); } } gpointer -rspamd_sqlite3_runtime (struct rspamd_task *task, - struct rspamd_statfile_config *stcf, gboolean learn, gpointer p, gint _id) +rspamd_sqlite3_runtime(struct rspamd_task *task, + struct rspamd_statfile_config *stcf, gboolean learn, gpointer p, gint _id) { struct rspamd_stat_sqlite3_rt *rt = NULL; struct rspamd_stat_sqlite3_db *bk = p; if (bk) { - rt = rspamd_mempool_alloc (task->task_pool, sizeof (*rt)); + rt = rspamd_mempool_alloc(task->task_pool, sizeof(*rt)); rt->db = bk; rt->task = task; rt->user_id = -1; @@ -679,9 +520,9 @@ rspamd_sqlite3_runtime (struct rspamd_task *task, } gboolean -rspamd_sqlite3_process_tokens (struct rspamd_task *task, - GPtrArray *tokens, - gint id, gpointer p) +rspamd_sqlite3_process_tokens(struct rspamd_task *task, + GPtrArray *tokens, + gint id, gpointer p) { struct rspamd_stat_sqlite3_db *bk; struct rspamd_stat_sqlite3_rt *rt = p; @@ -689,13 +530,13 @@ rspamd_sqlite3_process_tokens (struct rspamd_task *task, guint i; rspamd_token_t *tok; - g_assert (p != NULL); - g_assert (tokens != NULL); + g_assert(p != NULL); + g_assert(tokens != NULL); bk = rt->db; - for (i = 0; i < tokens->len; i ++) { - tok = g_ptr_array_index (tokens, i); + for (i = 0; i < tokens->len; i++) { + tok = g_ptr_array_index(tokens, i); if (bk == NULL) { /* Statfile is does not exist, so all values are zero */ @@ -704,14 +545,14 @@ rspamd_sqlite3_process_tokens (struct rspamd_task *task, } if (!bk->in_transaction) { - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_START_DEF); bk->in_transaction = TRUE; } if (rt->user_id == -1) { if (bk->enable_users) { - rt->user_id = rspamd_sqlite3_get_user (bk, task, FALSE); + rt->user_id = rspamd_sqlite3_get_user(bk, task, FALSE); } else { rt->user_id = 0; @@ -720,7 +561,7 @@ rspamd_sqlite3_process_tokens (struct rspamd_task *task, if (rt->lang_id == -1) { if (bk->enable_languages) { - rt->lang_id = rspamd_sqlite3_get_language (bk, task, FALSE); + rt->lang_id = rspamd_sqlite3_get_language(bk, task, FALSE); } else { rt->lang_id = 0; @@ -728,9 +569,9 @@ rspamd_sqlite3_process_tokens (struct rspamd_task *task, } if (bk->enable_languages || bk->enable_users) { - if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_GET_TOKEN_FULL, - tok->data, rt->user_id, rt->lang_id, &iv) == SQLITE_OK) { + if (rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_GET_TOKEN_FULL, + tok->data, rt->user_id, rt->lang_id, &iv) == SQLITE_OK) { tok->values[id] = iv; } else { @@ -738,9 +579,9 @@ rspamd_sqlite3_process_tokens (struct rspamd_task *task, } } else { - if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_GET_TOKEN_SIMPLE, - tok->data, &iv) == SQLITE_OK) { + if (rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_GET_TOKEN_SIMPLE, + tok->data, &iv) == SQLITE_OK) { tok->values[id] = iv; } else { @@ -761,18 +602,18 @@ rspamd_sqlite3_process_tokens (struct rspamd_task *task, } gboolean -rspamd_sqlite3_finalize_process (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_sqlite3_finalize_process(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { struct rspamd_stat_sqlite3_rt *rt = runtime; struct rspamd_stat_sqlite3_db *bk; - g_assert (rt != NULL); + g_assert(rt != NULL); bk = rt->db; if (bk->in_transaction) { - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); bk->in_transaction = FALSE; } @@ -783,8 +624,8 @@ rspamd_sqlite3_finalize_process (struct rspamd_task *task, gpointer runtime, } gboolean -rspamd_sqlite3_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, - gint id, gpointer p) +rspamd_sqlite3_learn_tokens(struct rspamd_task *task, GPtrArray *tokens, + gint id, gpointer p) { struct rspamd_stat_sqlite3_db *bk; struct rspamd_stat_sqlite3_rt *rt = p; @@ -792,27 +633,27 @@ rspamd_sqlite3_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, guint i; rspamd_token_t *tok; - g_assert (tokens != NULL); - g_assert (p != NULL); + g_assert(tokens != NULL); + g_assert(p != NULL); bk = rt->db; for (i = 0; i < tokens->len; i++) { - tok = g_ptr_array_index (tokens, i); + tok = g_ptr_array_index(tokens, i); if (bk == NULL) { /* Statfile is does not exist, so all values are zero */ return FALSE; } if (!bk->in_transaction) { - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_START_IM); bk->in_transaction = TRUE; } if (rt->user_id == -1) { if (bk->enable_users) { - rt->user_id = rspamd_sqlite3_get_user (bk, task, TRUE); + rt->user_id = rspamd_sqlite3_get_user(bk, task, TRUE); } else { rt->user_id = 0; @@ -821,7 +662,7 @@ rspamd_sqlite3_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, if (rt->lang_id == -1) { if (bk->enable_languages) { - rt->lang_id = rspamd_sqlite3_get_language (bk, task, TRUE); + rt->lang_id = rspamd_sqlite3_get_language(bk, task, TRUE); } else { rt->lang_id = 0; @@ -830,11 +671,11 @@ rspamd_sqlite3_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, iv = tok->values[id]; - if (rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_SET_TOKEN, - tok->data, rt->user_id, rt->lang_id, iv) != SQLITE_OK) { - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK); + if (rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_SET_TOKEN, + tok->data, rt->user_id, rt->lang_id, iv) != SQLITE_OK) { + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_ROLLBACK); bk->in_transaction = FALSE; return FALSE; @@ -845,19 +686,19 @@ rspamd_sqlite3_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, } gboolean -rspamd_sqlite3_finalize_learn (struct rspamd_task *task, gpointer runtime, - gpointer ctx, GError **err) +rspamd_sqlite3_finalize_learn(struct rspamd_task *task, gpointer runtime, + gpointer ctx, GError **err) { struct rspamd_stat_sqlite3_rt *rt = runtime; struct rspamd_stat_sqlite3_db *bk; gint wal_frames, wal_checkpointed, mode; - g_assert (rt != NULL); + g_assert(rt != NULL); bk = rt->db; if (bk->in_transaction) { - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); bk->in_transaction = FALSE; } @@ -870,17 +711,17 @@ rspamd_sqlite3_finalize_learn (struct rspamd_task *task, gpointer runtime, mode = SQLITE_CHECKPOINT_FULL; #endif /* Perform wal checkpoint (might be long) */ - if (sqlite3_wal_checkpoint_v2 (bk->sqlite, - NULL, - mode, - &wal_frames, - &wal_checkpointed) != SQLITE_OK) { - msg_warn_task ("cannot commit checkpoint: %s", - sqlite3_errmsg (bk->sqlite)); - - g_set_error (err, rspamd_sqlite3_backend_quark (), 500, - "cannot commit checkpoint: %s", - sqlite3_errmsg (bk->sqlite)); + if (sqlite3_wal_checkpoint_v2(bk->sqlite, + NULL, + mode, + &wal_frames, + &wal_checkpointed) != SQLITE_OK) { + msg_warn_task("cannot commit checkpoint: %s", + sqlite3_errmsg(bk->sqlite)); + + g_set_error(err, rspamd_sqlite3_backend_quark(), 500, + "cannot commit checkpoint: %s", + sqlite3_errmsg(bk->sqlite)); return FALSE; } #endif @@ -889,98 +730,98 @@ rspamd_sqlite3_finalize_learn (struct rspamd_task *task, gpointer runtime, } gulong -rspamd_sqlite3_total_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_sqlite3_total_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { struct rspamd_stat_sqlite3_rt *rt = runtime; struct rspamd_stat_sqlite3_db *bk; guint64 res; - g_assert (rt != NULL); + g_assert(rt != NULL); bk = rt->db; - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_GET_LEARNS, &res); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_GET_LEARNS, &res); return res; } gulong -rspamd_sqlite3_inc_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_sqlite3_inc_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { struct rspamd_stat_sqlite3_rt *rt = runtime; struct rspamd_stat_sqlite3_db *bk; guint64 res; - g_assert (rt != NULL); + g_assert(rt != NULL); bk = rt->db; - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_INC_LEARNS_LANG, - rt->lang_id); - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_INC_LEARNS_USER, - rt->user_id); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_INC_LEARNS_LANG, + rt->lang_id); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_INC_LEARNS_USER, + rt->user_id); if (bk->in_transaction) { - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); bk->in_transaction = FALSE; } - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_GET_LEARNS, &res); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_GET_LEARNS, &res); return res; } gulong -rspamd_sqlite3_dec_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_sqlite3_dec_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { struct rspamd_stat_sqlite3_rt *rt = runtime; struct rspamd_stat_sqlite3_db *bk; guint64 res; - g_assert (rt != NULL); + g_assert(rt != NULL); bk = rt->db; - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_DEC_LEARNS_LANG, - rt->lang_id); - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_DEC_LEARNS_USER, - rt->user_id); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_DEC_LEARNS_LANG, + rt->lang_id); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_DEC_LEARNS_USER, + rt->user_id); if (bk->in_transaction) { - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_TRANSACTION_COMMIT); bk->in_transaction = FALSE; } - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_GET_LEARNS, &res); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_GET_LEARNS, &res); return res; } gulong -rspamd_sqlite3_learns (struct rspamd_task *task, gpointer runtime, - gpointer ctx) +rspamd_sqlite3_learns(struct rspamd_task *task, gpointer runtime, + gpointer ctx) { struct rspamd_stat_sqlite3_rt *rt = runtime; struct rspamd_stat_sqlite3_db *bk; guint64 res; - g_assert (rt != NULL); + g_assert(rt != NULL); bk = rt->db; - rspamd_sqlite3_run_prstmt (task->task_pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_GET_LEARNS, &res); + rspamd_sqlite3_run_prstmt(task->task_pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_GET_LEARNS, &res); return res; } ucl_object_t * -rspamd_sqlite3_get_stat (gpointer runtime, - gpointer ctx) +rspamd_sqlite3_get_stat(gpointer runtime, + gpointer ctx) { ucl_object_t *res = NULL; struct rspamd_stat_sqlite3_rt *rt = runtime; @@ -989,73 +830,73 @@ rspamd_sqlite3_get_stat (gpointer runtime, struct stat st; gint64 rev; - g_assert (rt != NULL); + g_assert(rt != NULL); bk = rt->db; pool = bk->pool; - (void)stat (bk->fname, &st); - rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_GET_LEARNS, &rev); - - res = ucl_object_typed_new (UCL_OBJECT); - ucl_object_insert_key (res, ucl_object_fromint (rev), "revision", - 0, false); - ucl_object_insert_key (res, ucl_object_fromint (st.st_size), "size", - 0, false); - rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_NTOKENS, &rev); - ucl_object_insert_key (res, ucl_object_fromint (rev), "total", 0, false); - ucl_object_insert_key (res, ucl_object_fromint (rev), "used", 0, false); - ucl_object_insert_key (res, ucl_object_fromstring (rt->cf->symbol), - "symbol", 0, false); - ucl_object_insert_key (res, ucl_object_fromstring ("sqlite3"), - "type", 0, false); - rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_NLANGUAGES, &rev); - ucl_object_insert_key (res, ucl_object_fromint (rev), - "languages", 0, false); - rspamd_sqlite3_run_prstmt (pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_NUSERS, &rev); - ucl_object_insert_key (res, ucl_object_fromint (rev), - "users", 0, false); + (void) stat(bk->fname, &st); + rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_GET_LEARNS, &rev); + + res = ucl_object_typed_new(UCL_OBJECT); + ucl_object_insert_key(res, ucl_object_fromint(rev), "revision", + 0, false); + ucl_object_insert_key(res, ucl_object_fromint(st.st_size), "size", + 0, false); + rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_NTOKENS, &rev); + ucl_object_insert_key(res, ucl_object_fromint(rev), "total", 0, false); + ucl_object_insert_key(res, ucl_object_fromint(rev), "used", 0, false); + ucl_object_insert_key(res, ucl_object_fromstring(rt->cf->symbol), + "symbol", 0, false); + ucl_object_insert_key(res, ucl_object_fromstring("sqlite3"), + "type", 0, false); + rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_NLANGUAGES, &rev); + ucl_object_insert_key(res, ucl_object_fromint(rev), + "languages", 0, false); + rspamd_sqlite3_run_prstmt(pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_NUSERS, &rev); + ucl_object_insert_key(res, ucl_object_fromint(rev), + "users", 0, false); if (rt->cf->label) { - ucl_object_insert_key (res, ucl_object_fromstring (rt->cf->label), - "label", 0, false); + ucl_object_insert_key(res, ucl_object_fromstring(rt->cf->label), + "label", 0, false); } return res; } gpointer -rspamd_sqlite3_load_tokenizer_config (gpointer runtime, - gsize *len) +rspamd_sqlite3_load_tokenizer_config(gpointer runtime, + gsize *len) { gpointer tk_conf, copied_conf; guint64 sz; struct rspamd_stat_sqlite3_rt *rt = runtime; struct rspamd_stat_sqlite3_db *bk; - g_assert (rt != NULL); + g_assert(rt != NULL); bk = rt->db; - g_assert (rspamd_sqlite3_run_prstmt (rt->db->pool, bk->sqlite, bk->prstmt, - RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, &sz, &tk_conf) == SQLITE_OK); - g_assert (sz > 0); + g_assert(rspamd_sqlite3_run_prstmt(rt->db->pool, bk->sqlite, bk->prstmt, + RSPAMD_STAT_BACKEND_LOAD_TOKENIZER, &sz, &tk_conf) == SQLITE_OK); + g_assert(sz > 0); /* * Here we can have either decoded or undecoded version of tokenizer config * XXX: dirty hack to check if we have osb magic here */ - if (sz > 7 && memcmp (tk_conf, "osbtokv", 7) == 0) { - copied_conf = rspamd_mempool_alloc (rt->task->task_pool, sz); - memcpy (copied_conf, tk_conf, sz); - g_free (tk_conf); + if (sz > 7 && memcmp(tk_conf, "osbtokv", 7) == 0) { + copied_conf = rspamd_mempool_alloc(rt->task->task_pool, sz); + memcpy(copied_conf, tk_conf, sz); + g_free(tk_conf); } else { /* Need to decode */ - copied_conf = rspamd_decode_base32 (tk_conf, sz, len, RSPAMD_BASE32_DEFAULT); - g_free (tk_conf); - rspamd_mempool_add_destructor (rt->task->task_pool, g_free, copied_conf); + copied_conf = rspamd_decode_base32(tk_conf, sz, len, RSPAMD_BASE32_DEFAULT); + g_free(tk_conf); + rspamd_mempool_add_destructor(rt->task->task_pool, g_free, copied_conf); } if (len) { |