${CMAKE_CURRENT_SOURCE_DIR}/cfg_utils.c
${CMAKE_CURRENT_SOURCE_DIR}/cfg_rcl.c
${CMAKE_CURRENT_SOURCE_DIR}/composites/composites.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/composites/composites_manager.cxx
${CMAKE_CURRENT_SOURCE_DIR}/dkim.c
${CMAKE_CURRENT_SOURCE_DIR}/dns.c
${CMAKE_CURRENT_SOURCE_DIR}/dynamic_cfg.c
ucl_object_t *config_comments; /**< comments saved from the config */
ucl_object_t *doc_strings; /**< documentation strings for config options */
GPtrArray *c_modules; /**< list of C modules */
- GHashTable *composite_symbols; /**< hash of composite symbols indexed by its name */
+ void *composites_manager; /**< hash of composite symbols indexed by its name */
GList *classifiers; /**< list of all classifiers defined */
GList *statfiles; /**< list of all statfiles in config file order */
GHashTable *classifiers_symbols; /**< hashtable indexed by symbol name of classifiers */
struct rspamd_rcl_section *section,
GError **err)
{
- const ucl_object_t *val, *elt;
- struct rspamd_expression *expr;
struct rspamd_config *cfg = ud;
- struct rspamd_composite *composite;
- const gchar *composite_name, *composite_expression, *group,
- *description;
- gdouble score;
- gboolean new = TRUE;
+ void *composite;
+ const gchar *composite_name;
g_assert (key != NULL);
composite_name = key;
- val = ucl_object_lookup (obj, "enabled");
- if (val != NULL && !ucl_object_toboolean (val)) {
- msg_info_config ("composite %s is disabled", composite_name);
- return TRUE;
- }
-
- if (g_hash_table_lookup (cfg->composite_symbols, composite_name) != NULL) {
- msg_warn_config ("composite %s is redefined", composite_name);
- new = FALSE;
- }
-
- val = ucl_object_lookup (obj, "expression");
- if (val == NULL || !ucl_object_tostring_safe (val, &composite_expression)) {
- g_set_error (err,
- CFG_RCL_ERROR,
- EINVAL,
- "composite must have an expression defined");
- return FALSE;
- }
-
- if (!rspamd_parse_expression (composite_expression, 0, &composite_expr_subr,
- NULL, cfg->cfg_pool, err, &expr)) {
- if (err && *err) {
- msg_err_config ("cannot parse composite expression for %s: %e",
- composite_name, *err);
- }
- else {
- msg_err_config ("cannot parse composite expression for %s: unknown error",
- composite_name);
- }
-
- return FALSE;
- }
-
- composite =
- rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_composite));
- composite->expr = expr;
- composite->id = g_hash_table_size (cfg->composite_symbols);
- composite->str_expr = composite_expression;
- composite->sym = composite_name;
-
- val = ucl_object_lookup (obj, "score");
- if (val != NULL && ucl_object_todouble_safe (val, &score)) {
- /* Also set score in the metric */
-
- val = ucl_object_lookup (obj, "group");
- if (val != NULL) {
- group = ucl_object_tostring (val);
- }
- else {
- group = "composite";
- }
-
- val = ucl_object_lookup (obj, "description");
- if (val != NULL) {
- description = ucl_object_tostring (val);
- }
- else {
- description = composite_expression;
- }
-
- rspamd_config_add_symbol (cfg, composite_name, score,
- description, group,
- 0,
- ucl_object_get_priority (obj), /* No +1 as it is default... */
- 1);
-
- elt = ucl_object_lookup (obj, "groups");
-
- if (elt) {
- ucl_object_iter_t gr_it;
- const ucl_object_t *cur_gr;
-
- gr_it = ucl_object_iterate_new (elt);
-
- while ((cur_gr = ucl_object_iterate_safe (gr_it, true)) != NULL) {
- rspamd_config_add_symbol_group (cfg, key,
- ucl_object_tostring (cur_gr));
- }
-
- ucl_object_iterate_free (gr_it);
- }
- }
-
- val = ucl_object_lookup (obj, "policy");
-
- if (val) {
- composite->policy = rspamd_composite_policy_from_str (
- ucl_object_tostring (val));
-
- if (composite->policy == RSPAMD_COMPOSITE_POLICY_UNKNOWN) {
- g_set_error (err,
- CFG_RCL_ERROR,
- EINVAL,
- "composite %s has incorrect policy", composite_name);
- return FALSE;
- }
- }
-
- g_hash_table_insert (cfg->composite_symbols,
- (gpointer)composite_name,
- composite);
-
- if (new) {
+ if ((composite = rspamd_composites_manager_add_from_ucl(cfg->composites_manager, obj)) != NULL) {
rspamd_symcache_add_symbol (cfg->cache, composite_name, 0,
NULL, composite, SYMBOL_TYPE_COMPOSITE, -1);
}
- return TRUE;
+ return composite != NULL;
}
static gboolean
#include <sys/resource.h>
#endif
#include <math.h>
+#include "libserver/composites/composites.h"
#include "blas-config.h"
rspamd_config_init_metric (cfg);
- cfg->composite_symbols =
- g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
+ cfg->composites_manager = rspamd_composites_manager_create(cfg);
cfg->classifiers_symbols = g_hash_table_new (rspamd_str_hash,
rspamd_str_equal);
cfg->cfg_params = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
ucl_object_unref (cfg->config_comments);
ucl_object_unref (cfg->doc_strings);
ucl_object_unref (cfg->neighbours);
- g_hash_table_remove_all (cfg->composite_symbols);
- g_hash_table_unref (cfg->composite_symbols);
g_hash_table_remove_all (cfg->cfg_params);
g_hash_table_unref (cfg->cfg_params);
g_hash_table_unref (cfg->classifiers_symbols);
#include <variant>
#include "contrib/robin-hood/robin_hood.h"
+#include "composites_internal.hxx"
+
#define msg_err_composites(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
"composites", task->task_pool->tag.uid, \
G_STRFUNC, \
static gint rspamd_composite_expr_priority(rspamd_expression_atom_t *atom);
static void rspamd_composite_expr_destroy(rspamd_expression_atom_t *atom);
static void composites_foreach_callback(gpointer key, gpointer value, void *data);
-}
const struct rspamd_atom_subr composite_expr_subr = {
.parse = rspamd::composites::rspamd_composite_expr_parse,
.priority = rspamd::composites::rspamd_composite_expr_priority,
.destroy = rspamd::composites::rspamd_composite_expr_destroy
};
+}
namespace rspamd::composites {
static constexpr const double epsilon = 0.00001;
-enum class rspamd_composite_policy {
- RSPAMD_COMPOSITE_POLICY_REMOVE_ALL = 0,
- RSPAMD_COMPOSITE_POLICY_REMOVE_SYMBOL,
- RSPAMD_COMPOSITE_POLICY_REMOVE_WEIGHT,
- RSPAMD_COMPOSITE_POLICY_LEAVE,
- RSPAMD_COMPOSITE_POLICY_UNKNOWN
-};
-
-/**
- * Static composites structure
- */
-struct rspamd_composite {
- std::string str_expr;
- std::string sym;
- struct rspamd_expression *expr;
- gint id;
- rspamd_composite_policy policy;
-};
-
struct symbol_remove_data {
const char *sym;
struct rspamd_composite *comp;
explicit composites_data(struct rspamd_task *task, struct rspamd_scan_result *mres) :
task(task), composite(nullptr), metric_res(mres) {
- checked.resize(g_hash_table_size(task->cfg->composite_symbols) * 2);
+ checked.resize(rspamd_composites_manager_nelts(task->cfg->composites_manager) * 2);
}
};
struct rspamd_composite_atom {
std::string symbol;
rspamd_composite_atom_type comp_type = rspamd_composite_atom_type::ATOM_UNKNOWN;
- struct rspamd_composite *ncomp; /* underlying composite */
+ const struct rspamd_composite *ncomp; /* underlying composite */
std::vector<rspamd_composite_option_match> opts;
};
static auto
process_single_symbol(struct composites_data *cd,
- const gchar *sym,
+ std::string_view sym,
struct rspamd_symbol_result **pms,
struct rspamd_composite_atom *atom) -> double
{
gdouble rc = 0;
struct rspamd_task *task = cd->task;
- if ((ms = rspamd_task_find_symbol_result(cd->task, sym, cd->metric_res)) == nullptr) {
+ if ((ms = rspamd_task_find_symbol_result(cd->task, sym.data(), cd->metric_res)) == nullptr) {
msg_debug_composites ("not found symbol %s in composite %s", sym,
cd->composite->sym.c_str());
if (G_UNLIKELY(atom->comp_type == rspamd_composite_atom_type::ATOM_UNKNOWN)) {
- struct rspamd_composite *ncomp;
+ const struct rspamd_composite *ncomp;
- if ((ncomp =
- g_hash_table_lookup(cd->task->cfg->composite_symbols,
- sym)) != NULL) {
+ if ((ncomp = COMPOSITE_MANAGER_FROM_PTR(task->cfg->composites_manager)->find(sym)) != NULL) {
atom->comp_type = rspamd_composite_atom_type::ATOM_COMPOSITE;
atom->ncomp = ncomp;
}
cd->composite = saved;
cd->checked[cd->composite->id * 2] = false;
- ms = rspamd_task_find_symbol_result(cd->task, sym,
+ ms = rspamd_task_find_symbol_result(cd->task, sym.data(),
cd->metric_res);
}
else {
* XXX: in case of cyclic references this would return 0
*/
if (cd->checked[atom->ncomp->id * 2 + 1]) {
- ms = rspamd_task_find_symbol_result(cd->task, sym,
+ ms = rspamd_task_find_symbol_result(cd->task, sym.data(),
cd->metric_res);
}
}
if (cond(sdef->score)) {
rc = process_single_symbol(cd,
- sdef->name,
+ std::string_view(sdef->name),
&ms,
comp_atom);
rc = group_process_functor([](auto sc) { return sc < 0.; }, 3);
}
else {
- rc = process_single_symbol(cd, sym.data(), &ms, comp_atom);
+ rc = process_single_symbol(cd, sym, &ms, comp_atom);
if (rc) {
process_symbol_removal(atom,
}
}
else {
- rc = process_single_symbol(cd, sym.data(), &ms, comp_atom);
+ rc = process_single_symbol(cd, sym, &ms, comp_atom);
if (fabs(rc) > epsilon) {
process_symbol_removal(atom,
}
}
-
-enum rspamd_composite_policy
-rspamd_composite_policy_from_str (const gchar *string)
-{
- enum rspamd_composite_policy ret = RSPAMD_COMPOSITE_POLICY_UNKNOWN;
-
- if (strcmp (string, "remove") == 0 || strcmp (string, "remove_all") == 0 ||
- strcmp (string, "default") == 0) {
- ret = RSPAMD_COMPOSITE_POLICY_REMOVE_ALL;
- }
- else if (strcmp (string, "remove_symbol") == 0) {
- ret = RSPAMD_COMPOSITE_POLICY_REMOVE_SYMBOL;
- }
- else if (strcmp (string, "remove_weight") == 0) {
- ret = RSPAMD_COMPOSITE_POLICY_REMOVE_WEIGHT;
- }
- else if (strcmp (string, "leave") == 0 || strcmp (string, "remove_none") == 0) {
- ret = RSPAMD_COMPOSITE_POLICY_LEAVE;
- }
-
- return ret;
-}
#define SRC_LIBSERVER_COMPOSITES_H_
#include "config.h"
+#include "contrib/libucl/ucl.h"
#ifdef __cplusplus
extern "C" {
#endif
struct rspamd_task;
-
-/**
- * Subr for composite expressions
- */
-extern const struct rspamd_atom_subr composite_expr_subr;
+struct rspamd_config;
/**
* Process all results and form composite metrics from existent metrics as it is defined in config
* @param task worker's task that present message from user
*/
-void rspamd_composites_process_task (struct rspamd_task *task);
+void rspamd_composites_process_task(struct rspamd_task *task);
-enum rspamd_composite_policy rspamd_composite_policy_from_str (const gchar *string);
+/**
+ * Creates a composites manager
+ * @param cfg
+ * @return
+ */
+void* rspamd_composites_manager_create(struct rspamd_config *cfg);
+/**
+ * Returns number of elements in a composite manager
+ * @return
+ */
+gsize rspamd_composites_manager_nelts(void *);
+/**
+ * Adds a composite from config
+ * @return
+ */
+void* rspamd_composites_manager_add_from_ucl(void *, const ucl_object_t *);
#ifdef __cplusplus
}
--- /dev/null
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_COMPOSITES_INTERNAL_HXX
+#define RSPAMD_COMPOSITES_INTERNAL_HXX
+#pragma once
+
+#include <string>
+#include "libutil/expression.h"
+#include "libserver/cfg_file.h"
+
+namespace rspamd::composites {
+
+/**
+ * Subr for composite expressions
+ */
+extern const struct rspamd_atom_subr composite_expr_subr;
+
+enum class rspamd_composite_policy {
+ RSPAMD_COMPOSITE_POLICY_REMOVE_ALL = 0,
+ RSPAMD_COMPOSITE_POLICY_REMOVE_SYMBOL,
+ RSPAMD_COMPOSITE_POLICY_REMOVE_WEIGHT,
+ RSPAMD_COMPOSITE_POLICY_LEAVE,
+ RSPAMD_COMPOSITE_POLICY_UNKNOWN
+};
+
+/**
+ * Static composites structure
+ */
+struct rspamd_composite {
+ std::string str_expr;
+ std::string sym;
+ struct rspamd_expression *expr;
+ gint id;
+ rspamd_composite_policy policy;
+};
+
+#define COMPOSITE_MANAGER_FROM_PTR(ptr) (reinterpret_cast<rspamd::composites::composites_manager *>(ptr))
+
+class composites_manager {
+public:
+ composites_manager(struct rspamd_config *_cfg) : cfg(_cfg) {
+ rspamd_mempool_add_destructor(_cfg->cfg_pool, composites_manager_dtor, this);
+ }
+
+ auto size(void) const -> std::size_t {
+ return composites.size();
+ }
+
+ auto find(std::string_view name) const -> const rspamd_composite * {
+ auto found = composites.find(std::string(name));
+
+ if (found != composites.end()) {
+ return found->second.get();
+ }
+
+ return nullptr;
+ }
+
+ auto add_composite(std::string_view, const ucl_object_t *) -> rspamd_composite *;
+private:
+ ~composites_manager() = default;
+ static void composites_manager_dtor(void *ptr) {
+ delete COMPOSITE_MANAGER_FROM_PTR(ptr);
+ }
+
+ /* Enable lookup by string view */
+ struct smart_str_equal {
+ using is_transparent = void;
+ auto operator()(const std::string &a, const std::string &b) const {
+ return a == b;
+ }
+ auto operator()(const std::string_view &a, const std::string &b) const {
+ return a == b;
+ }
+ auto operator()(const std::string &a, const std::string_view &b) const {
+ return a == b;
+ }
+ };
+
+ struct smart_str_hash {
+ using is_transparent = void;
+ auto operator()(const std::string &a) const {
+ return robin_hood::hash<std::string>()(a);
+ }
+ auto operator()(const std::string_view &a) const {
+ return robin_hood::hash<std::string_view>()(a);
+ }
+ };
+
+ robin_hood::unordered_flat_map<std::string,
+ std::shared_ptr<rspamd_composite>, smart_str_hash, smart_str_equal> composites;
+ /* Store all composites here, even if we have duplicates */
+ std::vector<std::shared_ptr<rspamd_composite>> all_composites;
+ struct rspamd_config *cfg;
+};
+
+}
+
+#endif //RSPAMD_COMPOSITES_INTERNAL_HXX
--- /dev/null
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+#include <vector>
+#include "contrib/robin-hood/robin_hood.h"
+
+#include "composites.h"
+#include "composites_internal.hxx"
+#include "libserver/cfg_file.h"
+#include "libserver/logger.h"
+
+namespace rspamd::composites {
+
+static auto
+composite_policy_from_str(const std::string_view &inp) -> enum rspamd_composite_policy
+{
+ const static robin_hood::unordered_flat_map<std::string_view,
+ enum rspamd_composite_policy> names{
+ {"remove", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_ALL},
+ {"remove_all", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_ALL},
+ {"default", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_ALL},
+ {"remove_symbol", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_SYMBOL},
+ {"remove_weight", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_REMOVE_WEIGHT},
+ {"leave", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_LEAVE},
+ {"remove_none", rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_LEAVE},
+ };
+
+ auto found = names.find(inp);
+ if (found != names.end()) {
+ return found->second;
+ }
+
+ return rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_UNKNOWN;
+}
+
+auto
+composites_manager::add_composite(std::string_view composite_name, const ucl_object_t *obj) -> rspamd_composite *
+{
+
+ const auto *val = ucl_object_lookup (obj, "enabled");
+ if (val != nullptr && !ucl_object_toboolean (val)) {
+ msg_info_config ("composite %s is disabled", composite_name.data());
+ return nullptr;
+ }
+
+ if (composites.contains(composite_name)) {
+ msg_warn_config ("composite %s is redefined", composite_name.data());
+ }
+
+ const char *composite_expression = nullptr;
+ val = ucl_object_lookup (obj, "expression");
+
+ if (val == NULL || !ucl_object_tostring_safe (val, &composite_expression)) {
+ msg_err_config ("composite must have an expression defined in %s",
+ composite_name.data());
+ return nullptr;
+ }
+
+ GError *err = nullptr;
+ rspamd_expression *expr = nullptr;
+
+ if (!rspamd_parse_expression(composite_expression, 0, &composite_expr_subr,
+ NULL, cfg->cfg_pool, &err, &expr)) {
+ msg_err_config ("cannot parse composite expression for %s: %e",
+ composite_name.data(), err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return nullptr;
+ }
+
+ auto &composite = all_composites.emplace_back(std::make_shared<rspamd_composite>());
+ composite->expr = expr;
+ composite->id = all_composites.size();
+ composite->str_expr = composite_expression;
+ composite->sym = composite_name;
+
+ double score;
+ val = ucl_object_lookup (obj, "score");
+ if (val != nullptr && ucl_object_todouble_safe (val, &score)) {
+ /* Also set score in the metric */
+
+ const auto *group = "composite";
+ val = ucl_object_lookup (obj, "group");
+ if (val != nullptr) {
+ group = ucl_object_tostring (val);
+ }
+
+ const auto *description = composite_expression;
+ val = ucl_object_lookup (obj, "description");
+ if (val != nullptr) {
+ description = ucl_object_tostring (val);
+ }
+ else {
+ description = composite_expression;
+ }
+
+ rspamd_config_add_symbol(cfg, composite_name.data(), score,
+ description, group,
+ 0,
+ ucl_object_get_priority (obj), /* No +1 as it is default... */
+ 1);
+
+ const auto *elt = ucl_object_lookup (obj, "groups");
+ if (elt) {
+ const ucl_object_t *cur_gr;
+ auto *gr_it = ucl_object_iterate_new (elt);
+
+ while ((cur_gr = ucl_object_iterate_safe(gr_it, true)) != nullptr) {
+ rspamd_config_add_symbol_group(cfg, composite_name.data(),
+ ucl_object_tostring(cur_gr));
+ }
+
+ ucl_object_iterate_free(gr_it);
+ }
+ }
+
+ val = ucl_object_lookup(obj, "policy");
+ if (val) {
+ composite->policy = composite_policy_from_str(ucl_object_tostring(val));
+
+ if (composite->policy == rspamd_composite_policy::RSPAMD_COMPOSITE_POLICY_UNKNOWN) {
+ msg_err_config("composite %s has incorrect policy", composite_name.data());
+ return nullptr;
+ }
+ }
+
+ composites[std::string(composite_name)] = composite;
+
+ return composite.get();
+}
+
+}
+
+
+void*
+rspamd_composites_manager_create(struct rspamd_config *cfg)
+{
+ auto *cm = new rspamd::composites::composites_manager(cfg);
+
+ return reinterpret_cast<void *>(cm);
+}
+
+
+gsize
+rspamd_composites_manager_nelts(void *ptr)
+{
+ return COMPOSITE_MANAGER_FROM_PTR(ptr)->size();
+}
+
+void*
+rspamd_composites_manager_add_from_ucl(void *cm, const char *sym, const ucl_object_t *obj)
+{
+ return reinterpret_cast<void *>(COMPOSITE_MANAGER_FROM_PTR(cm)->add_composite(sym, obj));
+}
{
g_assert (re != NULL);
- return re->id;
+ return (gpointer)re->id;
}
gpointer
projects / rspamd.git / commitdiff