diff options
author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2017-11-04 15:42:15 +0000 |
---|---|---|
committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2017-11-04 15:42:15 +0000 |
commit | 70c5a7b68850e825b29b12c672fda71e366f58ff (patch) | |
tree | f100c3558bdf7cbcff45d7156ea5681c4ab4c60a /src/libserver | |
parent | cc2301c1517da93dfe0867a5a91bafae18001e1a (diff) | |
download | rspamd-70c5a7b68850e825b29b12c672fda71e366f58ff.tar.gz rspamd-70c5a7b68850e825b29b12c672fda71e366f58ff.zip |
[Rework] Kill metrics!11
Diffstat (limited to 'src/libserver')
-rw-r--r-- | src/libserver/cfg_file.h | 54 | ||||
-rw-r--r-- | src/libserver/cfg_rcl.c | 198 | ||||
-rw-r--r-- | src/libserver/cfg_rcl.h | 10 | ||||
-rw-r--r-- | src/libserver/cfg_utils.c | 147 | ||||
-rw-r--r-- | src/libserver/composites.c | 5 | ||||
-rw-r--r-- | src/libserver/dynamic_cfg.c | 12 | ||||
-rw-r--r-- | src/libserver/protocol.c | 16 | ||||
-rw-r--r-- | src/libserver/re_cache.c | 2 | ||||
-rw-r--r-- | src/libserver/symbols_cache.c | 79 |
9 files changed, 139 insertions, 384 deletions
diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index d91a4f06e..412cd4bb0 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -243,7 +243,7 @@ struct rspamd_log_format { struct rspamd_log_format *prev, *next; }; -enum rspamd_metric_action { +enum rspamd_action_type { METRIC_ACTION_REJECT = 0, METRIC_ACTION_SOFT_REJECT, METRIC_ACTION_REWRITE_SUBJECT, @@ -253,27 +253,12 @@ enum rspamd_metric_action { METRIC_ACTION_MAX }; -struct metric_action { - enum rspamd_metric_action action; +struct rspamd_action { + enum rspamd_action_type action; gdouble score; guint priority; }; -/** - * Common definition of metric - */ -struct rspamd_metric { - const gchar *name; /**< name of metric */ - gchar *func_name; /**< name of consolidation function */ - gboolean accept_unknown_symbols; /**< if true unknown symbols are registered here */ - gdouble unknown_weight; /**< weight of unknown symbols */ - gdouble grow_factor; /**< grow factor for metric */ - GHashTable *symbols; /**< weights of symbols in metric */ - const gchar *subject; /**< subject rewrite string */ - GHashTable * groups; /**< groups of symbols */ - struct metric_action actions[METRIC_ACTION_MAX]; /**< all actions of the metric */ -}; - struct rspamd_config_post_load_script { gint cbref; struct rspamd_config_post_load_script *prev, *next; @@ -295,6 +280,14 @@ struct rspamd_config { gchar *profile_path; #endif + gboolean accept_unknown_symbols; /**< if true unknown symbols are registered here */ + gdouble unknown_weight; /**< weight of unknown symbols */ + gdouble grow_factor; /**< grow factor for metric */ + GHashTable *symbols; /**< weights of symbols in metric */ + const gchar *subject; /**< subject rewrite string */ + GHashTable * groups; /**< groups of symbols */ + struct rspamd_action actions[METRIC_ACTION_MAX]; /**< all actions of the metric */ + gboolean raw_mode; /**< work in raw mode instead of utf one */ gboolean one_shot_mode; /**< rules add only one symbol */ gboolean check_text_attachements; /**< check text attachements as text */ @@ -359,9 +352,6 @@ struct rspamd_config { ucl_object_t *rcl_obj; /**< rcl object */ ucl_object_t *config_comments; /**< comments saved from the config */ ucl_object_t *doc_strings; /**< documentation strings for config options */ - GHashTable * metrics; /**< hash of metrics indexed by metric name */ - GList * metrics_list; /**< linked list of metrics */ - GHashTable * metrics_symbols; /**< hash table of metrics indexed by symbol */ GHashTable * c_modules; /**< hash of c modules indexed by module name */ GHashTable * composite_symbols; /**< hash of composite symbols indexed by its name */ GList *classifiers; /**< list of all classifiers defined */ @@ -386,8 +376,6 @@ struct rspamd_config { struct symbols_cache *cache; /**< symbols cache object */ gchar *cache_filename; /**< filename of cache file */ gdouble cache_reload_time; /**< how often cache reload should be performed */ - struct rspamd_metric *default_metric; /**< default metric */ - gchar * checksum; /**< real checksum of config file */ gchar * dump_checksum; /**< dump checksum of config file */ gpointer lua_state; /**< pointer to lua state */ @@ -537,14 +525,13 @@ struct rspamd_worker_conf * rspamd_config_new_worker (struct rspamd_config *cfg, /* * Return a new metric structure, setting default and non-conflicting attributes */ -struct rspamd_metric * rspamd_config_new_metric (struct rspamd_config *cfg, - struct rspamd_metric *c, const gchar *name); +void rspamd_config_init_metric (struct rspamd_config *cfg); /* * Return new symbols group definition */ struct rspamd_symbols_group * rspamd_config_new_group ( - struct rspamd_config *cfg, struct rspamd_metric *metric, + struct rspamd_config *cfg, const gchar *name); /* * Return a new statfile structure, setting default and non-conflicting attributes @@ -554,14 +541,6 @@ struct rspamd_statfile_config * rspamd_config_new_statfile ( struct rspamd_statfile_config *c); /* - * Read XML configuration file - */ -gboolean rspamd_config_read (struct rspamd_config *cfg, - const gchar *filename, const gchar *convert_to, - rspamd_rcl_section_fin_t logger_fin, gpointer logger_ud, - GHashTable *vars); - -/* * Register symbols of classifiers inside metrics */ void rspamd_config_insert_classify_symbols (struct rspamd_config *cfg); @@ -605,7 +584,7 @@ gboolean rspamd_init_filters (struct rspamd_config *cfg, bool reconfig); * @param nshots means maximum number of hits for a symbol in metric (-1 for unlimited) * @return TRUE if symbol has been inserted or FALSE if symbol already exists with higher priority */ -gboolean rspamd_config_add_metric_symbol (struct rspamd_config *cfg, +gboolean rspamd_config_add_symbol (struct rspamd_config *cfg, const gchar *symbol, gdouble score, const gchar *description, const gchar *group, guint flags, guint priority, @@ -621,7 +600,6 @@ gboolean rspamd_config_add_metric_symbol (struct rspamd_config *cfg, * @return TRUE if symbol has been inserted or FALSE if action already exists with higher priority */ gboolean rspamd_config_set_action_score (struct rspamd_config *cfg, - const gchar *metric, const gchar *action_name, gdouble score, guint priority); @@ -650,8 +628,8 @@ gboolean rspamd_action_from_str (const gchar *data, gint *result); /* * Return textual representation of action enumeration */ -const gchar * rspamd_action_to_str (enum rspamd_metric_action action); -const gchar * rspamd_action_to_str_alt (enum rspamd_metric_action action); +const gchar * rspamd_action_to_str (enum rspamd_action_type action); +const gchar * rspamd_action_to_str_alt (enum rspamd_action_type action); /** * Parse radix tree or radix map from ucl object diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index 0a60d45f1..3aa277e1a 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -290,7 +290,6 @@ rspamd_rcl_options_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, } struct rspamd_rcl_symbol_data { - struct rspamd_metric *metric; struct rspamd_symbols_group *gr; struct rspamd_config *cfg; }; @@ -300,37 +299,36 @@ rspamd_rcl_group_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, const gchar *key, gpointer ud, struct rspamd_rcl_section *section, GError **err) { - struct rspamd_rcl_symbol_data *sd = ud; - struct rspamd_metric *metric; + struct rspamd_config *cfg = ud; struct rspamd_symbols_group *gr; const ucl_object_t *val, *cur; struct rspamd_rcl_section *subsection; + struct rspamd_rcl_symbol_data sd; g_assert (key != NULL); - metric = sd->metric; - - gr = g_hash_table_lookup (metric->groups, key); + gr = g_hash_table_lookup (cfg->groups, key); if (gr == NULL) { - gr = rspamd_config_new_group (sd->cfg, metric, key); + gr = rspamd_config_new_group (cfg, key); } - if (!rspamd_rcl_section_parse_defaults (sd->cfg, section, pool, obj, + if (!rspamd_rcl_section_parse_defaults (cfg, section, pool, obj, gr, err)) { return FALSE; } - sd->gr = gr; + sd.gr = gr; + sd.cfg = cfg; /* Handle symbols */ - val = ucl_object_lookup (obj, "symbol"); + val = ucl_object_lookup (obj, "symbols"); if (val != NULL && ucl_object_type (val) == UCL_OBJECT) { - HASH_FIND_STR (section->subsections, "symbol", subsection); + HASH_FIND_STR (section->subsections, "symbols", subsection); g_assert (subsection != NULL); LL_FOREACH (val, cur) { - if (!rspamd_rcl_process_section (sd->cfg, subsection, sd, cur, + if (!rspamd_rcl_process_section (cfg, subsection, &sd, cur, pool, err)) { return FALSE; } @@ -404,37 +402,28 @@ rspamd_rcl_symbol_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, } if (sd->gr) { - rspamd_config_add_metric_symbol (cfg, key, score, + rspamd_config_add_symbol (cfg, key, score, description, sd->gr->name, flags, priority, nshots); } else { - rspamd_config_add_metric_symbol (cfg, key, score, + rspamd_config_add_symbol (cfg, key, score, description, NULL, flags, priority, nshots); } return TRUE; } -struct metric_actions_cbdata { - struct rspamd_config *cfg; - struct rspamd_metric *metric; -}; - static gboolean rspamd_rcl_actions_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, const gchar *key, gpointer ud, struct rspamd_rcl_section *section, GError **err) { gdouble action_score; - struct metric_actions_cbdata *cbdata = ud; + struct rspamd_config *cfg = ud; gint action_value; const ucl_object_t *cur; ucl_object_iter_t it; - struct rspamd_metric *metric; - struct rspamd_config *cfg; - metric = cbdata->metric; - cfg = cbdata->cfg; it = ucl_object_iterate_new (obj); while ((cur = ucl_object_iterate_safe (it, true)) != NULL) { @@ -450,8 +439,9 @@ rspamd_rcl_actions_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, return FALSE; } else { - rspamd_config_set_action_score (cfg, metric->name, - ucl_object_key (cur), action_score, + rspamd_config_set_action_score (cfg, + ucl_object_key (cur), + action_score, ucl_object_get_priority (cur)); } } @@ -462,118 +452,6 @@ rspamd_rcl_actions_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, } static gboolean -rspamd_rcl_metric_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, - const gchar *key, gpointer ud, - struct rspamd_rcl_section *section, GError **err) -{ - const ucl_object_t *val, *cur, *elt; - ucl_object_iter_t it; - struct rspamd_config *cfg = ud; - struct rspamd_metric *metric; - struct rspamd_rcl_section *subsection; - struct rspamd_rcl_symbol_data sd; - struct rspamd_symbol *sym_def; - struct metric_actions_cbdata acts_cbdata; - - g_assert (key != NULL); - - metric = g_hash_table_lookup (cfg->metrics, key); - if (metric == NULL) { - metric = rspamd_config_new_metric (cfg, metric, key); - } - - if (!rspamd_rcl_section_parse_defaults (cfg, section, cfg->cfg_pool, obj, - metric, err)) { - return FALSE; - } - - if (metric->unknown_weight > 0) { - metric->accept_unknown_symbols = TRUE; - } - - /* Handle actions */ - val = ucl_object_lookup (obj, "actions"); - if (val != NULL) { - if (val->type != UCL_OBJECT) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, - "actions must be an object"); - return FALSE; - } - - HASH_FIND_STR (section->subsections, "actions", subsection); - g_assert (subsection != NULL); - acts_cbdata.cfg = cfg; - acts_cbdata.metric = metric; - - if (!rspamd_rcl_process_section (cfg, subsection, &acts_cbdata, val, - cfg->cfg_pool, err)) { - return FALSE; - } - } - - /* No more legacy mode */ - - /* Handle grouped symbols */ - val = ucl_object_lookup (obj, "group"); - if (val != NULL && ucl_object_type (val) == UCL_OBJECT) { - HASH_FIND_STR (section->subsections, "group", subsection); - g_assert (subsection != NULL); - sd.gr = NULL; - sd.cfg = cfg; - sd.metric = metric; - - LL_FOREACH (val, cur) { - if (!rspamd_rcl_process_section (cfg, subsection, &sd, cur, - cfg->cfg_pool, err)) { - return FALSE; - } - } - } - - /* Handle symbols */ - val = ucl_object_lookup (obj, "symbol"); - if (val != NULL && ucl_object_type (val) == UCL_OBJECT) { - HASH_FIND_STR (section->subsections, "symbol", subsection); - g_assert (subsection != NULL); - sd.gr = NULL; - sd.cfg = cfg; - sd.metric = metric; - - LL_FOREACH (val, cur) { - if (!rspamd_rcl_process_section (cfg, subsection, &sd, cur, - cfg->cfg_pool, err)) { - return FALSE; - } - } - } - - /* Handle ignored symbols */ - val = ucl_object_lookup (obj, "ignore"); - if (val != NULL && ucl_object_type (val) == UCL_ARRAY) { - LL_FOREACH (val, cur) { - it = NULL; - - while ((elt = ucl_object_iterate (cur, &it, true)) != NULL) { - if (ucl_object_type (elt) == UCL_STRING) { - sym_def = g_hash_table_lookup (metric->symbols, - ucl_object_tostring (elt)); - - if (sym_def != NULL) { - sym_def->flags |= RSPAMD_SYMBOL_FLAG_IGNORE; - } - else { - msg_warn ("cannot find symbol %s to set ignore flag", - ucl_object_tostring (elt)); - } - } - } - } - } - - return TRUE; -} - -static gboolean rspamd_rcl_worker_handler (rspamd_mempool_t *pool, const ucl_object_t *obj, const gchar *key, gpointer ud, struct rspamd_rcl_section *section, GError **err) @@ -1500,7 +1378,7 @@ rspamd_rcl_composite_handler (rspamd_mempool_t *pool, description = composite_expression; } - rspamd_config_add_metric_symbol (cfg, composite_name, score, + rspamd_config_add_symbol (cfg, composite_name, score, description, group, FALSE, FALSE, 1); } @@ -2307,27 +2185,26 @@ rspamd_rcl_config_init (struct rspamd_config *cfg) "Time before attempting to recover upstream after an error"); /** - * Metric section + * Symbols and actions sections */ sub = rspamd_rcl_add_section_doc (&new, - "metric", "name", - rspamd_rcl_metric_handler, + "actions", NULL, + rspamd_rcl_actions_handler, UCL_OBJECT, FALSE, TRUE, cfg->doc_strings, - "Metrics configuration"); - sub->default_key = DEFAULT_METRIC; + "Actions configuration"); rspamd_rcl_add_default_handler (sub, "unknown_weight", rspamd_rcl_parse_struct_double, - G_STRUCT_OFFSET (struct rspamd_metric, unknown_weight), + G_STRUCT_OFFSET (struct rspamd_config, unknown_weight), 0, "Accept unknown symbols with the specified weight"); rspamd_rcl_add_default_handler (sub, "grow_factor", rspamd_rcl_parse_struct_double, - G_STRUCT_OFFSET (struct rspamd_metric, grow_factor), + G_STRUCT_OFFSET (struct rspamd_config, grow_factor), 0, "Multiply the subsequent symbols by this number " "(does not affect symbols with score less or " @@ -2335,34 +2212,23 @@ rspamd_rcl_config_init (struct rspamd_config *cfg) rspamd_rcl_add_default_handler (sub, "subject", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_metric, subject), + G_STRUCT_OFFSET (struct rspamd_config, subject), 0, "Rewrite subject with this value"); - /* Ungrouped symbols */ - ssub = rspamd_rcl_add_section_doc (&sub->subsections, - "symbol", "name", - rspamd_rcl_symbol_handler, - UCL_OBJECT, - TRUE, - TRUE, - sub->doc_ref, - "Symbols settings"); - - /* Actions part */ - ssub = rspamd_rcl_add_section_doc (&sub->subsections, - "actions", NULL, - rspamd_rcl_actions_handler, + sub = rspamd_rcl_add_section_doc (&new, + "groups", "name", + rspamd_rcl_group_handler, UCL_OBJECT, + FALSE, TRUE, - TRUE, - sub->doc_ref, - "Actions settings"); + cfg->doc_strings, + "Symbol groups configuration"); /* Group part */ ssub = rspamd_rcl_add_section_doc (&sub->subsections, - "group", "name", - rspamd_rcl_group_handler, + "symbols", "name", + rspamd_rcl_symbol_handler, UCL_OBJECT, TRUE, TRUE, diff --git a/src/libserver/cfg_rcl.h b/src/libserver/cfg_rcl.h index e115fda97..51b348e9c 100644 --- a/src/libserver/cfg_rcl.h +++ b/src/libserver/cfg_rcl.h @@ -17,6 +17,7 @@ #define CFG_RCL_H_ #include "config.h" +#include "cfg_file.h" #include "ucl.h" #include "mem_pool.h" @@ -57,6 +58,7 @@ struct rspamd_rcl_struct_parser { enum rspamd_rcl_flag flags; }; + /** * Common handler type * @param cfg configuration @@ -457,4 +459,12 @@ ucl_object_t *rspamd_rcl_add_doc_by_example (struct rspamd_config *cfg, gboolean rspamd_rcl_add_lua_plugins_path (struct rspamd_config *cfg, const gchar *path, GError **err); + +/* + * Read configuration file + */ +gboolean rspamd_config_read (struct rspamd_config *cfg, + const gchar *filename, const gchar *convert_to, + rspamd_rcl_section_fin_t logger_fin, gpointer logger_ud, + GHashTable *vars); #endif /* CFG_RCL_H_ */ diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index 691482773..00c7e0db4 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -122,16 +122,13 @@ rspamd_config_new (void) /* 20 Kb */ cfg->max_diff = 20480; - cfg->metrics = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); - rspamd_config_new_metric (cfg, NULL, DEFAULT_METRIC); + rspamd_config_init_metric (cfg); cfg->c_modules = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cfg->composite_symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cfg->classifiers_symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cfg->cfg_params = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); - cfg->metrics_symbols = g_hash_table_new_full (rspamd_str_hash, rspamd_str_equal, - NULL, (GDestroyNotify)g_list_free); cfg->debug_modules = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cfg->explicit_modules = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cfg->wrk_parsers = g_hash_table_new (g_int_hash, g_int_equal); @@ -215,7 +212,6 @@ rspamd_config_free (struct rspamd_config *cfg) } g_list_free (cfg->classifiers); - g_list_free (cfg->metrics_list); g_list_free (cfg->workers); rspamd_symbols_cache_destroy (cfg->cache); #ifdef WITH_HIREDIS @@ -227,14 +223,11 @@ rspamd_config_free (struct rspamd_config *cfg) ucl_object_unref (cfg->config_comments); ucl_object_unref (cfg->doc_strings); ucl_object_unref (cfg->neighbours); - g_hash_table_remove_all (cfg->metrics); - g_hash_table_unref (cfg->metrics); g_hash_table_unref (cfg->c_modules); g_hash_table_remove_all (cfg->composite_symbols); g_hash_table_unref (cfg->composite_symbols); g_hash_table_remove_all (cfg->cfg_params); g_hash_table_unref (cfg->cfg_params); - g_hash_table_destroy (cfg->metrics_symbols); g_hash_table_unref (cfg->classifiers_symbols); g_hash_table_unref (cfg->debug_modules); g_hash_table_unref (cfg->explicit_modules); @@ -764,25 +757,24 @@ rspamd_config_post_load (struct rspamd_config *cfg, if (opts & RSPAMD_CONFIG_INIT_VALIDATE) { /* Check for actions sanity */ int i, prev_act = 0; - struct rspamd_metric *metric = cfg->default_metric; gdouble prev_score = NAN; gboolean seen_controller = FALSE; GList *cur; struct rspamd_worker_conf *wcf; for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i ++) { - if (!isnan (prev_score) && !isnan (metric->actions[i].score)) { - if (prev_score <= isnan (metric->actions[i].score)) { + if (!isnan (prev_score) && !isnan (cfg->actions[i].score)) { + if (prev_score <= isnan (cfg->actions[i].score)) { msg_warn_config ("incorrect metrics scores: action %s" " has lower score: %.2f than action %s: %.2f", rspamd_action_to_str (prev_act), prev_score, - rspamd_action_to_str (i), metric->actions[i].score); + rspamd_action_to_str (i), cfg->actions[i].score); ret = FALSE; } } - if (!isnan (metric->actions[i].score)) { - prev_score = metric->actions[i].score; + if (!isnan (cfg->actions[i].score)) { + prev_score = cfg->actions[i].score; prev_act = i; } } @@ -940,47 +932,32 @@ rspamd_config_new_statfile (struct rspamd_config *cfg, return c; } -struct rspamd_metric * -rspamd_config_new_metric (struct rspamd_config *cfg, struct rspamd_metric *c, - const gchar *name) +void +rspamd_config_init_metric (struct rspamd_config *cfg) { int i; - if (c == NULL) { - c = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_metric)); - c->grow_factor = 1.0; - c->symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); - c->groups = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); - - for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) { - c->actions[i].score = NAN; - c->actions[i].action = i; - c->actions[i].priority = 0; - } - - c->subject = SPAM_SUBJECT; - c->name = rspamd_mempool_strdup (cfg->cfg_pool, name); - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t) g_hash_table_unref, - c->symbols); - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t) g_hash_table_unref, - c->groups); - - g_hash_table_insert (cfg->metrics, (void *)c->name, c); - cfg->metrics_list = g_list_prepend (cfg->metrics_list, c); + cfg->grow_factor = 1.0; + cfg->symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); + cfg->groups = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); - if (strcmp (c->name, DEFAULT_METRIC) == 0) { - cfg->default_metric = c; - } + for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) { + cfg->actions[i].score = NAN; + cfg->actions[i].action = i; + cfg->actions[i].priority = 0; } - return c; + cfg->subject = SPAM_SUBJECT; + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t) g_hash_table_unref, + cfg->symbols); + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t) g_hash_table_unref, + cfg->groups); } struct rspamd_symbols_group * -rspamd_config_new_group (struct rspamd_config *cfg, struct rspamd_metric *metric, - const gchar *name) +rspamd_config_new_group (struct rspamd_config *cfg, const gchar *name) { struct rspamd_symbols_group *gr; @@ -991,7 +968,7 @@ rspamd_config_new_group (struct rspamd_config *cfg, struct rspamd_metric *metric (rspamd_mempool_destruct_t)g_hash_table_unref, gr->symbols); gr->name = rspamd_mempool_strdup (cfg->cfg_pool, name); - g_hash_table_insert (metric->groups, gr->name, gr); + g_hash_table_insert (cfg->groups, gr->name, gr); return gr; } @@ -1431,14 +1408,12 @@ rspamd_init_filters (struct rspamd_config *cfg, bool reconfig) } static void -rspamd_config_new_metric_symbol (struct rspamd_config *cfg, - struct rspamd_metric *metric, const gchar *symbol, +rspamd_config_new_symbol (struct rspamd_config *cfg, const gchar *symbol, gdouble score, const gchar *description, const gchar *group, guint flags, guint priority, gint nshots) { struct rspamd_symbols_group *sym_group; struct rspamd_symbol *sym_def; - GList *metric_list; gdouble *score_ptr; sym_def = @@ -1457,32 +1432,20 @@ rspamd_config_new_metric_symbol (struct rspamd_config *cfg, sym_def->description = rspamd_mempool_strdup (cfg->cfg_pool, description); } - msg_debug_config ("registered symbol %s with weight %.2f in metric %s and group %s", - sym_def->name, score, metric->name, group); + msg_debug_config ("registered symbol %s with weight %.2f in and group %s", + sym_def->name, score, group); - g_hash_table_insert (metric->symbols, sym_def->name, sym_def); - - if ((metric_list = - g_hash_table_lookup (cfg->metrics_symbols, sym_def->name)) == NULL) { - metric_list = g_list_prepend (NULL, metric); - g_hash_table_insert (cfg->metrics_symbols, sym_def->name, metric_list); - } - else { - /* Slow but keep start element of list in safe */ - if (!g_list_find (metric_list, metric)) { - metric_list = g_list_append (metric_list, metric); - } - } + g_hash_table_insert (cfg->symbols, sym_def->name, sym_def); /* Search for symbol group */ if (group == NULL) { group = "ungrouped"; } - sym_group = g_hash_table_lookup (metric->groups, group); + sym_group = g_hash_table_lookup (cfg->groups, group); if (sym_group == NULL) { /* Create new group */ - sym_group = rspamd_config_new_group (cfg, metric, group); + sym_group = rspamd_config_new_group (cfg, group); } sym_def->gr = sym_group; @@ -1491,19 +1454,16 @@ rspamd_config_new_metric_symbol (struct rspamd_config *cfg, gboolean -rspamd_config_add_metric_symbol (struct rspamd_config *cfg, +rspamd_config_add_symbol (struct rspamd_config *cfg, const gchar *symbol, gdouble score, const gchar *description, const gchar *group, guint flags, guint priority, gint nshots) { struct rspamd_symbol *sym_def; - struct rspamd_metric *metric; - g_assert (cfg != NULL); g_assert (symbol != NULL); - metric = cfg->default_metric; - sym_def = g_hash_table_lookup (metric->symbols, symbol); + sym_def = g_hash_table_lookup (cfg->symbols, symbol); if (sym_def != NULL) { if (sym_def->priority > priority) { @@ -1546,7 +1506,7 @@ rspamd_config_add_metric_symbol (struct rspamd_config *cfg, } } - rspamd_config_new_metric_symbol (cfg, metric, symbol, score, description, + rspamd_config_new_symbol (cfg, symbol, score, description, group, flags, priority, nshots); return TRUE; @@ -1557,12 +1517,10 @@ rspamd_config_is_module_enabled (struct rspamd_config *cfg, const gchar *module_name) { gboolean is_c = FALSE; - struct rspamd_metric *metric; const ucl_object_t *conf, *enabled; GList *cur; struct rspamd_symbols_group *gr; - metric = cfg->default_metric; if (g_hash_table_lookup (cfg->c_modules, module_name)) { is_c = TRUE; @@ -1618,17 +1576,15 @@ rspamd_config_is_module_enabled (struct rspamd_config *cfg, } } - if (metric) { - /* Now we check symbols group */ - gr = g_hash_table_lookup (metric->groups, module_name); + /* Now we check symbols group */ + gr = g_hash_table_lookup (cfg->groups, module_name); - if (gr) { - if (gr->disabled) { - msg_info_config ("%s module %s is disabled in the configuration as " - "its group has been disabled", - is_c ? "internal" : "lua", module_name); - return FALSE; - } + if (gr) { + if (gr->disabled) { + msg_info_config ("%s module %s is disabled in the configuration as " + "its group has been disabled", + is_c ? "internal" : "lua", module_name); + return FALSE; } } @@ -1637,29 +1593,16 @@ rspamd_config_is_module_enabled (struct rspamd_config *cfg, gboolean rspamd_config_set_action_score (struct rspamd_config *cfg, - const gchar *metric_name, const gchar *action_name, gdouble score, guint priority) { - struct metric_action *act; - struct rspamd_metric *metric; + struct rspamd_action *act; gint act_num; g_assert (cfg != NULL); g_assert (action_name != NULL); - if (metric_name == NULL) { - metric_name = DEFAULT_METRIC; - } - - metric = g_hash_table_lookup (cfg->metrics, metric_name); - - if (metric == NULL) { - msg_err_config ("metric %s has not been found", metric_name); - return FALSE; - } - if (!rspamd_action_from_str (action_name, &act_num)) { msg_err_config ("invalid action name: %s", action_name); return FALSE; @@ -1667,7 +1610,7 @@ rspamd_config_set_action_score (struct rspamd_config *cfg, g_assert (act_num >= METRIC_ACTION_REJECT && act_num < METRIC_ACTION_MAX); - act = &metric->actions[act_num]; + act = &cfg->actions[act_num]; if (isnan (act->score)) { act->score = score; @@ -1827,7 +1770,7 @@ rspamd_action_from_str (const gchar *data, gint *result) } const gchar * -rspamd_action_to_str (enum rspamd_metric_action action) +rspamd_action_to_str (enum rspamd_action_type action) { switch (action) { case METRIC_ACTION_REJECT: @@ -1850,7 +1793,7 @@ rspamd_action_to_str (enum rspamd_metric_action action) } const gchar * -rspamd_action_to_str_alt (enum rspamd_metric_action action) +rspamd_action_to_str_alt (enum rspamd_action_type action) { switch (action) { case METRIC_ACTION_REJECT: diff --git a/src/libserver/composites.c b/src/libserver/composites.c index 41dd60b0c..cfb8968ff 100644 --- a/src/libserver/composites.c +++ b/src/libserver/composites.c @@ -142,7 +142,6 @@ rspamd_composite_expr_process (gpointer input, rspamd_expression_atom_t *atom) struct rspamd_symbol_result *ms = NULL; struct rspamd_symbols_group *gr; struct rspamd_symbol *sdef; - struct rspamd_metric *metric; GHashTableIter it; gpointer k, v; gint rc = 0; @@ -160,9 +159,7 @@ rspamd_composite_expr_process (gpointer input, rspamd_expression_atom_t *atom) } if (strncmp (sym, "g:", 2) == 0) { - metric = g_hash_table_lookup (cd->task->cfg->metrics, DEFAULT_METRIC); - g_assert (metric != NULL); - gr = g_hash_table_lookup (metric->groups, sym + 2); + gr = g_hash_table_lookup (cd->task->cfg->groups, sym + 2); if (gr != NULL) { g_hash_table_iter_init (&it, gr->symbols); diff --git a/src/libserver/dynamic_cfg.c b/src/libserver/dynamic_cfg.c index d576e055a..3f0d490f4 100644 --- a/src/libserver/dynamic_cfg.c +++ b/src/libserver/dynamic_cfg.c @@ -37,7 +37,6 @@ apply_dynamic_conf (const ucl_object_t *top, struct rspamd_config *cfg) gint test_act; const ucl_object_t *cur_elt, *cur_nm, *it_val; ucl_object_iter_t it = NULL; - struct rspamd_metric *real_metric; const gchar *name; gdouble nscore; static const guint priority = 3; @@ -54,12 +53,6 @@ apply_dynamic_conf (const ucl_object_t *top, struct rspamd_config *cfg) "loaded json metric object element has no 'metric' attribute"); continue; } - real_metric = g_hash_table_lookup (cfg->metrics, - ucl_object_tostring (cur_nm)); - if (real_metric == NULL) { - msg_warn ("cannot find metric %s", ucl_object_tostring (cur_nm)); - continue; - } cur_nm = ucl_object_lookup (cur_elt, "symbols"); /* Parse symbols */ @@ -79,7 +72,7 @@ apply_dynamic_conf (const ucl_object_t *top, struct rspamd_config *cfg) /* * We use priority = 3 here */ - rspamd_config_add_metric_symbol (cfg, + rspamd_config_add_symbol (cfg, ucl_object_tostring (n), nscore, NULL, NULL, 0, priority, cfg->default_max_shots); } @@ -114,8 +107,7 @@ apply_dynamic_conf (const ucl_object_t *top, struct rspamd_config *cfg) } nscore = ucl_object_todouble (ucl_object_lookup (it_val, "value")); - rspamd_config_set_action_score (cfg, real_metric->name, - name, nscore, priority); + rspamd_config_set_action_score (cfg, name, nscore, priority); } else { msg_info ( diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c index 25fb56a0f..5e5132e1d 100644 --- a/src/libserver/protocol.c +++ b/src/libserver/protocol.c @@ -740,7 +740,7 @@ rspamd_emails_tree_ucl (GHashTable *input, struct rspamd_task *task) /* Write new subject */ static const gchar * -make_rewritten_subject (struct rspamd_metric *metric, struct rspamd_task *task) +make_rewritten_subject (struct rspamd_task *task) { GString *subj_buf; gchar *res; @@ -750,7 +750,7 @@ make_rewritten_subject (struct rspamd_metric *metric, struct rspamd_task *task) c = rspamd_mempool_get_variable (task->task_pool, "metric_subject"); if (c == NULL) { - c = metric->subject; + c = task->cfg->subject; } if (c == NULL) { @@ -797,8 +797,7 @@ make_rewritten_subject (struct rspamd_metric *metric, struct rspamd_task *task) } static ucl_object_t * -rspamd_metric_symbol_ucl (struct rspamd_task *task, struct rspamd_metric *m, - struct rspamd_symbol_result *sym) +rspamd_metric_symbol_ucl (struct rspamd_task *task, struct rspamd_symbol_result *sym) { ucl_object_t *obj = NULL, *ar; const gchar *description = NULL; @@ -849,15 +848,12 @@ rspamd_metric_result_ucl (struct rspamd_task *task, { GHashTableIter hiter; struct rspamd_symbol_result *sym; - struct rspamd_metric *m; gboolean is_spam; - enum rspamd_metric_action action = METRIC_ACTION_NOACTION; + enum rspamd_action_type action = METRIC_ACTION_NOACTION; ucl_object_t *obj = NULL, *sobj; gpointer h, v; const gchar *subject; - m = mres->metric; - if (mres->action == METRIC_ACTION_MAX) { mres->action = rspamd_check_action_metric (task, mres); } @@ -895,7 +891,7 @@ rspamd_metric_result_ucl (struct rspamd_task *task, "action", 0, false); if (action == METRIC_ACTION_REWRITE_SUBJECT) { - subject = make_rewritten_subject (m, task); + subject = make_rewritten_subject (task); if (subject) { ucl_object_insert_key (obj, ucl_object_fromstring (subject), @@ -912,7 +908,7 @@ rspamd_metric_result_ucl (struct rspamd_task *task, while (g_hash_table_iter_next (&hiter, &h, &v)) { sym = (struct rspamd_symbol_result *)v; - sobj = rspamd_metric_symbol_ucl (task, m, sym); + sobj = rspamd_metric_symbol_ucl (task, sym); ucl_object_insert_key (obj, sobj, h, 0, false); } diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c index e74de4e73..9aad6b591 100644 --- a/src/libserver/re_cache.c +++ b/src/libserver/re_cache.c @@ -1389,7 +1389,7 @@ rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache, /* Read number of regexps */ g_assert (fd != -1); lseek (fd, RSPAMD_HS_MAGIC_LEN + sizeof (cache->plt), SEEK_SET); - read (fd, &n, sizeof (n)); + g_assert (read (fd, &n, sizeof (n)) == sizeof (n)); close (fd); if (re_class->type_len > 0) { diff --git a/src/libserver/symbols_cache.c b/src/libserver/symbols_cache.c index 4cf1cab5b..9b8fba25b 100644 --- a/src/libserver/symbols_cache.c +++ b/src/libserver/symbols_cache.c @@ -1013,48 +1013,35 @@ static void rspamd_symbols_cache_validate_cb (gpointer k, gpointer v, gpointer ud) { struct cache_item *item = v, *parent; + struct rspamd_config *cfg; struct symbols_cache *cache = (struct symbols_cache *)ud; - GList *cur; - struct rspamd_metric *m; struct rspamd_symbol *s; gboolean skipped, ghost; gint p1, p2; ghost = item->st->weight == 0 ? TRUE : FALSE; + cfg = cache->cfg; /* Check whether this item is skipped */ skipped = !ghost; - g_assert (cache->cfg != NULL); + g_assert (cfg != NULL); if ((item->type & (SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_COMPOSITE|SYMBOL_TYPE_CLASSIFIER)) - && g_hash_table_lookup (cache->cfg->metrics_symbols, item->symbol) == NULL) { - cur = g_list_first (cache->cfg->metrics_list); - while (cur) { - m = cur->data; - - if (m->accept_unknown_symbols) { - GList *mlist; - - skipped = FALSE; - item->st->weight = m->unknown_weight; - s = rspamd_mempool_alloc0 (cache->static_pool, - sizeof (*s)); - s->name = item->symbol; - s->weight_ptr = &item->st->weight; - g_hash_table_insert (m->symbols, item->symbol, s); - mlist = g_hash_table_lookup (cache->cfg->metrics_symbols, - item->symbol); - mlist = g_list_append (mlist, m); - g_hash_table_insert (cache->cfg->metrics_symbols, - item->symbol, mlist); - - msg_info_cache ("adding unknown symbol %s to metric %s", item->symbol, - m->name); - ghost = FALSE; - } + && g_hash_table_lookup (cfg->symbols, item->symbol) == NULL) { + + if (cfg->accept_unknown_symbols) { - cur = g_list_next (cur); + skipped = FALSE; + item->st->weight = cfg->unknown_weight; + s = rspamd_mempool_alloc0 (cache->static_pool, + sizeof (*s)); + s->name = item->symbol; + s->weight_ptr = &item->st->weight; + g_hash_table_insert (cfg->symbols, item->symbol, s); + + msg_info_cache ("adding unknown symbol %s", item->symbol); + ghost = FALSE; } } else { @@ -1120,10 +1107,8 @@ rspamd_symbols_cache_validate (struct symbols_cache *cache, { struct cache_item *item; GHashTableIter it; - GList *cur; gpointer k, v; struct rspamd_symbol *sym_def; - struct rspamd_metric *metric; gboolean ignore_symbol = FALSE, ret = TRUE; if (cache == NULL) { @@ -1132,32 +1117,23 @@ rspamd_symbols_cache_validate (struct symbols_cache *cache, } /* Now adjust symbol weights according to default metric */ - if (cfg->default_metric != NULL) { - g_hash_table_foreach (cfg->default_metric->symbols, + g_hash_table_foreach (cfg->symbols, rspamd_symbols_cache_metric_validate_cb, cache); - } g_hash_table_foreach (cache->items_by_symbol, rspamd_symbols_cache_validate_cb, cache); /* Now check each metric item and find corresponding symbol in a cache */ - g_hash_table_iter_init (&it, cfg->metrics_symbols); + g_hash_table_iter_init (&it, cfg->symbols); while (g_hash_table_iter_next (&it, &k, &v)) { ignore_symbol = FALSE; - cur = v; + sym_def = v; - while (cur) { - metric = cur->data; - sym_def = g_hash_table_lookup (metric->symbols, k); - - if (sym_def && (sym_def->flags & RSPAMD_SYMBOL_FLAG_IGNORE)) { - ignore_symbol = TRUE; - break; - } - - cur = g_list_next (cur); + if (sym_def && (sym_def->flags & RSPAMD_SYMBOL_FLAG_IGNORE)) { + ignore_symbol = TRUE; + break; } if (!ignore_symbol) { @@ -1518,7 +1494,6 @@ rspamd_symbols_cache_process_settings (struct rspamd_task *task, struct symbols_cache *cache) { const ucl_object_t *wl, *cur, *disabled, *enabled; - struct rspamd_metric *def; struct rspamd_symbols_group *gr; GHashTableIter gr_it; ucl_object_iter_t it = NULL; @@ -1547,15 +1522,14 @@ rspamd_symbols_cache_process_settings (struct rspamd_task *task, /* Enable groups of symbols */ enabled = ucl_object_lookup (task->settings, "groups_enabled"); - def = g_hash_table_lookup (task->cfg->metrics, DEFAULT_METRIC); - if (def && enabled) { + if (enabled) { it = NULL; rspamd_symbols_cache_disable_all_symbols (task, cache); while ((cur = ucl_iterate_object (enabled, &it, true)) != NULL) { if (ucl_object_type (cur) == UCL_STRING) { - gr = g_hash_table_lookup (def->groups, + gr = g_hash_table_lookup (task->cfg->groups, ucl_object_tostring (cur)); if (gr) { @@ -1582,14 +1556,13 @@ rspamd_symbols_cache_process_settings (struct rspamd_task *task, /* Disable groups of symbols */ disabled = ucl_object_lookup (task->settings, "groups_disabled"); - def = g_hash_table_lookup (task->cfg->metrics, DEFAULT_METRIC); - if (def && disabled) { + if (disabled) { it = NULL; while ((cur = ucl_iterate_object (disabled, &it, true)) != NULL) { if (ucl_object_type (cur) == UCL_STRING) { - gr = g_hash_table_lookup (def->groups, + gr = g_hash_table_lookup (task->cfg->groups, ucl_object_tostring (cur)); if (gr) { |