From 67761b737080a3e8f4a5ac8417258d6fcda05240 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Mon, 30 Jul 2018 15:40:31 +0100 Subject: [PATCH] [Project] Implement fast module ctx lookup --- src/controller.c | 5 ++--- src/libserver/cfg_file.h | 2 +- src/libserver/cfg_utils.c | 46 ++++++++++++++++++++++++--------------- src/plugins/chartable.c | 13 ++++++----- src/plugins/dkim_check.c | 13 ++++++----- src/plugins/fuzzy_check.c | 13 ++++++----- src/plugins/regexp.c | 37 ++++++++++++++++--------------- src/plugins/spf.c | 28 +++++++++++++----------- src/plugins/surbl.c | 26 ++++++++++++---------- src/rspamadm/confighelp.c | 11 ++++++---- src/rspamd.c | 2 -- src/rspamd.h | 1 + 12 files changed, 109 insertions(+), 88 deletions(-) diff --git a/src/controller.c b/src/controller.c index 0f415210c..d41860937 100644 --- a/src/controller.c +++ b/src/controller.c @@ -3633,6 +3633,7 @@ start_controller_worker (struct rspamd_worker *worker) struct module_ctx *mctx; GHashTableIter iter; gpointer key, value; + guint i; struct rspamd_keypair_cache *cache; struct timeval stv; const guint save_stats_interval = 60 * 1000; /* 1 minute */ @@ -3792,9 +3793,7 @@ start_controller_worker (struct rspamd_worker *worker) rspamd_http_router_set_key (ctx->http, ctx->key); } - g_hash_table_iter_init (&iter, ctx->cfg->c_modules); - while (g_hash_table_iter_next (&iter, &key, &value)) { - mctx = value; + PTR_ARRAY_FOREACH (ctx->cfg->c_modules, i, mctx) { if (mctx->mod->module_attach_controller_func != NULL) { mctx->mod->module_attach_controller_func (mctx, ctx->custom_commands); diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index 12919c198..75b404530 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -352,7 +352,7 @@ struct rspamd_config { ucl_object_t *rcl_obj; /**< rcl object */ ucl_object_t *config_comments; /**< comments saved from the config */ ucl_object_t *doc_strings; /**< documentation strings for config options */ - GHashTable * c_modules; /**< hash of c modules indexed by module name */ + GPtrArray *c_modules; /**< list of C modules */ GHashTable * composite_symbols; /**< hash of composite symbols indexed by its name */ GList *classifiers; /**< list of all classifiers defined */ GList *statfiles; /**< list of all statfiles in config file order */ diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index ab01a5403..b7b9dfdee 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -134,7 +134,6 @@ rspamd_config_new (enum rspamd_config_init_flags flags) cfg->max_diff = 20480; rspamd_config_init_metric (cfg); - cfg->c_modules = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cfg->composite_symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cfg->classifiers_symbols = g_hash_table_new (rspamd_str_hash, @@ -199,6 +198,7 @@ rspamd_config_new (enum rspamd_config_init_flags flags) cfg->default_max_shots = DEFAULT_MAX_SHOTS; cfg->max_sessions_cache = DEFAULT_MAX_SESSIONS; cfg->maps_cache_dir = rspamd_mempool_strdup (cfg->cfg_pool, RSPAMD_DBDIR); + cfg->c_modules = g_ptr_array_new (); REF_INIT_RETAIN (cfg, rspamd_config_free); @@ -239,7 +239,6 @@ rspamd_config_free (struct rspamd_config *cfg) ucl_object_unref (cfg->config_comments); ucl_object_unref (cfg->doc_strings); ucl_object_unref (cfg->neighbours); - g_hash_table_unref (cfg->c_modules); g_hash_table_remove_all (cfg->composite_symbols); g_hash_table_unref (cfg->composite_symbols); g_hash_table_remove_all (cfg->cfg_params); @@ -257,6 +256,7 @@ rspamd_config_free (struct rspamd_config *cfg) rspamd_re_cache_unref (cfg->re_cache); rspamd_upstreams_library_unref (cfg->ups_ctx); rspamd_mempool_delete (cfg->cfg_pool); + g_ptr_array_free (cfg->c_modules, TRUE); if (cfg->lua_state && cfg->own_lua_state) { lua_close (cfg->lua_state); @@ -1455,21 +1455,19 @@ rspamd_init_filters (struct rspamd_config *cfg, bool reconfig) { GList *cur; module_t *mod, **pmod; - struct module_ctx *mod_ctx; + guint i = 0; + struct module_ctx *mod_ctx, *cur_ctx; /* Init all compiled modules */ - if (!reconfig) { - for (pmod = cfg->compiled_modules; pmod != NULL && *pmod != NULL; pmod ++) { - mod = *pmod; - - if (rspamd_check_module (cfg, mod)) { - if (mod->module_init_func (cfg, &mod_ctx) == 0) { - g_assert (mod_ctx != NULL); - g_hash_table_insert (cfg->c_modules, - (gpointer) mod->name, - mod_ctx); - mod_ctx->mod = mod; - } + + for (pmod = cfg->compiled_modules; pmod != NULL && *pmod != NULL; pmod ++) { + mod = *pmod; + if (rspamd_check_module (cfg, mod)) { + if (mod->module_init_func (cfg, &mod_ctx) == 0) { + g_assert (mod_ctx != NULL); + g_ptr_array_add (cfg->c_modules, mod_ctx); + mod_ctx->mod = mod; + mod->ctx_offset = i ++; } } } @@ -1479,7 +1477,14 @@ rspamd_init_filters (struct rspamd_config *cfg, bool reconfig) while (cur) { /* Perform modules configuring */ - mod_ctx = g_hash_table_lookup (cfg->c_modules, cur->data); + mod_ctx = NULL; + PTR_ARRAY_FOREACH (cfg->c_modules, i, cur_ctx) { + if (g_ascii_strcasecmp (cur_ctx->mod->name, + (const gchar *)cur->data) == 0) { + mod_ctx = cur_ctx; + break; + } + } if (mod_ctx) { mod = mod_ctx->mod; @@ -1728,9 +1733,14 @@ rspamd_config_is_module_enabled (struct rspamd_config *cfg, GList *cur; struct rspamd_symbols_group *gr; lua_State *L = cfg->lua_state; + struct module_ctx *cur_ctx; + guint i; - if (g_hash_table_lookup (cfg->c_modules, module_name)) { - is_c = TRUE; + PTR_ARRAY_FOREACH (cfg->c_modules, i, cur_ctx) { + if (g_ascii_strcasecmp (cur_ctx->mod->name, module_name) == 0) { + is_c = TRUE; + break; + } } if (g_hash_table_lookup (cfg->explicit_modules, module_name) != NULL) { diff --git a/src/plugins/chartable.c b/src/plugins/chartable.c index c9df2269a..d07e7fc00 100644 --- a/src/plugins/chartable.c +++ b/src/plugins/chartable.c @@ -61,12 +61,13 @@ gint chartable_module_config (struct rspamd_config *cfg); gint chartable_module_reconfig (struct rspamd_config *cfg); module_t chartable_module = { - "chartable", - chartable_module_init, - chartable_module_config, - chartable_module_reconfig, - NULL, - RSPAMD_MODULE_VER + "chartable", + chartable_module_init, + chartable_module_config, + chartable_module_reconfig, + NULL, + RSPAMD_MODULE_VER, + (guint)-1, }; struct chartable_ctx { diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index 3993d7bc4..7d930b2b4 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -106,12 +106,13 @@ gint dkim_module_config (struct rspamd_config *cfg); gint dkim_module_reconfig (struct rspamd_config *cfg); module_t dkim_module = { - "dkim", - dkim_module_init, - dkim_module_config, - dkim_module_reconfig, - NULL, - RSPAMD_MODULE_VER + "dkim", + dkim_module_init, + dkim_module_config, + dkim_module_reconfig, + NULL, + RSPAMD_MODULE_VER, + (guint)-1, }; static void diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 7557d9bf7..1b0fa2844 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -188,12 +188,13 @@ static gint fuzzy_lua_learn_handler (lua_State *L); static gint fuzzy_lua_unlearn_handler (lua_State *L); module_t fuzzy_check_module = { - "fuzzy_check", - fuzzy_check_module_init, - fuzzy_check_module_config, - fuzzy_check_module_reconfig, - fuzzy_attach_controller, - RSPAMD_MODULE_VER + "fuzzy_check", + fuzzy_check_module_init, + fuzzy_check_module_config, + fuzzy_check_module_reconfig, + fuzzy_attach_controller, + RSPAMD_MODULE_VER, + (guint)-1, }; static void diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c index 1ffa7d88b..915305aa3 100644 --- a/src/plugins/regexp.c +++ b/src/plugins/regexp.c @@ -39,8 +39,6 @@ struct regexp_ctx { gsize max_size; }; -static struct regexp_ctx *regexp_module_ctx = NULL; - static void process_regexp_item (struct rspamd_task *task, void *user_data); @@ -50,14 +48,23 @@ gint regexp_module_config (struct rspamd_config *cfg); gint regexp_module_reconfig (struct rspamd_config *cfg); module_t regexp_module = { - "regexp", - regexp_module_init, - regexp_module_config, - regexp_module_reconfig, - NULL, - RSPAMD_MODULE_VER + "regexp", + regexp_module_init, + regexp_module_config, + regexp_module_reconfig, + NULL, + RSPAMD_MODULE_VER, + (guint)-1, }; + +static inline struct regexp_ctx * +regexp_get_context (struct rspamd_config *cfg) +{ + return (struct regexp_ctx *)g_ptr_array_index (cfg->c_modules, + regexp_module.ctx_offset); +} + /* Process regexp expression */ static gboolean read_regexp_expression (rspamd_mempool_t * pool, @@ -90,9 +97,10 @@ read_regexp_expression (rspamd_mempool_t * pool, gint regexp_module_init (struct rspamd_config *cfg, struct module_ctx **ctx) { - if (regexp_module_ctx == NULL) { - regexp_module_ctx = g_malloc0 (sizeof (struct regexp_ctx)); - } + struct regexp_ctx *regexp_module_ctx; + + regexp_module_ctx = rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (*regexp_module_ctx)); *ctx = (struct module_ctx *)regexp_module_ctx; @@ -122,6 +130,7 @@ regexp_module_init (struct rspamd_config *cfg, struct module_ctx **ctx) gint regexp_module_config (struct rspamd_config *cfg) { + struct regexp_ctx *regexp_module_ctx = regexp_get_context (cfg); struct regexp_module_item *cur_item = NULL; const ucl_object_t *sec, *value, *elt; ucl_object_iter_t it = NULL; @@ -336,12 +345,6 @@ regexp_module_config (struct rspamd_config *cfg) gint regexp_module_reconfig (struct rspamd_config *cfg) { - struct module_ctx saved_ctx; - - saved_ctx = regexp_module_ctx->ctx; - memset (regexp_module_ctx, 0, sizeof (*regexp_module_ctx)); - regexp_module_ctx->ctx = saved_ctx; - return regexp_module_config (cfg); } diff --git a/src/plugins/spf.c b/src/plugins/spf.c index 683683a55..5ec5bfcfc 100644 --- a/src/plugins/spf.c +++ b/src/plugins/spf.c @@ -62,13 +62,6 @@ struct spf_ctx { gboolean check_authed; }; -static inline struct spf_ctx * -spf_get_context (struct rspamd_config *cfg) -{ - return (struct spf_ctx *)g_hash_table_lookup (cfg->c_modules, "spf"); -} - - static void spf_symbol_callback (struct rspamd_task *task, void *unused); /* Initialization */ @@ -77,14 +70,23 @@ gint spf_module_config (struct rspamd_config *cfg); gint spf_module_reconfig (struct rspamd_config *cfg); module_t spf_module = { - "spf", - spf_module_init, - spf_module_config, - spf_module_reconfig, - NULL, - RSPAMD_MODULE_VER + "spf", + spf_module_init, + spf_module_config, + spf_module_reconfig, + NULL, + RSPAMD_MODULE_VER, + (guint)-1, }; +static inline struct spf_ctx * +spf_get_context (struct rspamd_config *cfg) +{ + return (struct spf_ctx *)g_ptr_array_index (cfg->c_modules, + spf_module.ctx_offset); +} + + gint spf_module_init (struct rspamd_config *cfg, struct module_ctx **ctx) { diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c index 07c6d06c1..d3ed6da74 100644 --- a/src/plugins/surbl.c +++ b/src/plugins/surbl.c @@ -159,26 +159,28 @@ surbl_error_quark (void) return g_quark_from_static_string ("surbl-error-quark"); } -static inline struct surbl_ctx * -surbl_get_context (struct rspamd_config *cfg) -{ - return (struct surbl_ctx *)g_hash_table_lookup (cfg->c_modules, "surbl"); -} - /* Initialization */ gint surbl_module_init (struct rspamd_config *cfg, struct module_ctx **ctx); gint surbl_module_config (struct rspamd_config *cfg); gint surbl_module_reconfig (struct rspamd_config *cfg); module_t surbl_module = { - "surbl", - surbl_module_init, - surbl_module_config, - surbl_module_reconfig, - NULL, - RSPAMD_MODULE_VER + "surbl", + surbl_module_init, + surbl_module_config, + surbl_module_reconfig, + NULL, + RSPAMD_MODULE_VER, + (guint)-1, }; +static inline struct surbl_ctx * +surbl_get_context (struct rspamd_config *cfg) +{ + return (struct surbl_ctx *)g_ptr_array_index (cfg->c_modules, + surbl_module.ctx_offset); +} + static void exceptions_free_value (gpointer v) { diff --git a/src/rspamadm/confighelp.c b/src/rspamadm/confighelp.c index f4c5176ac..85564cf4c 100644 --- a/src/rspamadm/confighelp.c +++ b/src/rspamadm/confighelp.c @@ -202,7 +202,7 @@ rspamadm_confighelp (gint argc, gchar **argv, const struct rspamadm_command *cmd module_t *mod, **pmod; worker_t **pworker; struct module_ctx *mod_ctx; - gint i = 1, ret = 0, processed_args = 0; + gint i, ret = 0, processed_args = 0; context = g_option_context_new ( "confighelp - displays help for the configuration options"); @@ -238,16 +238,19 @@ rspamadm_confighelp (gint argc, gchar **argv, const struct rspamadm_command *cmd rspamd_rcl_add_lua_plugins_path (cfg, plugins_path, NULL); /* Init modules to get documentation strings */ + i = 0; for (pmod = cfg->compiled_modules; pmod != NULL && *pmod != NULL; pmod++) { mod = *pmod; mod_ctx = g_malloc0 (sizeof (struct module_ctx)); if (mod->module_init_func (cfg, &mod_ctx) == 0) { - g_hash_table_insert (cfg->c_modules, - (gpointer) mod->name, - mod_ctx); + g_ptr_array_add (cfg->c_modules, mod_ctx); + mod_ctx->mod = mod; + mod->ctx_offset = i++; mod_ctx->mod = mod; } + + } /* Also init all workers */ for (pworker = cfg->compiled_workers; *pworker != NULL; pworker ++) { diff --git a/src/rspamd.c b/src/rspamd.c index 4b001948d..9339e0a3f 100644 --- a/src/rspamd.c +++ b/src/rspamd.c @@ -279,8 +279,6 @@ reread_config (struct rspamd_main *rspamd_main) rspamd_symbols_cache_save (rspamd_main->cfg->cache); tmp_cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT); - g_hash_table_unref (tmp_cfg->c_modules); - tmp_cfg->c_modules = g_hash_table_ref (rspamd_main->cfg->c_modules); tmp_cfg->libs_ctx = rspamd_main->cfg->libs_ctx; REF_RETAIN (tmp_cfg->libs_ctx); cfg_file = rspamd_mempool_strdup (tmp_cfg->cfg_pool, diff --git a/src/rspamd.h b/src/rspamd.h index a993238a9..409c051b3 100644 --- a/src/rspamd.h +++ b/src/rspamd.h @@ -191,6 +191,7 @@ typedef struct module_s { guint module_version; guint64 rspamd_version; const gchar *rspamd_features; + guint ctx_offset; } module_t; enum rspamd_worker_socket_type { -- 2.39.5