From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 30 Jul 2018 14:40:31 +0000 (+0100)
Subject: [Project] Implement fast module ctx lookup
X-Git-Tag: 1.7.9~16
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=67761b737080a3e8f4a5ac8417258d6fcda05240;p=rspamd.git

[Project] Implement fast module ctx lookup
---

diff --git a/src/controller.c b/src/controller.c
index 0f415210c..d41860937 100644
--- a/src/controller.c
+++ b/src/controller.c
@@ -3633,6 +3633,7 @@ start_controller_worker (struct rspamd_worker *worker)
 	struct module_ctx *mctx;
 	GHashTableIter iter;
 	gpointer key, value;
+	guint i;
 	struct rspamd_keypair_cache *cache;
 	struct timeval stv;
 	const guint save_stats_interval = 60 * 1000; /* 1 minute */
@@ -3792,9 +3793,7 @@ start_controller_worker (struct rspamd_worker *worker)
 		rspamd_http_router_set_key (ctx->http, ctx->key);
 	}
 
-	g_hash_table_iter_init (&iter, ctx->cfg->c_modules);
-	while (g_hash_table_iter_next (&iter, &key, &value)) {
-		mctx = value;
+	PTR_ARRAY_FOREACH (ctx->cfg->c_modules, i, mctx) {
 		if (mctx->mod->module_attach_controller_func != NULL) {
 			mctx->mod->module_attach_controller_func (mctx,
 					ctx->custom_commands);
diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h
index 12919c198..75b404530 100644
--- a/src/libserver/cfg_file.h
+++ b/src/libserver/cfg_file.h
@@ -352,7 +352,7 @@ struct rspamd_config {
 	ucl_object_t *rcl_obj;                          /**< rcl object											*/
 	ucl_object_t *config_comments;                  /**< comments saved from the config						*/
 	ucl_object_t *doc_strings;                      /**< documentation strings for config options			*/
-	GHashTable * c_modules;                         /**< hash of c modules indexed by module name			*/
+	GPtrArray *c_modules;                           /**< list of C modules			*/
 	GHashTable * composite_symbols;                 /**< hash of composite symbols indexed by its name		*/
 	GList *classifiers;                             /**< list of all classifiers defined                    */
 	GList *statfiles;                               /**< list of all statfiles in config file order         */
diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c
index ab01a5403..b7b9dfdee 100644
--- a/src/libserver/cfg_utils.c
+++ b/src/libserver/cfg_utils.c
@@ -134,7 +134,6 @@ rspamd_config_new (enum rspamd_config_init_flags flags)
 	cfg->max_diff = 20480;
 
 	rspamd_config_init_metric (cfg);
-	cfg->c_modules = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
 	cfg->composite_symbols =
 		g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
 	cfg->classifiers_symbols = g_hash_table_new (rspamd_str_hash,
@@ -199,6 +198,7 @@ rspamd_config_new (enum rspamd_config_init_flags flags)
 	cfg->default_max_shots = DEFAULT_MAX_SHOTS;
 	cfg->max_sessions_cache = DEFAULT_MAX_SESSIONS;
 	cfg->maps_cache_dir = rspamd_mempool_strdup (cfg->cfg_pool, RSPAMD_DBDIR);
+	cfg->c_modules = g_ptr_array_new ();
 
 	REF_INIT_RETAIN (cfg, rspamd_config_free);
 
@@ -239,7 +239,6 @@ rspamd_config_free (struct rspamd_config *cfg)
 	ucl_object_unref (cfg->config_comments);
 	ucl_object_unref (cfg->doc_strings);
 	ucl_object_unref (cfg->neighbours);
-	g_hash_table_unref (cfg->c_modules);
 	g_hash_table_remove_all (cfg->composite_symbols);
 	g_hash_table_unref (cfg->composite_symbols);
 	g_hash_table_remove_all (cfg->cfg_params);
@@ -257,6 +256,7 @@ rspamd_config_free (struct rspamd_config *cfg)
 	rspamd_re_cache_unref (cfg->re_cache);
 	rspamd_upstreams_library_unref (cfg->ups_ctx);
 	rspamd_mempool_delete (cfg->cfg_pool);
+	g_ptr_array_free (cfg->c_modules, TRUE);
 
 	if (cfg->lua_state && cfg->own_lua_state) {
 		lua_close (cfg->lua_state);
@@ -1455,21 +1455,19 @@ rspamd_init_filters (struct rspamd_config *cfg, bool reconfig)
 {
 	GList *cur;
 	module_t *mod, **pmod;
-	struct module_ctx *mod_ctx;
+	guint i = 0;
+	struct module_ctx *mod_ctx, *cur_ctx;
 
 	/* Init all compiled modules */
-	if (!reconfig) {
-		for (pmod = cfg->compiled_modules; pmod != NULL && *pmod != NULL; pmod ++) {
-			mod = *pmod;
-
-			if (rspamd_check_module (cfg, mod)) {
-				if (mod->module_init_func (cfg, &mod_ctx) == 0) {
-					g_assert (mod_ctx != NULL);
-					g_hash_table_insert (cfg->c_modules,
-							(gpointer) mod->name,
-							mod_ctx);
-					mod_ctx->mod = mod;
-				}
+
+	for (pmod = cfg->compiled_modules; pmod != NULL && *pmod != NULL; pmod ++) {
+		mod = *pmod;
+		if (rspamd_check_module (cfg, mod)) {
+			if (mod->module_init_func (cfg, &mod_ctx) == 0) {
+				g_assert (mod_ctx != NULL);
+				g_ptr_array_add (cfg->c_modules, mod_ctx);
+				mod_ctx->mod = mod;
+				mod->ctx_offset = i ++;
 			}
 		}
 	}
@@ -1479,7 +1477,14 @@ rspamd_init_filters (struct rspamd_config *cfg, bool reconfig)
 
 	while (cur) {
 		/* Perform modules configuring */
-		mod_ctx = g_hash_table_lookup (cfg->c_modules, cur->data);
+		mod_ctx = NULL;
+		PTR_ARRAY_FOREACH (cfg->c_modules, i, cur_ctx) {
+			if (g_ascii_strcasecmp (cur_ctx->mod->name,
+					(const gchar *)cur->data) == 0) {
+				mod_ctx = cur_ctx;
+				break;
+			}
+		}
 
 		if (mod_ctx) {
 			mod = mod_ctx->mod;
@@ -1728,9 +1733,14 @@ rspamd_config_is_module_enabled (struct rspamd_config *cfg,
 	GList *cur;
 	struct rspamd_symbols_group *gr;
 	lua_State *L = cfg->lua_state;
+	struct module_ctx *cur_ctx;
+	guint i;
 
-	if (g_hash_table_lookup (cfg->c_modules, module_name)) {
-		is_c = TRUE;
+	PTR_ARRAY_FOREACH (cfg->c_modules, i, cur_ctx) {
+		if (g_ascii_strcasecmp (cur_ctx->mod->name, module_name) == 0) {
+			is_c = TRUE;
+			break;
+		}
 	}
 
 	if (g_hash_table_lookup (cfg->explicit_modules, module_name) != NULL) {
diff --git a/src/plugins/chartable.c b/src/plugins/chartable.c
index c9df2269a..d07e7fc00 100644
--- a/src/plugins/chartable.c
+++ b/src/plugins/chartable.c
@@ -61,12 +61,13 @@ gint chartable_module_config (struct rspamd_config *cfg);
 gint chartable_module_reconfig (struct rspamd_config *cfg);
 
 module_t chartable_module = {
-	"chartable",
-	chartable_module_init,
-	chartable_module_config,
-	chartable_module_reconfig,
-	NULL,
-	RSPAMD_MODULE_VER
+		"chartable",
+		chartable_module_init,
+		chartable_module_config,
+		chartable_module_reconfig,
+		NULL,
+		RSPAMD_MODULE_VER,
+		(guint)-1,
 };
 
 struct chartable_ctx {
diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c
index 3993d7bc4..7d930b2b4 100644
--- a/src/plugins/dkim_check.c
+++ b/src/plugins/dkim_check.c
@@ -106,12 +106,13 @@ gint dkim_module_config (struct rspamd_config *cfg);
 gint dkim_module_reconfig (struct rspamd_config *cfg);
 
 module_t dkim_module = {
-	"dkim",
-	dkim_module_init,
-	dkim_module_config,
-	dkim_module_reconfig,
-	NULL,
-	RSPAMD_MODULE_VER
+		"dkim",
+		dkim_module_init,
+		dkim_module_config,
+		dkim_module_reconfig,
+		NULL,
+		RSPAMD_MODULE_VER,
+		(guint)-1,
 };
 
 static void
diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index 7557d9bf7..1b0fa2844 100644
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -188,12 +188,13 @@ static gint fuzzy_lua_learn_handler (lua_State *L);
 static gint fuzzy_lua_unlearn_handler (lua_State *L);
 
 module_t fuzzy_check_module = {
-	"fuzzy_check",
-	fuzzy_check_module_init,
-	fuzzy_check_module_config,
-	fuzzy_check_module_reconfig,
-	fuzzy_attach_controller,
-	RSPAMD_MODULE_VER
+		"fuzzy_check",
+		fuzzy_check_module_init,
+		fuzzy_check_module_config,
+		fuzzy_check_module_reconfig,
+		fuzzy_attach_controller,
+		RSPAMD_MODULE_VER,
+		(guint)-1,
 };
 
 static void
diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c
index 1ffa7d88b..915305aa3 100644
--- a/src/plugins/regexp.c
+++ b/src/plugins/regexp.c
@@ -39,8 +39,6 @@ struct regexp_ctx {
 	gsize max_size;
 };
 
-static struct regexp_ctx *regexp_module_ctx = NULL;
-
 static void process_regexp_item (struct rspamd_task *task, void *user_data);
 
 
@@ -50,14 +48,23 @@ gint regexp_module_config (struct rspamd_config *cfg);
 gint regexp_module_reconfig (struct rspamd_config *cfg);
 
 module_t regexp_module = {
-	"regexp",
-	regexp_module_init,
-	regexp_module_config,
-	regexp_module_reconfig,
-	NULL,
-	RSPAMD_MODULE_VER
+		"regexp",
+		regexp_module_init,
+		regexp_module_config,
+		regexp_module_reconfig,
+		NULL,
+		RSPAMD_MODULE_VER,
+		(guint)-1,
 };
 
+
+static inline struct regexp_ctx *
+regexp_get_context (struct rspamd_config *cfg)
+{
+	return (struct regexp_ctx *)g_ptr_array_index (cfg->c_modules,
+			regexp_module.ctx_offset);
+}
+
 /* Process regexp expression */
 static gboolean
 read_regexp_expression (rspamd_mempool_t * pool,
@@ -90,9 +97,10 @@ read_regexp_expression (rspamd_mempool_t * pool,
 gint
 regexp_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
 {
-	if (regexp_module_ctx == NULL) {
-		regexp_module_ctx = g_malloc0 (sizeof (struct regexp_ctx));
-	}
+	struct regexp_ctx *regexp_module_ctx;
+
+	regexp_module_ctx = rspamd_mempool_alloc0 (cfg->cfg_pool,
+			sizeof (*regexp_module_ctx));
 
 	*ctx = (struct module_ctx *)regexp_module_ctx;
 
@@ -122,6 +130,7 @@ regexp_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
 gint
 regexp_module_config (struct rspamd_config *cfg)
 {
+	struct regexp_ctx *regexp_module_ctx = regexp_get_context (cfg);
 	struct regexp_module_item *cur_item = NULL;
 	const ucl_object_t *sec, *value, *elt;
 	ucl_object_iter_t it = NULL;
@@ -336,12 +345,6 @@ regexp_module_config (struct rspamd_config *cfg)
 gint
 regexp_module_reconfig (struct rspamd_config *cfg)
 {
-	struct module_ctx saved_ctx;
-
-	saved_ctx = regexp_module_ctx->ctx;
-	memset (regexp_module_ctx, 0, sizeof (*regexp_module_ctx));
-	regexp_module_ctx->ctx = saved_ctx;
-
 	return regexp_module_config (cfg);
 }
 
diff --git a/src/plugins/spf.c b/src/plugins/spf.c
index 683683a55..5ec5bfcfc 100644
--- a/src/plugins/spf.c
+++ b/src/plugins/spf.c
@@ -62,13 +62,6 @@ struct spf_ctx {
 	gboolean check_authed;
 };
 
-static inline struct spf_ctx *
-spf_get_context (struct rspamd_config *cfg)
-{
-	return (struct spf_ctx *)g_hash_table_lookup (cfg->c_modules, "spf");
-}
-
-
 static void spf_symbol_callback (struct rspamd_task *task, void *unused);
 
 /* Initialization */
@@ -77,14 +70,23 @@ gint spf_module_config (struct rspamd_config *cfg);
 gint spf_module_reconfig (struct rspamd_config *cfg);
 
 module_t spf_module = {
-	"spf",
-	spf_module_init,
-	spf_module_config,
-	spf_module_reconfig,
-	NULL,
-	RSPAMD_MODULE_VER
+		"spf",
+		spf_module_init,
+		spf_module_config,
+		spf_module_reconfig,
+		NULL,
+		RSPAMD_MODULE_VER,
+		(guint)-1,
 };
 
+static inline struct spf_ctx *
+spf_get_context (struct rspamd_config *cfg)
+{
+	return (struct spf_ctx *)g_ptr_array_index (cfg->c_modules,
+			spf_module.ctx_offset);
+}
+
+
 gint
 spf_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
 {
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 07c6d06c1..d3ed6da74 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -159,26 +159,28 @@ surbl_error_quark (void)
 	return g_quark_from_static_string ("surbl-error-quark");
 }
 
-static inline struct surbl_ctx *
-surbl_get_context (struct rspamd_config *cfg)
-{
-	return (struct surbl_ctx *)g_hash_table_lookup (cfg->c_modules, "surbl");
-}
-
 /* Initialization */
 gint surbl_module_init (struct rspamd_config *cfg, struct module_ctx **ctx);
 gint surbl_module_config (struct rspamd_config *cfg);
 gint surbl_module_reconfig (struct rspamd_config *cfg);
 
 module_t surbl_module = {
-	"surbl",
-	surbl_module_init,
-	surbl_module_config,
-	surbl_module_reconfig,
-	NULL,
-	RSPAMD_MODULE_VER
+		"surbl",
+		surbl_module_init,
+		surbl_module_config,
+		surbl_module_reconfig,
+		NULL,
+		RSPAMD_MODULE_VER,
+		(guint)-1,
 };
 
+static inline struct surbl_ctx *
+surbl_get_context (struct rspamd_config *cfg)
+{
+	return (struct surbl_ctx *)g_ptr_array_index (cfg->c_modules,
+			surbl_module.ctx_offset);
+}
+
 static void
 exceptions_free_value (gpointer v)
 {
diff --git a/src/rspamadm/confighelp.c b/src/rspamadm/confighelp.c
index f4c5176ac..85564cf4c 100644
--- a/src/rspamadm/confighelp.c
+++ b/src/rspamadm/confighelp.c
@@ -202,7 +202,7 @@ rspamadm_confighelp (gint argc, gchar **argv, const struct rspamadm_command *cmd
 	module_t *mod, **pmod;
 	worker_t **pworker;
 	struct module_ctx *mod_ctx;
-	gint i = 1, ret = 0, processed_args = 0;
+	gint i, ret = 0, processed_args = 0;
 
 	context = g_option_context_new (
 			"confighelp - displays help for the configuration options");
@@ -238,16 +238,19 @@ rspamadm_confighelp (gint argc, gchar **argv, const struct rspamadm_command *cmd
 	rspamd_rcl_add_lua_plugins_path (cfg, plugins_path, NULL);
 
 	/* Init modules to get documentation strings */
+	i = 0;
 	for (pmod = cfg->compiled_modules; pmod != NULL && *pmod != NULL; pmod++) {
 		mod = *pmod;
 		mod_ctx = g_malloc0 (sizeof (struct module_ctx));
 
 		if (mod->module_init_func (cfg, &mod_ctx) == 0) {
-			g_hash_table_insert (cfg->c_modules,
-					(gpointer) mod->name,
-					mod_ctx);
+			g_ptr_array_add (cfg->c_modules, mod_ctx);
+			mod_ctx->mod = mod;
+			mod->ctx_offset = i++;
 			mod_ctx->mod = mod;
 		}
+
+
 	}
 	/* Also init all workers */
 	for (pworker = cfg->compiled_workers; *pworker != NULL; pworker ++) {
diff --git a/src/rspamd.c b/src/rspamd.c
index 4b001948d..9339e0a3f 100644
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -279,8 +279,6 @@ reread_config (struct rspamd_main *rspamd_main)
 
 	rspamd_symbols_cache_save (rspamd_main->cfg->cache);
 	tmp_cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_DEFAULT);
-	g_hash_table_unref (tmp_cfg->c_modules);
-	tmp_cfg->c_modules = g_hash_table_ref (rspamd_main->cfg->c_modules);
 	tmp_cfg->libs_ctx = rspamd_main->cfg->libs_ctx;
 	REF_RETAIN (tmp_cfg->libs_ctx);
 	cfg_file = rspamd_mempool_strdup (tmp_cfg->cfg_pool,
diff --git a/src/rspamd.h b/src/rspamd.h
index a993238a9..409c051b3 100644
--- a/src/rspamd.h
+++ b/src/rspamd.h
@@ -191,6 +191,7 @@ typedef struct module_s {
 	guint module_version;
 	guint64 rspamd_version;
 	const gchar *rspamd_features;
+	guint ctx_offset;
 } module_t;
 
 enum rspamd_worker_socket_type {