From 413e08bb4f8dd7e85e5bc86b15751d1789553fe3 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 24 Jun 2016 16:47:22 +0100 Subject: [PATCH] [Feature] Allow to skip some initialization phases to speed up rspamadm --- src/libserver/cfg_file.h | 11 +++++- src/libserver/cfg_utils.c | 83 +++++++++++++++++++++------------------ src/lua/lua_util.c | 4 +- src/rspamadm/configdump.c | 2 +- src/rspamadm/configtest.c | 2 +- src/rspamadm/rspamadm.c | 2 +- src/rspamd.c | 14 ++++--- 7 files changed, 68 insertions(+), 50 deletions(-) diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index 93470c0f3..eaaafc318 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -453,12 +453,21 @@ const ucl_object_t * rspamd_config_get_module_opt (struct rspamd_config *cfg, */ gchar rspamd_config_parse_flag (const gchar *str, guint len); +enum rspamd_post_load_options { + RSPAMD_CONFIG_INIT_URL = 1 << 0, + RSPAMD_CONFIG_INIT_LIBS = 1 << 1, + RSPAMD_CONFIG_INIT_SYMCACHE = 1 << 2, + RSPAMD_CONFIG_INIT_VALIDATE = 1 << 3 +}; + +#define RSPAMD_CONFIG_LOAD_ALL (RSPAMD_CONFIG_INIT_URL|RSPAMD_CONFIG_INIT_LIBS|RSPAMD_CONFIG_INIT_SYMCACHE|RSPAMD_CONFIG_INIT_VALIDATE) + /** * Do post load actions for config * @param cfg config file */ gboolean rspamd_config_post_load (struct rspamd_config *cfg, - gboolean validate_cache); + enum rspamd_post_load_options opts); /** * Calculate checksum for config file diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index 8f8fb551c..bbbb062bb 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -611,7 +611,8 @@ rspamd_config_parse_log_format (struct rspamd_config *cfg) * Perform post load actions */ gboolean -rspamd_config_post_load (struct rspamd_config *cfg, gboolean validate_cache) +rspamd_config_post_load (struct rspamd_config *cfg, + enum rspamd_post_load_options opts) { #ifdef HAVE_CLOCK_GETTIME struct timespec ts; @@ -651,46 +652,48 @@ rspamd_config_post_load (struct rspamd_config *cfg, gboolean validate_cache) def_metric->actions[METRIC_ACTION_REJECT].score = DEFAULT_SCORE; } - if (cfg->tld_file == NULL) { - /* Try to guess tld file */ - GString *fpath = g_string_new (NULL); + if (opts & RSPAMD_CONFIG_INIT_URL) { + if (cfg->tld_file == NULL) { + /* Try to guess tld file */ + GString *fpath = g_string_new (NULL); - rspamd_printf_gstring (fpath, "%s%c%s", RSPAMD_PLUGINSDIR, - G_DIR_SEPARATOR, "effective_tld_names.dat"); + rspamd_printf_gstring (fpath, "%s%c%s", RSPAMD_PLUGINSDIR, + G_DIR_SEPARATOR, "effective_tld_names.dat"); - if (access (fpath->str, R_OK) != -1) { - msg_debug_config ("url_tld option is not specified but %s is available," - " therefore this file is assumed as TLD file for URL" - " extraction", fpath->str); - cfg->tld_file = rspamd_mempool_strdup (cfg->cfg_pool, fpath->str); + if (access (fpath->str, R_OK) != -1) { + msg_debug_config ("url_tld option is not specified but %s is available," + " therefore this file is assumed as TLD file for URL" + " extraction", fpath->str); + cfg->tld_file = rspamd_mempool_strdup (cfg->cfg_pool, fpath->str); + } + else { + if (opts & RSPAMD_CONFIG_INIT_VALIDATE) { + msg_err_config ("no url_tld option has been specified"); + ret = FALSE; + } + } + + g_string_free (fpath, TRUE); } else { - if (validate_cache) { - msg_err_config ("no url_tld option has been specified"); - ret = FALSE; + if (access (cfg->tld_file, R_OK) == -1) { + if (opts & RSPAMD_CONFIG_INIT_VALIDATE) { + ret = FALSE; + msg_err_config ("cannot access tld file %s: %s", cfg->tld_file, + strerror (errno)); + } + else { + msg_debug_config ("cannot access tld file %s: %s", cfg->tld_file, + strerror (errno)); + cfg->tld_file = NULL; + } } } - g_string_free (fpath, TRUE); - } - else { - if (access (cfg->tld_file, R_OK) == -1) { - if (validate_cache) { - ret = FALSE; - msg_err_config ("cannot access tld file %s: %s", cfg->tld_file, - strerror (errno)); - } - else { - msg_debug_config ("cannot access tld file %s: %s", cfg->tld_file, - strerror (errno)); - cfg->tld_file = NULL; - } - } + rspamd_url_init (cfg->tld_file); } init_dynamic_config (cfg); - rspamd_url_init (cfg->tld_file); - /* Insert classifiers symbols */ rspamd_config_insert_classify_symbols (cfg); @@ -699,14 +702,18 @@ rspamd_config_post_load (struct rspamd_config *cfg, gboolean validate_cache) msg_err_config ("cannot parse log format, task logging will not be available"); } - /* Init config cache */ - rspamd_symbols_cache_init (cfg->cache); + if (opts & RSPAMD_CONFIG_INIT_SYMCACHE) { + /* Init config cache */ + rspamd_symbols_cache_init (cfg->cache); - /* Init re cache */ - rspamd_re_cache_init (cfg->re_cache, cfg); + /* Init re cache */ + rspamd_re_cache_init (cfg->re_cache, cfg); + } - /* Config other libraries */ - rspamd_config_libs (cfg->libs_ctx, cfg); + if (opts & RSPAMD_CONFIG_INIT_LIBS) { + /* Config other libraries */ + rspamd_config_libs (cfg->libs_ctx, cfg); + } /* Execute post load scripts */ LL_FOREACH (cfg->on_load, sc) { @@ -725,7 +732,7 @@ rspamd_config_post_load (struct rspamd_config *cfg, gboolean validate_cache) } /* Validate cache */ - if (validate_cache) { + if (opts & RSPAMD_CONFIG_INIT_VALIDATE) { return rspamd_symbols_cache_validate (cfg->cache, cfg, FALSE) && ret; } diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c index 272e39463..e50e704ee 100644 --- a/src/lua/lua_util.c +++ b/src/lua/lua_util.c @@ -399,7 +399,7 @@ lua_util_load_rspamd_config (lua_State *L) lua_pushnil (L); } else { - rspamd_config_post_load (cfg, FALSE); + rspamd_config_post_load (cfg, 0); pcfg = lua_newuserdata (L, sizeof (struct rspamd_config *)); rspamd_lua_setclass (L, "rspamd{config}", -1); *pcfg = cfg; @@ -433,7 +433,7 @@ lua_util_config_from_ucl (lua_State *L) lua_pushnil (L); } else { - rspamd_config_post_load (cfg, FALSE); + rspamd_config_post_load (cfg, 0); pcfg = lua_newuserdata (L, sizeof (struct rspamd_config *)); rspamd_lua_setclass (L, "rspamd{config}", -1); *pcfg = cfg; diff --git a/src/rspamadm/configdump.c b/src/rspamadm/configdump.c index e7d51b442..1cc85676c 100644 --- a/src/rspamadm/configdump.c +++ b/src/rspamadm/configdump.c @@ -290,7 +290,7 @@ rspamadm_configdump (gint argc, gchar **argv) } if (ret) { - ret = rspamd_config_post_load (cfg, FALSE); + ret = rspamd_config_post_load (cfg, RSPAMD_CONFIG_INIT_SYMCACHE); } } diff --git a/src/rspamadm/configtest.c b/src/rspamadm/configtest.c index 7ba9af3fe..967bca2ba 100644 --- a/src/rspamadm/configtest.c +++ b/src/rspamadm/configtest.c @@ -150,7 +150,7 @@ rspamadm_configtest (gint argc, gchar **argv) } if (ret) { - ret = rspamd_config_post_load (cfg, FALSE); + ret = rspamd_config_post_load (cfg, RSPAMD_CONFIG_INIT_SYMCACHE); } if (!rspamd_symbols_cache_validate (rspamd_main->cfg->cache, diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index 523011a64..38d5e77eb 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -285,7 +285,7 @@ main (gint argc, gchar **argv, gchar **env) (void) rspamd_log_open (rspamd_main->logger); g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger); g_set_printerr_handler (rspamd_glib_printerr_function); - rspamd_config_post_load (cfg, FALSE); + rspamd_config_post_load (cfg, RSPAMD_CONFIG_INIT_LIBS); /* Setup logger */ if (verbose) { diff --git a/src/rspamd.c b/src/rspamd.c index eaf6c6d30..cb8eb1806 100644 --- a/src/rspamd.c +++ b/src/rspamd.c @@ -68,7 +68,7 @@ static gboolean load_rspamd_config (struct rspamd_main *rspamd_main, struct rspamd_config *cfg, gboolean init_modules, - gboolean validate); + enum rspamd_post_load_options opts); /* Control socket */ static gint control_fd; @@ -280,7 +280,7 @@ reread_config (struct rspamd_main *rspamd_main) /* Save some variables */ tmp_cfg->cfg_name = cfg_file; - if (!load_rspamd_config (rspamd_main, tmp_cfg, FALSE, TRUE)) { + if (!load_rspamd_config (rspamd_main, tmp_cfg, FALSE, RSPAMD_CONFIG_INIT_VALIDATE)) { rspamd_set_logger (rspamd_main->cfg, g_quark_try_string ( "main"), rspamd_main); msg_err_main ("cannot parse new config file, revert to old one"); @@ -749,7 +749,8 @@ reopen_log_handler (gpointer key, gpointer value, gpointer unused) static gboolean load_rspamd_config (struct rspamd_main *rspamd_main, - struct rspamd_config *cfg, gboolean init_modules, gboolean validate) + struct rspamd_config *cfg, gboolean init_modules, + enum rspamd_post_load_options opts) { cfg->compiled_modules = modules; cfg->compiled_workers = workers; @@ -782,7 +783,7 @@ load_rspamd_config (struct rspamd_main *rspamd_main, } /* Do post-load actions */ - rspamd_config_post_load (cfg, validate); + rspamd_config_post_load (cfg, opts); return TRUE; } @@ -1120,7 +1121,7 @@ main (gint argc, gchar **argv, gchar **env) } if (config_test || dump_cache) { - if (!load_rspamd_config (rspamd_main, rspamd_main->cfg, FALSE, FALSE)) { + if (!load_rspamd_config (rspamd_main, rspamd_main->cfg, FALSE, 0)) { exit (EXIT_FAILURE); } @@ -1146,7 +1147,8 @@ main (gint argc, gchar **argv, gchar **env) } /* Load config */ - if (!load_rspamd_config (rspamd_main, rspamd_main->cfg, TRUE, TRUE)) { + if (!load_rspamd_config (rspamd_main, rspamd_main->cfg, TRUE, + RSPAMD_CONFIG_LOAD_ALL)) { exit (EXIT_FAILURE); } -- 2.39.5