From: Vsevolod Stakhov Date: Tue, 29 May 2018 15:58:54 +0000 (+0100) Subject: [Minor] Rework config load routines X-Git-Tag: 1.7.6~86 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=e8366d156e09eba81ae38f57c41c74ec203c27bc;p=rspamd.git [Minor] Rework config load routines --- diff --git a/lualib/rspamadm/corpus_test.lua b/lualib/rspamadm/corpus_test.lua index 59b02f649..60aa2d7a9 100644 --- a/lualib/rspamadm/corpus_test.lua +++ b/lualib/rspamadm/corpus_test.lua @@ -74,16 +74,16 @@ local function encoded_json_to_log(result) -- Returns table containing score, action, list of symbols local filtered_result = {} - local parser = ucl.parser() + local ucl_parser = ucl.parser() - local is_good, err = parser:parse_string(result) + local is_good, err = ucl_parser:parse_string(result) if not is_good then rspamd_logger.errx("Parser error: %1", err) return nil end - result = parser:get_object() + result = ucl_parser:get_object() filtered_result.score = result.score if not result.action then diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index abb0e9eef..ad6bbe8a3 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -3229,7 +3229,7 @@ rspamd_rcl_section_free (gpointer p) * it is changed, then rcl_obj is imported from lua. Old config is dereferenced. * @param cfg */ -static void +void rspamd_rcl_maybe_apply_lua_transform (struct rspamd_config *cfg) { lua_State *L = cfg->lua_state; @@ -3318,37 +3318,56 @@ rspamd_rcl_decrypt_free (unsigned char *data, size_t len, void *user_data) g_free (data); } +void +rspamd_config_calculate_cksum (struct rspamd_config *cfg) +{ + rspamd_cryptobox_hash_state_t hs; + unsigned char cksumbuf[rspamd_cryptobox_HASHBYTES]; + struct ucl_emitter_functions f; + + /* Calculate checksum */ + rspamd_cryptobox_hash_init (&hs, NULL, 0); + f.ucl_emitter_append_character = rspamd_rcl_emitter_append_c; + f.ucl_emitter_append_double = rspamd_rcl_emitter_append_double; + f.ucl_emitter_append_int = rspamd_rcl_emitter_append_int; + f.ucl_emitter_append_len = rspamd_rcl_emitter_append_len; + f.ucl_emitter_free_func = NULL; + f.ud = &hs; + ucl_object_emit_full (cfg->rcl_obj, UCL_EMIT_MSGPACK, + &f, cfg->config_comments); + rspamd_cryptobox_hash_final (&hs, cksumbuf); + cfg->checksum = rspamd_encode_base32 (cksumbuf, sizeof (cksumbuf)); + /* Also change the tag of cfg pool to be equal to the checksum */ + rspamd_strlcpy (cfg->cfg_pool->tag.uid, cfg->checksum, + MIN (sizeof (cfg->cfg_pool->tag.uid), strlen (cfg->checksum))); +} + gboolean -rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, - const gchar *convert_to, rspamd_rcl_section_fin_t logger_fin, - gpointer logger_ud, GHashTable *vars) +rspamd_config_parse_ucl (struct rspamd_config *cfg, const gchar *filename, + GHashTable *vars, GError **err) { struct stat st; gint fd; - gchar *data; - GError *err = NULL; - struct rspamd_rcl_section *top, *logger_section; struct ucl_parser *parser; - const ucl_object_t *logger_obj; - rspamd_cryptobox_hash_state_t hs; - unsigned char cksumbuf[rspamd_cryptobox_HASHBYTES]; gchar keypair_path[PATH_MAX]; struct rspamd_cryptobox_keypair *decrypt_keypair = NULL; - struct ucl_emitter_functions f; + gchar *data; if (stat (filename, &st) == -1) { - msg_err_config_forced ("cannot stat %s: %s", filename, strerror (errno)); + g_set_error (err, cfg_rcl_error_quark (), errno, + "cannot stat %s: %s", filename, strerror (errno)); return FALSE; } if ((fd = open (filename, O_RDONLY)) == -1) { - msg_err_config_forced ("cannot open %s: %s", filename, strerror (errno)); + g_set_error (err, cfg_rcl_error_quark (), errno, + "cannot open %s: %s", filename, strerror (errno)); return FALSE; } /* Now mmap this file to simplify reading process */ - if ((data = - mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { - msg_err_config_forced ("cannot mmap %s: %s", filename, strerror (errno)); + if ((data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { + g_set_error (err, cfg_rcl_error_quark (), errno, + "cannot mmap %s: %s", filename, strerror (errno)); close (fd); return FALSE; } @@ -3393,7 +3412,6 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, ucl_parser_free (kp_parser); } - rspamd_cryptobox_hash_init (&hs, NULL, 0); parser = ucl_parser_new (UCL_PARSER_SAVE_COMMENTS); rspamd_ucl_add_conf_variables (parser, vars); rspamd_ucl_add_conf_macros (parser, cfg); @@ -3424,6 +3442,25 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, cfg->config_comments = ucl_object_ref (ucl_parser_get_comments (parser)); ucl_parser_free (parser); + return TRUE; +} + +gboolean +rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, + rspamd_rcl_section_fin_t logger_fin, + gpointer logger_ud, GHashTable *vars) +{ + GError *err = NULL; + struct rspamd_rcl_section *top, *logger_section; + const ucl_object_t *logger_obj; + + if (!rspamd_config_parse_ucl (cfg, filename, vars, &err)) { + msg_err_config_forced ("failed to load config: %e", err); + g_error_free (err); + + return FALSE; + } + top = rspamd_rcl_config_init (cfg); rspamd_lua_set_path (cfg->lua_state, cfg->rcl_obj, vars); rspamd_lua_set_globals (cfg, cfg->lua_state, vars); @@ -3459,28 +3496,13 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, /* Transform config if needed */ rspamd_rcl_maybe_apply_lua_transform (cfg); - /* Calculate checksum */ - f.ucl_emitter_append_character = rspamd_rcl_emitter_append_c; - f.ucl_emitter_append_double = rspamd_rcl_emitter_append_double; - f.ucl_emitter_append_int = rspamd_rcl_emitter_append_int; - f.ucl_emitter_append_len = rspamd_rcl_emitter_append_len; - f.ucl_emitter_free_func = NULL; - f.ud = &hs; - ucl_object_emit_full (cfg->rcl_obj, UCL_EMIT_MSGPACK, - &f, cfg->config_comments); - rspamd_cryptobox_hash_final (&hs, cksumbuf); - cfg->checksum = rspamd_encode_base32 (cksumbuf, sizeof (cksumbuf)); - /* Also change the tag of cfg pool to be equal to the checksum */ - rspamd_strlcpy (cfg->cfg_pool->tag.uid, cfg->checksum, - MIN (sizeof (cfg->cfg_pool->tag.uid), strlen (cfg->checksum))); - - if (!rspamd_rcl_parse (top, cfg, cfg, cfg->cfg_pool, cfg->rcl_obj, &err)) { msg_err_config ("rcl parse error: %e", err); g_error_free (err); return FALSE; } + rspamd_config_calculate_cksum (cfg); cfg->lang_det = rspamd_language_detector_init (cfg); return TRUE; diff --git a/src/libserver/cfg_rcl.h b/src/libserver/cfg_rcl.h index c1d958efc..8661687a3 100644 --- a/src/libserver/cfg_rcl.h +++ b/src/libserver/cfg_rcl.h @@ -446,11 +446,40 @@ gboolean rspamd_rcl_add_lua_plugins_path (struct rspamd_config *cfg, const gchar *path, GError **err); + +/** + * Calls for an external lua function to apply potential config transformations + * if needed. This function can change the cfg->rcl_obj. + * + * Example of transformation function: + * + * function(obj) + * if obj.something == 'foo' then + * obj.something = "bla" + * return true, obj + * end + * + * return false, nil + * end + * + * If function returns 'false' then rcl_obj is not touched. Otherwise, + * it is changed, then rcl_obj is imported from lua. Old config is dereferenced. + * @param cfg + */ +void rspamd_rcl_maybe_apply_lua_transform (struct rspamd_config *cfg); + +void rspamd_config_calculate_cksum (struct rspamd_config *cfg); + /* * Read configuration file */ +gboolean rspamd_config_parse_ucl (struct rspamd_config *cfg, + const gchar *filename, + GHashTable *vars, + GError **err); gboolean rspamd_config_read (struct rspamd_config *cfg, - const gchar *filename, const gchar *convert_to, - rspamd_rcl_section_fin_t logger_fin, gpointer logger_ud, - GHashTable *vars); + const gchar *filename, + rspamd_rcl_section_fin_t logger_fin, + gpointer logger_ud, + GHashTable *vars); #endif /* CFG_RCL_H_ */ diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c index 47b7106b6..578b5e3c9 100644 --- a/src/lua/lua_util.c +++ b/src/lua/lua_util.c @@ -629,7 +629,7 @@ lua_util_load_rspamd_config (lua_State *L) cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA); cfg->lua_state = L; - if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL, NULL)) { + if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL)) { msg_err_config ("cannot load config from %s", cfg_name); lua_pushnil (L); } diff --git a/src/rspamadm/configdump.c b/src/rspamadm/configdump.c index 486a74cef..bd776e586 100644 --- a/src/rspamadm/configdump.c +++ b/src/rspamadm/configdump.c @@ -283,8 +283,7 @@ rspamadm_configdump (gint argc, gchar **argv, const struct rspamadm_command *cmd cfg->compiled_workers = workers; cfg->cfg_name = config; - if (!rspamd_config_read (cfg, cfg->cfg_name, NULL, - config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { ret = FALSE; } else { diff --git a/src/rspamadm/configtest.c b/src/rspamadm/configtest.c index 94914e817..72a8f4945 100644 --- a/src/rspamadm/configtest.c +++ b/src/rspamadm/configtest.c @@ -141,8 +141,7 @@ rspamadm_configtest (gint argc, gchar **argv, const struct rspamadm_command *cmd cfg->compiled_workers = workers; cfg->cfg_name = config; - if (!rspamd_config_read (cfg, cfg->cfg_name, NULL, - config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { ret = FALSE; } else { diff --git a/src/rspamadm/configwizard.c b/src/rspamadm/configwizard.c index 5aaa94914..bfa35bb71 100644 --- a/src/rspamadm/configwizard.c +++ b/src/rspamadm/configwizard.c @@ -132,8 +132,7 @@ rspamadm_configwizard (gint argc, gchar **argv, cfg->compiled_workers = workers; cfg->cfg_name = config; - if (!rspamd_config_read (cfg, cfg->cfg_name, NULL, - config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { ret = FALSE; } else { diff --git a/src/rspamadm/rescore.c b/src/rspamadm/rescore.c index e30c6613a..346c49c68 100644 --- a/src/rspamadm/rescore.c +++ b/src/rspamadm/rescore.c @@ -150,8 +150,7 @@ rspamadm_rescore (gint argc, gchar **argv, const struct rspamadm_command *cmd) cfg->compiled_workers = workers; cfg->cfg_name = config; - if (!rspamd_config_read (cfg, cfg->cfg_name, NULL, - config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { ret = FALSE; } else { diff --git a/src/rspamd.c b/src/rspamd.c index d6490d6aa..643d249b1 100644 --- a/src/rspamd.c +++ b/src/rspamd.c @@ -874,8 +874,7 @@ load_rspamd_config (struct rspamd_main *rspamd_main, cfg->compiled_modules = modules; cfg->compiled_workers = workers; - if (!rspamd_config_read (cfg, cfg->cfg_name, NULL, - config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { return FALSE; }