From 812dfbbd064daf5f60260b5718bbe3ba4173eabf Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 27 Mar 2019 14:18:22 +0000 Subject: [PATCH] [Feature] Preprocess config files using jinja templates --- src/libserver/cfg_rcl.c | 93 +++++++++++++++++++-- src/libserver/cfg_rcl.h | 5 +- src/lua/lua_common.c | 171 +++++++++++++++++++++++++------------- src/lua/lua_common.h | 3 +- src/lua/lua_config.c | 5 +- src/lua/lua_util.c | 2 +- src/rspamadm/configdump.c | 6 +- src/rspamadm/configtest.c | 6 +- src/rspamadm/rspamadm.c | 12 ++- src/rspamadm/rspamadm.h | 1 + src/rspamd.c | 9 +- test/rspamd_lua_test.c | 2 +- 12 files changed, 241 insertions(+), 74 deletions(-) diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index c6d2f3572..5c99aed59 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -3506,7 +3506,7 @@ rspamd_rcl_maybe_apply_lua_transform (struct rspamd_config *cfg) } static bool -rspamd_rcl_decrypt_handler(struct ucl_parser *parser, +rspamd_rcl_decrypt_handler (struct ucl_parser *parser, const unsigned char *source, size_t source_len, unsigned char **destination, size_t *dest_len, void *user_data) @@ -3525,6 +3525,64 @@ rspamd_rcl_decrypt_handler(struct ucl_parser *parser, return true; } +static bool +rspamd_rcl_jinja_handler (struct ucl_parser *parser, + const unsigned char *source, size_t source_len, + unsigned char **destination, size_t *dest_len, + void *user_data) +{ + struct rspamd_config *cfg = (struct rspamd_config *)user_data; + lua_State *L = cfg->lua_state; + gint err_idx; + + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + + /* Obtain function */ + if (!rspamd_lua_require_function (L, "lua_util", "jinja_template")) { + msg_err_config ("cannot require lua_util.jinja_template"); + lua_settop (L, err_idx - 1); + + return false; + } + + lua_pushlstring (L, source, source_len); + lua_getglobal (L, "rspamd_env"); + lua_pushboolean (L, false); + + if (lua_pcall (L, 3, 1, err_idx) != 0) { + GString *tb; + + tb = lua_touserdata (L, -1); + msg_err_config ("cannot call lua try_load_redis_servers script: %s", tb->str); + g_string_free (tb, TRUE); + lua_settop (L, err_idx - 1); + + return false; + } + + if (lua_type (L, -1) == LUA_TSTRING) { + const char *ndata; + gsize nsize; + + ndata = lua_tolstring (L, -1, &nsize); + *destination = UCL_ALLOC (nsize); + memcpy (*destination, ndata, nsize); + *dest_len = nsize; + } + else { + msg_err_config ("invalid return type when templating jinja %s", + lua_typename (L, lua_type (L, -1))); + lua_settop (L, err_idx - 1); + + return false; + } + + lua_settop (L, err_idx - 1); + + return true; +} + static void rspamd_rcl_decrypt_free (unsigned char *data, size_t len, void *user_data) { @@ -3561,6 +3619,7 @@ rspamd_config_parse_ucl (struct rspamd_config *cfg, GHashTable *vars, ucl_include_trace_func_t inc_trace, void *trace_data, + gboolean skip_jinja, GError **err) { struct stat st; @@ -3652,6 +3711,18 @@ rspamd_config_parse_ucl (struct rspamd_config *cfg, ucl_parser_add_special_handler (parser, decrypt_handler); } + if (!skip_jinja) { + struct ucl_parser_special_handler *jinja_handler; + + jinja_handler = rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (*jinja_handler)); + jinja_handler->user_data = cfg; + jinja_handler->flags = UCL_SPECIAL_HANDLER_PREPROCESS_ALL; + jinja_handler->handler = rspamd_rcl_jinja_handler; + + ucl_parser_add_special_handler (parser, jinja_handler); + } + if (!ucl_parser_add_chunk (parser, data, st.st_size)) { g_set_error (err, cfg_rcl_error_quark (), errno, "ucl parser error: %s", ucl_parser_get_error (parser)); @@ -3670,17 +3741,28 @@ rspamd_config_parse_ucl (struct rspamd_config *cfg, } gboolean -rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, +rspamd_config_read (struct rspamd_config *cfg, + const gchar *filename, rspamd_rcl_section_fin_t logger_fin, - gpointer logger_ud, GHashTable *vars) + gpointer logger_ud, + GHashTable *vars, + gboolean skip_jinja, + gchar **lua_env) { GError *err = NULL; struct rspamd_rcl_section *top, *logger_section; const ucl_object_t *logger_obj; - rspamd_lua_set_env (cfg->lua_state, vars); + rspamd_lua_set_path (cfg->lua_state, NULL, vars); + + if (!rspamd_lua_set_env (cfg->lua_state, vars, lua_env, &err)) { + msg_err_config_forced ("failed to set up environment: %e", err); + g_error_free (err); + + return FALSE; + } - if (!rspamd_config_parse_ucl (cfg, filename, vars, NULL, NULL, &err)) { + if (!rspamd_config_parse_ucl (cfg, filename, vars, NULL, NULL, skip_jinja, &err)) { msg_err_config_forced ("failed to load config: %e", err); g_error_free (err); @@ -3688,6 +3770,7 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, } top = rspamd_rcl_config_init (cfg, NULL); + /* Add new paths if defined in options */ rspamd_lua_set_path (cfg->lua_state, cfg->rcl_obj, vars); rspamd_lua_set_globals (cfg, cfg->lua_state); rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_rcl_section_free, top); diff --git a/src/libserver/cfg_rcl.h b/src/libserver/cfg_rcl.h index e2477481e..12a89a673 100644 --- a/src/libserver/cfg_rcl.h +++ b/src/libserver/cfg_rcl.h @@ -481,10 +481,13 @@ gboolean rspamd_config_parse_ucl (struct rspamd_config *cfg, GHashTable *vars, ucl_include_trace_func_t inc_trace, void *trace_data, + gboolean skip_jinja, GError **err); gboolean rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, rspamd_rcl_section_fin_t logger_fin, gpointer logger_ud, - GHashTable *vars); + GHashTable *vars, + gboolean skip_jinja, + gchar **lua_env); #endif /* CFG_RCL_H_ */ diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c index 1c687fb48..5f1be424d 100644 --- a/src/lua/lua_common.c +++ b/src/lua/lua_common.c @@ -263,83 +263,76 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars } } - /* Try environment */ - t = getenv ("SHAREDIR"); - if (t) { - sharedir = t; - } - - t = getenv ("PLUGINSDIR"); - if (t) { - pluginsdir = t; - } - - t = getenv ("RULESDIR"); - if (t) { - rulesdir = t; - } - - t = getenv ("LUALIBDIR"); - if (t) { - lualibdir = t; - } - - t = getenv ("LIBDIR"); - if (t) { - libdir = t; - } - - t = getenv ("RSPAMD_LIBDIR"); - if (t) { - libdir = t; + if (additional_path) { + rspamd_snprintf (path_buf, sizeof (path_buf), + "%s;" + "%s", + additional_path, old_path); } - - if (vars) { - t = g_hash_table_lookup (vars, "PLUGINSDIR"); + else { + /* Try environment */ + t = getenv ("SHAREDIR"); if (t) { - pluginsdir = t; + sharedir = t; } - t = g_hash_table_lookup (vars, "SHAREDIR"); + t = getenv ("PLUGINSDIR"); if (t) { - sharedir = t; + pluginsdir = t; } - t = g_hash_table_lookup (vars, "RULESDIR"); + t = getenv ("RULESDIR"); if (t) { rulesdir = t; } - t = g_hash_table_lookup (vars, "LUALIBDIR"); + t = getenv ("LUALIBDIR"); if (t) { lualibdir = t; } - t = g_hash_table_lookup (vars, "LIBDIR"); + t = getenv ("LIBDIR"); if (t) { libdir = t; } - t = g_hash_table_lookup (vars, "RSPAMD_LIBDIR"); + t = getenv ("RSPAMD_LIBDIR"); if (t) { libdir = t; } - } - if (additional_path) { - rspamd_snprintf (path_buf, sizeof (path_buf), - "%s/lua/?.lua;" - "%s/?.lua;" - "%s/?.lua;" - "%s/?/init.lua;" - "%s;" - "%s", - RSPAMD_CONFDIR, - rulesdir, - lualibdir, lualibdir, - additional_path, old_path); - } - else { + if (vars) { + t = g_hash_table_lookup (vars, "PLUGINSDIR"); + if (t) { + pluginsdir = t; + } + + t = g_hash_table_lookup (vars, "SHAREDIR"); + if (t) { + sharedir = t; + } + + t = g_hash_table_lookup (vars, "RULESDIR"); + if (t) { + rulesdir = t; + } + + t = g_hash_table_lookup (vars, "LUALIBDIR"); + if (t) { + lualibdir = t; + } + + t = g_hash_table_lookup (vars, "LIBDIR"); + if (t) { + libdir = t; + } + + t = g_hash_table_lookup (vars, "RSPAMD_LIBDIR"); + if (t) { + libdir = t; + } + } + rspamd_snprintf (path_buf, sizeof (path_buf), "%s/lua/?.lua;" "%s/?.lua;" @@ -372,11 +365,9 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars if (additional_path) { rspamd_snprintf (path_buf, sizeof (path_buf), "%s/?%s;" - "%s;" "%s", - libdir, - OS_SO_SUFFIX, additional_path, + OS_SO_SUFFIX, old_path); } else { @@ -387,6 +378,7 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars OS_SO_SUFFIX, old_path); } + lua_pop (L, 1); lua_pushstring (L, path_buf); lua_setfield (L, -2, "cpath"); @@ -537,8 +529,56 @@ rspamd_lua_rspamd_version (lua_State *L) return 1; } -void -rspamd_lua_set_env (lua_State *L, GHashTable *vars) +static gboolean +rspamd_lua_load_env (lua_State *L, const char *fname, gint tbl_pos, GError **err) +{ + gint orig_top = lua_gettop (L), err_idx; + gboolean ret = TRUE; + + lua_pushcfunction (L, &rspamd_lua_traceback); + err_idx = lua_gettop (L); + + if (luaL_loadfile (L, fname) != 0) { + g_set_error (err, g_quark_from_static_string ("lua_env"), errno, + "cannot load lua file %s: %s", + fname, + lua_tostring (L, -1)); + ret = FALSE; + } + + if (ret && lua_pcall (L, 0, 1, err_idx) != 0) { + GString *tb = lua_touserdata (L, -1); + g_set_error (err, g_quark_from_static_string ("lua_env"), errno, + "cannot init lua file %s: %s", + fname, + tb->str); + g_string_free (tb, TRUE); + + ret = FALSE; + } + + if (ret && lua_type (L, -1) == LUA_TTABLE) { + for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) { + lua_pushvalue (L, -2); /* Store key */ + lua_pushvalue (L, -2); /* Store value */ + lua_settable (L, tbl_pos); + } + } + else if (ret) { + g_set_error (err, g_quark_from_static_string ("lua_env"), errno, + "invalid return type when loading env from %s: %s", + fname, + lua_typename (L, lua_type (L, -1))); + ret = FALSE; + } + + lua_settop (L, orig_top); + + return ret; +} + +gboolean +rspamd_lua_set_env (lua_State *L, GHashTable *vars, char **lua_env, GError **err) { gint orig_top = lua_gettop (L); gchar **env = g_get_environ (); @@ -742,10 +782,23 @@ rspamd_lua_set_env (lua_State *L, GHashTable *vars) } } + if (lua_env) { + gint lim = g_strv_length (lua_env); + + for (gint i = 0; i < lim; i ++) { + if (!rspamd_lua_load_env (L, lua_env[i], lua_gettop (L), err)) { + return FALSE; + } + } + } + lua_setglobal (L, "rspamd_env"); } lua_settop (L, orig_top); + g_strfreev (env); + + return TRUE; } void diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h index 520f5920d..01b7d3581 100644 --- a/src/lua/lua_common.h +++ b/src/lua/lua_common.h @@ -306,7 +306,8 @@ void rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars); /* Set some lua globals */ -void rspamd_lua_set_env (lua_State *L, GHashTable *vars); +gboolean rspamd_lua_set_env (lua_State *L, GHashTable *vars, char **lua_env, + GError **err); void rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L); struct memory_pool_s * rspamd_lua_check_mempool (lua_State * L, gint pos); diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index 3990e014e..f60fe6995 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -3819,7 +3819,7 @@ lua_config_load_ucl (lua_State *L) cbd.L = L; if (!rspamd_config_parse_ucl (cfg, filename, paths, - lua_include_trace_cb, &cbd, &err)) { + lua_include_trace_cb, &cbd, lua_toboolean (L, 4), &err)) { luaL_unref (L, LUA_REGISTRYINDEX, cbd.cbref); lua_pushboolean (L, false); lua_pushfstring (L, "failed to load config: %s", err->message); @@ -3832,7 +3832,8 @@ lua_config_load_ucl (lua_State *L) luaL_unref (L, LUA_REGISTRYINDEX, cbd.cbref); } else { - if (!rspamd_config_parse_ucl (cfg, filename, paths, NULL, NULL, &err)) { + if (!rspamd_config_parse_ucl (cfg, filename, paths, NULL, NULL, + lua_toboolean (L, 3), &err)) { lua_pushboolean (L, false); lua_pushfstring (L, "failed to load config: %s", err->message); g_error_free (err); diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c index af4673af8..84958ebc0 100644 --- a/src/lua/lua_util.c +++ b/src/lua/lua_util.c @@ -717,7 +717,7 @@ lua_util_load_rspamd_config (lua_State *L) cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA); cfg->lua_state = L; - if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL)) { + if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL, FALSE, NULL)) { msg_err_config ("cannot load config from %s", cfg_name); lua_pushnil (L); } diff --git a/src/rspamadm/configdump.c b/src/rspamadm/configdump.c index a255994a5..8f38cdf61 100644 --- a/src/rspamadm/configdump.c +++ b/src/rspamadm/configdump.c @@ -27,6 +27,7 @@ static gboolean compact = FALSE; static gboolean show_help = FALSE; static gboolean show_comments = FALSE; static gboolean modules_state = FALSE; +static gboolean skip_template = FALSE; static gchar *config = NULL; extern struct rspamd_main *rspamd_main; /* Defined in modules.c */ @@ -57,6 +58,8 @@ static GOptionEntry entries[] = { "Show saved comments from the configuration file", NULL }, {"modules-state", 'm', 0, G_OPTION_ARG_NONE, &modules_state, "Show modules state only", NULL}, + {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template, + "Do not apply Jinja templates", NULL}, {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL} }; @@ -284,7 +287,8 @@ rspamadm_configdump (gint argc, gchar **argv, const struct rspamadm_command *cmd cfg->compiled_workers = workers; cfg->cfg_name = config; - if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, + ucl_vars, skip_template, lua_env)) { ret = FALSE; } else { diff --git a/src/rspamadm/configtest.c b/src/rspamadm/configtest.c index db9a8d604..3b7a6b5b0 100644 --- a/src/rspamadm/configtest.c +++ b/src/rspamadm/configtest.c @@ -23,6 +23,7 @@ static gboolean quiet = FALSE; static gchar *config = NULL; static gboolean strict = FALSE; +static gboolean skip_template = FALSE; extern struct rspamd_main *rspamd_main; /* Defined in modules.c */ extern module_t *modules[]; @@ -48,6 +49,8 @@ static GOptionEntry entries[] = { "Config file to test", NULL}, {"strict", 's', 0, G_OPTION_ARG_NONE, &strict, "Stop on any error in config", NULL}, + {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template, + "Do not apply Jinja templates", NULL}, {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL} }; @@ -141,7 +144,8 @@ rspamadm_configtest (gint argc, gchar **argv, const struct rspamadm_command *cmd cfg->compiled_workers = workers; cfg->cfg_name = config; - if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, + ucl_vars, skip_template, lua_env)) { ret = FALSE; } else { diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index 281690132..762a74c1f 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -31,6 +31,7 @@ static gboolean list_commands = FALSE; static gboolean show_help = FALSE; static gboolean show_version = FALSE; GHashTable *ucl_vars = NULL; +gchar **lua_env = NULL; struct rspamd_main *rspamd_main = NULL; struct rspamd_async_session *rspamadm_session = NULL; lua_State *L = NULL; @@ -65,6 +66,8 @@ static GOptionEntry entries[] = { "Show help", NULL}, {"version", 'V', 0, G_OPTION_ARG_NONE, &show_version, "Show version", NULL}, + {"lua-env", '\0', 0, G_OPTION_ARG_FILENAME_ARRAY, &lua_env, + "Load lua environment from the specified files", NULL}, {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL} }; @@ -458,8 +461,15 @@ main (gint argc, gchar **argv, gchar **env) setproctitle ("rspamdadm"); L = cfg->lua_state; - rspamd_lua_set_env (L, ucl_vars); rspamd_lua_set_path (L, NULL, ucl_vars); + + if (!rspamd_lua_set_env (L, ucl_vars, lua_env, &error)) { + rspamd_fprintf (stderr, "Cannot load lua environment: %e", error); + g_error_free (error); + + exit (EXIT_FAILURE); + } + rspamd_lua_set_globals (cfg, L); rspamadm_add_lua_globals (); diff --git a/src/rspamadm/rspamadm.h b/src/rspamadm/rspamadm.h index cd01cc86b..e8ed9c873 100644 --- a/src/rspamadm/rspamadm.h +++ b/src/rspamadm/rspamadm.h @@ -23,6 +23,7 @@ #include extern GHashTable *ucl_vars; +extern gchar **lua_env; extern struct rspamd_main *rspamd_main; GQuark rspamadm_error (void); diff --git a/src/rspamd.c b/src/rspamd.c index 631759f05..8b12fa48e 100644 --- a/src/rspamd.c +++ b/src/rspamd.c @@ -95,6 +95,8 @@ static gboolean is_insecure = FALSE; static gboolean gen_keypair = FALSE; static gboolean encrypt_password = FALSE; static GHashTable *ucl_vars = NULL; +static gchar **lua_env = NULL; +static gboolean skip_template = FALSE; static gint term_attempts = 0; @@ -146,6 +148,10 @@ static GOptionEntry entries[] = "Show version and exit", NULL }, {"var", 0, 0, G_OPTION_ARG_CALLBACK, (gpointer)&rspamd_parse_var, "Redefine/define environment variable", NULL}, + {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template, + "Do not apply Jinja templates", NULL}, + {"lua-env", '\0', 0, G_OPTION_ARG_FILENAME_ARRAY, &lua_env, + "Load lua environment from the specified files", NULL}, { NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL } }; @@ -923,7 +929,8 @@ load_rspamd_config (struct rspamd_main *rspamd_main, cfg->compiled_modules = modules; cfg->compiled_workers = workers; - if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) { + if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, + ucl_vars, skip_template, lua_env)) { return FALSE; } diff --git a/test/rspamd_lua_test.c b/test/rspamd_lua_test.c index 2af0d9aab..ad40ef488 100644 --- a/test/rspamd_lua_test.c +++ b/test/rspamd_lua_test.c @@ -64,7 +64,7 @@ rspamd_lua_test_func (void) glob_t globbuf; gint i, len; - rspamd_lua_set_env (L, NULL); + rspamd_lua_set_env (L, NULL, NULL, NULL); rspamd_lua_set_globals (rspamd_main->cfg, L); if (lua_test_case) { -- 2.39.5