]> source.dussan.org Git - rspamd.git/commitdiff
[Feature] Preprocess config files using jinja templates
authorVsevolod Stakhov <vsevolod@highsecure.ru>
Wed, 27 Mar 2019 14:18:22 +0000 (14:18 +0000)
committerVsevolod Stakhov <vsevolod@highsecure.ru>
Wed, 27 Mar 2019 14:18:22 +0000 (14:18 +0000)
12 files changed:
src/libserver/cfg_rcl.c
src/libserver/cfg_rcl.h
src/lua/lua_common.c
src/lua/lua_common.h
src/lua/lua_config.c
src/lua/lua_util.c
src/rspamadm/configdump.c
src/rspamadm/configtest.c
src/rspamadm/rspamadm.c
src/rspamadm/rspamadm.h
src/rspamd.c
test/rspamd_lua_test.c

index c6d2f3572bd37f501a32cf635819ea7fa25fdbda..5c99aed59fe3ce8ec2837d5fb6da44782f3d1881 100644 (file)
@@ -3506,7 +3506,7 @@ rspamd_rcl_maybe_apply_lua_transform (struct rspamd_config *cfg)
 }
 
 static bool
-rspamd_rcl_decrypt_handler(struct ucl_parser *parser,
+rspamd_rcl_decrypt_handler (struct ucl_parser *parser,
                                                   const unsigned char *source, size_t source_len,
                                                   unsigned char **destination, size_t *dest_len,
                                                   void *user_data)
@@ -3525,6 +3525,64 @@ rspamd_rcl_decrypt_handler(struct ucl_parser *parser,
        return true;
 }
 
+static bool
+rspamd_rcl_jinja_handler (struct ucl_parser *parser,
+                                                  const unsigned char *source, size_t source_len,
+                                                  unsigned char **destination, size_t *dest_len,
+                                                  void *user_data)
+{
+       struct rspamd_config *cfg = (struct rspamd_config *)user_data;
+       lua_State *L = cfg->lua_state;
+       gint err_idx;
+
+       lua_pushcfunction (L, &rspamd_lua_traceback);
+       err_idx = lua_gettop (L);
+
+       /* Obtain function */
+       if (!rspamd_lua_require_function (L, "lua_util", "jinja_template")) {
+               msg_err_config ("cannot require lua_util.jinja_template");
+               lua_settop (L, err_idx - 1);
+
+               return false;
+       }
+
+       lua_pushlstring (L, source, source_len);
+       lua_getglobal (L, "rspamd_env");
+       lua_pushboolean (L, false);
+
+       if (lua_pcall (L, 3, 1, err_idx) != 0) {
+               GString *tb;
+
+               tb = lua_touserdata (L, -1);
+               msg_err_config ("cannot call lua try_load_redis_servers script: %s", tb->str);
+               g_string_free (tb, TRUE);
+               lua_settop (L, err_idx - 1);
+
+               return false;
+       }
+
+       if (lua_type (L, -1) == LUA_TSTRING) {
+               const char *ndata;
+               gsize nsize;
+
+               ndata = lua_tolstring (L, -1, &nsize);
+               *destination = UCL_ALLOC (nsize);
+               memcpy (*destination, ndata, nsize);
+               *dest_len = nsize;
+       }
+       else {
+               msg_err_config ("invalid return type when templating jinja %s",
+                               lua_typename (L, lua_type (L, -1)));
+               lua_settop (L, err_idx - 1);
+
+               return false;
+       }
+
+       lua_settop (L, err_idx - 1);
+
+       return true;
+}
+
 static void
 rspamd_rcl_decrypt_free (unsigned char *data, size_t len, void *user_data)
 {
@@ -3561,6 +3619,7 @@ rspamd_config_parse_ucl (struct rspamd_config *cfg,
                                                 GHashTable *vars,
                                                 ucl_include_trace_func_t inc_trace,
                                                 void *trace_data,
+                                                gboolean skip_jinja,
                                                 GError **err)
 {
        struct stat st;
@@ -3652,6 +3711,18 @@ rspamd_config_parse_ucl (struct rspamd_config *cfg,
                ucl_parser_add_special_handler (parser, decrypt_handler);
        }
 
+       if (!skip_jinja) {
+               struct ucl_parser_special_handler *jinja_handler;
+
+               jinja_handler = rspamd_mempool_alloc0 (cfg->cfg_pool,
+                               sizeof (*jinja_handler));
+               jinja_handler->user_data = cfg;
+               jinja_handler->flags = UCL_SPECIAL_HANDLER_PREPROCESS_ALL;
+               jinja_handler->handler = rspamd_rcl_jinja_handler;
+
+               ucl_parser_add_special_handler (parser, jinja_handler);
+       }
+
        if (!ucl_parser_add_chunk (parser, data, st.st_size)) {
                g_set_error (err, cfg_rcl_error_quark (), errno,
                                "ucl parser error: %s", ucl_parser_get_error (parser));
@@ -3670,17 +3741,28 @@ rspamd_config_parse_ucl (struct rspamd_config *cfg,
 }
 
 gboolean
-rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
+rspamd_config_read (struct rspamd_config *cfg,
+                                       const gchar *filename,
                                        rspamd_rcl_section_fin_t logger_fin,
-                                       gpointer logger_ud, GHashTable *vars)
+                                       gpointer logger_ud,
+                                       GHashTable *vars,
+                                       gboolean skip_jinja,
+                                       gchar **lua_env)
 {
        GError *err = NULL;
        struct rspamd_rcl_section *top, *logger_section;
        const ucl_object_t *logger_obj;
 
-       rspamd_lua_set_env (cfg->lua_state, vars);
+       rspamd_lua_set_path (cfg->lua_state, NULL, vars);
+
+       if (!rspamd_lua_set_env (cfg->lua_state, vars, lua_env, &err)) {
+               msg_err_config_forced ("failed to set up environment: %e", err);
+               g_error_free (err);
+
+               return FALSE;
+       }
 
-       if (!rspamd_config_parse_ucl (cfg, filename, vars, NULL, NULL, &err)) {
+       if (!rspamd_config_parse_ucl (cfg, filename, vars, NULL, NULL, skip_jinja, &err)) {
                msg_err_config_forced ("failed to load config: %e", err);
                g_error_free (err);
 
@@ -3688,6 +3770,7 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
        }
 
        top = rspamd_rcl_config_init (cfg, NULL);
+       /* Add new paths if defined in options */
        rspamd_lua_set_path (cfg->lua_state, cfg->rcl_obj, vars);
        rspamd_lua_set_globals (cfg, cfg->lua_state);
        rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_rcl_section_free, top);
index e2477481eaf014ad0a47ed01bb909fa050ac1aeb..12a89a673579990822f9ff4a2a09299997d69320 100644 (file)
@@ -481,10 +481,13 @@ gboolean rspamd_config_parse_ucl (struct rspamd_config *cfg,
                                                                  GHashTable *vars,
                                                                  ucl_include_trace_func_t inc_trace,
                                                                  void *trace_data,
+                                                                 gboolean skip_jinja,
                                                                  GError **err);
 gboolean rspamd_config_read (struct rspamd_config *cfg,
                                                         const gchar *filename,
                                                         rspamd_rcl_section_fin_t logger_fin,
                                                         gpointer logger_ud,
-                                                        GHashTable *vars);
+                                                        GHashTable *vars,
+                                                        gboolean skip_jinja,
+                                                        gchar **lua_env);
 #endif /* CFG_RCL_H_ */
index 1c687fb48097923ddd0f2c2e99747ae8c1d0fcaa..5f1be424dd9faaff85a9951b94946466cca6ddd6 100644 (file)
@@ -263,83 +263,76 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars
                }
        }
 
-       /* Try environment */
-       t = getenv ("SHAREDIR");
-       if (t) {
-               sharedir = t;
-       }
-
-       t = getenv ("PLUGINSDIR");
-       if (t) {
-               pluginsdir = t;
-       }
-
-       t = getenv ("RULESDIR");
-       if (t) {
-               rulesdir = t;
-       }
-
-       t = getenv ("LUALIBDIR");
-       if (t) {
-               lualibdir = t;
-       }
-
-       t = getenv ("LIBDIR");
-       if (t) {
-               libdir = t;
-       }
-
-       t = getenv ("RSPAMD_LIBDIR");
-       if (t) {
-               libdir = t;
+       if (additional_path) {
+               rspamd_snprintf (path_buf, sizeof (path_buf),
+                               "%s;"
+                               "%s",
+                               additional_path, old_path);
        }
-
-       if (vars) {
-               t = g_hash_table_lookup (vars, "PLUGINSDIR");
+       else {
+               /* Try environment */
+               t = getenv ("SHAREDIR");
                if (t) {
-                       pluginsdir = t;
+                       sharedir = t;
                }
 
-               t = g_hash_table_lookup (vars, "SHAREDIR");
+               t = getenv ("PLUGINSDIR");
                if (t) {
-                       sharedir = t;
+                       pluginsdir = t;
                }
 
-               t = g_hash_table_lookup (vars, "RULESDIR");
+               t = getenv ("RULESDIR");
                if (t) {
                        rulesdir = t;
                }
 
-               t = g_hash_table_lookup (vars, "LUALIBDIR");
+               t = getenv ("LUALIBDIR");
                if (t) {
                        lualibdir = t;
                }
 
-               t = g_hash_table_lookup (vars, "LIBDIR");
+               t = getenv ("LIBDIR");
                if (t) {
                        libdir = t;
                }
 
-               t = g_hash_table_lookup (vars, "RSPAMD_LIBDIR");
+               t = getenv ("RSPAMD_LIBDIR");
                if (t) {
                        libdir = t;
                }
-       }
 
-       if (additional_path) {
-               rspamd_snprintf (path_buf, sizeof (path_buf),
-                               "%s/lua/?.lua;"
-                               "%s/?.lua;"
-                               "%s/?.lua;"
-                               "%s/?/init.lua;"
-                               "%s;"
-                               "%s",
-                               RSPAMD_CONFDIR,
-                               rulesdir,
-                               lualibdir, lualibdir,
-                               additional_path, old_path);
-       }
-       else {
+               if (vars) {
+                       t = g_hash_table_lookup (vars, "PLUGINSDIR");
+                       if (t) {
+                               pluginsdir = t;
+                       }
+
+                       t = g_hash_table_lookup (vars, "SHAREDIR");
+                       if (t) {
+                               sharedir = t;
+                       }
+
+                       t = g_hash_table_lookup (vars, "RULESDIR");
+                       if (t) {
+                               rulesdir = t;
+                       }
+
+                       t = g_hash_table_lookup (vars, "LUALIBDIR");
+                       if (t) {
+                               lualibdir = t;
+                       }
+
+                       t = g_hash_table_lookup (vars, "LIBDIR");
+                       if (t) {
+                               libdir = t;
+                       }
+
+                       t = g_hash_table_lookup (vars, "RSPAMD_LIBDIR");
+                       if (t) {
+                               libdir = t;
+                       }
+               }
+
                rspamd_snprintf (path_buf, sizeof (path_buf),
                                "%s/lua/?.lua;"
                                "%s/?.lua;"
@@ -372,11 +365,9 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars
        if (additional_path) {
                rspamd_snprintf (path_buf, sizeof (path_buf),
                                "%s/?%s;"
-                               "%s;"
                                "%s",
-                               libdir,
-                               OS_SO_SUFFIX,
                                additional_path,
+                               OS_SO_SUFFIX,
                                old_path);
        }
        else {
@@ -387,6 +378,7 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars
                                OS_SO_SUFFIX,
                                old_path);
        }
+
        lua_pop (L, 1);
        lua_pushstring (L, path_buf);
        lua_setfield (L, -2, "cpath");
@@ -537,8 +529,56 @@ rspamd_lua_rspamd_version (lua_State *L)
        return 1;
 }
 
-void
-rspamd_lua_set_env (lua_State *L, GHashTable *vars)
+static gboolean
+rspamd_lua_load_env (lua_State *L, const char *fname, gint tbl_pos, GError **err)
+{
+       gint orig_top = lua_gettop (L), err_idx;
+       gboolean ret = TRUE;
+
+       lua_pushcfunction (L, &rspamd_lua_traceback);
+       err_idx = lua_gettop (L);
+
+       if (luaL_loadfile (L, fname) != 0) {
+               g_set_error (err, g_quark_from_static_string ("lua_env"), errno,
+                               "cannot load lua file %s: %s",
+                               fname,
+                               lua_tostring (L, -1));
+               ret = FALSE;
+       }
+
+       if (ret && lua_pcall (L, 0, 1, err_idx) != 0) {
+               GString *tb = lua_touserdata (L, -1);
+               g_set_error (err, g_quark_from_static_string ("lua_env"), errno,
+                               "cannot init lua file %s: %s",
+                               fname,
+                               tb->str);
+               g_string_free (tb, TRUE);
+
+               ret = FALSE;
+       }
+
+       if (ret && lua_type (L, -1) == LUA_TTABLE) {
+               for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) {
+                       lua_pushvalue (L, -2); /* Store key */
+                       lua_pushvalue (L, -2); /* Store value */
+                       lua_settable (L, tbl_pos);
+               }
+       }
+       else if (ret) {
+               g_set_error (err, g_quark_from_static_string ("lua_env"), errno,
+                               "invalid return type when loading env from %s: %s",
+                               fname,
+                               lua_typename (L, lua_type (L, -1)));
+               ret = FALSE;
+       }
+
+       lua_settop (L, orig_top);
+
+       return ret;
+}
+
+gboolean
+rspamd_lua_set_env (lua_State *L, GHashTable *vars, char **lua_env, GError **err)
 {
        gint orig_top = lua_gettop (L);
        gchar **env = g_get_environ ();
@@ -742,10 +782,23 @@ rspamd_lua_set_env (lua_State *L, GHashTable *vars)
                        }
                }
 
+               if (lua_env) {
+                       gint lim = g_strv_length (lua_env);
+
+                       for (gint i = 0; i < lim; i ++) {
+                               if (!rspamd_lua_load_env (L, lua_env[i], lua_gettop (L), err)) {
+                                       return FALSE;
+                               }
+                       }
+               }
+
                lua_setglobal (L, "rspamd_env");
        }
 
        lua_settop (L, orig_top);
+       g_strfreev (env);
+
+       return TRUE;
 }
 
 void
index 520f5920de69a44415adf1ea28105ef534d5e855..01b7d35816570b316c9e9f8ea726a2a61c072d04 100644 (file)
@@ -306,7 +306,8 @@ void rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj,
                GHashTable *vars);
 
 /* Set some lua globals */
-void rspamd_lua_set_env (lua_State *L, GHashTable *vars);
+gboolean rspamd_lua_set_env (lua_State *L, GHashTable *vars, char **lua_env,
+               GError **err);
 void rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L);
 
 struct memory_pool_s * rspamd_lua_check_mempool (lua_State * L, gint pos);
index 3990e014edbe96b38b8e8a0a9de1de3b0a18fce8..f60fe699513d45144ba7ece153f495958856ad2c 100644 (file)
@@ -3819,7 +3819,7 @@ lua_config_load_ucl (lua_State *L)
                        cbd.L = L;
 
                        if (!rspamd_config_parse_ucl (cfg, filename, paths,
-                                       lua_include_trace_cb, &cbd, &err)) {
+                                       lua_include_trace_cb, &cbd, lua_toboolean (L, 4), &err)) {
                                luaL_unref (L, LUA_REGISTRYINDEX, cbd.cbref);
                                lua_pushboolean (L, false);
                                lua_pushfstring (L, "failed to load config: %s", err->message);
@@ -3832,7 +3832,8 @@ lua_config_load_ucl (lua_State *L)
                        luaL_unref (L, LUA_REGISTRYINDEX, cbd.cbref);
                }
                else {
-                       if (!rspamd_config_parse_ucl (cfg, filename, paths, NULL, NULL, &err)) {
+                       if (!rspamd_config_parse_ucl (cfg, filename, paths, NULL, NULL,
+                                       lua_toboolean (L, 3), &err)) {
                                lua_pushboolean (L, false);
                                lua_pushfstring (L, "failed to load config: %s", err->message);
                                g_error_free (err);
index af4673af8309d29a7e2a7dc5faed83e20bc4d99d..84958ebc0b0acc68ccf20ffa8ec64fb6fd912ec3 100644 (file)
@@ -717,7 +717,7 @@ lua_util_load_rspamd_config (lua_State *L)
                cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA);
                cfg->lua_state = L;
 
-               if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL)) {
+               if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL, FALSE, NULL)) {
                        msg_err_config ("cannot load config from %s", cfg_name);
                        lua_pushnil (L);
                }
index a255994a568842f0fd463829893cb5f0a0dab136..8f38cdf61efad50fc1ee653093d4ef8f24b5c468 100644 (file)
@@ -27,6 +27,7 @@ static gboolean compact = FALSE;
 static gboolean show_help = FALSE;
 static gboolean show_comments = FALSE;
 static gboolean modules_state = FALSE;
+static gboolean skip_template = FALSE;
 static gchar *config = NULL;
 extern struct rspamd_main *rspamd_main;
 /* Defined in modules.c */
@@ -57,6 +58,8 @@ static GOptionEntry entries[] = {
                                "Show saved comments from the configuration file", NULL },
                {"modules-state", 'm', 0, G_OPTION_ARG_NONE, &modules_state,
                                "Show modules state only", NULL},
+               {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template,
+                               "Do not apply Jinja templates", NULL},
                {NULL,  0,   0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
 };
 
@@ -284,7 +287,8 @@ rspamadm_configdump (gint argc, gchar **argv, const struct rspamadm_command *cmd
        cfg->compiled_workers = workers;
        cfg->cfg_name = config;
 
-       if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) {
+       if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main,
+                       ucl_vars, skip_template, lua_env)) {
                ret = FALSE;
        }
        else {
index db9a8d604b239f8c6d36c0e3d3bad647f0b4455e..3b7a6b5b0545bda4fa866b6c2b86c90b4524ebab 100644 (file)
@@ -23,6 +23,7 @@
 static gboolean quiet = FALSE;
 static gchar *config = NULL;
 static gboolean strict = FALSE;
+static gboolean skip_template = FALSE;
 extern struct rspamd_main *rspamd_main;
 /* Defined in modules.c */
 extern module_t *modules[];
@@ -48,6 +49,8 @@ static GOptionEntry entries[] = {
                                "Config file to test",     NULL},
                {"strict", 's', 0, G_OPTION_ARG_NONE, &strict,
                                "Stop on any error in config", NULL},
+               {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template,
+                               "Do not apply Jinja templates", NULL},
                {NULL,  0,   0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
 };
 
@@ -141,7 +144,8 @@ rspamadm_configtest (gint argc, gchar **argv, const struct rspamadm_command *cmd
        cfg->compiled_workers = workers;
        cfg->cfg_name = config;
 
-       if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) {
+       if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main,
+                       ucl_vars, skip_template, lua_env)) {
                ret = FALSE;
        }
        else {
index 28169013224ebf4314719b9cf9a457cfcef363de..762a74c1f0a065a4518a3f761828e976e431fd47 100644 (file)
@@ -31,6 +31,7 @@ static gboolean list_commands = FALSE;
 static gboolean show_help = FALSE;
 static gboolean show_version = FALSE;
 GHashTable *ucl_vars = NULL;
+gchar **lua_env = NULL;
 struct rspamd_main *rspamd_main = NULL;
 struct rspamd_async_session *rspamadm_session = NULL;
 lua_State *L = NULL;
@@ -65,6 +66,8 @@ static GOptionEntry entries[] = {
                        "Show help", NULL},
        {"version", 'V', 0, G_OPTION_ARG_NONE, &show_version,
                        "Show version", NULL},
+       {"lua-env", '\0', 0, G_OPTION_ARG_FILENAME_ARRAY, &lua_env,
+                       "Load lua environment from the specified files", NULL},
        {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
 };
 
@@ -458,8 +461,15 @@ main (gint argc, gchar **argv, gchar **env)
        setproctitle ("rspamdadm");
 
        L = cfg->lua_state;
-       rspamd_lua_set_env (L, ucl_vars);
        rspamd_lua_set_path (L, NULL, ucl_vars);
+
+       if (!rspamd_lua_set_env (L, ucl_vars, lua_env, &error)) {
+               rspamd_fprintf (stderr, "Cannot load lua environment: %e", error);
+               g_error_free (error);
+
+               exit (EXIT_FAILURE);
+       }
+
        rspamd_lua_set_globals (cfg, L);
        rspamadm_add_lua_globals ();
 
index cd01cc86bf03742dcb6b454ab75f9b157d15b0a7..e8ed9c873e9af74b37f224a41c7828238d4db54a 100644 (file)
@@ -23,6 +23,7 @@
 #include <lualib.h>
 
 extern GHashTable *ucl_vars;
+extern gchar **lua_env;
 extern struct rspamd_main *rspamd_main;
 
 GQuark rspamadm_error (void);
index 631759f05abb4a22a02a6815a47b4db0942c899c..8b12fa48e683b198750492b9d607821022a89b75 100644 (file)
@@ -95,6 +95,8 @@ static gboolean is_insecure = FALSE;
 static gboolean gen_keypair = FALSE;
 static gboolean encrypt_password = FALSE;
 static GHashTable *ucl_vars = NULL;
+static gchar **lua_env = NULL;
+static gboolean skip_template = FALSE;
 
 static gint term_attempts = 0;
 
@@ -146,6 +148,10 @@ static GOptionEntry entries[] =
          "Show version and exit", NULL },
        {"var", 0, 0, G_OPTION_ARG_CALLBACK, (gpointer)&rspamd_parse_var,
                        "Redefine/define environment variable", NULL},
+       {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template,
+                       "Do not apply Jinja templates", NULL},
+       {"lua-env", '\0', 0, G_OPTION_ARG_FILENAME_ARRAY, &lua_env,
+                       "Load lua environment from the specified files", NULL},
        { NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL }
 };
 
@@ -923,7 +929,8 @@ load_rspamd_config (struct rspamd_main *rspamd_main,
        cfg->compiled_modules = modules;
        cfg->compiled_workers = workers;
 
-       if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) {
+       if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main,
+                       ucl_vars, skip_template, lua_env)) {
                return FALSE;
        }
 
index 2af0d9aab787721a6450edfb153afeedd9249482..ad40ef488ad5b1dab36e6c1a5f328906c40a4d3e 100644 (file)
@@ -64,7 +64,7 @@ rspamd_lua_test_func (void)
        glob_t globbuf;
        gint i, len;
 
-       rspamd_lua_set_env (L, NULL);
+       rspamd_lua_set_env (L, NULL, NULL, NULL);
        rspamd_lua_set_globals (rspamd_main->cfg, L);
 
        if (lua_test_case) {