Merge pull request #2461 from negram/rspamadm-coroutines

[Project] Rspamadm coroutines

16 files changed, 374 insertions, 164 deletions

diff --git a/.luacheckrc b/.luacheckrc
index 324c45985..a5819a46b 100644
--- a/.luacheckrc
+++ b/.luacheckrc
@@ -5,6 +5,7 @@ exclude_files = {
   '/**/contrib/**',
   '/**/test/lua/**',
   '/**/test/functional/lua/miltertest/**',
+  '/**/test/functional/lua/rspamadm/**',
 }
 
 globals = {
diff --git a/src/rspamadm/commands.c b/src/rspamadm/commands.c
index 9216c79b6..81aafcdc9 100644
--- a/src/rspamadm/commands.c
+++ b/src/rspamadm/commands.c
@@ -17,6 +17,7 @@
 #include "libutil/util.h"
 #include "libutil/logger.h"
 #include "lua/lua_common.h"
+#include "lua/lua_thread_pool.h"
 
 extern struct rspamadm_command pw_command;
 extern struct rspamadm_command configtest_command;
@@ -88,15 +89,24 @@ rspamadm_fill_internal_commands (GPtrArray *dest)
 }
 
 static void
+lua_thread_str_error_cb (struct thread_entry *thread, int ret, const char *msg)
+{
+	const struct rspamadm_command *cmd = thread->cd;
+
+	msg_err ("call to rspamadm lua script %s failed (%d): %s", cmd->name,
+			ret, msg);
+}
+
+static void
 rspamadm_lua_command_run (gint argc, gchar **argv,
 						  const struct rspamadm_command *cmd)
 {
-	gint table_idx = GPOINTER_TO_INT (cmd->command_data);
-	gint i, err_idx, ret;
-	GString *tb;
+	struct thread_entry *thread = lua_thread_pool_get_for_config (rspamd_main->cfg);
 
-	lua_pushcfunction (L, &rspamd_lua_traceback);
-	err_idx = lua_gettop (L);
+	lua_State *L = thread->lua_state;
+
+	gint table_idx = GPOINTER_TO_INT (cmd->command_data);
+	gint i;
 
 	/* Function */
 	lua_rawgeti (L, LUA_REGISTRYINDEX, table_idx);
@@ -111,17 +121,7 @@ rspamadm_lua_command_run (gint argc, gchar **argv,
 		lua_rawseti (L, -2, i); /* Starting from zero ! */
 	}
 
-	if ((ret = lua_pcall (L, 1, 0, err_idx)) != 0) {
-		tb = lua_touserdata (L, -1);
-		msg_err ("call to rspamadm lua script %s failed (%d): %v", cmd->name,
-				ret, tb);
-
-		if (tb) {
-			g_string_free (tb, TRUE);
-		}
-
-		lua_settop (L, 0);
-
+	if (lua_repl_thread_call (thread, 1, (void *)cmd, lua_thread_str_error_cb) != 0) {
 		exit (EXIT_FAILURE);
 	}
 
@@ -132,13 +132,13 @@ static const gchar *
 rspamadm_lua_command_help (gboolean full_help,
 						  const struct rspamadm_command *cmd)
 {
+	struct thread_entry *thread = lua_thread_pool_get_for_config (rspamd_main->cfg);
+
+	lua_State *L = thread->lua_state;
+
 	gint table_idx = GPOINTER_TO_INT (cmd->command_data);
-	gint err_idx, ret;
-	GString *tb;
 
 	if (full_help) {
-		lua_pushcfunction (L, &rspamd_lua_traceback);
-		err_idx = lua_gettop (L);
 
 		lua_rawgeti (L, LUA_REGISTRYINDEX, table_idx);
 		/* Function */
@@ -153,17 +153,7 @@ rspamadm_lua_command_help (gboolean full_help,
 		lua_pushstring (L, "--help");
 		lua_rawseti (L, -2, 1);
 
-		if ((ret = lua_pcall (L, 1, 0, err_idx)) != 0) {
-			tb = lua_touserdata (L, -1);
-			msg_err ("call to rspamadm lua script %s failed (%d): %v", cmd->name,
-					ret, tb);
-
-			if (tb) {
-				g_string_free (tb, TRUE);
-			}
-
-			lua_settop (L, 0);
-
+		if (lua_repl_thread_call (thread, 1, (void *)cmd, lua_thread_str_error_cb) != 0) {
 			exit (EXIT_FAILURE);
 		}
 	}
diff --git a/src/rspamadm/configdump.c b/src/rspamadm/configdump.c
index 8e26ef0af..962089437 100644
--- a/src/rspamadm/configdump.c
+++ b/src/rspamadm/configdump.c
@@ -302,17 +302,13 @@ rspamadm_configdump (gint argc, gchar **argv, const struct rspamadm_command *cmd
 
 	if (ret) {
 		if (modules_state) {
-			lua_State *L = cfg->lua_state;
 
-			rspamadm_execute_lua_ucl_subr (L,
-					argc,
+			rspamadm_execute_lua_ucl_subr (argc,
 					argv,
 					cfg->rcl_obj,
 					"plugins_stats",
 					FALSE);
 
-			lua_close (L);
-
 			exit (EXIT_SUCCESS);
 		}
 		/* Output configuration */
diff --git a/src/rspamadm/confighelp.c b/src/rspamadm/confighelp.c
index e0ca2a3fc..d3461489e 100644
--- a/src/rspamadm/confighelp.c
+++ b/src/rspamadm/confighelp.c
@@ -106,8 +106,7 @@ rspamadm_confighelp_show (struct rspamd_config *cfg, gint argc, gchar **argv,
 			rspamd_fprintf (stdout, "Showing help for all options:\n");
 		}
 
-		rspamadm_execute_lua_ucl_subr (cfg->lua_state,
-				argc,
+		rspamadm_execute_lua_ucl_subr (argc,
 				argv,
 				obj,
 				"confighelp",
@@ -228,7 +227,7 @@ rspamadm_confighelp (gint argc, gchar **argv, const struct rspamadm_command *cmd
 	}
 
 	cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA);
-	cfg->lua_state = L;
+	cfg->lua_state = rspamd_main->cfg->lua_state;
 	cfg->compiled_modules = modules;
 	cfg->compiled_workers = workers;
 
diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c
index 6d2849cc7..50f515a5d 100644
--- a/src/rspamadm/control.c
+++ b/src/rspamadm/control.c
@@ -45,7 +45,6 @@ struct rspamadm_command control_command = {
 };
 
 struct rspamadm_control_cbdata {
-	lua_State *L;
 	const gchar *path;
 	gint argc;
 	gchar **argv;
@@ -133,8 +132,7 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn,
 		}
 		else {
 			if (strcmp (cbdata->path, "/fuzzystat") == 0) {
-				rspamadm_execute_lua_ucl_subr (cbdata->L,
-						cbdata->argc,
+				rspamadm_execute_lua_ucl_subr (cbdata->argc,
 						cbdata->argv,
 						obj,
 						"fuzzy_stat",
@@ -172,7 +170,6 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
 	rspamd_inet_addr_t *addr;
 	struct timeval tv;
 	static struct rspamadm_control_cbdata cbdata;
-	lua_State *L;
 	gint sock;
 
 	context = g_option_context_new (
@@ -237,9 +234,6 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
 		exit (1);
 	}
 
-	L = rspamd_lua_init ();
-	rspamd_lua_set_path (L, NULL, ucl_vars);
-
 	conn = rspamd_http_connection_new (NULL,
 			rspamd_control_error_handler,
 			rspamd_control_finish_handler,
@@ -251,7 +245,6 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
 	msg->url = rspamd_fstring_new_init (path, strlen (path));
 	double_to_tv (timeout, &tv);
 
-	cbdata.L = L;
 	cbdata.argc = argc;
 	cbdata.argv = argv;
 	cbdata.path = path;
diff --git a/src/rspamadm/fuzzy_convert.c b/src/rspamadm/fuzzy_convert.c
index 1c5620730..c59200bc7 100644
--- a/src/rspamadm/fuzzy_convert.c
+++ b/src/rspamadm/fuzzy_convert.c
@@ -78,7 +78,6 @@ rspamadm_fuzzyconvert (gint argc, gchar **argv, const struct rspamadm_command *c
 {
 	GOptionContext *context;
 	GError *error = NULL;
-	lua_State *L;
 	ucl_object_t *obj;
 
 	context = g_option_context_new (
@@ -110,9 +109,6 @@ rspamadm_fuzzyconvert (gint argc, gchar **argv, const struct rspamadm_command *c
 		exit (1);
 	}
 
-	L = rspamd_lua_init ();
-	rspamd_lua_set_path (L, NULL, ucl_vars);
-
 	obj = ucl_object_typed_new (UCL_OBJECT);
 	ucl_object_insert_key (obj, ucl_object_fromstring (source_db),
 			"source_db", 0, false);
@@ -131,8 +127,7 @@ rspamadm_fuzzyconvert (gint argc, gchar **argv, const struct rspamadm_command *c
 				"redis_db", 0, false);
 	}
 
-	rspamadm_execute_lua_ucl_subr (L,
-			argc,
+	rspamadm_execute_lua_ucl_subr (argc,
 			argv,
 			obj,
 			"fuzzy_convert",
diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c
index 26b642871..710b9c5ef 100644
--- a/src/rspamadm/lua_repl.c
+++ b/src/rspamadm/lua_repl.c
@@ -20,6 +20,7 @@
 #include "libutil/http_private.h"
 #include "printf.h"
 #include "lua/lua_common.h"
+#include "lua/lua_thread_pool.h"
 #include "message.h"
 #include "unix-std.h"
 #include "linenoise.h"
@@ -37,6 +38,7 @@ static gchar *serve = NULL;
 static gchar *exec_line = NULL;
 static gint batch = -1;
 static gboolean per_line = FALSE;
+extern struct rspamd_async_session *rspamadm_session;
 
 static const char *default_history_file = ".rspamd_repl.hist";
 
@@ -75,6 +77,9 @@ static void rspamadm_lua_load_handler (lua_State *L, gint argc, gchar **argv);
 static void rspamadm_lua_exec_handler (lua_State *L, gint argc, gchar **argv);
 static void rspamadm_lua_message_handler (lua_State *L, gint argc, gchar **argv);
 
+static void lua_thread_error_cb (struct thread_entry *thread, int ret, const char *msg);
+static void lua_thread_finish_cb (struct thread_entry *thread, int ret);
+
 static struct rspamadm_lua_dot_command cmds[] = {
 	{
 		.name = "help",
@@ -172,16 +177,41 @@ rspamadm_lua_add_path (lua_State *L, const gchar *path)
 	g_string_free (new_path, TRUE);
 }
 
+
+static void
+lua_thread_finish_cb (struct thread_entry *thread, int ret)
+{
+	struct lua_call_data *cd = thread->cd;
+
+	cd->ret = ret;
+}
+
+static void
+lua_thread_error_cb (struct thread_entry *thread, int ret, const char *msg)
+{
+	struct lua_call_data *cd = thread->cd;
+
+	rspamd_fprintf (stderr, "call failed: %s\n", msg);
+
+	cd->ret = ret;
+}
+
+static void
+lua_thread_str_error_cb (struct thread_entry *thread, int ret, const char *msg)
+{
+	struct lua_call_data *cd = thread->cd;
+	const char *what = cd->ud;
+
+	rspamd_fprintf (stderr, "call to %s failed: %s\n", what, msg);
+
+	cd->ret = ret;
+}
+
 static gboolean
 rspamadm_lua_load_script (lua_State *L, const gchar *path)
 {
-	GString *tb;
-	gint err_idx = 0;
-
-	if (!per_line) {
-		lua_pushcfunction (L, &rspamd_lua_traceback);
-		err_idx = lua_gettop (L);
-	}
+	struct thread_entry *thread = lua_thread_pool_get_for_config (rspamd_main->cfg);
+	L = thread->lua_state;
 
 	if (luaL_loadfile (L, path) != 0) {
 		rspamd_fprintf (stderr, "cannot load script %s: %s\n",
@@ -192,12 +222,8 @@ rspamadm_lua_load_script (lua_State *L, const gchar *path)
 	}
 
 	if (!per_line) {
-		if (lua_pcall (L, 0, 0, err_idx) != 0) {
-			tb = lua_touserdata (L, -1);
-			rspamd_fprintf (stderr, "call to %s failed: %v", path, tb);
-			g_string_free (tb, TRUE);
-			lua_settop (L, 0);
 
+		if (lua_repl_thread_call (thread, 0, (void *)path, lua_thread_str_error_cb) != 0) {
 			return FALSE;
 		}
 
@@ -211,27 +237,29 @@ static void
 rspamadm_exec_input (lua_State *L, const gchar *input)
 {
 	GString *tb;
-	gint err_idx, i, cbref;
+	gint i, cbref;
+	int top = 0;
 	gchar outbuf[8192];
 
-	lua_pushcfunction (L, &rspamd_lua_traceback);
-	err_idx = lua_gettop (L);
+	struct thread_entry *thread = lua_thread_pool_get_for_config (rspamd_main->cfg);
+	L = thread->lua_state;
 
 	/* First try return + input */
 	tb = g_string_sized_new (strlen (input) + sizeof ("return "));
 	rspamd_printf_gstring (tb, "return %s", input);
 
-	if (luaL_loadstring (L, tb->str) != 0) {
+	int r = luaL_loadstring (L, tb->str);
+	if (r != 0) {
 		/* Reset stack */
 		lua_settop (L, 0);
-		lua_pushcfunction (L, &rspamd_lua_traceback);
-		err_idx = lua_gettop (L);
 		/* Try with no return */
 		if (luaL_loadstring (L, input) != 0) {
 			rspamd_fprintf (stderr, "cannot load string %s\n",
 					input);
 			g_string_free (tb, TRUE);
 			lua_settop (L, 0);
+
+			lua_thread_pool_return (rspamd_main->cfg->lua_thread_pool, thread);
 			return;
 		}
 	}
@@ -239,29 +267,61 @@ rspamadm_exec_input (lua_State *L, const gchar *input)
 	g_string_free (tb, TRUE);
 
 	if (!per_line) {
-		if (lua_pcall (L, 0, LUA_MULTRET, err_idx) != 0) {
-			tb = lua_touserdata (L, -1);
-			rspamd_fprintf (stderr, "call failed: %v\n", tb);
-			g_string_free (tb, TRUE);
-			lua_settop (L, 0);
-			return;
-		}
 
-		/* Print output */
-		for (i = err_idx + 1; i <= lua_gettop (L); i++) {
-			if (lua_isfunction (L, i)) {
-				lua_pushvalue (L, i);
-				cbref = luaL_ref (L, LUA_REGISTRYINDEX);
+		top = lua_gettop (L);
+
+		if (lua_repl_thread_call (thread, 0, NULL, NULL) == 0) {
+			/* Print output */
+			for (i = top; i <= lua_gettop (L); i++) {
+				if (lua_isfunction (L, i)) {
+					lua_pushvalue (L, i);
+					cbref = luaL_ref (L, LUA_REGISTRYINDEX);
 
-				rspamd_printf ("local function: %d\n", cbref);
-			} else {
-				lua_logger_out_type (L, i, outbuf, sizeof (outbuf));
-				rspamd_printf ("%s\n", outbuf);
+					rspamd_printf ("local function: %d\n", cbref);
+				} else {
+					lua_logger_out_type (L, i, outbuf, sizeof (outbuf));
+					rspamd_printf ("%s\n", outbuf);
+				}
 			}
 		}
+	}
+}
 
-		lua_settop (L, 0);
+void
+wait_session_events ()
+{
+	/* XXX: it's probably worth to add timeout here - not to wait forever */
+	while (rspamd_session_events_pending (rspamadm_session) > 0) {
+		event_base_loop (rspamd_main->ev_base, EVLOOP_ONCE);
+	}
+}
+
+gint
+lua_repl_thread_call (struct thread_entry *thread, gint narg, gpointer ud, lua_thread_error_t error_func)
+{
+	int ret;
+	struct lua_call_data *cd = g_new0 (struct lua_call_data, 1);
+	cd->top = lua_gettop (thread->lua_state);
+	cd->ud = ud;
+
+	thread->finish_callback = lua_thread_finish_cb;
+	if (error_func) {
+		thread->error_callback = error_func;
+	}
+	else {
+		thread->error_callback = lua_thread_error_cb;
 	}
+	thread->cd = cd;
+
+	lua_thread_call (thread, narg);
+
+	wait_session_events ();
+
+	ret = cd->ret;
+
+	g_free (cd);
+
+	return ret;
 }
 
 static void
@@ -308,12 +368,12 @@ rspamadm_lua_load_handler (lua_State *L, gint argc, gchar **argv)
 static void
 rspamadm_lua_exec_handler (lua_State *L, gint argc, gchar **argv)
 {
-	GString *tb;
-	gint err_idx, i;
+	gint i;
+
+	struct thread_entry *thread = lua_thread_pool_get_for_config (rspamd_main->cfg);
+	L = thread->lua_state;
 
 	for (i = 1; argv[i] != NULL; i ++) {
-		lua_pushcfunction (L, &rspamd_lua_traceback);
-		err_idx = lua_gettop (L);
 
 		if (luaL_loadfile (L, argv[i]) != 0) {
 			rspamd_fprintf (stderr, "cannot load script %s: %s\n",
@@ -323,16 +383,7 @@ rspamadm_lua_exec_handler (lua_State *L, gint argc, gchar **argv)
 			return;
 		}
 
-		if (lua_pcall (L, 0, 0, err_idx) != 0) {
-			tb = lua_touserdata (L, -1);
-			rspamd_fprintf (stderr, "call to %s failed: %v", argv[i], tb);
-			g_string_free (tb, TRUE);
-			lua_settop (L, 0);
-
-			return;
-		}
-
-		lua_settop (L, 0);
+		lua_repl_thread_call (thread, 0, argv[i], lua_thread_str_error_cb);
 	}
 }
 
@@ -340,11 +391,10 @@ static void
 rspamadm_lua_message_handler (lua_State *L, gint argc, gchar **argv)
 {
 	gulong cbref;
-	gint err_idx, func_idx, i, j;
+	gint old_top, func_idx, i, j;
 	struct rspamd_task *task, **ptask;
 	gpointer map;
 	gsize len;
-	GString *tb;
 	gchar outbuf[8192];
 
 	if (argv[1] == NULL) {
@@ -352,22 +402,26 @@ rspamadm_lua_message_handler (lua_State *L, gint argc, gchar **argv)
 		return;
 	}
 
-	if (rspamd_strtoul (argv[1], strlen (argv[1]), &cbref)) {
-		lua_rawgeti (L, LUA_REGISTRYINDEX, cbref);
-	}
-	else {
-		lua_getglobal (L, argv[1]);
-	}
+	for (i = 2; argv[i] != NULL; i ++) {
+		struct thread_entry *thread = lua_thread_pool_get_for_config (rspamd_main->cfg);
+		L = thread->lua_state;
 
-	if (lua_type (L, -1) != LUA_TFUNCTION) {
-		rspamd_printf ("bad callback type: %s\n", lua_typename (L, lua_type (L, -1)));
-		return;
-	}
+		if (rspamd_strtoul (argv[1], strlen (argv[1]), &cbref)) {
+			lua_rawgeti (L, LUA_REGISTRYINDEX, cbref);
+		}
+		else {
+			lua_getglobal (L, argv[1]);
+		}
 
-	/* Save index to reuse */
-	func_idx = lua_gettop (L);
+		if (lua_type (L, -1) != LUA_TFUNCTION) {
+			rspamd_printf ("bad callback type: %s\n", lua_typename (L, lua_type (L, -1)));
+			lua_thread_pool_return (rspamd_main->cfg->lua_thread_pool, thread);
+			return;
+		}
+
+		/* Save index to reuse */
+		func_idx = lua_gettop (L);
 
-	for (i = 2; argv[i] != NULL; i ++) {
 		map = rspamd_file_xmap (argv[i], PROT_READ, &len, TRUE);
 
 		if (map == NULL) {
@@ -391,23 +445,18 @@ rspamadm_lua_message_handler (lua_State *L, gint argc, gchar **argv)
 			}
 
 			rspamd_message_process (task);
-			lua_pushcfunction (L, &rspamd_lua_traceback);
-			err_idx = lua_gettop (L);
+			old_top = lua_gettop (L);
 
 			lua_pushvalue (L, func_idx);
 			ptask = lua_newuserdata (L, sizeof (*ptask));
 			*ptask = task;
 			rspamd_lua_setclass (L, "rspamd{task}", -1);
 
-			if (lua_pcall (L, 1, LUA_MULTRET, err_idx) != 0) {
-				tb = lua_touserdata (L, -1);
-				rspamd_printf ("lua callback for %s failed: %v\n", argv[i], tb);
-				g_string_free (tb, TRUE);
-			}
-			else {
+
+			if (lua_repl_thread_call (thread, 1, argv[i], lua_thread_str_error_cb) == 0) {
 				rspamd_printf ("lua callback for %s returned:\n", argv[i]);
 
-				for (j = err_idx + 1; j <= lua_gettop (L); j ++) {
+				for (j = old_top + 1; j <= lua_gettop (L); j ++) {
 					lua_logger_out_type (L, j, outbuf, sizeof (outbuf));
 					rspamd_printf ("%s\n", outbuf);
 				}
@@ -578,6 +627,18 @@ rspamadm_lua_finish_handler (struct rspamd_http_connection_entry *conn_ent)
 	g_free (session);
 }
 
+static void
+lua_thread_http_error_cb (struct thread_entry *thread, int ret, const char *msg)
+{
+	struct lua_call_data *cd = thread->cd;
+	struct rspamd_http_connection_entry *conn_ent = cd->ud;
+
+	rspamd_controller_send_error (conn_ent, 500, "call failed: %s\n", msg);
+
+	cd->ret = ret;
+}
+
+
 /*
  * Exec command handler:
  * request: /exec
@@ -598,7 +659,10 @@ rspamadm_lua_handle_exec (struct rspamd_http_connection_entry *conn_ent,
 	gsize body_len;
 
 	ctx = session->ctx;
-	L = ctx->L;
+
+	struct thread_entry *thread = lua_thread_pool_get_for_config (rspamd_main->cfg);
+	L = thread->lua_state;
+
 	body = rspamd_http_message_get_body (msg, &body_len);
 
 	if (body == NULL) {
@@ -629,12 +693,7 @@ rspamadm_lua_handle_exec (struct rspamd_http_connection_entry *conn_ent,
 
 	g_string_free (tb, TRUE);
 
-	if (lua_pcall (L, 0, LUA_MULTRET, err_idx) != 0) {
-		tb = lua_touserdata (L, -1);
-		rspamd_controller_send_error (conn_ent, 500, "call failed: %v\n", tb);
-		g_string_free (tb, TRUE);
-		lua_settop (L, 0);
-
+	if (lua_repl_thread_call (thread, 0, conn_ent, lua_thread_http_error_cb) != 0) {
 		return 0;
 	}
 
@@ -667,6 +726,7 @@ rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd)
 	GError *error = NULL;
 	gchar **elt;
 	guint i;
+	lua_State *L = rspamd_main->cfg->lua_state;
 
 	context = g_option_context_new ("lua - run lua interpreter");
 	g_option_context_set_summary (context,
@@ -830,12 +890,10 @@ again:
 			lua_pushlstring (L, buf->str, MIN (buf->len, end_pos));
 			lua_setglobal (L, "input");
 
-			if (lua_pcall (L, 0, 0, 0) != 0) {
-				rspamd_fprintf (stderr, "call to script failed: %s",
-						lua_tostring (L, -1));
-				lua_settop (L, 0);
-				break;
-			}
+			struct thread_entry *thread = lua_thread_pool_get_for_config (rspamd_main->cfg);
+			L = thread->lua_state;
+
+			lua_repl_thread_call (thread, 0, NULL, NULL);
 
 			lua_settop (L, old_top);
 		}
diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c
index d5a656a51..fad9b2fcd 100644
--- a/src/rspamadm/rspamadm.c
+++ b/src/rspamadm/rspamadm.c
@@ -18,6 +18,7 @@
 #include "rspamd.h"
 #include "ottery.h"
 #include "lua/lua_common.h"
+#include "lua/lua_thread_pool.h"
 #include "lua_ucl.h"
 #include "unix-std.h"
 
@@ -31,6 +32,7 @@ static gboolean show_help = FALSE;
 static gboolean show_version = FALSE;
 GHashTable *ucl_vars = NULL;
 struct rspamd_main *rspamd_main = NULL;
+struct rspamd_async_session *rspamadm_session = NULL;
 lua_State *L = NULL;
 
 /* Defined in modules.c */
@@ -204,15 +206,27 @@ rspamadm_parse_ucl_var (const gchar *option_name,
 	return TRUE;
 }
 
+static void
+lua_thread_str_error_cb (struct thread_entry *thread, int ret, const char *msg)
+{
+	struct lua_call_data *cd = thread->cd;
+
+	msg_err ("call to rspamadm lua script failed (%d): %s", ret, msg);
+
+	cd->ret = ret;
+}
+
 gboolean
-rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv,
+rspamadm_execute_lua_ucl_subr (gint argc, gchar **argv,
 							   const ucl_object_t *res,
 							   const gchar *script_name,
 							   gboolean rspamadm_subcommand)
 {
-	lua_State *L = pL;
-	gint err_idx, i, ret;
-	GString *tb;
+	struct thread_entry *thread = lua_thread_pool_get_for_config (rspamd_main->cfg);
+
+	lua_State *L = thread->lua_state;
+
+	gint i;
 	gchar str[PATH_MAX];
 
 	g_assert (script_name != NULL);
@@ -250,32 +264,21 @@ rspamadm_execute_lua_ucl_subr (gpointer pL, gint argc, gchar **argv,
 		}
 	}
 
-	lua_pushcfunction (L, &rspamd_lua_traceback);
-	err_idx = lua_gettop (L);
-
 	/* Push function */
-	lua_pushvalue (L, -2);
+	lua_pushvalue (L, -1);
 
 	/* Push argv */
 	lua_newtable (L);
 
 	for (i = 1; i < argc; i ++) {
 		lua_pushstring (L, argv[i]);
-		lua_rawseti (L, -2, i);
+		lua_rawseti (L, -1, i);
 	}
 
 	/* Push results */
 	ucl_object_push_lua (L, res, TRUE);
 
-	if ((ret = lua_pcall (L, 2, 0, err_idx)) != 0) {
-		tb = lua_touserdata (L, -1);
-		msg_err ("call to rspamadm lua script failed (%d): %v", ret, tb);
-
-		if (tb) {
-			g_string_free (tb, TRUE);
-		}
-
-		lua_settop (L, 0);
+	if (lua_repl_thread_call (thread, 2, NULL, lua_thread_str_error_cb) != 0) {
 
 		return FALSE;
 	}
@@ -317,6 +320,28 @@ rspamadm_command_maybe_match_name (const gchar *cmd, const gchar *input)
 	return FALSE;
 }
 
+
+
+static void
+rspamadm_add_lua_globals()
+{
+	struct rspamd_async_session  **psession;
+	struct event_base **pev_base;
+
+	rspamadm_session = rspamd_session_create (rspamd_main->cfg->cfg_pool, NULL,
+			NULL, (event_finalizer_t )NULL, NULL);
+
+	psession = lua_newuserdata (L, sizeof (struct rspamd_async_session*));
+	rspamd_lua_setclass (L, "rspamd{session}", -1);
+	*psession = rspamadm_session;
+	lua_setglobal (L, "rspamadm_session");
+
+	pev_base = lua_newuserdata (L, sizeof (struct event_base *));
+	rspamd_lua_setclass (L, "rspamd{ev_base}", -1);
+	*pev_base = rspamd_main->ev_base;
+	lua_setglobal (L, "rspamadm_ev_base");
+}
+
 gint
 main (gint argc, gchar **argv, gchar **env)
 {
@@ -343,6 +368,8 @@ main (gint argc, gchar **argv, gchar **env)
 	rspamd_main->type = process_quark;
 	rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
 			"rspamadm");
+	rspamd_main->ev_base = event_init ();
+
 	rspamadm_fill_internal_commands (all_commands);
 	help_command.command_data = all_commands;
 
@@ -417,6 +444,11 @@ main (gint argc, gchar **argv, gchar **env)
 	L = cfg->lua_state;
 	rspamd_lua_set_path (L, NULL, ucl_vars);
 	rspamd_lua_set_globals (cfg, L, ucl_vars);
+	rspamadm_add_lua_globals();
+
+#ifdef WITH_HIREDIS
+	rspamd_redis_pool_config (cfg->redis_pool, cfg, rspamd_main->ev_base);
+#endif
 
 	/* Init rspamadm global */
 	lua_newtable (L);
@@ -498,6 +530,8 @@ main (gint argc, gchar **argv, gchar **env)
 		cmd->run (0, NULL, cmd);
 	}
 
+	event_base_loopexit (rspamd_main->ev_base, NULL);
+
 	REF_RELEASE (rspamd_main->cfg);
 	rspamd_log_close (rspamd_main->logger, TRUE);
 	g_free (rspamd_main);
diff --git a/src/rspamadm/rspamadm.h b/src/rspamadm/rspamadm.h
index 02ecb2f47..cd01cc86b 100644
--- a/src/rspamadm/rspamadm.h
+++ b/src/rspamadm/rspamadm.h
@@ -23,7 +23,7 @@
 #include <lualib.h>
 
 extern GHashTable *ucl_vars;
-extern lua_State *L;
+extern struct rspamd_main *rspamd_main;
 
 GQuark rspamadm_error (void);
 
@@ -55,9 +55,21 @@ const struct rspamadm_command *rspamadm_search_command (const gchar *name,
 void rspamadm_fill_internal_commands (GPtrArray *dest);
 void rspamadm_fill_lua_commands (lua_State *L, GPtrArray *dest);
 
-gboolean rspamadm_execute_lua_ucl_subr (gpointer L, gint argc, gchar **argv,
+gboolean rspamadm_execute_lua_ucl_subr (gint argc, gchar **argv,
 										const ucl_object_t *res,
 										const gchar *script_name,
 										gboolean rspamadm_subcommand);
 
+struct thread_entry;
+typedef void (*lua_thread_error_t) (struct thread_entry *thread, int ret, const char *msg);
+
+
+struct lua_call_data {
+	gint top;
+	gint ret;
+	gpointer ud;
+};
+gint lua_repl_thread_call (struct thread_entry *thread, gint narg,
+		gpointer ud, lua_thread_error_t error_func);
+
 #endif
diff --git a/src/rspamadm/stat_convert.c b/src/rspamadm/stat_convert.c
index acbe11550..e3a3e179b 100644
--- a/src/rspamadm/stat_convert.c
+++ b/src/rspamadm/stat_convert.c
@@ -113,7 +113,6 @@ rspamadm_statconvert (gint argc, gchar **argv, const struct rspamadm_command *cm
 {
 	GOptionContext *context;
 	GError *error = NULL;
-	lua_State *L;
 	ucl_object_t *obj;
 
 	context = g_option_context_new (
@@ -237,8 +236,6 @@ rspamadm_statconvert (gint argc, gchar **argv, const struct rspamadm_command *cm
 		}
 	}
 
-	L = rspamd_lua_init ();
-	rspamd_lua_set_path (L, obj, ucl_vars);
 	ucl_object_insert_key (obj, ucl_object_frombool (reset_previous),
 			"reset_previous", 0, false);
 
@@ -247,8 +244,7 @@ rspamadm_statconvert (gint argc, gchar **argv, const struct rspamadm_command *cm
 				"expire", 0, false);
 	}
 
-	rspamadm_execute_lua_ucl_subr (L,
-			argc,
+	rspamadm_execute_lua_ucl_subr (argc,
 			argv,
 			obj,
 			"stat_convert",
diff --git a/test/functional/cases/150_rspamadm.robot b/test/functional/cases/150_rspamadm.robot
index 4e7b3c8aa..82532ddab 100644
--- a/test/functional/cases/150_rspamadm.robot
+++ b/test/functional/cases/150_rspamadm.robot
@@ -1,5 +1,8 @@
 *** Settings ***
 Library         Process
+Library         ../lib/rspamd.py
+
+Suite Teardown    Terminate All Processes    kill=True
 
 *** Test Cases ***
 Config Test
@@ -12,3 +15,26 @@ Config Help
   ${result} =  Run Process  ${RSPAMADM}  confighelp
   Should Match Regexp  ${result.stderr}  ^$
   Should Be Equal As Integers  ${result.rc}  0
+
+Simple interpreter
+  ${handle} =  Start Process  ${RSPAMADM}  lua
+  ${result} =  Write to stdin  ${handle}  1+1
+  Should Be Equal As Strings  ${result}  2\n
+
+Simple interpreter, two results
+  ${handle} =  Start Process  ${RSPAMADM}  lua
+  ${result} =  Write to stdin  ${handle}  1+1, 2 * 5
+  Should Be Equal  ${result}  2\n10\n
+
+Process message callback
+  ${handle} =  Start Process  ${RSPAMADM}  lua
+  ${result} =  Write to stdin  ${handle}  .load ${TESTDIR}/lua/rspamadm/test_message_callback.lua\n.message message_callback ${TESTDIR}/messages/empty_part.eml
+  Should Contain  ${result}  n parts = 2
+  Should Contain  ${result}  1\n2\n4\n6
+
+Lua batch mode
+  ${result} =  Run Process  ${RSPAMADM}  lua  -b  ${TESTDIR}/lua/rspamadm/test_batch.lua
+  Should Match Regexp  ${result.stderr}  ^$
+  Should Be Equal As Integers  ${result.rc}  0
+  Should Be Equal  ${result.stdout}  hello world
+
diff --git a/test/functional/cases/151_rspamadm_async.robot b/test/functional/cases/151_rspamadm_async.robot
new file mode 100644
index 000000000..5a848d274
--- /dev/null
+++ b/test/functional/cases/151_rspamadm_async.robot
@@ -0,0 +1,34 @@
+*** Settings ***
+Test Setup      Http Setup
+Test Teardown   Http Teardown
+Library         Process
+Library         ${TESTDIR}/lib/rspamd.py
+Resource        ${TESTDIR}/lib/rspamd.robot
+Variables       ${TESTDIR}/lib/vars.py
+Suite Teardown  Terminate All Processes    kill=True
+
+*** Variables ***
+${REDIS_SCOPE}   Test
+
+
+*** Test Cases ***
+Tcp client
+  ${result} =  Run Process  ${RSPAMADM}  lua  -b  ${TESTDIR}/lua/rspamadm/test_tcp_client.lua
+  Should Match Regexp  ${result.stderr}  ^$
+  Should Be Equal As Integers  ${result.rc}  0
+  Should Be Equal  ${result.stdout}  hello post
+
+*** Keywords ***
+
+Http Setup
+  Run Dummy Http
+
+Http Teardown
+  ${http_pid} =  Get File  /tmp/dummy_http.pid
+  Shutdown Process With Children  ${http_pid}
+  Remove file  /tmp/dummy_http.pid
+
+Run Dummy Http
+  [Arguments]
+  ${result} =  Start Process  ${TESTDIR}/util/dummy_http.py
+  Wait Until Created  /tmp/dummy_http.pid
diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py
index 5673a10d9..eeb5f2799 100644
--- a/test/functional/lib/rspamd.py
+++ b/test/functional/lib/rspamd.py
@@ -212,8 +212,18 @@ def shutdown_process_with_children(pid):
         except:
             pass
 
+def write_to_stdin(process_handle, text):
+    lib = BuiltIn().get_library_instance('Process')
+    obj = lib.get_process_object()
+    obj.stdin.write(text + "\n")
+    obj.stdin.flush()
+    obj.stdin.close()
+    out = obj.stdout.read(4096)
+    return out
+
 def get_file_if_exists(file_path):
     if os.path.exists(file_path):
         with open(file_path, 'r') as myfile:
             return myfile.read()
     return None
+
diff --git a/test/functional/lua/rspamadm/test_batch.lua b/test/functional/lua/rspamadm/test_batch.lua
new file mode 100644
index 000000000..e75154b7c
--- /dev/null
+++ b/test/functional/lua/rspamadm/test_batch.lua
@@ -0,0 +1 @@
+print("hello world")
\ No newline at end of file
diff --git a/test/functional/lua/rspamadm/test_message_callback.lua b/test/functional/lua/rspamadm/test_message_callback.lua
new file mode 100644
index 000000000..6be512ac0
--- /dev/null
+++ b/test/functional/lua/rspamadm/test_message_callback.lua
@@ -0,0 +1,5 @@
+function message_callback(task)
+  local parts = task:get_text_parts()
+  print("n parts = " .. tostring(#parts))
+  return 1,2,4,6
+end
diff --git a/test/functional/lua/rspamadm/test_tcp_client.lua b/test/functional/lua/rspamadm/test_tcp_client.lua
new file mode 100644
index 000000000..796fe913b
--- /dev/null
+++ b/test/functional/lua/rspamadm/test_tcp_client.lua
@@ -0,0 +1,60 @@
+local logger = require "rspamd_logger"
+local tcp_sync = require "lua_tcp_sync"
+
+local is_ok, connection = tcp_sync.connect {
+  config = rspamd_config,
+  ev_base = rspamadm_ev_base,
+  session = rspamadm_session,
+  host = '127.0.0.1',
+  timeout = 20,
+  port = 18080,
+}
+local err
+is_ok, err = connection:write(string.format('POST /request HTTP/1.1\r\nConnection: close\r\n\r\n'))
+
+logger.info('write %1, %2', is_ok, err)
+if not is_ok then
+  logger.errx(rspamd_config, 'write error: %1', err)
+  return
+end
+
+local content_length, content
+
+while true do
+  local header_line
+  is_ok, header_line = connection:read_until("\r\n")
+  if not is_ok then
+    logger.errx(rspamd_config, 'failed to get header: %1', header_line)
+    return
+  end
+
+  if header_line == "" then
+    logger.info('headers done')
+    break
+  end
+
+  local value
+  local header = header_line:gsub("([%w-]+): (.*)", 
+      function (h, v) value = v; return h:lower() end)
+
+  logger.info('parsed header: %1 -> "%2"', header, value)
+
+  if header == "content-length" then
+    content_length = tonumber(value)
+  end
+
+end
+
+if content_length then
+  is_ok, content = connection:read_bytes(content_length)
+  if is_ok then
+  end
+else
+  is_ok, content = connection:read_until_eof()
+  if is_ok then
+  end
+end
+logger.info('(is_ok: %1) content [%2 bytes] %3', is_ok, content_length, content)
+
+
+print(content)