From 957e21258d23679a0b4b9f46a720750ff86d8cf8 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 17 May 2019 15:46:20 +0100 Subject: [PATCH] [Rework] Rework expression API --- src/libmime/mime_expressions.c | 6 +- src/libserver/composites.c | 15 +-- src/libutil/expression.c | 38 ++++-- src/libutil/expression.h | 28 ++--- src/lua/lua_expression.c | 204 ++++++++++++++++++++++++--------- src/plugins/regexp.c | 13 +-- 6 files changed, 203 insertions(+), 101 deletions(-) diff --git a/src/libmime/mime_expressions.c b/src/libmime/mime_expressions.c index d45dafe5c..4a0071f28 100644 --- a/src/libmime/mime_expressions.c +++ b/src/libmime/mime_expressions.c @@ -87,7 +87,7 @@ static gboolean rspamd_has_flag_expr (struct rspamd_task *task, static rspamd_expression_atom_t * rspamd_mime_expr_parse (const gchar *line, gsize len, rspamd_mempool_t *pool, gpointer ud, GError **err); -static gdouble rspamd_mime_expr_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom); +static gdouble rspamd_mime_expr_process (void *ud, rspamd_expression_atom_t *atom); static gint rspamd_mime_expr_priority (rspamd_expression_atom_t *atom); static void rspamd_mime_expr_destroy (rspamd_expression_atom_t *atom); @@ -1070,9 +1070,9 @@ rspamd_mime_expr_process_function (struct rspamd_function_atom * func, } static gdouble -rspamd_mime_expr_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom) +rspamd_mime_expr_process (void *ud, rspamd_expression_atom_t *atom) { - struct rspamd_task *task = process_data->task; + struct rspamd_task *task = (struct rspamd_task *)ud; struct rspamd_mime_atom *mime_atom; lua_State *L; gdouble ret = 0; diff --git a/src/libserver/composites.c b/src/libserver/composites.c index b1bb2f694..4d8329376 100644 --- a/src/libserver/composites.c +++ b/src/libserver/composites.c @@ -68,7 +68,7 @@ struct symbol_remove_data { static rspamd_expression_atom_t * rspamd_composite_expr_parse (const gchar *line, gsize len, rspamd_mempool_t *pool, gpointer ud, GError **err); -static gdouble rspamd_composite_expr_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom); +static gdouble rspamd_composite_expr_process (void *ud, rspamd_expression_atom_t *atom); static gint rspamd_composite_expr_priority (rspamd_expression_atom_t *atom); static void rspamd_composite_expr_destroy (rspamd_expression_atom_t *atom); static void composites_foreach_callback (gpointer key, gpointer value, void *data); @@ -253,10 +253,10 @@ rspamd_composite_process_symbol_removal (rspamd_expression_atom_t *atom, } static gdouble -rspamd_composite_expr_process (struct rspamd_expr_process_data *process_data, +rspamd_composite_expr_process (void *ud, rspamd_expression_atom_t *atom) { - struct composites_data *cd = process_data->cd; + struct composites_data *cd = (struct composites_data *)ud; const gchar *beg = atom->data, *sym = NULL; struct rspamd_symbol_result *ms = NULL; @@ -442,13 +442,8 @@ composites_foreach_callback (gpointer key, gpointer value, void *data) return; } - struct rspamd_expr_process_data process_data; - memset (&process_data, 0, sizeof process_data); - - process_data.flags = RSPAMD_EXPRESSION_FLAG_NOOPT; - process_data.cd = cd; - - rc = rspamd_process_expression (comp->expr, &process_data); + rc = rspamd_process_expression (comp->expr, RSPAMD_EXPRESSION_FLAG_NOOPT, + cd); /* Checked bit */ setbit (cd->checked, comp->id * 2); diff --git a/src/libutil/expression.c b/src/libutil/expression.c index ecdb2e87a..71414dc66 100644 --- a/src/libutil/expression.c +++ b/src/libutil/expression.c @@ -57,6 +57,14 @@ struct rspamd_expression { guint evals; }; +struct rspamd_expr_process_data { + gpointer *ud; + gint flags; + /* != NULL if trace is collected */ + GPtrArray *trace; + rspamd_expression_process_cb process_closure; +}; + static GQuark rspamd_expr_quark (void) { @@ -1021,7 +1029,7 @@ rspamd_ast_process_node (struct rspamd_expression *expr, GNode *node, t1 = rspamd_get_ticks (TRUE); } - elt->value = process_data->process_closure (process_data, elt->p.atom); + elt->value = process_data->process_closure (process_data->ud, elt->p.atom); if (fabs (elt->value) > 1e-9) { elt->p.atom->hits ++; @@ -1103,8 +1111,11 @@ rspamd_ast_cleanup_traverse (GNode *n, gpointer d) gdouble rspamd_process_expression_closure (struct rspamd_expression *expr, rspamd_expression_process_cb cb, - struct rspamd_expr_process_data *process_data) + gint flags, + gpointer runtime_ud, + GPtrArray **track) { + struct rspamd_expr_process_data pd; gdouble ret = 0; g_assert (expr != NULL); @@ -1113,8 +1124,16 @@ rspamd_process_expression_closure (struct rspamd_expression *expr, expr->evals ++; - process_data->process_closure = cb; - ret = rspamd_ast_process_node (expr, expr->ast, process_data); + pd.process_closure = cb; + pd.flags = flags; + pd.ud = runtime_ud; + + if (track) { + pd.trace = g_ptr_array_sized_new (32); + *track = pd.trace; + } + + ret = rspamd_ast_process_node (expr, expr->ast, &pd); /* Cleanup */ g_node_traverse (expr->ast, G_IN_ORDER, G_TRAVERSE_ALL, -1, @@ -1138,18 +1157,21 @@ rspamd_process_expression_closure (struct rspamd_expression *expr, gdouble rspamd_process_expression_track (struct rspamd_expression *expr, - struct rspamd_expr_process_data *process_data) + gint flags, + gpointer runtime_ud, + GPtrArray **track) { return rspamd_process_expression_closure (expr, - expr->subr->process, process_data); + expr->subr->process, flags, runtime_ud, track); } gdouble rspamd_process_expression (struct rspamd_expression *expr, - struct rspamd_expr_process_data *process_data) + gint flags, + gpointer runtime_ud) { return rspamd_process_expression_closure (expr, - expr->subr->process, process_data); + expr->subr->process, flags, runtime_ud, NULL); } static gboolean diff --git a/src/libutil/expression.h b/src/libutil/expression.h index 4f16812e6..bd336b87b 100644 --- a/src/libutil/expression.h +++ b/src/libutil/expression.h @@ -56,24 +56,9 @@ typedef struct rspamd_expression_atom_s { gint priority; } rspamd_expression_atom_t; -struct rspamd_expr_process_data; - -typedef gdouble (*rspamd_expression_process_cb)(struct rspamd_expr_process_data *process_data, +typedef gdouble (*rspamd_expression_process_cb)(gpointer runtime_data, rspamd_expression_atom_t *atom); -struct rspamd_expr_process_data { - /* Current Lua state to run atom processing */ - struct lua_State *L; - /* Parameter of lua-function processing atom*/ - gint stack_item; - gint flags; - /* != NULL if trace is collected */ - GPtrArray *trace; - struct composites_data *cd; - struct rspamd_task *task; - rspamd_expression_process_cb process_closure; -}; - struct rspamd_atom_subr { /* Parses atom from string and returns atom structure */ rspamd_expression_atom_t * (*parse)(const gchar *line, gsize len, @@ -111,7 +96,8 @@ gboolean rspamd_parse_expression (const gchar *line, gsize len, * @return the value of expression */ gdouble rspamd_process_expression (struct rspamd_expression *expr, - struct rspamd_expr_process_data *process_data); + gint flags, + gpointer runtime_ud); /** * Process the expression and return its value using atom 'process' functions with the specified data pointer. @@ -122,7 +108,9 @@ gdouble rspamd_process_expression (struct rspamd_expression *expr, * @return the value of expression */ gdouble rspamd_process_expression_track (struct rspamd_expression *expr, - struct rspamd_expr_process_data *process_data); + gint flags, + gpointer runtime_ud, + GPtrArray **track); /** * Process the expression with the custom processor @@ -133,7 +121,9 @@ gdouble rspamd_process_expression_track (struct rspamd_expression *expr, */ gdouble rspamd_process_expression_closure (struct rspamd_expression *expr, rspamd_expression_process_cb cb, - struct rspamd_expr_process_data *process_data); + gint flags, + gpointer runtime_ud, + GPtrArray **track); /** * Shows string representation of an expression * @param expr expression to show diff --git a/src/lua/lua_expression.c b/src/lua/lua_expression.c index 0d57a3bd8..d65f8c454 100644 --- a/src/lua/lua_expression.c +++ b/src/lua/lua_expression.c @@ -23,10 +23,10 @@ */ /*** - * @function rspamd_expression.create(line, {parse_func, process_func}, pool) + * @function rspamd_expression.create(line, {parse_func, [process_func]}, pool) * Create expression from the line using atom parsing routines and the specified memory pool * @param {string} line expression line - * @param {table} atom_functions parse_atom function and process_atom function + * @param {table} atom_functions parse_atom function and optional process_atom function * @param {rspamd_mempool} memory pool to use for this function * @return {expr, err} expression object and error message of `expr` is nil * @example @@ -60,16 +60,18 @@ LUA_FUNCTION_DEF (expr, create); LUA_FUNCTION_DEF (expr, to_string); /*** - * @method rspamd_expression:process(input) + * @method rspamd_expression:process([callback, ]input[, flags]) * Executes the expression and pass input to process atom callbacks + * @param {function} callback if not specified on process, then callback must be here * @param {any} input input data for processing callbacks * @return {number} result of the expression evaluation */ LUA_FUNCTION_DEF (expr, process); /*** - * @method rspamd_expression:process_traced(input) + * @method rspamd_expression:process_traced([callback, ]input[, flags]) * Executes the expression and pass input to process atom callbacks. This function also saves the full trace + * @param {function} callback if not specified on process, then callback must be here * @param {any} input input data for processing callbacks * @return {number, table of matched atoms} result of the expression evaluation */ @@ -98,7 +100,7 @@ static const struct luaL_reg exprlib_f[] = { static rspamd_expression_atom_t * lua_atom_parse (const gchar *line, gsize len, rspamd_mempool_t *pool, gpointer ud, GError **err); -static gdouble lua_atom_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom); +static gdouble lua_atom_process (gpointer runtime_ud, rspamd_expression_atom_t *atom); static const struct rspamd_atom_subr lua_atom_subr = { .parse = lua_atom_parse, @@ -129,6 +131,19 @@ rspamd_lua_expression (lua_State * L, gint pos) return ud ? *((struct lua_expression **)ud) : NULL; } +static void +lua_expr_dtor (gpointer p) +{ + struct lua_expression *e = (struct lua_expression *)p; + + if (e->parse_idx != -1) { + luaL_unref (e->L, LUA_REGISTRYINDEX, e->parse_idx); + } + + if (e->process_idx != -1) { + luaL_unref (e->L, LUA_REGISTRYINDEX, e->process_idx); + } +} static rspamd_expression_atom_t * lua_atom_parse (const gchar *line, gsize len, @@ -165,25 +180,52 @@ lua_atom_parse (const gchar *line, gsize len, return atom; } +struct lua_atom_process_data { + lua_State *L; + struct lua_expression *e; + gint process_cb_pos; + gint stack_item; +}; + static gdouble -lua_atom_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom) +lua_atom_process (gpointer runtime_ud, rspamd_expression_atom_t *atom) { - struct lua_expression *e = (struct lua_expression *)atom->data; + struct lua_atom_process_data *pd = (struct lua_atom_process_data *)runtime_ud; gdouble ret = 0; + guint nargs; + gint err_idx; + + if (pd->stack_item != -1) { + nargs = 2; + } + else { + nargs = 1; + } - lua_rawgeti (process_data->L, LUA_REGISTRYINDEX, e->process_idx); - lua_pushlstring (process_data->L, atom->str, atom->len); - lua_pushvalue (process_data->L, process_data->stack_item); + lua_pushcfunction (pd->L, &rspamd_lua_traceback); + err_idx = lua_gettop (pd->L); - if (lua_pcall (process_data->L, 2, 1, 0) != 0) { - msg_info ("callback call failed: %s", lua_tostring (process_data->L, -1)); - lua_pop (process_data->L, 1); + /* Function position */ + lua_pushvalue (pd->L, pd->process_cb_pos); + /* Atom name */ + lua_pushlstring (pd->L, atom->str, atom->len); + + /* If we have data passed */ + if (pd->stack_item != -1) { + lua_pushvalue (pd->L, pd->stack_item); + } + + if (lua_pcall (pd->L, nargs, 1, err_idx) != 0) { + GString *tb = lua_touserdata (pd->L, -1); + msg_info ("expression process callback failed: %s", tb->str); + g_string_free (tb, TRUE); } else { - ret = lua_tonumber (process_data->L, -1); - lua_pop (process_data->L, 1); + ret = lua_tonumber (pd->L, -1); } + lua_settop (pd->L, err_idx - 1); + return ret; } @@ -192,20 +234,51 @@ lua_expr_process (lua_State *L) { LUA_TRACE_POINT; struct lua_expression *e = rspamd_lua_expression (L, 1); + struct lua_atom_process_data pd; gdouble res; - gint flags = 0; + gint flags = 0, old_top; + + pd.L = L; + pd.e = e; + old_top = lua_gettop (L); + + if (e->process_idx == -1) { + if (!lua_isfunction (L, 2)) { + return luaL_error (L, "expression process is called with no callback function"); + } - struct rspamd_expr_process_data process_data; - memset (&process_data, 0, sizeof process_data); - process_data.L = L; - process_data.stack_item = 2; + pd.process_cb_pos = 2; - if (lua_gettop (L) >= 3) { - process_data.flags = flags; + if (lua_type (L, 3) != LUA_TNONE && lua_type (L, 3) != LUA_TNIL) { + pd.stack_item = 3; + } + else { + pd.stack_item = -1; + } + + if (lua_isnumber (L, 4)) { + flags = lua_tointeger (L, 4); + } } + else { + lua_rawgeti (L, LUA_REGISTRYINDEX, e->process_idx); + pd.process_cb_pos = lua_gettop (L); - res = rspamd_process_expression (e->expr, &process_data); + if (lua_type (L, 2) != LUA_TNONE && lua_type (L, 2) != LUA_TNIL) { + pd.stack_item = 2; + } + else { + pd.stack_item = -1; + } + if (lua_isnumber (L, 3)) { + flags = lua_tointeger (L, 3); + } + } + + res = rspamd_process_expression (e->expr, flags, &pd); + + lua_settop (L, old_top); lua_pushnumber (L, res); return 1; @@ -216,39 +289,52 @@ lua_expr_process_traced (lua_State *L) { LUA_TRACE_POINT; struct lua_expression *e = rspamd_lua_expression (L, 1); - rspamd_expression_atom_t *atom; - gint res; - guint i; - struct rspamd_expr_process_data process_data; - memset (&process_data, 0, sizeof process_data); - - process_data.L = L; - /* - * stack:1 - self - * stack:2 - data, see process_traced() definition for details - */ - process_data.stack_item = 2; - - if (lua_gettop (L) >= 3) { - process_data.flags = lua_tonumber (L, 3); + struct lua_atom_process_data pd; + gdouble res; + gint flags = 0, old_top; + GPtrArray *trace; + + pd.L = L; + pd.e = e; + old_top = lua_gettop (L); + + if (e->process_idx == -1) { + if (!lua_isfunction (L, 2)) { + return luaL_error (L, "expression process is called with no callback function"); + } + + pd.process_cb_pos = 2; + pd.stack_item = 3; + + if (lua_isnumber (L, 4)) { + flags = lua_tointeger (L, 4); + } } + else { + lua_rawgeti (L, LUA_REGISTRYINDEX, e->process_idx); + pd.process_cb_pos = lua_gettop (L); + pd.stack_item = 2; - process_data.trace = g_ptr_array_sized_new (32); + if (lua_isnumber (L, 3)) { + flags = lua_tointeger (L, 3); + } + } - res = rspamd_process_expression_track (e->expr, &process_data); + res = rspamd_process_expression_track (e->expr, flags, &pd, &trace); + lua_settop (L, old_top); lua_pushnumber (L, res); - lua_createtable (L, process_data.trace->len, 0); + lua_createtable (L, trace->len, 0); - for (i = 0; i < process_data.trace->len; i ++) { - atom = g_ptr_array_index (process_data.trace, i); + for (guint i = 0; i < trace->len; i ++) { + struct rspamd_expression_atom_s *atom = g_ptr_array_index (trace, i); lua_pushlstring (L, atom->str, atom->len); lua_rawseti (L, -2, i + 1); } - g_ptr_array_free (process_data.trace, TRUE); + g_ptr_array_free (trace, TRUE); return 2; } @@ -260,6 +346,7 @@ lua_expr_create (lua_State *L) struct lua_expression *e, **pe; const char *line; gsize len; + gboolean no_process = FALSE; GError *err = NULL; rspamd_mempool_t *pool; @@ -293,11 +380,16 @@ lua_expr_create (lua_State *L) lua_gettable (L, -2); if (lua_type (L, -1) != LUA_TFUNCTION) { - lua_pop (L, 2); - lua_pushnil (L); - lua_pushstring (L, "bad process callback"); - - return 2; + if (lua_type (L, -1) != LUA_TNIL && lua_type (L, -1) != LUA_TNONE) { + lua_pop (L, 2); + lua_pushnil (L); + lua_pushstring (L, "bad process callback"); + + return 2; + } + else { + no_process = TRUE; + } } lua_pop (L, 1); @@ -312,9 +404,15 @@ lua_expr_create (lua_State *L) lua_gettable (L, -2); e->parse_idx = luaL_ref (L, LUA_REGISTRYINDEX); - lua_pushnumber (L, 2); - lua_gettable (L, -2); - e->process_idx = luaL_ref (L, LUA_REGISTRYINDEX); + if (!no_process) { + lua_pushnumber (L, 2); + lua_gettable (L, -2); + e->process_idx = luaL_ref (L, LUA_REGISTRYINDEX); + } + else { + e->process_idx = -1; + } + lua_pop (L, 1); /* Table */ if (!rspamd_parse_expression (line, len, &lua_atom_subr, e, pool, &err, @@ -322,10 +420,12 @@ lua_expr_create (lua_State *L) lua_pushnil (L); lua_pushstring (L, err->message); g_error_free (err); + lua_expr_dtor (e); return 2; } + rspamd_mempool_add_destructor (pool, lua_expr_dtor, e); pe = lua_newuserdata (L, sizeof (struct lua_expression *)); rspamd_lua_setclass (L, "rspamd{expr}", -1); *pe = e; diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c index 2fea0be97..6841e452a 100644 --- a/src/plugins/regexp.c +++ b/src/plugins/regexp.c @@ -442,7 +442,7 @@ regexp_module_reconfig (struct rspamd_config *cfg) static gboolean rspamd_lua_call_expression_func (struct ucl_lua_funcdata *lua_data, struct rspamd_task *task, - GArray *args, gint *res, + GArray *args, gdouble *res, const gchar *symbol) { lua_State *L = lua_data->L; @@ -511,7 +511,7 @@ process_regexp_item (struct rspamd_task *task, void *user_data) { struct regexp_module_item *item = user_data; - gint res = FALSE; + gdouble res = FALSE; /* Non-threaded version */ if (item->lua_function) { @@ -526,12 +526,7 @@ process_regexp_item (struct rspamd_task *task, else { /* Process expression */ if (item->expr) { - struct rspamd_expr_process_data process_data; - memset (&process_data, 0, sizeof process_data); - - process_data.task = task; - - res = rspamd_process_expression (item->expr, &process_data); + res = rspamd_process_expression (item->expr, 0, task); } else { msg_warn_task ("FIXME: %s symbol is broken with new expressions", @@ -539,7 +534,7 @@ process_regexp_item (struct rspamd_task *task, } } - if (res) { + if (res != 0) { rspamd_task_insert_result (task, item->symbol, res, NULL); } -- 2.39.5