]> source.dussan.org Git - rspamd.git/commitdiff
[Rework] Rework expression API
authorVsevolod Stakhov <vsevolod@highsecure.ru>
Fri, 17 May 2019 14:46:20 +0000 (15:46 +0100)
committerVsevolod Stakhov <vsevolod@highsecure.ru>
Fri, 17 May 2019 14:46:20 +0000 (15:46 +0100)
src/libmime/mime_expressions.c
src/libserver/composites.c
src/libutil/expression.c
src/libutil/expression.h
src/lua/lua_expression.c
src/plugins/regexp.c

index d45dafe5c778914d191b9164bb9e4c94108d3b94..4a0071f282527422eab754eaa354d7e839eb4685 100644 (file)
@@ -87,7 +87,7 @@ static gboolean rspamd_has_flag_expr (struct rspamd_task *task,
 
 static rspamd_expression_atom_t * rspamd_mime_expr_parse (const gchar *line, gsize len,
                rspamd_mempool_t *pool, gpointer ud, GError **err);
-static gdouble rspamd_mime_expr_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom);
+static gdouble rspamd_mime_expr_process (void *ud, rspamd_expression_atom_t *atom);
 static gint rspamd_mime_expr_priority (rspamd_expression_atom_t *atom);
 static void rspamd_mime_expr_destroy (rspamd_expression_atom_t *atom);
 
@@ -1070,9 +1070,9 @@ rspamd_mime_expr_process_function (struct rspamd_function_atom * func,
 }
 
 static gdouble
-rspamd_mime_expr_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom)
+rspamd_mime_expr_process (void *ud, rspamd_expression_atom_t *atom)
 {
-       struct rspamd_task *task = process_data->task;
+       struct rspamd_task *task = (struct rspamd_task *)ud;
        struct rspamd_mime_atom *mime_atom;
        lua_State *L;
        gdouble ret = 0;
index b1bb2f694731604f62552e27bb5899af01f239a0..4d83293761f0abaf66f24d1aeacf069ae3963bb0 100644 (file)
@@ -68,7 +68,7 @@ struct symbol_remove_data {
 
 static rspamd_expression_atom_t * rspamd_composite_expr_parse (const gchar *line, gsize len,
                rspamd_mempool_t *pool, gpointer ud, GError **err);
-static gdouble rspamd_composite_expr_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom);
+static gdouble rspamd_composite_expr_process (void *ud, rspamd_expression_atom_t *atom);
 static gint rspamd_composite_expr_priority (rspamd_expression_atom_t *atom);
 static void rspamd_composite_expr_destroy (rspamd_expression_atom_t *atom);
 static void composites_foreach_callback (gpointer key, gpointer value, void *data);
@@ -253,10 +253,10 @@ rspamd_composite_process_symbol_removal (rspamd_expression_atom_t *atom,
 }
 
 static gdouble
-rspamd_composite_expr_process (struct rspamd_expr_process_data *process_data,
+rspamd_composite_expr_process (void *ud,
                rspamd_expression_atom_t *atom)
 {
-       struct composites_data *cd = process_data->cd;
+       struct composites_data *cd = (struct composites_data *)ud;
        const gchar *beg = atom->data, *sym = NULL;
 
        struct rspamd_symbol_result *ms = NULL;
@@ -442,13 +442,8 @@ composites_foreach_callback (gpointer key, gpointer value, void *data)
                                return;
                        }
 
-                       struct rspamd_expr_process_data process_data;
-                       memset (&process_data, 0, sizeof process_data);
-
-                       process_data.flags = RSPAMD_EXPRESSION_FLAG_NOOPT;
-                       process_data.cd = cd;
-
-                       rc = rspamd_process_expression (comp->expr, &process_data);
+                       rc = rspamd_process_expression (comp->expr, RSPAMD_EXPRESSION_FLAG_NOOPT,
+                                       cd);
 
                        /* Checked bit */
                        setbit (cd->checked, comp->id * 2);
index ecdb2e87a6aae139c2d2e6da5e6873c21d8f2e3a..71414dc66d8746f03b05b68c158577ec5e9d209e 100644 (file)
@@ -57,6 +57,14 @@ struct rspamd_expression {
        guint evals;
 };
 
+struct rspamd_expr_process_data {
+       gpointer *ud;
+       gint flags;
+       /* != NULL if trace is collected */
+       GPtrArray *trace;
+       rspamd_expression_process_cb process_closure;
+};
+
 static GQuark
 rspamd_expr_quark (void)
 {
@@ -1021,7 +1029,7 @@ rspamd_ast_process_node (struct rspamd_expression *expr, GNode *node,
                                t1 = rspamd_get_ticks (TRUE);
                        }
 
-                       elt->value = process_data->process_closure (process_data, elt->p.atom);
+                       elt->value = process_data->process_closure (process_data->ud, elt->p.atom);
 
                        if (fabs (elt->value) > 1e-9) {
                                elt->p.atom->hits ++;
@@ -1103,8 +1111,11 @@ rspamd_ast_cleanup_traverse (GNode *n, gpointer d)
 gdouble
 rspamd_process_expression_closure (struct rspamd_expression *expr,
                                                                   rspamd_expression_process_cb cb,
-                                                                  struct rspamd_expr_process_data *process_data)
+                                                                  gint flags,
+                                                                  gpointer runtime_ud,
+                                                                  GPtrArray **track)
 {
+       struct rspamd_expr_process_data pd;
        gdouble ret = 0;
 
        g_assert (expr != NULL);
@@ -1113,8 +1124,16 @@ rspamd_process_expression_closure (struct rspamd_expression *expr,
 
        expr->evals ++;
 
-       process_data->process_closure = cb;
-       ret = rspamd_ast_process_node (expr, expr->ast, process_data);
+       pd.process_closure = cb;
+       pd.flags = flags;
+       pd.ud = runtime_ud;
+
+       if (track) {
+               pd.trace = g_ptr_array_sized_new (32);
+               *track = pd.trace;
+       }
+
+       ret = rspamd_ast_process_node (expr, expr->ast, &pd);
 
        /* Cleanup */
        g_node_traverse (expr->ast, G_IN_ORDER, G_TRAVERSE_ALL, -1,
@@ -1138,18 +1157,21 @@ rspamd_process_expression_closure (struct rspamd_expression *expr,
 
 gdouble
 rspamd_process_expression_track (struct rspamd_expression *expr,
-                                                  struct rspamd_expr_process_data *process_data)
+                                                                gint flags,
+                                                                gpointer runtime_ud,
+                                                                GPtrArray **track)
 {
        return rspamd_process_expression_closure (expr,
-                       expr->subr->process, process_data);
+                       expr->subr->process, flags, runtime_ud, track);
 }
 
 gdouble
 rspamd_process_expression (struct rspamd_expression *expr,
-               struct rspamd_expr_process_data *process_data)
+                                                  gint flags,
+                                                  gpointer runtime_ud)
 {
        return rspamd_process_expression_closure (expr,
-                       expr->subr->process, process_data);
+                       expr->subr->process, flags, runtime_ud, NULL);
 }
 
 static gboolean
index 4f16812e6761ce23a3eaf043fcee7b429f9d694c..bd336b87bf98a9ace321c16d340d41ab1f5b8556 100644 (file)
@@ -56,24 +56,9 @@ typedef struct rspamd_expression_atom_s {
        gint priority;
 } rspamd_expression_atom_t;
 
-struct rspamd_expr_process_data;
-
-typedef gdouble (*rspamd_expression_process_cb)(struct rspamd_expr_process_data *process_data,
+typedef gdouble (*rspamd_expression_process_cb)(gpointer runtime_data,
                                                                                                rspamd_expression_atom_t *atom);
 
-struct rspamd_expr_process_data {
-       /* Current Lua state to run atom processing */
-       struct lua_State *L;
-       /* Parameter of lua-function processing atom*/
-       gint stack_item;
-       gint flags;
-       /* != NULL if trace is collected */
-       GPtrArray *trace;
-       struct composites_data *cd;
-       struct rspamd_task *task;
-       rspamd_expression_process_cb process_closure;
-};
-
 struct rspamd_atom_subr {
        /* Parses atom from string and returns atom structure */
        rspamd_expression_atom_t * (*parse)(const gchar *line, gsize len,
@@ -111,7 +96,8 @@ gboolean rspamd_parse_expression (const gchar *line, gsize len,
  * @return the value of expression
  */
 gdouble rspamd_process_expression (struct rspamd_expression *expr,
-               struct rspamd_expr_process_data *process_data);
+                                                                  gint flags,
+                                                                  gpointer runtime_ud);
 
 /**
  * Process the expression and return its value using atom 'process' functions with the specified data pointer.
@@ -122,7 +108,9 @@ gdouble rspamd_process_expression (struct rspamd_expression *expr,
  * @return the value of expression
  */
 gdouble rspamd_process_expression_track (struct rspamd_expression *expr,
-               struct rspamd_expr_process_data *process_data);
+                                                                                gint flags,
+                                                                                gpointer runtime_ud,
+                                                                                GPtrArray **track);
 
 /**
  * Process the expression with the custom processor
@@ -133,7 +121,9 @@ gdouble rspamd_process_expression_track (struct rspamd_expression *expr,
  */
 gdouble rspamd_process_expression_closure (struct rspamd_expression *expr,
                                                                                   rspamd_expression_process_cb cb,
-                                                                                  struct rspamd_expr_process_data *process_data);
+                                                                                  gint flags,
+                                                                                  gpointer runtime_ud,
+                                                                                  GPtrArray **track);
 /**
  * Shows string representation of an expression
  * @param expr expression to show
index 0d57a3bd8b0db166c3c5f1d77f58f19a70143ffd..d65f8c45424e9f50b3035585e0a63d1d11edc2a2 100644 (file)
  */
 
 /***
- * @function rspamd_expression.create(line, {parse_func, process_func}, pool)
+ * @function rspamd_expression.create(line, {parse_func, [process_func]}, pool)
  * Create expression from the line using atom parsing routines and the specified memory pool
  * @param {string} line expression line
- * @param {table} atom_functions parse_atom function and process_atom function
+ * @param {table} atom_functions parse_atom function and optional process_atom function
  * @param {rspamd_mempool} memory pool to use for this function
  * @return {expr, err} expression object and error message of `expr` is nil
  * @example
@@ -60,16 +60,18 @@ LUA_FUNCTION_DEF (expr, create);
 LUA_FUNCTION_DEF (expr, to_string);
 
 /***
- * @method rspamd_expression:process(input)
+ * @method rspamd_expression:process([callback, ]input[, flags])
  * Executes the expression and pass input to process atom callbacks
+ * @param {function} callback if not specified on process, then callback must be here
  * @param {any} input input data for processing callbacks
  * @return {number} result of the expression evaluation
  */
 LUA_FUNCTION_DEF (expr, process);
 
 /***
- * @method rspamd_expression:process_traced(input)
+ * @method rspamd_expression:process_traced([callback, ]input[, flags])
  * Executes the expression and pass input to process atom callbacks. This function also saves the full trace
+ * @param {function} callback if not specified on process, then callback must be here
  * @param {any} input input data for processing callbacks
  * @return {number, table of matched atoms} result of the expression evaluation
  */
@@ -98,7 +100,7 @@ static const struct luaL_reg exprlib_f[] = {
 
 static rspamd_expression_atom_t * lua_atom_parse (const gchar *line, gsize len,
                        rspamd_mempool_t *pool, gpointer ud, GError **err);
-static gdouble lua_atom_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom);
+static gdouble lua_atom_process (gpointer runtime_ud, rspamd_expression_atom_t *atom);
 
 static const struct rspamd_atom_subr lua_atom_subr = {
        .parse = lua_atom_parse,
@@ -129,6 +131,19 @@ rspamd_lua_expression (lua_State * L, gint pos)
        return ud ? *((struct lua_expression **)ud) : NULL;
 }
 
+static void
+lua_expr_dtor (gpointer p)
+{
+       struct lua_expression *e = (struct lua_expression *)p;
+
+       if (e->parse_idx != -1) {
+               luaL_unref (e->L, LUA_REGISTRYINDEX, e->parse_idx);
+       }
+
+       if (e->process_idx != -1) {
+               luaL_unref (e->L, LUA_REGISTRYINDEX, e->process_idx);
+       }
+}
 
 static rspamd_expression_atom_t *
 lua_atom_parse (const gchar *line, gsize len,
@@ -165,25 +180,52 @@ lua_atom_parse (const gchar *line, gsize len,
        return atom;
 }
 
+struct lua_atom_process_data {
+       lua_State *L;
+       struct lua_expression *e;
+       gint process_cb_pos;
+       gint stack_item;
+};
+
 static gdouble
-lua_atom_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom)
+lua_atom_process (gpointer runtime_ud, rspamd_expression_atom_t *atom)
 {
-       struct lua_expression *e = (struct lua_expression *)atom->data;
+       struct lua_atom_process_data *pd = (struct lua_atom_process_data *)runtime_ud;
        gdouble ret = 0;
+       guint nargs;
+       gint err_idx;
+
+       if (pd->stack_item != -1) {
+               nargs = 2;
+       }
+       else {
+               nargs = 1;
+       }
 
-       lua_rawgeti (process_data->L, LUA_REGISTRYINDEX, e->process_idx);
-       lua_pushlstring (process_data->L, atom->str, atom->len);
-       lua_pushvalue (process_data->L, process_data->stack_item);
+       lua_pushcfunction (pd->L, &rspamd_lua_traceback);
+       err_idx = lua_gettop (pd->L);
 
-       if (lua_pcall (process_data->L, 2, 1, 0) != 0) {
-               msg_info ("callback call failed: %s", lua_tostring (process_data->L, -1));
-               lua_pop (process_data->L, 1);
+       /* Function position */
+       lua_pushvalue (pd->L, pd->process_cb_pos);
+       /* Atom name */
+       lua_pushlstring (pd->L, atom->str, atom->len);
+
+       /* If we have data passed */
+       if (pd->stack_item != -1) {
+               lua_pushvalue (pd->L, pd->stack_item);
+       }
+
+       if (lua_pcall (pd->L, nargs, 1, err_idx) != 0) {
+               GString *tb = lua_touserdata (pd->L, -1);
+               msg_info ("expression process callback failed: %s", tb->str);
+               g_string_free (tb, TRUE);
        }
        else {
-               ret = lua_tonumber (process_data->L, -1);
-               lua_pop (process_data->L, 1);
+               ret = lua_tonumber (pd->L, -1);
        }
 
+       lua_settop (pd->L, err_idx - 1);
+
        return ret;
 }
 
@@ -192,20 +234,51 @@ lua_expr_process (lua_State *L)
 {
        LUA_TRACE_POINT;
        struct lua_expression *e = rspamd_lua_expression (L, 1);
+       struct lua_atom_process_data pd;
        gdouble res;
-       gint flags = 0;
+       gint flags = 0, old_top;
+
+       pd.L = L;
+       pd.e = e;
+       old_top = lua_gettop (L);
+
+       if (e->process_idx == -1) {
+               if (!lua_isfunction (L, 2)) {
+                       return luaL_error (L, "expression process is called with no callback function");
+               }
 
-       struct rspamd_expr_process_data process_data;
-       memset (&process_data, 0, sizeof process_data);
-       process_data.L = L;
-       process_data.stack_item = 2;
+               pd.process_cb_pos = 2;
 
-       if (lua_gettop (L) >= 3) {
-               process_data.flags = flags;
+               if (lua_type (L, 3) != LUA_TNONE && lua_type (L, 3) != LUA_TNIL) {
+                       pd.stack_item = 3;
+               }
+               else {
+                       pd.stack_item = -1;
+               }
+
+               if (lua_isnumber (L, 4)) {
+                       flags = lua_tointeger (L, 4);
+               }
        }
+       else {
+               lua_rawgeti (L, LUA_REGISTRYINDEX, e->process_idx);
+               pd.process_cb_pos = lua_gettop (L);
 
-       res = rspamd_process_expression (e->expr, &process_data);
+               if (lua_type (L, 2) != LUA_TNONE && lua_type (L, 2) != LUA_TNIL) {
+                       pd.stack_item = 2;
+               }
+               else {
+                       pd.stack_item = -1;
+               }
 
+               if (lua_isnumber (L, 3)) {
+                       flags = lua_tointeger (L, 3);
+               }
+       }
+
+       res = rspamd_process_expression (e->expr, flags, &pd);
+
+       lua_settop (L, old_top);
        lua_pushnumber (L, res);
 
        return 1;
@@ -216,39 +289,52 @@ lua_expr_process_traced (lua_State *L)
 {
        LUA_TRACE_POINT;
        struct lua_expression *e = rspamd_lua_expression (L, 1);
-       rspamd_expression_atom_t *atom;
-       gint res;
-       guint i;
-       struct rspamd_expr_process_data process_data;
-       memset (&process_data, 0, sizeof process_data);
-
-       process_data.L = L;
-       /*
-        * stack:1 - self
-        * stack:2 - data, see process_traced() definition for details
-        */
-       process_data.stack_item = 2;
-
-       if (lua_gettop (L) >= 3) {
-               process_data.flags = lua_tonumber (L, 3);
+       struct lua_atom_process_data pd;
+       gdouble res;
+       gint flags = 0, old_top;
+       GPtrArray *trace;
+
+       pd.L = L;
+       pd.e = e;
+       old_top = lua_gettop (L);
+
+       if (e->process_idx == -1) {
+               if (!lua_isfunction (L, 2)) {
+                       return luaL_error (L, "expression process is called with no callback function");
+               }
+
+               pd.process_cb_pos = 2;
+               pd.stack_item = 3;
+
+               if (lua_isnumber (L, 4)) {
+                       flags = lua_tointeger (L, 4);
+               }
        }
+       else {
+               lua_rawgeti (L, LUA_REGISTRYINDEX, e->process_idx);
+               pd.process_cb_pos = lua_gettop (L);
+               pd.stack_item = 2;
 
-       process_data.trace = g_ptr_array_sized_new (32);
+               if (lua_isnumber (L, 3)) {
+                       flags = lua_tointeger (L, 3);
+               }
+       }
 
-       res = rspamd_process_expression_track (e->expr, &process_data);
+       res = rspamd_process_expression_track (e->expr, flags, &pd, &trace);
 
+       lua_settop (L, old_top);
        lua_pushnumber (L, res);
 
-       lua_createtable (L, process_data.trace->len, 0);
+       lua_createtable (L, trace->len, 0);
 
-       for (i = 0; i < process_data.trace->len; i ++) {
-               atom = g_ptr_array_index (process_data.trace, i);
+       for (guint i = 0; i < trace->len; i ++) {
+               struct rspamd_expression_atom_s *atom = g_ptr_array_index (trace, i);
 
                lua_pushlstring (L, atom->str, atom->len);
                lua_rawseti (L, -2, i + 1);
        }
 
-       g_ptr_array_free (process_data.trace, TRUE);
+       g_ptr_array_free (trace, TRUE);
 
        return 2;
 }
@@ -260,6 +346,7 @@ lua_expr_create (lua_State *L)
        struct lua_expression *e, **pe;
        const char *line;
        gsize len;
+       gboolean no_process = FALSE;
        GError *err = NULL;
        rspamd_mempool_t *pool;
 
@@ -293,11 +380,16 @@ lua_expr_create (lua_State *L)
                lua_gettable (L, -2);
 
                if (lua_type (L, -1) != LUA_TFUNCTION) {
-                       lua_pop (L, 2);
-                       lua_pushnil (L);
-                       lua_pushstring (L, "bad process callback");
-
-                       return 2;
+                       if (lua_type (L, -1) != LUA_TNIL && lua_type (L, -1) != LUA_TNONE) {
+                               lua_pop (L, 2);
+                               lua_pushnil (L);
+                               lua_pushstring (L, "bad process callback");
+
+                               return 2;
+                       }
+                       else {
+                               no_process = TRUE;
+                       }
                }
 
                lua_pop (L, 1);
@@ -312,9 +404,15 @@ lua_expr_create (lua_State *L)
                lua_gettable (L, -2);
                e->parse_idx = luaL_ref (L, LUA_REGISTRYINDEX);
 
-               lua_pushnumber (L, 2);
-               lua_gettable (L, -2);
-               e->process_idx = luaL_ref (L, LUA_REGISTRYINDEX);
+               if (!no_process) {
+                       lua_pushnumber (L, 2);
+                       lua_gettable (L, -2);
+                       e->process_idx = luaL_ref (L, LUA_REGISTRYINDEX);
+               }
+               else {
+                       e->process_idx = -1;
+               }
+
                lua_pop (L, 1); /* Table */
 
                if (!rspamd_parse_expression (line, len, &lua_atom_subr, e, pool, &err,
@@ -322,10 +420,12 @@ lua_expr_create (lua_State *L)
                        lua_pushnil (L);
                        lua_pushstring (L, err->message);
                        g_error_free (err);
+                       lua_expr_dtor (e);
 
                        return 2;
                }
 
+               rspamd_mempool_add_destructor (pool, lua_expr_dtor, e);
                pe = lua_newuserdata (L, sizeof (struct lua_expression *));
                rspamd_lua_setclass (L, "rspamd{expr}", -1);
                *pe = e;
index 2fea0be97a31bb27dc8eaa397b7d1b972f19163e..6841e452a283294b849405e25b6762d4e03a434c 100644 (file)
@@ -442,7 +442,7 @@ regexp_module_reconfig (struct rspamd_config *cfg)
 static gboolean
 rspamd_lua_call_expression_func (struct ucl_lua_funcdata *lua_data,
                struct rspamd_task *task,
-               GArray *args, gint *res,
+               GArray *args, gdouble *res,
                const gchar *symbol)
 {
        lua_State *L = lua_data->L;
@@ -511,7 +511,7 @@ process_regexp_item (struct rspamd_task *task,
                void *user_data)
 {
        struct regexp_module_item *item = user_data;
-       gint res = FALSE;
+       gdouble res = FALSE;
 
        /* Non-threaded version */
        if (item->lua_function) {
@@ -526,12 +526,7 @@ process_regexp_item (struct rspamd_task *task,
        else {
                /* Process expression */
                if (item->expr) {
-                       struct rspamd_expr_process_data process_data;
-                       memset (&process_data, 0, sizeof process_data);
-
-                       process_data.task = task;
-
-                       res = rspamd_process_expression (item->expr, &process_data);
+                       res = rspamd_process_expression (item->expr, 0, task);
                }
                else {
                        msg_warn_task ("FIXME: %s symbol is broken with new expressions",
@@ -539,7 +534,7 @@ process_regexp_item (struct rspamd_task *task,
                }
        }
 
-       if (res) {
+       if (res != 0) {
                rspamd_task_insert_result (task, item->symbol, res, NULL);
        }