[Feature] Allow to use postfilters in composites

We actually process composites twice now so it somehow slows down
processing by extra hash lookup. Additional logic to regenerate metric
reply if something is changed by this extra run of composites is also added.

Issue: #1726

diff --git a/src/libmime/filter.c b/src/libmime/filter.c
index b16018507eea72e3db421cf460e3cc49369f9509..51a2d5c47fe9290ce179c213122f17cb1fdc0a36 100644 (file)
--- a/src/libmime/filter.c
+++ b/src/libmime/filter.c
@@ -59,6 +59,7 @@ rspamd_create_metric_result (struct rspamd_task *task)
        metric_res->metric = metric;
        metric_res->grow_factor = 0;
        metric_res->score = 0;
+       metric_res->changes = 0;
 
        for (i = 0; i < METRIC_ACTION_MAX; i++) {
                metric_res->actions_limits[i] = metric->actions[i].score;
@@ -251,6 +252,7 @@ insert_metric_result (struct rspamd_task *task,
                s->score,
                metric->name,
                w);
+       metric_res->changes ++;
 
        return s;
 }

diff --git a/src/libmime/filter.h b/src/libmime/filter.h
index 9a4379fab21e460dd35c42823e83c26ea34f0478..139034723832800c43bc5879abe12dcb5980671c 100644 (file)
--- a/src/libmime/filter.h
+++ b/src/libmime/filter.h
@@ -41,6 +41,7 @@ struct rspamd_metric_result {
        GHashTable *symbols;                            /**< symbols of metric                                          */
        GHashTable *sym_groups;                                                 /**< groups of symbols                                          */
        gdouble actions_limits[METRIC_ACTION_MAX];              /**< set of actions for this metric                     */
+       guint changes;
        enum rspamd_metric_action action;               /**< the current action                                         */
 };
 

diff --git a/src/libserver/composites.c b/src/libserver/composites.c
index 89281b117684817d9c097c39f66e6574a6c361f3..7aee6a230f8574bce6f72ce43fbbfde1882ee2ed 100644 (file)
--- a/src/libserver/composites.c
+++ b/src/libserver/composites.c
@@ -279,6 +279,14 @@ composites_foreach_callback (gpointer key, gpointer value, void *data)
                        clrbit (cd->checked, comp->id * 2 + 1);
                }
                else {
+                       if (g_hash_table_lookup (cd->metric_res->symbols, key) != NULL) {
+                               /* Already set, no need to check */
+                               setbit (cd->checked, comp->id * 2);
+                               clrbit (cd->checked, comp->id * 2 + 1);
+
+                               return;
+                       }
+
                        rc = rspamd_process_expression (comp->expr,
                                        RSPAMD_EXPRESSION_FLAG_NOOPT, cd);
 
@@ -358,10 +366,13 @@ composites_remove_symbols (gpointer key, gpointer value, gpointer data)
                if (want_remove_symbol || want_forced) {
                        g_hash_table_remove (cd->metric_res->symbols, key);
                }
+
                if (want_remove_score || want_forced) {
                        cd->metric_res->score -= rd->ms->score;
                        rd->ms->score = 0.0;
                }
+
+               cd->metric_res->changes ++;
        }
 }
 
@@ -373,7 +384,7 @@ composites_metric_callback (struct rspamd_metric_result *metric_res,
                rspamd_mempool_alloc (task->task_pool, sizeof (struct composites_data));
 
        cd->task = task;
-       cd->metric_res = (struct rspamd_metric_result *)metric_res;
+       cd->metric_res = metric_res;
        cd->symbols_to_remove = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
        cd->checked =
                rspamd_mempool_alloc0 (task->task_pool,

diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c
index 6ccb832298501ad900cd7a4be3a58a82f87a3563..93ea62816b525aa0198f6e1c1b4ad8893241dc6b 100644 (file)
--- a/src/libserver/protocol.c
+++ b/src/libserver/protocol.c
@@ -1067,6 +1067,7 @@ rspamd_protocol_output_profiling (struct rspamd_task *task,
 
 struct rspamd_saved_protocol_reply {
        ucl_object_t *obj;
+       guint metric_changes;
        enum rspamd_protocol_flags flags;
 };
 
@@ -1101,12 +1102,43 @@ rspamd_protocol_write_ucl (struct rspamd_task *task,
                flags ^= cached->flags;
                cached->flags |= flags;
 
+               if (task->result &&
+                               cached->metric_changes != task->result->changes) {
+                       msg_info_task ("found metric modifications (%d) before we have "
+                                       "generated protocol results (%d), regenerate them",
+                                       task->result->changes, cached->metric_changes);
+
+                       flags |= RSPAMD_PROTOCOL_METRICS;
+
+                       if (task->cmd == CMD_CHECK_V2) {
+                               ucl_object_delete_key (top, "symbols");
+                       }
+                       else {
+                               ucl_object_delete_key (top, DEFAULT_METRIC);
+                       }
+
+                       /* That all is related to metric unfortunately */
+                       ucl_object_delete_key (top, "is_spam");
+                       ucl_object_delete_key (top, "is_skipped");
+                       ucl_object_delete_key (top, "score");
+                       ucl_object_delete_key (top, "required_score");
+                       ucl_object_delete_key (top, "action");
+                       ucl_object_delete_key (top, "subject");
+               }
+               if (task->result) {
+                       cached->metric_changes = task->result->changes;
+               }
        }
        else {
                top = ucl_object_typed_new (UCL_OBJECT);
                cached = rspamd_mempool_alloc (task->task_pool, sizeof (*cached));
                cached->obj = top;
                cached->flags = flags;
+
+               if (task->result) {
+                       cached->metric_changes = task->result->changes;
+               }
+
                rspamd_mempool_set_variable (task->task_pool, varname,
                                cached, rspamd_protocol_cached_dtor);
 

diff --git a/src/libserver/task.c b/src/libserver/task.c
index 3c8eeb24ec6895aacb0e12c6ef4bbbfd28eb052b..876ab51caff9a34cd78b31aed32261989fe26408 100644 (file)
--- a/src/libserver/task.c
+++ b/src/libserver/task.c
@@ -780,6 +780,8 @@ rspamd_task_process (struct rspamd_task *task, guint stages)
                break;
 
        case RSPAMD_TASK_STAGE_DONE:
+               /* Second run of composites processing */
+               rspamd_make_composites (task);
                task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
                break;