From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Sat, 5 Aug 2017 22:08:18 +0000 (+0100)
Subject: [Feature] Allow to use postfilters in composites
X-Git-Tag: 1.7.0~762
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=9632c62a868e9086c5a016eb6b259ffe6f233770;p=rspamd.git

[Feature] Allow to use postfilters in composites

We actually process composites twice now so it somehow slows down
processing by extra hash lookup. Additional logic to regenerate metric
reply if something is changed by this extra run of composites is also added.

Issue: #1726
---

diff --git a/src/libmime/filter.c b/src/libmime/filter.c
index b16018507..51a2d5c47 100644
--- a/src/libmime/filter.c
+++ b/src/libmime/filter.c
@@ -59,6 +59,7 @@ rspamd_create_metric_result (struct rspamd_task *task)
 	metric_res->metric = metric;
 	metric_res->grow_factor = 0;
 	metric_res->score = 0;
+	metric_res->changes = 0;
 
 	for (i = 0; i < METRIC_ACTION_MAX; i++) {
 		metric_res->actions_limits[i] = metric->actions[i].score;
@@ -251,6 +252,7 @@ insert_metric_result (struct rspamd_task *task,
 		s->score,
 		metric->name,
 		w);
+	metric_res->changes ++;
 
 	return s;
 }
diff --git a/src/libmime/filter.h b/src/libmime/filter.h
index 9a4379fab..139034723 100644
--- a/src/libmime/filter.h
+++ b/src/libmime/filter.h
@@ -41,6 +41,7 @@ struct rspamd_metric_result {
 	GHashTable *symbols;                            /**< symbols of metric						*/
 	GHashTable *sym_groups;							/**< groups of symbols						*/
 	gdouble actions_limits[METRIC_ACTION_MAX];		/**< set of actions for this metric			*/
+	guint changes;
 	enum rspamd_metric_action action;               /**< the current action						*/
 };
 
diff --git a/src/libserver/composites.c b/src/libserver/composites.c
index 89281b117..7aee6a230 100644
--- a/src/libserver/composites.c
+++ b/src/libserver/composites.c
@@ -279,6 +279,14 @@ composites_foreach_callback (gpointer key, gpointer value, void *data)
 			clrbit (cd->checked, comp->id * 2 + 1);
 		}
 		else {
+			if (g_hash_table_lookup (cd->metric_res->symbols, key) != NULL) {
+				/* Already set, no need to check */
+				setbit (cd->checked, comp->id * 2);
+				clrbit (cd->checked, comp->id * 2 + 1);
+
+				return;
+			}
+
 			rc = rspamd_process_expression (comp->expr,
 					RSPAMD_EXPRESSION_FLAG_NOOPT, cd);
 
@@ -358,10 +366,13 @@ composites_remove_symbols (gpointer key, gpointer value, gpointer data)
 		if (want_remove_symbol || want_forced) {
 			g_hash_table_remove (cd->metric_res->symbols, key);
 		}
+
 		if (want_remove_score || want_forced) {
 			cd->metric_res->score -= rd->ms->score;
 			rd->ms->score = 0.0;
 		}
+
+		cd->metric_res->changes ++;
 	}
 }
 
@@ -373,7 +384,7 @@ composites_metric_callback (struct rspamd_metric_result *metric_res,
 		rspamd_mempool_alloc (task->task_pool, sizeof (struct composites_data));
 
 	cd->task = task;
-	cd->metric_res = (struct rspamd_metric_result *)metric_res;
+	cd->metric_res = metric_res;
 	cd->symbols_to_remove = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
 	cd->checked =
 		rspamd_mempool_alloc0 (task->task_pool,
diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c
index 6ccb83229..93ea62816 100644
--- a/src/libserver/protocol.c
+++ b/src/libserver/protocol.c
@@ -1067,6 +1067,7 @@ rspamd_protocol_output_profiling (struct rspamd_task *task,
 
 struct rspamd_saved_protocol_reply {
 	ucl_object_t *obj;
+	guint metric_changes;
 	enum rspamd_protocol_flags flags;
 };
 
@@ -1101,12 +1102,43 @@ rspamd_protocol_write_ucl (struct rspamd_task *task,
 		flags ^= cached->flags;
 		cached->flags |= flags;
 
+		if (task->result &&
+				cached->metric_changes != task->result->changes) {
+			msg_info_task ("found metric modifications (%d) before we have "
+					"generated protocol results (%d), regenerate them",
+					task->result->changes, cached->metric_changes);
+
+			flags |= RSPAMD_PROTOCOL_METRICS;
+
+			if (task->cmd == CMD_CHECK_V2) {
+				ucl_object_delete_key (top, "symbols");
+			}
+			else {
+				ucl_object_delete_key (top, DEFAULT_METRIC);
+			}
+
+			/* That all is related to metric unfortunately */
+			ucl_object_delete_key (top, "is_spam");
+			ucl_object_delete_key (top, "is_skipped");
+			ucl_object_delete_key (top, "score");
+			ucl_object_delete_key (top, "required_score");
+			ucl_object_delete_key (top, "action");
+			ucl_object_delete_key (top, "subject");
+		}
+		if (task->result) {
+			cached->metric_changes = task->result->changes;
+		}
 	}
 	else {
 		top = ucl_object_typed_new (UCL_OBJECT);
 		cached = rspamd_mempool_alloc (task->task_pool, sizeof (*cached));
 		cached->obj = top;
 		cached->flags = flags;
+
+		if (task->result) {
+			cached->metric_changes = task->result->changes;
+		}
+
 		rspamd_mempool_set_variable (task->task_pool, varname,
 				cached, rspamd_protocol_cached_dtor);
 
diff --git a/src/libserver/task.c b/src/libserver/task.c
index 3c8eeb24e..876ab51ca 100644
--- a/src/libserver/task.c
+++ b/src/libserver/task.c
@@ -780,6 +780,8 @@ rspamd_task_process (struct rspamd_task *task, guint stages)
 		break;
 
 	case RSPAMD_TASK_STAGE_DONE:
+		/* Second run of composites processing */
+		rspamd_make_composites (task);
 		task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
 		break;