[Rework] Another bunch of changes to the dependencies processing

8 files changed, 72 insertions, 68 deletions

diff --git a/src/libserver/composites/composites.cxx b/src/libserver/composites/composites.cxx
index 64f8f2daf..0e7adfb8a 100644
--- a/src/libserver/composites/composites.cxx
+++ b/src/libserver/composites/composites.cxx
@@ -1,11 +1,11 @@
-/*-
- * Copyright 2021 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/libserver/composites/composites_internal.hxx b/src/libserver/composites/composites_internal.hxx
index 55aaa2ee1..2ae4219eb 100644
--- a/src/libserver/composites/composites_internal.hxx
+++ b/src/libserver/composites/composites_internal.hxx
@@ -1,11 +1,11 @@
-/*-
- * Copyright 2021 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/libserver/html/html_tag_defs.hxx b/src/libserver/html/html_tag_defs.hxx
index 4cff79855..05a5e3cde 100644
--- a/src/libserver/html/html_tag_defs.hxx
+++ b/src/libserver/html/html_tag_defs.hxx
@@ -1,11 +1,11 @@
-/*-
- * Copyright 2021 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/libserver/symcache/symcache_impl.cxx b/src/libserver/symcache/symcache_impl.cxx
index 3a1c187cf..56d3602ae 100644
--- a/src/libserver/symcache/symcache_impl.cxx
+++ b/src/libserver/symcache/symcache_impl.cxx
@@ -112,29 +112,39 @@ auto symcache::init() -> bool
 	/* Deal with the delayed dependencies */
 	msg_debug_cache("resolving delayed dependencies: %d in list", (int) delayed_deps->size());
 	for (const auto &delayed_dep: *delayed_deps) {
-		auto virt_item = get_item_by_name(delayed_dep.from, false);
-		auto real_item = get_item_by_name(delayed_dep.from, true);
-
-		if (virt_item == nullptr || real_item == nullptr) {
-			msg_err_cache("cannot register delayed dependency between %s and %s: "
-						  "%s is missing",
-						  delayed_dep.from.data(),
-						  delayed_dep.to.data(), delayed_dep.from.data());
+		auto virt_source = get_item_by_name(delayed_dep.from, false);
+		auto real_source = get_item_by_name(delayed_dep.from, true);
+
+		auto real_destination = get_item_by_name(delayed_dep.to, true);
+
+		if (virt_source == nullptr || real_source == nullptr || real_destination == nullptr) {
+			if (real_destination != nullptr) {
+				msg_err_cache("cannot register delayed dependency %s -> %s: "
+							  "source %s is missing",
+							  delayed_dep.from.data(),
+							  delayed_dep.to.data(), delayed_dep.from.data());
+			}
+			else {
+				msg_err_cache("cannot register delayed dependency %s -> %s: "
+							  "destionation %s is missing",
+							  delayed_dep.from.data(),
+							  delayed_dep.to.data(), delayed_dep.to.data());
+			}
 		}
 		else {
 
-			if (!disabled_ids.contains(real_item->id)) {
+			if (!disabled_ids.contains(real_source->id)) {
 				msg_debug_cache("delayed between %s(%d:%d) -> %s",
 								delayed_dep.from.data(),
-								real_item->id, virt_item->id,
+								real_source->id, virt_source->id,
 								delayed_dep.to.data());
-				add_dependency(real_item->id, delayed_dep.to,
-							   virt_item != real_item ? virt_item->id : -1);
+				add_dependency(real_source->id, delayed_dep.to, real_destination->id,
+							   virt_source != real_source ? virt_source->id : -1);
 			}
 			else {
 				msg_debug_cache("no delayed between %s(%d:%d) -> %s; %s is disabled",
 								delayed_dep.from.data(),
-								real_item->id, virt_item->id,
+								real_source->id, virt_source->id,
 								delayed_dep.to.data(),
 								delayed_dep.from.data());
 			}
@@ -529,19 +539,21 @@ auto symcache::get_item_by_name_mut(std::string_view name, bool resolve_parent)
 	return it->second;
 }
 
-auto symcache::add_dependency(int id_from, std::string_view to, int virtual_id_from) -> void
+auto symcache::add_dependency(int id_from, std::string_view to, int id_to, int virtual_id_from) -> void
 {
 	g_assert(id_from >= 0 && id_from < (int) items_by_id.size());
+	g_assert(id_to >= 0 && id_to < (int) items_by_id.size());
 	const auto &source = items_by_id[id_from];
+	const auto &dest = items_by_id[id_to];
 	g_assert(source.get() != nullptr);
-
-	if (!source->deps.contains(id_from)) {
-		msg_debug_cache("add dependency %s -> %s",
-						source->symbol.c_str(), to.data());
-		source->deps.emplace(id_from, cache_dependency{nullptr,
-													   std::string(to),
-													   id_from,
-													   -1});
+	g_assert(dest.get() != nullptr);
+
+	if (!source->deps.contains(id_to)) {
+		msg_debug_cache("add dependency %s(%d) -> %s(%d)",
+						source->symbol.c_str(), source->id, to.data(), dest->id);
+		source->deps.emplace(id_to, cache_dependency{dest.get(),
+													 std::string(to),
+													 -1});
 	}
 	else {
 		msg_debug_cache("duplicate dependency %s -> %s",
@@ -556,13 +568,12 @@ auto symcache::add_dependency(int id_from, std::string_view to, int virtual_id_f
 		const auto &vsource = items_by_id[virtual_id_from];
 		g_assert(vsource.get() != nullptr);
 
-		if (!vsource->deps.contains(virtual_id_from)) {
+		if (!vsource->deps.contains(id_to)) {
 			msg_debug_cache("add virtual dependency %s -> %s",
 							vsource->symbol.c_str(), to.data());
-			vsource->deps.emplace(virtual_id_from, cache_dependency{nullptr,
-																	std::string(to),
-																	-1,
-																	virtual_id_from});
+			vsource->deps.emplace(id_to, cache_dependency{dest.get(),
+														  std::string(to),
+														  virtual_id_from});
 		}
 		else {
 			msg_debug_cache("duplicate virtual dependency %s -> %s",
diff --git a/src/libserver/symcache/symcache_internal.hxx b/src/libserver/symcache/symcache_internal.hxx
index 255a4b1c1..c7dda51d1 100644
--- a/src/libserver/symcache/symcache_internal.hxx
+++ b/src/libserver/symcache/symcache_internal.hxx
@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Vsevolod Stakhov
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -365,7 +365,7 @@ public:
 	 * @param virtual_id_from
 	 * @return
 	 */
-	auto add_dependency(int id_from, std::string_view to, int virtual_id_from) -> void;
+	auto add_dependency(int id_from, std::string_view to, int id_to, int virtual_id_from) -> void;
 
 	/**
 	 * Add a delayed dependency between symbols that will be resolved on the init stage
diff --git a/src/libserver/symcache/symcache_item.cxx b/src/libserver/symcache/symcache_item.cxx
index cbbb5c03f..0b5de9f3a 100644
--- a/src/libserver/symcache/symcache_item.cxx
+++ b/src/libserver/symcache/symcache_item.cxx
@@ -82,7 +82,7 @@ auto cache_item::process_deps(const symcache &cache) -> void
 		msg_debug_cache("process real dependency %s on %s", symbol.c_str(), dep.sym.c_str());
 		auto *dit = cache.get_item_by_name_mut(dep.sym, true);
 
-		if (dep.vid >= 0) {
+		if (dep.virtual_source_id >= 0) {
 			/* Case of the virtual symbol that depends on another (maybe virtual) symbol */
 			const auto *vdit = cache.get_item_by_name(dep.sym, false);
 
@@ -94,7 +94,7 @@ auto cache_item::process_deps(const symcache &cache) -> void
 			}
 			else {
 				msg_debug_cache("process virtual dependency %s(%d) on %s(%d)", symbol.c_str(),
-								dep.vid, vdit->symbol.c_str(), vdit->id);
+								dep.virtual_source_id, vdit->symbol.c_str(), vdit->id);
 
 				unsigned nids = 0;
 
@@ -164,7 +164,7 @@ auto cache_item::process_deps(const symcache &cache) -> void
 
 						if (parent) {
 							if (!dit->rdeps.contains(parent->id)) {
-								dit->rdeps.emplace(parent->id, cache_dependency{parent, parent->symbol, parent->id, -1});
+								dit->rdeps.emplace(parent->id, cache_dependency{parent, parent->symbol, -1});
 								msg_debug_cache("added reverse dependency from %d on %d", parent->id,
 												dit->id);
 							}
@@ -173,7 +173,6 @@ auto cache_item::process_deps(const symcache &cache) -> void
 												parent->id, dit->id);
 							}
 							dep.item = dit;
-							dep.id = dit->id;
 						}
 						else {
 							msg_err_cache("cannot find parent for virtual symbol %s, when resolving dependency %s",
@@ -182,9 +181,8 @@ auto cache_item::process_deps(const symcache &cache) -> void
 					}
 					else {
 						dep.item = dit;
-						dep.id = dit->id;
 						if (!dit->rdeps.contains(id)) {
-							dit->rdeps.emplace(id, cache_dependency{this, symbol, id, -1});
+							dit->rdeps.emplace(id, cache_dependency{this, symbol, -1});
 							msg_debug_cache("added reverse dependency from %d on %d", id,
 											dit->id);
 						}
@@ -196,12 +194,6 @@ auto cache_item::process_deps(const symcache &cache) -> void
 				}
 			}
 		}
-		else if (dep.id >= 0) {
-			msg_err_cache("cannot find dependency on symbol %s for symbol %s",
-						  dep.sym.c_str(), symbol.c_str());
-
-			continue;
-		}
 		else {
 			msg_err_cache("cannot find dependency named %s for symbol %s",
 						  dep.sym.c_str(), symbol.c_str());
diff --git a/src/libserver/symcache/symcache_item.hxx b/src/libserver/symcache/symcache_item.hxx
index acbcf68ff..e446eb9ba 100644
--- a/src/libserver/symcache/symcache_item.hxx
+++ b/src/libserver/symcache/symcache_item.hxx
@@ -177,14 +177,13 @@ public:
 };
 
 struct cache_dependency {
-	cache_item *item; /* Real dependency */
-	std::string sym;  /* Symbolic dep name */
-	int id;           /* Real from */
-	int vid;          /* Virtual from */
+	cache_item *item;      /* Real dependency */
+	std::string sym;       /* Symbolic dep name */
+	int virtual_source_id; /* Virtual source */
 public:
 	/* Default piecewise constructor */
-	explicit cache_dependency(cache_item *_item, std::string _sym, int _id, int _vid)
-		: item(_item), sym(std::move(_sym)), id(_id), vid(_vid)
+	explicit cache_dependency(cache_item *_item, std::string _sym, int _vid)
+		: item(_item), sym(std::move(_sym)), virtual_source_id(_vid)
 	{
 	}
 };
diff --git a/src/libserver/symcache/symcache_runtime.cxx b/src/libserver/symcache/symcache_runtime.cxx
index bbf3b3d00..148346cb5 100644
--- a/src/libserver/symcache/symcache_runtime.cxx
+++ b/src/libserver/symcache/symcache_runtime.cxx
@@ -489,6 +489,8 @@ auto symcache_runtime::process_symbol(struct rspamd_task *task, symcache &cache,
 		cur_item = dyn_item;
 		items_inflight++;
 		/* Callback now must finalize itself */
+
+
 		if (item->call(task, dyn_item)) {
 			cur_item = nullptr;
 
@@ -505,10 +507,10 @@ auto symcache_runtime::process_symbol(struct rspamd_task *task, symcache &cache,
 								   item->symbol.data());
 				g_assert_not_reached();
 			}
-
-			msg_debug_cache_task("item %s, %d is now pending", item->symbol.data(),
-								 item->id);
-			dyn_item->status = cache_item_status::pending;
+			else if (dyn_item->async_events > 0) {
+				msg_debug_cache_task("item %s, %d is now pending with %d async events", item->symbol.data(),
+									 item->id, dyn_item->async_events);
+			}
 
 			return false;
 		}
@@ -588,11 +590,11 @@ auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache
 
 		auto ret = true;
 
-		for (const auto &[id, dep]: item->deps) {
+		for (const auto &[dest_id, dep]: item->deps) {
 			if (!dep.item) {
 				/* Assume invalid deps as done */
 				msg_debug_cache_task_lambda("symbol %d(%s) has invalid dependencies on %d(%s)",
-											item->id, item->symbol.c_str(), dep.id, dep.sym.c_str());
+											item->id, item->symbol.c_str(), dest_id, dep.sym.c_str());
 				continue;
 			}
 
@@ -610,7 +612,7 @@ auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache
 							ret = false;
 							msg_debug_cache_task_lambda("delayed dependency %d(%s) for "
 														"symbol %d(%s)",
-														dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+														dest_id, dep.sym.c_str(), item->id, item->symbol.c_str());
 						}
 						else if (!process_symbol(task, cache, dep.item, dep_dyn_item)) {
 							/* Now started, but has events pending */
@@ -618,18 +620,18 @@ auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache
 							msg_debug_cache_task_lambda("started check of %d(%s) symbol "
 														"as dep for "
 														"%d(%s)",
-														dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+														dest_id, dep.sym.c_str(), item->id, item->symbol.c_str());
 						}
 						else {
 							msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) is "
 														"already processed",
-														dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+														dest_id, dep.sym.c_str(), item->id, item->symbol.c_str());
 						}
 					}
 					else {
 						msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) "
 													"cannot be started now",
-													dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+													dest_id, dep.sym.c_str(), item->id, item->symbol.c_str());
 						ret = false;
 					}
 				}
@@ -637,7 +639,7 @@ auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache
 					/* Started but not finished */
 					msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) is "
 												"still executing (%d events pending)",
-												dep.id, dep.sym.c_str(),
+												dest_id, dep.sym.c_str(),
 												item->id, item->symbol.c_str(),
 												dep_dyn_item->async_events);
 					g_assert(dep_dyn_item->async_events > 0);
@@ -647,7 +649,7 @@ auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache
 			else {
 				msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) is already "
 											"finished",
-											dep.id, dep.sym.c_str(), item->id, item->symbol.c_str());
+											dest_id, dep.sym.c_str(), item->id, item->symbol.c_str());
 			}
 		}