From e0856b113ba7e3f66d4cdbb66d63429777acc89f Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 19 Jun 2024 15:41:12 +0100 Subject: [PATCH] [Rework] Another bunch of changes to the dependencies processing --- src/libserver/composites/composites.cxx | 6 +- .../composites/composites_internal.hxx | 6 +- src/libserver/html/html_tag_defs.hxx | 6 +- src/libserver/symcache/symcache_impl.cxx | 65 +++++++++++-------- src/libserver/symcache/symcache_internal.hxx | 4 +- src/libserver/symcache/symcache_item.cxx | 16 ++--- src/libserver/symcache/symcache_item.hxx | 11 ++-- src/libserver/symcache/symcache_runtime.cxx | 26 ++++---- 8 files changed, 72 insertions(+), 68 deletions(-) diff --git a/src/libserver/composites/composites.cxx b/src/libserver/composites/composites.cxx index 64f8f2daf..0e7adfb8a 100644 --- a/src/libserver/composites/composites.cxx +++ b/src/libserver/composites/composites.cxx @@ -1,11 +1,11 @@ -/*- - * Copyright 2021 Vsevolod Stakhov +/* + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/src/libserver/composites/composites_internal.hxx b/src/libserver/composites/composites_internal.hxx index 55aaa2ee1..2ae4219eb 100644 --- a/src/libserver/composites/composites_internal.hxx +++ b/src/libserver/composites/composites_internal.hxx @@ -1,11 +1,11 @@ -/*- - * Copyright 2021 Vsevolod Stakhov +/* + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/src/libserver/html/html_tag_defs.hxx b/src/libserver/html/html_tag_defs.hxx index 4cff79855..05a5e3cde 100644 --- a/src/libserver/html/html_tag_defs.hxx +++ b/src/libserver/html/html_tag_defs.hxx @@ -1,11 +1,11 @@ -/*- - * Copyright 2021 Vsevolod Stakhov +/* + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/src/libserver/symcache/symcache_impl.cxx b/src/libserver/symcache/symcache_impl.cxx index 3a1c187cf..56d3602ae 100644 --- a/src/libserver/symcache/symcache_impl.cxx +++ b/src/libserver/symcache/symcache_impl.cxx @@ -112,29 +112,39 @@ auto symcache::init() -> bool /* Deal with the delayed dependencies */ msg_debug_cache("resolving delayed dependencies: %d in list", (int) delayed_deps->size()); for (const auto &delayed_dep: *delayed_deps) { - auto virt_item = get_item_by_name(delayed_dep.from, false); - auto real_item = get_item_by_name(delayed_dep.from, true); - - if (virt_item == nullptr || real_item == nullptr) { - msg_err_cache("cannot register delayed dependency between %s and %s: " - "%s is missing", - delayed_dep.from.data(), - delayed_dep.to.data(), delayed_dep.from.data()); + auto virt_source = get_item_by_name(delayed_dep.from, false); + auto real_source = get_item_by_name(delayed_dep.from, true); + + auto real_destination = get_item_by_name(delayed_dep.to, true); + + if (virt_source == nullptr || real_source == nullptr || real_destination == nullptr) { + if (real_destination != nullptr) { + msg_err_cache("cannot register delayed dependency %s -> %s: " + "source %s is missing", + delayed_dep.from.data(), + delayed_dep.to.data(), delayed_dep.from.data()); + } + else { + msg_err_cache("cannot register delayed dependency %s -> %s: " + "destionation %s is missing", + delayed_dep.from.data(), + delayed_dep.to.data(), delayed_dep.to.data()); + } } else { - if (!disabled_ids.contains(real_item->id)) { + if (!disabled_ids.contains(real_source->id)) { msg_debug_cache("delayed between %s(%d:%d) -> %s", delayed_dep.from.data(), - real_item->id, virt_item->id, + real_source->id, virt_source->id, delayed_dep.to.data()); - add_dependency(real_item->id, delayed_dep.to, - virt_item != real_item ? virt_item->id : -1); + add_dependency(real_source->id, delayed_dep.to, real_destination->id, + virt_source != real_source ? virt_source->id : -1); } else { msg_debug_cache("no delayed between %s(%d:%d) -> %s; %s is disabled", delayed_dep.from.data(), - real_item->id, virt_item->id, + real_source->id, virt_source->id, delayed_dep.to.data(), delayed_dep.from.data()); } @@ -529,19 +539,21 @@ auto symcache::get_item_by_name_mut(std::string_view name, bool resolve_parent) return it->second; } -auto symcache::add_dependency(int id_from, std::string_view to, int virtual_id_from) -> void +auto symcache::add_dependency(int id_from, std::string_view to, int id_to, int virtual_id_from) -> void { g_assert(id_from >= 0 && id_from < (int) items_by_id.size()); + g_assert(id_to >= 0 && id_to < (int) items_by_id.size()); const auto &source = items_by_id[id_from]; + const auto &dest = items_by_id[id_to]; g_assert(source.get() != nullptr); - - if (!source->deps.contains(id_from)) { - msg_debug_cache("add dependency %s -> %s", - source->symbol.c_str(), to.data()); - source->deps.emplace(id_from, cache_dependency{nullptr, - std::string(to), - id_from, - -1}); + g_assert(dest.get() != nullptr); + + if (!source->deps.contains(id_to)) { + msg_debug_cache("add dependency %s(%d) -> %s(%d)", + source->symbol.c_str(), source->id, to.data(), dest->id); + source->deps.emplace(id_to, cache_dependency{dest.get(), + std::string(to), + -1}); } else { msg_debug_cache("duplicate dependency %s -> %s", @@ -556,13 +568,12 @@ auto symcache::add_dependency(int id_from, std::string_view to, int virtual_id_f const auto &vsource = items_by_id[virtual_id_from]; g_assert(vsource.get() != nullptr); - if (!vsource->deps.contains(virtual_id_from)) { + if (!vsource->deps.contains(id_to)) { msg_debug_cache("add virtual dependency %s -> %s", vsource->symbol.c_str(), to.data()); - vsource->deps.emplace(virtual_id_from, cache_dependency{nullptr, - std::string(to), - -1, - virtual_id_from}); + vsource->deps.emplace(id_to, cache_dependency{dest.get(), + std::string(to), + virtual_id_from}); } else { msg_debug_cache("duplicate virtual dependency %s -> %s", diff --git a/src/libserver/symcache/symcache_internal.hxx b/src/libserver/symcache/symcache_internal.hxx index 255a4b1c1..c7dda51d1 100644 --- a/src/libserver/symcache/symcache_internal.hxx +++ b/src/libserver/symcache/symcache_internal.hxx @@ -1,5 +1,5 @@ /* - * Copyright 2023 Vsevolod Stakhov + * Copyright 2024 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -365,7 +365,7 @@ public: * @param virtual_id_from * @return */ - auto add_dependency(int id_from, std::string_view to, int virtual_id_from) -> void; + auto add_dependency(int id_from, std::string_view to, int id_to, int virtual_id_from) -> void; /** * Add a delayed dependency between symbols that will be resolved on the init stage diff --git a/src/libserver/symcache/symcache_item.cxx b/src/libserver/symcache/symcache_item.cxx index cbbb5c03f..0b5de9f3a 100644 --- a/src/libserver/symcache/symcache_item.cxx +++ b/src/libserver/symcache/symcache_item.cxx @@ -82,7 +82,7 @@ auto cache_item::process_deps(const symcache &cache) -> void msg_debug_cache("process real dependency %s on %s", symbol.c_str(), dep.sym.c_str()); auto *dit = cache.get_item_by_name_mut(dep.sym, true); - if (dep.vid >= 0) { + if (dep.virtual_source_id >= 0) { /* Case of the virtual symbol that depends on another (maybe virtual) symbol */ const auto *vdit = cache.get_item_by_name(dep.sym, false); @@ -94,7 +94,7 @@ auto cache_item::process_deps(const symcache &cache) -> void } else { msg_debug_cache("process virtual dependency %s(%d) on %s(%d)", symbol.c_str(), - dep.vid, vdit->symbol.c_str(), vdit->id); + dep.virtual_source_id, vdit->symbol.c_str(), vdit->id); unsigned nids = 0; @@ -164,7 +164,7 @@ auto cache_item::process_deps(const symcache &cache) -> void if (parent) { if (!dit->rdeps.contains(parent->id)) { - dit->rdeps.emplace(parent->id, cache_dependency{parent, parent->symbol, parent->id, -1}); + dit->rdeps.emplace(parent->id, cache_dependency{parent, parent->symbol, -1}); msg_debug_cache("added reverse dependency from %d on %d", parent->id, dit->id); } @@ -173,7 +173,6 @@ auto cache_item::process_deps(const symcache &cache) -> void parent->id, dit->id); } dep.item = dit; - dep.id = dit->id; } else { msg_err_cache("cannot find parent for virtual symbol %s, when resolving dependency %s", @@ -182,9 +181,8 @@ auto cache_item::process_deps(const symcache &cache) -> void } else { dep.item = dit; - dep.id = dit->id; if (!dit->rdeps.contains(id)) { - dit->rdeps.emplace(id, cache_dependency{this, symbol, id, -1}); + dit->rdeps.emplace(id, cache_dependency{this, symbol, -1}); msg_debug_cache("added reverse dependency from %d on %d", id, dit->id); } @@ -196,12 +194,6 @@ auto cache_item::process_deps(const symcache &cache) -> void } } } - else if (dep.id >= 0) { - msg_err_cache("cannot find dependency on symbol %s for symbol %s", - dep.sym.c_str(), symbol.c_str()); - - continue; - } else { msg_err_cache("cannot find dependency named %s for symbol %s", dep.sym.c_str(), symbol.c_str()); diff --git a/src/libserver/symcache/symcache_item.hxx b/src/libserver/symcache/symcache_item.hxx index acbcf68ff..e446eb9ba 100644 --- a/src/libserver/symcache/symcache_item.hxx +++ b/src/libserver/symcache/symcache_item.hxx @@ -177,14 +177,13 @@ public: }; struct cache_dependency { - cache_item *item; /* Real dependency */ - std::string sym; /* Symbolic dep name */ - int id; /* Real from */ - int vid; /* Virtual from */ + cache_item *item; /* Real dependency */ + std::string sym; /* Symbolic dep name */ + int virtual_source_id; /* Virtual source */ public: /* Default piecewise constructor */ - explicit cache_dependency(cache_item *_item, std::string _sym, int _id, int _vid) - : item(_item), sym(std::move(_sym)), id(_id), vid(_vid) + explicit cache_dependency(cache_item *_item, std::string _sym, int _vid) + : item(_item), sym(std::move(_sym)), virtual_source_id(_vid) { } }; diff --git a/src/libserver/symcache/symcache_runtime.cxx b/src/libserver/symcache/symcache_runtime.cxx index bbf3b3d00..148346cb5 100644 --- a/src/libserver/symcache/symcache_runtime.cxx +++ b/src/libserver/symcache/symcache_runtime.cxx @@ -489,6 +489,8 @@ auto symcache_runtime::process_symbol(struct rspamd_task *task, symcache &cache, cur_item = dyn_item; items_inflight++; /* Callback now must finalize itself */ + + if (item->call(task, dyn_item)) { cur_item = nullptr; @@ -505,10 +507,10 @@ auto symcache_runtime::process_symbol(struct rspamd_task *task, symcache &cache, item->symbol.data()); g_assert_not_reached(); } - - msg_debug_cache_task("item %s, %d is now pending", item->symbol.data(), - item->id); - dyn_item->status = cache_item_status::pending; + else if (dyn_item->async_events > 0) { + msg_debug_cache_task("item %s, %d is now pending with %d async events", item->symbol.data(), + item->id, dyn_item->async_events); + } return false; } @@ -588,11 +590,11 @@ auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache auto ret = true; - for (const auto &[id, dep]: item->deps) { + for (const auto &[dest_id, dep]: item->deps) { if (!dep.item) { /* Assume invalid deps as done */ msg_debug_cache_task_lambda("symbol %d(%s) has invalid dependencies on %d(%s)", - item->id, item->symbol.c_str(), dep.id, dep.sym.c_str()); + item->id, item->symbol.c_str(), dest_id, dep.sym.c_str()); continue; } @@ -610,7 +612,7 @@ auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache ret = false; msg_debug_cache_task_lambda("delayed dependency %d(%s) for " "symbol %d(%s)", - dep.id, dep.sym.c_str(), item->id, item->symbol.c_str()); + dest_id, dep.sym.c_str(), item->id, item->symbol.c_str()); } else if (!process_symbol(task, cache, dep.item, dep_dyn_item)) { /* Now started, but has events pending */ @@ -618,18 +620,18 @@ auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache msg_debug_cache_task_lambda("started check of %d(%s) symbol " "as dep for " "%d(%s)", - dep.id, dep.sym.c_str(), item->id, item->symbol.c_str()); + dest_id, dep.sym.c_str(), item->id, item->symbol.c_str()); } else { msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) is " "already processed", - dep.id, dep.sym.c_str(), item->id, item->symbol.c_str()); + dest_id, dep.sym.c_str(), item->id, item->symbol.c_str()); } } else { msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) " "cannot be started now", - dep.id, dep.sym.c_str(), item->id, item->symbol.c_str()); + dest_id, dep.sym.c_str(), item->id, item->symbol.c_str()); ret = false; } } @@ -637,7 +639,7 @@ auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache /* Started but not finished */ msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) is " "still executing (%d events pending)", - dep.id, dep.sym.c_str(), + dest_id, dep.sym.c_str(), item->id, item->symbol.c_str(), dep_dyn_item->async_events); g_assert(dep_dyn_item->async_events > 0); @@ -647,7 +649,7 @@ auto symcache_runtime::check_item_deps(struct rspamd_task *task, symcache &cache else { msg_debug_cache_task_lambda("dependency %d(%s) for symbol %d(%s) is already " "finished", - dep.id, dep.sym.c_str(), item->id, item->symbol.c_str()); + dest_id, dep.sym.c_str(), item->id, item->symbol.c_str()); } } -- 2.39.5