[Project] Implement dynamic items lookup and processing

4 files changed, 162 insertions, 21 deletions

diff --git a/src/libserver/symcache/symcache_c.cxx b/src/libserver/symcache/symcache_c.cxx
index 14fead7e4..4547f23a4 100644
--- a/src/libserver/symcache/symcache_c.cxx
+++ b/src/libserver/symcache/symcache_c.cxx
@@ -335,9 +335,9 @@ rspamd_symcache_get_allowed_settings_ids(struct rspamd_symcache *cache,
 }
 
 const guint32 *
-rspamd_symcache_get_forbidden_settings_ids (struct rspamd_symcache *cache,
-											const gchar *symbol,
-											guint *nids)
+rspamd_symcache_get_forbidden_settings_ids(struct rspamd_symcache *cache,
+										   const gchar *symbol,
+										   guint *nids)
 {
 	auto *real_cache = C_API_SYMCACHE(cache);
 
@@ -433,5 +433,65 @@ rspamd_symcache_set_cur_item(struct rspamd_task *task, struct rspamd_symcache_it
 	auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
 	auto *real_item = C_API_SYMCACHE_ITEM(item);
 
-	return (struct rspamd_symcache_item *) cache_runtime->set_cur_item(real_item);
+	return (struct rspamd_symcache_item *)cache_runtime->set_cur_item(real_item);
+}
+
+void
+rspamd_symcache_enable_profile(struct rspamd_task *task)
+{
+	auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+
+	cache_runtime->set_profile_mode(true);
+}
+
+guint
+rspamd_symcache_item_async_inc_full(struct rspamd_task *task,
+									struct rspamd_symcache_item *item,
+									const gchar *subsystem,
+									const gchar *loc)
+{
+	auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+	auto *real_item = C_API_SYMCACHE_ITEM(item);
+
+	auto *dyn_item = cache_runtime->get_dynamic_item(real_item->id, true);
+	msg_debug_cache_task("increase async events counter for %s(%d) = %d + 1; "
+						 "subsystem %s (%s)",
+			real_item->symbol.c_str(), real_item->id,
+			dyn_item->async_events, subsystem, loc);
+
+	return ++dyn_item->async_events;
+}
+
+guint
+rspamd_symcache_item_async_dec_full(struct rspamd_task *task,
+									struct rspamd_symcache_item *item,
+									const gchar *subsystem,
+									const gchar *loc)
+{
+	auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+	auto *real_item = C_API_SYMCACHE_ITEM(item);
+
+	auto *dyn_item = cache_runtime->get_dynamic_item(real_item->id, true);
+	msg_debug_cache_task("increase async events counter for %s(%d) = %d + 1; "
+						 "subsystem %s (%s)",
+			real_item->symbol.c_str(), real_item->id,
+			dyn_item->async_events, subsystem, loc);
+	g_assert(dyn_item->async_events > 0);
+
+	return --dyn_item->async_events;
+}
+
+gboolean
+rspamd_symcache_item_async_dec_check_full(struct rspamd_task *task,
+										  struct rspamd_symcache_item *item,
+										  const gchar *subsystem,
+										  const gchar *loc)
+{
+	if (rspamd_symcache_item_async_dec_full(task, item, subsystem, loc) == 0) {
+		rspamd_symcache_finalize_item(task, item);
+
+		return TRUE;
+	}
+
+	return FALSE;
 }
\ No newline at end of file
diff --git a/src/libserver/symcache/symcache_internal.hxx b/src/libserver/symcache/symcache_internal.hxx
index 25eaa4f40..457dc5848 100644
--- a/src/libserver/symcache/symcache_internal.hxx
+++ b/src/libserver/symcache/symcache_internal.hxx
@@ -53,11 +53,11 @@
         RSPAMD_LOG_FUNC, \
         __VA_ARGS__)
 #define msg_debug_cache(...)  rspamd_conditional_debug_fast (NULL, NULL, \
-        rspamd_symcache_log_id, "symcache", log_tag(), \
+        ::rspamd::symcache::rspamd_symcache_log_id, "symcache", log_tag(), \
         RSPAMD_LOG_FUNC, \
         __VA_ARGS__)
 #define msg_debug_cache_task(...)  rspamd_conditional_debug_fast (NULL, NULL, \
-        rspamd_symcache_log_id, "symcache", task->task_pool->tag.uid, \
+        ::rspamd::symcache::rspamd_symcache_log_id, "symcache", task->task_pool->tag.uid, \
         RSPAMD_LOG_FUNC, \
         __VA_ARGS__)
 
diff --git a/src/libserver/symcache/symcache_runtime.cxx b/src/libserver/symcache/symcache_runtime.cxx
index a231bda31..a6ccd5509 100644
--- a/src/libserver/symcache/symcache_runtime.cxx
+++ b/src/libserver/symcache/symcache_runtime.cxx
@@ -43,6 +43,11 @@ symcache_runtime::create_savepoint(struct rspamd_task *task, symcache &cache) ->
 	rspamd_mempool_add_destructor(task->task_pool,
 			symcache_runtime::savepoint_dtor, checkpoint);
 
+	for (auto &pair : checkpoint->last_id_mappings) {
+		pair.first = -1;
+		pair.second = -1;
+	}
+
 	/* Calculate profile probability */
 	ev_now_update_if_cheap(task->event_loop);
 	ev_tstamp now = ev_now(task->event_loop);
@@ -120,7 +125,6 @@ symcache_runtime::process_settings(struct rspamd_task *task, const symcache &cac
 		}
 	}
 
-
 	/* Enable groups of symbols */
 	enabled = ucl_object_lookup(task->settings, "groups_enabled");
 	if (enabled && !already_disabled) {
@@ -132,7 +136,6 @@ symcache_runtime::process_settings(struct rspamd_task *task, const symcache &cac
 
 	const auto *disabled = ucl_object_lookup(task->settings, "symbols_disabled");
 
-
 	if (disabled) {
 		it = nullptr;
 
@@ -170,10 +173,9 @@ symcache_runtime::disable_symbol(struct rspamd_task *task, const symcache &cache
 
 	if (item != nullptr) {
 
-		auto our_id_maybe = rspamd::find_map(order->by_cache_id, item->id);
+		auto *dyn_item = get_dynamic_item(item->id, false);
 
-		if (our_id_maybe) {
-			auto *dyn_item = &dynamic_items[our_id_maybe.value()];
+		if (dyn_item) {
 			dyn_item->finished = true;
 			dyn_item->started = true;
 			msg_debug_cache_task("disable execution of %s", name.data());
@@ -198,10 +200,9 @@ symcache_runtime::enable_symbol(struct rspamd_task *task, const symcache &cache,
 
 	if (item != nullptr) {
 
-		auto our_id_maybe = rspamd::find_map(order->by_cache_id, item->id);
+		auto *dyn_item = get_dynamic_item(item->id, false);
 
-		if (our_id_maybe) {
-			auto *dyn_item = &dynamic_items[our_id_maybe.value()];
+		if (dyn_item) {
 			dyn_item->finished = false;
 			dyn_item->started = false;
 			msg_debug_cache_task("enable execution of %s", name.data());
@@ -226,10 +227,9 @@ symcache_runtime::is_symbol_checked(const symcache &cache, std::string_view name
 
 	if (item != nullptr) {
 
-		auto our_id_maybe = rspamd::find_map(order->by_cache_id, item->id);
+		auto *dyn_item = get_dynamic_item(item->id, true);
 
-		if (our_id_maybe) {
-			auto *dyn_item = &dynamic_items[our_id_maybe.value()];
+		if (dyn_item) {
 			return dyn_item->started;
 		}
 	}
@@ -248,10 +248,9 @@ symcache_runtime::is_symbol_enabled(struct rspamd_task *task, const symcache &ca
 			return false;
 		}
 		else {
-			auto our_id_maybe = rspamd::find_map(order->by_cache_id, item->id);
+			auto *dyn_item = get_dynamic_item(item->id, true);
 
-			if (our_id_maybe) {
-				auto *dyn_item = &dynamic_items[our_id_maybe.value()];
+			if (dyn_item) {
 				if (dyn_item->started) {
 					/* Already started */
 					return false;
@@ -271,5 +270,59 @@ symcache_runtime::is_symbol_enabled(struct rspamd_task *task, const symcache &ca
 	return true;
 }
 
+auto symcache_runtime::get_dynamic_item(int id, bool save_in_cache) const -> cache_dynamic_item *
+{
+	/* Lookup in cache */
+	if (save_in_cache) {
+		for (const auto &cache_id : last_id_mappings) {
+			if (cache_id.first == -1) {
+				break;
+			}
+			if (cache_id.first == id) {
+				auto *dyn_item = &dynamic_items[cache_id.second];
+
+				return dyn_item;
+			}
+		}
+	}
+
+	/* Not found in the cache, do a hash lookup */
+	auto our_id_maybe = rspamd::find_map(order->by_cache_id, id);
+
+	if (our_id_maybe) {
+		auto *dyn_item = &dynamic_items[our_id_maybe.value()];
+
+		if (!save_in_cache) {
+			return dyn_item;
+		}
+
+		/* Insert in the cache, swapping the first item with the last empty item */
+		auto first_known = last_id_mappings[0];
+		last_id_mappings[0].first = id;
+		last_id_mappings[0].second = our_id_maybe.value();
+
+		if (first_known.first != -1) {
+			/* This loop is guaranteed to finish as we have just inserted one item */
+
+			constexpr const auto cache_size = sizeof(last_id_mappings) / sizeof(last_id_mappings[0]);
+			int i = cache_size - 1;
+			for (;; --i) {
+				if (last_id_mappings[i].first != -1) {
+					if (i < cache_size - 1) {
+						i++;
+					}
+					break;
+				}
+			}
+
+			last_id_mappings[i] = first_known;
+		}
+
+		return dyn_item;
+	}
+
+	return nullptr;
+}
+
 }
 
diff --git a/src/libserver/symcache/symcache_runtime.hxx b/src/libserver/symcache/symcache_runtime.hxx
index 1f237b5b3..a505f22e8 100644
--- a/src/libserver/symcache/symcache_runtime.hxx
+++ b/src/libserver/symcache/symcache_runtime.hxx
@@ -56,8 +56,10 @@ class symcache_runtime {
 
 	struct cache_item *cur_item;
 	order_generation_ptr order;
+	/* Cache of the last items to speed up lookups */
+	mutable std::pair<int, int> last_id_mappings[8];
 	/* Dynamically expanded as needed */
-	struct cache_dynamic_item dynamic_items[];
+	mutable struct cache_dynamic_item dynamic_items[];
 	/* We allocate this structure merely in memory pool, so destructor is absent */
 	~symcache_runtime() = delete;
 	/* Dropper for a shared ownership */
@@ -119,14 +121,40 @@ public:
 	 */
 	auto is_symbol_enabled(struct rspamd_task *task, const symcache &cache, std::string_view name) -> bool;
 
+	/**
+	 * Get the current processed item
+	 * @return
+	 */
 	auto get_cur_item() const -> auto {
 		return cur_item;
 	}
 
+	/**
+	 * Set the current processed item
+	 * @param item
+	 * @return
+	 */
 	auto set_cur_item(cache_item *item) -> auto {
 		std::swap(item, cur_item);
 		return item;
 	}
+
+	/**
+	 * Set profile mode for the runtime
+	 * @param enable
+	 * @return
+	 */
+	auto set_profile_mode(bool enable) -> auto {
+		std::swap(profile, enable);
+		return enable;
+	}
+
+	/**
+	 * Returns the dynamic item by static item id
+	 * @param id
+	 * @return
+	 */
+	auto get_dynamic_item(int id, bool save_in_cache) const -> cache_dynamic_item *;
 };