]> source.dussan.org Git - rspamd.git/commitdiff
[Project] Implement dynamic items lookup and processing
authorVsevolod Stakhov <vsevolod@rspamd.com>
Sat, 30 Apr 2022 12:14:25 +0000 (13:14 +0100)
committerVsevolod Stakhov <vsevolod@rspamd.com>
Sat, 30 Apr 2022 12:14:25 +0000 (13:14 +0100)
src/libserver/symcache/symcache_c.cxx
src/libserver/symcache/symcache_internal.hxx
src/libserver/symcache/symcache_runtime.cxx
src/libserver/symcache/symcache_runtime.hxx

index 14fead7e4987777903093af90bd4a2f1daae58a2..4547f23a4a8a94539b560570c5af757ce63527dc 100644 (file)
@@ -335,9 +335,9 @@ rspamd_symcache_get_allowed_settings_ids(struct rspamd_symcache *cache,
 }
 
 const guint32 *
-rspamd_symcache_get_forbidden_settings_ids (struct rspamd_symcache *cache,
-                                                                                       const gchar *symbol,
-                                                                                       guint *nids)
+rspamd_symcache_get_forbidden_settings_ids(struct rspamd_symcache *cache,
+                                                                                  const gchar *symbol,
+                                                                                  guint *nids)
 {
        auto *real_cache = C_API_SYMCACHE(cache);
 
@@ -433,5 +433,65 @@ rspamd_symcache_set_cur_item(struct rspamd_task *task, struct rspamd_symcache_it
        auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
        auto *real_item = C_API_SYMCACHE_ITEM(item);
 
-       return (struct rspamd_symcache_item *) cache_runtime->set_cur_item(real_item);
+       return (struct rspamd_symcache_item *)cache_runtime->set_cur_item(real_item);
+}
+
+void
+rspamd_symcache_enable_profile(struct rspamd_task *task)
+{
+       auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+
+       cache_runtime->set_profile_mode(true);
+}
+
+guint
+rspamd_symcache_item_async_inc_full(struct rspamd_task *task,
+                                                                       struct rspamd_symcache_item *item,
+                                                                       const gchar *subsystem,
+                                                                       const gchar *loc)
+{
+       auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+       auto *real_item = C_API_SYMCACHE_ITEM(item);
+
+       auto *dyn_item = cache_runtime->get_dynamic_item(real_item->id, true);
+       msg_debug_cache_task("increase async events counter for %s(%d) = %d + 1; "
+                                                "subsystem %s (%s)",
+                       real_item->symbol.c_str(), real_item->id,
+                       dyn_item->async_events, subsystem, loc);
+
+       return ++dyn_item->async_events;
+}
+
+guint
+rspamd_symcache_item_async_dec_full(struct rspamd_task *task,
+                                                                       struct rspamd_symcache_item *item,
+                                                                       const gchar *subsystem,
+                                                                       const gchar *loc)
+{
+       auto *cache_runtime = C_API_SYMCACHE_RUNTIME(task->symcache_runtime);
+       auto *real_item = C_API_SYMCACHE_ITEM(item);
+
+       auto *dyn_item = cache_runtime->get_dynamic_item(real_item->id, true);
+       msg_debug_cache_task("increase async events counter for %s(%d) = %d + 1; "
+                                                "subsystem %s (%s)",
+                       real_item->symbol.c_str(), real_item->id,
+                       dyn_item->async_events, subsystem, loc);
+       g_assert(dyn_item->async_events > 0);
+
+       return --dyn_item->async_events;
+}
+
+gboolean
+rspamd_symcache_item_async_dec_check_full(struct rspamd_task *task,
+                                                                                 struct rspamd_symcache_item *item,
+                                                                                 const gchar *subsystem,
+                                                                                 const gchar *loc)
+{
+       if (rspamd_symcache_item_async_dec_full(task, item, subsystem, loc) == 0) {
+               rspamd_symcache_finalize_item(task, item);
+
+               return TRUE;
+       }
+
+       return FALSE;
 }
\ No newline at end of file
index 25eaa4f401f59faa8ff1688036008e4c37c89b8c..457dc58483c3f8f1984a71b453df98d2d3dc6be1 100644 (file)
         RSPAMD_LOG_FUNC, \
         __VA_ARGS__)
 #define msg_debug_cache(...)  rspamd_conditional_debug_fast (NULL, NULL, \
-        rspamd_symcache_log_id, "symcache", log_tag(), \
+        ::rspamd::symcache::rspamd_symcache_log_id, "symcache", log_tag(), \
         RSPAMD_LOG_FUNC, \
         __VA_ARGS__)
 #define msg_debug_cache_task(...)  rspamd_conditional_debug_fast (NULL, NULL, \
-        rspamd_symcache_log_id, "symcache", task->task_pool->tag.uid, \
+        ::rspamd::symcache::rspamd_symcache_log_id, "symcache", task->task_pool->tag.uid, \
         RSPAMD_LOG_FUNC, \
         __VA_ARGS__)
 
index a231bda3158639be79d2bb9df299f72e65f080cf..a6ccd550952b8f05c539239abc3e83f8ae51dd0f 100644 (file)
@@ -43,6 +43,11 @@ symcache_runtime::create_savepoint(struct rspamd_task *task, symcache &cache) ->
        rspamd_mempool_add_destructor(task->task_pool,
                        symcache_runtime::savepoint_dtor, checkpoint);
 
+       for (auto &pair : checkpoint->last_id_mappings) {
+               pair.first = -1;
+               pair.second = -1;
+       }
+
        /* Calculate profile probability */
        ev_now_update_if_cheap(task->event_loop);
        ev_tstamp now = ev_now(task->event_loop);
@@ -120,7 +125,6 @@ symcache_runtime::process_settings(struct rspamd_task *task, const symcache &cac
                }
        }
 
-
        /* Enable groups of symbols */
        enabled = ucl_object_lookup(task->settings, "groups_enabled");
        if (enabled && !already_disabled) {
@@ -132,7 +136,6 @@ symcache_runtime::process_settings(struct rspamd_task *task, const symcache &cac
 
        const auto *disabled = ucl_object_lookup(task->settings, "symbols_disabled");
 
-
        if (disabled) {
                it = nullptr;
 
@@ -170,10 +173,9 @@ symcache_runtime::disable_symbol(struct rspamd_task *task, const symcache &cache
 
        if (item != nullptr) {
 
-               auto our_id_maybe = rspamd::find_map(order->by_cache_id, item->id);
+               auto *dyn_item = get_dynamic_item(item->id, false);
 
-               if (our_id_maybe) {
-                       auto *dyn_item = &dynamic_items[our_id_maybe.value()];
+               if (dyn_item) {
                        dyn_item->finished = true;
                        dyn_item->started = true;
                        msg_debug_cache_task("disable execution of %s", name.data());
@@ -198,10 +200,9 @@ symcache_runtime::enable_symbol(struct rspamd_task *task, const symcache &cache,
 
        if (item != nullptr) {
 
-               auto our_id_maybe = rspamd::find_map(order->by_cache_id, item->id);
+               auto *dyn_item = get_dynamic_item(item->id, false);
 
-               if (our_id_maybe) {
-                       auto *dyn_item = &dynamic_items[our_id_maybe.value()];
+               if (dyn_item) {
                        dyn_item->finished = false;
                        dyn_item->started = false;
                        msg_debug_cache_task("enable execution of %s", name.data());
@@ -226,10 +227,9 @@ symcache_runtime::is_symbol_checked(const symcache &cache, std::string_view name
 
        if (item != nullptr) {
 
-               auto our_id_maybe = rspamd::find_map(order->by_cache_id, item->id);
+               auto *dyn_item = get_dynamic_item(item->id, true);
 
-               if (our_id_maybe) {
-                       auto *dyn_item = &dynamic_items[our_id_maybe.value()];
+               if (dyn_item) {
                        return dyn_item->started;
                }
        }
@@ -248,10 +248,9 @@ symcache_runtime::is_symbol_enabled(struct rspamd_task *task, const symcache &ca
                        return false;
                }
                else {
-                       auto our_id_maybe = rspamd::find_map(order->by_cache_id, item->id);
+                       auto *dyn_item = get_dynamic_item(item->id, true);
 
-                       if (our_id_maybe) {
-                               auto *dyn_item = &dynamic_items[our_id_maybe.value()];
+                       if (dyn_item) {
                                if (dyn_item->started) {
                                        /* Already started */
                                        return false;
@@ -271,5 +270,59 @@ symcache_runtime::is_symbol_enabled(struct rspamd_task *task, const symcache &ca
        return true;
 }
 
+auto symcache_runtime::get_dynamic_item(int id, bool save_in_cache) const -> cache_dynamic_item *
+{
+       /* Lookup in cache */
+       if (save_in_cache) {
+               for (const auto &cache_id : last_id_mappings) {
+                       if (cache_id.first == -1) {
+                               break;
+                       }
+                       if (cache_id.first == id) {
+                               auto *dyn_item = &dynamic_items[cache_id.second];
+
+                               return dyn_item;
+                       }
+               }
+       }
+
+       /* Not found in the cache, do a hash lookup */
+       auto our_id_maybe = rspamd::find_map(order->by_cache_id, id);
+
+       if (our_id_maybe) {
+               auto *dyn_item = &dynamic_items[our_id_maybe.value()];
+
+               if (!save_in_cache) {
+                       return dyn_item;
+               }
+
+               /* Insert in the cache, swapping the first item with the last empty item */
+               auto first_known = last_id_mappings[0];
+               last_id_mappings[0].first = id;
+               last_id_mappings[0].second = our_id_maybe.value();
+
+               if (first_known.first != -1) {
+                       /* This loop is guaranteed to finish as we have just inserted one item */
+
+                       constexpr const auto cache_size = sizeof(last_id_mappings) / sizeof(last_id_mappings[0]);
+                       int i = cache_size - 1;
+                       for (;; --i) {
+                               if (last_id_mappings[i].first != -1) {
+                                       if (i < cache_size - 1) {
+                                               i++;
+                                       }
+                                       break;
+                               }
+                       }
+
+                       last_id_mappings[i] = first_known;
+               }
+
+               return dyn_item;
+       }
+
+       return nullptr;
+}
+
 }
 
index 1f237b5b3cbfa82977d95a5a4e667c61735ab00e..a505f22e8e0000af3ad570076300e82a34da1990 100644 (file)
@@ -56,8 +56,10 @@ class symcache_runtime {
 
        struct cache_item *cur_item;
        order_generation_ptr order;
+       /* Cache of the last items to speed up lookups */
+       mutable std::pair<int, int> last_id_mappings[8];
        /* Dynamically expanded as needed */
-       struct cache_dynamic_item dynamic_items[];
+       mutable struct cache_dynamic_item dynamic_items[];
        /* We allocate this structure merely in memory pool, so destructor is absent */
        ~symcache_runtime() = delete;
        /* Dropper for a shared ownership */
@@ -119,14 +121,40 @@ public:
         */
        auto is_symbol_enabled(struct rspamd_task *task, const symcache &cache, std::string_view name) -> bool;
 
+       /**
+        * Get the current processed item
+        * @return
+        */
        auto get_cur_item() const -> auto {
                return cur_item;
        }
 
+       /**
+        * Set the current processed item
+        * @param item
+        * @return
+        */
        auto set_cur_item(cache_item *item) -> auto {
                std::swap(item, cur_item);
                return item;
        }
+
+       /**
+        * Set profile mode for the runtime
+        * @param enable
+        * @return
+        */
+       auto set_profile_mode(bool enable) -> auto {
+               std::swap(profile, enable);
+               return enable;
+       }
+
+       /**
+        * Returns the dynamic item by static item id
+        * @param id
+        * @return
+        */
+       auto get_dynamic_item(int id, bool save_in_cache) const -> cache_dynamic_item *;
 };