From 291383ca94f38ff208b634d0723a163c53f43cd9 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 1 Apr 2022 18:33:28 +0100 Subject: [PATCH] [Project] Further rework --- src/libserver/rspamd_symcache.cxx | 144 +++++++++++++++++++++++------- 1 file changed, 114 insertions(+), 30 deletions(-) diff --git a/src/libserver/rspamd_symcache.cxx b/src/libserver/rspamd_symcache.cxx index a1aa8c504..8474b2f0e 100644 --- a/src/libserver/rspamd_symcache.cxx +++ b/src/libserver/rspamd_symcache.cxx @@ -25,6 +25,7 @@ #include "libserver/worker_util.h" #include +#include #include #include #include @@ -84,15 +85,15 @@ struct rspamd_symcache_header { }; struct cache_item; -using cache_item_ptr = rspamd::local_shared_ptr; -using cache_item_weak_ptr = rspamd::local_weak_ptr; +using cache_item_ptr = std::shared_ptr; +using cache_item_weak_ptr = std::weak_ptr; struct order_generation { std::vector d; unsigned int generation_id; }; -using order_generation_ptr = rspamd::local_shared_ptr; +using order_generation_ptr = std::shared_ptr; /* * This structure is optimised to store ids list: @@ -101,14 +102,89 @@ using order_generation_ptr = rspamd::local_shared_ptr; */ struct id_list { union { - guint32 st[4]; + std::uint32_t st[4]; struct { - guint32 e; /* First element */ - guint16 len; - guint16 allocated; - guint *n; + std::uint32_t e; /* First element */ + std::uint16_t len; + std::uint16_t allocated; + std::uint32_t *n; } dyn; - }; + } data; + + id_list() { + std::memset((void *)&data, 0, sizeof(data)); + } + /** + * Returns ids from a compressed list, accepting a mutable reference for number of elements + * @param nids output of the number of elements + * @return + */ + auto get_ids(std::size_t &nids) const -> const std::uint32_t * { + if (data.dyn.e == -1) { + /* Dynamic list */ + nids = data.dyn.len; + + return data.dyn.n; + } + else { + auto cnt = 0; + + while (data.st[cnt] != 0 && cnt < G_N_ELEMENTS(data.st)) { + cnt ++; + } + + nids = cnt; + + return data.st; + } + } + + auto add_id(std::uint32_t id, rspamd_mempool_t *pool) -> void { + if (data.st[0] == -1) { + /* Dynamic array */ + if (data.dyn.len < data.dyn.allocated) { + /* Trivial, append + sort */ + data.dyn.n[data.dyn.len++] = id; + } + else { + /* Reallocate */ + g_assert (data.dyn.allocated <= G_MAXINT16); + data.dyn.allocated *= 2; + + auto *new_array = rspamd_mempool_alloc_array_type(pool, + data.dyn.allocated, std::uint32_t); + memcpy(new_array, data.dyn.n, data.dyn.len * sizeof(std::uint32_t)); + data.dyn.n = new_array; + data.dyn.n[data.dyn.len++] = id; + } + + std::sort(data.dyn.n, data.dyn.n + data.dyn.len); + } + else { + /* Static part */ + auto cnt = 0u; + while (data.st[cnt] != 0 && cnt < G_N_ELEMENTS (data.st)) { + cnt ++; + } + + if (cnt < G_N_ELEMENTS (data.st)) { + data.st[cnt] = id; + } + else { + /* Switch to dynamic */ + data.dyn.allocated = G_N_ELEMENTS (data.st) * 2; + auto *new_array = rspamd_mempool_alloc_array_type(pool, + data.dyn.allocated, std::uint32_t); + memcpy (new_array, data.st, sizeof(data.st)); + data.dyn.n = new_array; + data.dyn.e = -1; /* Marker */ + data.dyn.len = G_N_ELEMENTS (data.st); + + /* Recursively jump to dynamic branch that will handle insertion + sorting */ + add_id(id, pool); // tail call + } + } + } }; struct item_condition { @@ -198,7 +274,7 @@ struct delayed_cache_condition { lua_State *L; }; -struct rspamd_symcache { +struct symcache { /* Map indexed by symbol name: all symbols must have unique names, so this map holds ownership */ robin_hood::unordered_flat_map items_by_symbol; std::vector items_by_id; @@ -215,8 +291,9 @@ struct rspamd_symcache { std::vector idempotent; std::vector virtual_symbols; - std::vector delayed_deps; - std::vector delayed_conditions; + /* These are stored within pointer to clean up after init */ + std::unique_ptr> delayed_deps; + std::unique_ptr> delayed_conditions; rspamd_mempool_t *static_pool; std::uint64_t cksum; @@ -231,44 +308,51 @@ struct rspamd_symcache { int peak_cb; }; +/* + * These items are saved within task structure and are used to track + * symbols execution + */ struct cache_dynamic_item { - guint16 start_msec; /* Relative to task time */ + std::uint16_t start_msec; /* Relative to task time */ unsigned started: 1; unsigned finished: 1; /* unsigned pad:14; */ - guint32 async_events; + std::uint32_t async_events; }; struct cache_dependency { - struct rspamd_symcache_item *item; /* Real dependency */ - gchar *sym; /* Symbolic dep name */ - gint id; /* Real from */ - gint vid; /* Virtual from */ + cache_item_ptr item; /* Owning pointer to the real dep */ + std::string_view sym; /* Symbolic dep name */ + int id; /* Real from */ + int vid; /* Virtual from */ }; struct cache_savepoint { - guint version; - guint items_inflight; - gboolean profile; - gboolean has_slow; - gdouble profile_start; + unsigned order_gen; + unsigned items_inflight; + bool profile; + bool has_slow; + + double profile_start; + double lim; struct rspamd_scan_result *rs; - gdouble lim; - struct rspamd_symcache_item *cur_item; - struct symcache_order *order; - struct rspamd_symcache_dynamic_item dynamic_items[]; + struct cache_item *cur_item; + order_generation_ptr order; + /* Dynamically expanded as needed */ + struct cache_dynamic_item dynamic_items[]; }; -struct rspamd_cache_refresh_cbdata { - gdouble last_resort; +struct cache_refresh_cbdata { + double last_resort; ev_timer resort_ev; - struct rspamd_symcache *cache; + struct symcache *cache; struct rspamd_worker *w; struct ev_loop *event_loop; }; + } // namespace rspamd /* At least once per minute */ -- 2.39.5