From d78c3739088d2701ea0546132c8036018dcfe068 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Tue, 24 Jan 2017 15:13:41 +0000 Subject: [PATCH] [Rework] Multiple fixes for symbols cache statistics --- src/libserver/symbols_cache.c | 82 ++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 26 deletions(-) diff --git a/src/libserver/symbols_cache.c b/src/libserver/symbols_cache.c index 9292df7c0..367948403 100644 --- a/src/libserver/symbols_cache.c +++ b/src/libserver/symbols_cache.c @@ -88,8 +88,10 @@ struct counter_data { struct item_stat { gdouble avg_time; gdouble weight; - guint64 frequency; - guint64 avg_counter; + guint hits; + guint64 total_hits; + gdouble avg_frequency; + gdouble stddev_frequency; }; struct cache_item { @@ -116,6 +118,7 @@ struct cache_item { /* Priority */ gint priority; gint id; + gint frequency_peaks; /* Dependencies */ GPtrArray *deps; @@ -296,8 +299,8 @@ cache_logic_cmp (const void *p1, const void *p2, gpointer ud) else if (i1->priority == i2->priority) { avg_freq = (cache->total_freq / cache->used_items); avg_weight = (cache->total_weight / cache->used_items); - f1 = (double)i1->st->frequency / avg_freq; - f2 = (double)i2->st->frequency / avg_freq; + f1 = (double)i1->st->hits / avg_freq; + f2 = (double)i2->st->hits / avg_freq; weight1 = fabs (i1->st->weight) / avg_weight; weight2 = fabs (i2->st->weight) / avg_weight; t1 = i1->st->avg_time; @@ -586,12 +589,23 @@ rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name) elt = ucl_object_lookup (cur, "count"); if (elt) { - item->st->avg_counter = ucl_object_toint (elt); + item->st->total_hits = ucl_object_toint (elt); } elt = ucl_object_lookup (cur, "frequency"); - if (elt) { - item->st->frequency = ucl_object_toint (elt); + if (elt && ucl_object_type (elt) == UCL_OBJECT) { + const ucl_object_t *cur; + + cur = ucl_object_lookup (elt, "avg"); + + if (cur) { + item->st->avg_frequency = ucl_object_todouble (cur); + } + cur = ucl_object_lookup (elt, "stddev"); + + if (cur) { + item->st->stddev_frequency = ucl_object_todouble (cur); + } } if ((item->type & SYMBOL_TYPE_VIRTUAL) && item->parent != -1) { @@ -607,11 +621,11 @@ rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name) * parent item avg_time */ parent->st->avg_time = item->st->avg_time; - parent->st->avg_counter = item->st->avg_counter; + parent->st->total_hits = item->st->total_hits; } cache->total_weight += fabs (item->st->weight); - cache->total_freq += item->st->frequency; + cache->total_freq += item->st->hits; } } @@ -625,7 +639,7 @@ static gboolean rspamd_symbols_cache_save_items (struct symbols_cache *cache, const gchar *name) { struct rspamd_symbols_cache_header hdr; - ucl_object_t *top, *elt; + ucl_object_t *top, *elt, *freq; GHashTableIter it; struct cache_item *item; struct ucl_emitter_functions *efunc; @@ -667,10 +681,15 @@ rspamd_symbols_cache_save_items (struct symbols_cache *cache, const gchar *name) "weight", 0, false); ucl_object_insert_key (elt, ucl_object_fromdouble (item->st->avg_time), "time", 0, false); - ucl_object_insert_key (elt, ucl_object_fromdouble (item->st->avg_counter), + ucl_object_insert_key (elt, ucl_object_fromdouble (item->st->total_hits), "count", 0, false); - ucl_object_insert_key (elt, ucl_object_fromint (item->st->frequency), - "frequency", 0, false); + + freq = ucl_object_typed_new (UCL_OBJECT); + ucl_object_insert_key (freq, ucl_object_fromdouble (item->st->avg_frequency), + "avg", 0, false); + ucl_object_insert_key (freq, ucl_object_fromdouble (item->st->stddev_frequency), + "stddev", 0, false); + ucl_object_insert_key (elt, freq, "frequency", 0, false); ucl_object_insert_key (top, elt, k, 0, false); } @@ -754,7 +773,6 @@ rspamd_symbols_cache_add_symbol (struct symbols_cache *cache, cache->cksum); } - rspamd_set_counter (item, 0); g_ptr_array_add (cache->items_by_id, item); item->deps = g_ptr_array_new (); item->rdeps = g_ptr_array_new (); @@ -1789,16 +1807,20 @@ rspamd_symbols_cache_counters_cb (gpointer v, gpointer ud) item->parent); ucl_object_insert_key (obj, ucl_object_fromdouble (item->st->weight), "weight", 0, false); - ucl_object_insert_key (obj, ucl_object_fromint (item->st->frequency), + ucl_object_insert_key (obj, ucl_object_fromdouble (parent->st->avg_frequency), "frequency", 0, false); + ucl_object_insert_key (obj, ucl_object_fromint (parent->st->total_hits), + "hits", 0, false); ucl_object_insert_key (obj, ucl_object_fromdouble (parent->st->avg_time), "time", 0, false); } else { ucl_object_insert_key (obj, ucl_object_fromdouble (item->st->weight), "weight", 0, false); - ucl_object_insert_key (obj, ucl_object_fromint (item->st->frequency), + ucl_object_insert_key (obj, ucl_object_fromdouble (item->st->avg_frequency), "frequency", 0, false); + ucl_object_insert_key (obj, ucl_object_fromint (item->st->total_hits), + "hits", 0, false); ucl_object_insert_key (obj, ucl_object_fromdouble (item->st->avg_time), "time", 0, false); } @@ -1851,18 +1873,20 @@ rspamd_symbols_cache_resort_cb (gint fd, short what, gpointer ud) /* Gather stats from shared execution times */ for (i = 0; i < cache->items_by_id->len; i ++) { item = g_ptr_array_index (cache->items_by_id, i); + if (item->cd->number > 0) { + item->st->total_hits += item->cd->number; - if (item->type & (SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_NORMAL)) { - if (item->cd->number > 0) { - item->st->avg_counter += item->cd->number + 1; + if (item->type & (SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_NORMAL)) { item->st->avg_time = item->st->avg_time + (item->cd->mean - item->st->avg_time) / - (gdouble)item->st->avg_counter; + (gdouble)item->st->total_hits; item->cd->mean = item->st->avg_time; - item->cd->number = item->st->avg_counter; } + + item->cd->number = item->st->total_hits; } } + /* Sync virtual symbols */ for (i = 0; i < cache->items_by_id->len; i ++) { item = g_ptr_array_index (cache->items_by_id, i); @@ -1872,11 +1896,18 @@ rspamd_symbols_cache_resort_cb (gint fd, short what, gpointer ud) if (parent) { item->st->avg_time = parent->st->avg_time; - item->st->avg_counter = parent->st->avg_counter; } } } + if (cbdata->w->index == 0) { + /* We also calculate frequencies */ + for (i = 0; i < cache->items_by_id->len; i ++) { + item = g_ptr_array_index (cache->items_by_id, i); + + } + } + rspamd_mempool_unlock_mutex (cache->mtx); } @@ -1916,14 +1947,13 @@ rspamd_symbols_cache_inc_frequency (struct symbols_cache *cache, item = g_hash_table_lookup (cache->items_by_symbol, symbol); if (item != NULL) { - /* We assume ++ as atomic op */ - item->st->frequency ++; + g_atomic_int_inc (&item->st->hits); cache->total_freq ++; /* For virtual symbols we also increase counter for parent */ if (item->parent != -1) { parent = g_ptr_array_index (cache->items_by_id, item->parent); - parent->st->frequency ++; + g_atomic_int_inc (&parent->st->hits); } } } @@ -1999,7 +2029,7 @@ rspamd_symbols_cache_stat_symbol (struct symbols_cache *cache, item = g_hash_table_lookup (cache->items_by_symbol, name); if (item != NULL) { - *frequency = item->st->frequency; + *frequency = item->st->hits; *tm = item->st->avg_time; return TRUE; -- 2.39.5