Rework saving and load of symbols cache.

diff --git a/src/libserver/symbols_cache.c b/src/libserver/symbols_cache.c
index cf40bf614b399ea692f86c88b1497fb2db15fd1a..5d4be988974b0d1d0ca35c0460715d882c0cc50e 100644 (file)
--- a/src/libserver/symbols_cache.c
+++ b/src/libserver/symbols_cache.c
@@ -28,6 +28,7 @@
 #include "message.h"
 #include "symbols_cache.h"
 #include "cfg_file.h"
+#include "blake2.h"
 
 /* After which number of messages try to resort cache */
 #define MAX_USES 100
@@ -39,6 +40,14 @@
 
 static guint64 total_frequency = 0;
 static guint32 nsymbols = 0;
+static const guchar rspamd_symbols_cache_magic[] = {'r', 's', 'c', 1, 0, 0 };
+
+struct rspamd_symbols_cache_header {
+       guchar magic;
+       guint nitems;
+       guchar checksum[BLAKE2B_OUTBYTES];
+       guchar unused[128];
+};
 
 gint
 cache_cmp (const void *p1, const void *p2)
@@ -301,6 +310,173 @@ create_cache_file (struct symbols_cache *cache,
        return mmap_cache_file (cache, fd, pool);
 }
 
+static gboolean
+rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name)
+{
+       struct rspamd_symbols_cache_header *hdr;
+       struct stat st;
+       struct ucl_parser *parser;
+       ucl_object_t *top;
+       const ucl_object_t *cur, *elt;
+       ucl_object_iter_t it;
+       struct cache_item *item;
+       const guchar *p;
+       gint fd;
+       gpointer map;
+
+       fd = open (name, O_RDONLY);
+
+       if (fd == -1) {
+               msg_info ("cannot open file %s, error %d, %s", name,
+                       errno, strerror (errno));
+               return FALSE;
+       }
+
+       if (fstat (fd, &st) == -1) {
+               close (fd);
+               msg_info ("cannot stat file %s, error %d, %s", name,
+                               errno, strerror (errno));
+               return FALSE;
+       }
+
+       if (st.st_size < sizeof (*hdr)) {
+               close (fd);
+               errno = EINVAL;
+               msg_info ("cannot use file %s, error %d, %s", name,
+                               errno, strerror (errno));
+               return FALSE;
+       }
+
+       map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
+
+       if (map == MAP_FAILED) {
+               close (fd);
+               msg_info ("cannot mmap file %s, error %d, %s", name,
+                               errno, strerror (errno));
+               return FALSE;
+       }
+
+       close (fd);
+       hdr = map;
+
+       if (memcmp (hdr->magic, rspamd_symbols_cache_magic,
+                       sizeof (rspamd_symbols_cache_magic)) == NULL) {
+               msg_info ("cannot use file %s, bad magic", name);
+               munmap (map, st.st_size);
+               return FALSE;
+       }
+
+       parser = ucl_parser_new (0);
+       p = hdr + 1;
+
+       if (!ucl_parser_add_chunk (parser, p, st.st_size - sizeof (*hdr))) {
+               msg_info ("cannot use file %s, cannot parse: %s", name,
+                               ucl_parser_get_error (parser));
+               munmap (map, st.st_size);
+               ucl_parser_free (parser);
+               return FALSE;
+       }
+
+       top = ucl_parser_get_object (parser);
+       munmap (map, st.st_size);
+       ucl_parser_free (parser);
+
+       if (top == NULL || ucl_object_type (top) != UCL_OBJECT) {
+               msg_info ("cannot use file %s, bad object", name);
+               ucl_object_unref (top);
+               return FALSE;
+       }
+
+       it = ucl_object_iterate_new (top);
+
+       while ((cur = ucl_object_iterate_safe (it, true))) {
+               item = g_hash_table_lookup (cache->items_by_symbol, ucl_object_key (cur));
+
+               if (item) {
+                       /* Copy saved info */
+                       elt = ucl_object_find_key (cur, "weight");
+
+                       if (elt) {
+                               item->weight = ucl_object_todouble (cur);
+                       }
+
+                       elt = ucl_object_find_key (cur, "time");
+
+                       if (elt) {
+                               item->avg_time = ucl_object_todouble (cur);
+                       }
+
+                       elt = ucl_object_find_key (cur, "frequency");
+
+                       if (elt) {
+                               item->frequency = ucl_object_toint (cur);
+                       }
+               }
+       }
+
+       ucl_object_iterate_free (it);
+       ucl_object_unref (top);
+
+       return TRUE;
+}
+
+static gboolean
+rspamd_symbols_cache_save_items (struct symbols_cache *cache, const gchar *name)
+{
+       struct rspamd_symbols_cache_header hdr;
+       ucl_object_t *top, *elt;
+       GHashTableIter it;
+       struct cache_item *item;
+       struct ucl_emitter_functions *efunc;
+       gpointer k, v;
+       gint fd;
+       bool ret;
+
+       fd = open (name, O_CREAT | O_TRUNC | O_WRONLY | O_EXCL, 00644);
+
+       if (fd == -1) {
+               msg_info ("cannot open file %s, error %d, %s", name,
+                               errno, strerror (errno));
+               return FALSE;
+       }
+
+       memset (&hdr, 0, sizeof (hdr));
+       memcpy (hdr->magic, rspamd_symbols_cache_magic,
+                       sizeof (rspamd_symbols_cache_magic));
+
+       if (write (fd, &hdr, sizeof (hdr)) == -1) {
+               msg_info ("cannot write to file %s, error %d, %s", name,
+                               errno, strerror (errno));
+               close (fd);
+
+               return FALSE;
+       }
+
+       top = ucl_object_typed_new (UCL_OBJECT);
+
+       g_hash_table_iter_init (&it, cache->items_by_symbol);
+
+       while (g_hash_table_iter_next (&it, &k, &v)) {
+               item = v;
+               elt = ucl_object_typed_new (UCL_OBJECT);
+               ucl_object_insert_key (elt, ucl_object_fromdouble (item->weight),
+                               "weight", 0, false);
+               ucl_object_insert_key (elt, ucl_object_fromdouble (item->avg_time),
+                               "time", 0, false);
+               ucl_object_insert_key (elt, ucl_object_fromint (item->frequency),
+                               "frequency", 0, false);
+
+               ucl_object_insert_key (top, elt, k, 0, false);
+       }
+
+       efunc = ucl_object_emit_fd_funcs (fd);
+       ret = ucl_object_emit_full (top, UCL_EMIT_JSON_COMPACT, efunc);
+       ucl_object_emit_funcs_free (efunc);
+       close (fd);
+
+       return ret;
+}
+
 void
 register_symbol_common (struct symbols_cache **cache,
        const gchar *name,

diff --git a/src/libserver/symbols_cache.h b/src/libserver/symbols_cache.h
index 2d194ab48c57a7e8d9d104ff87bab67f61e40c5e..5af74d090c6ec0d56437acd14218babdbe39749f 100644 (file)
--- a/src/libserver/symbols_cache.h
+++ b/src/libserver/symbols_cache.h
@@ -56,6 +56,9 @@ struct cache_item {
        /* Static item's data */
        struct saved_cache_item *s;
        struct counter_data *cd;
+       gdouble weight;
+       guint32 frequency;
+       gdouble avg_time;
 
        rspamd_mempool_mutex_t *mtx;