[Feature] Improve maps cached data lifetime

diff --git a/src/libutil/map.c b/src/libutil/map.c
index 77103ac00197c8671fc092e7eea15dced0850663..c0953c90f2303ed88afdfb1e18cd1b0a323db9cb 100644 (file)
--- a/src/libutil/map.c
+++ b/src/libutil/map.c
@@ -61,6 +61,9 @@ struct rspamd_http_map_cached_cbdata {
        struct event timeout;
        struct rspamd_storage_shmem *shm;
        struct rspamd_map *map;
+       struct http_map_data *data;
+       guint64 gen;
+       time_t last_checked;
 };
 
 /**
@@ -97,9 +100,9 @@ write_http_request (struct http_callback_data *cbd)
                                        cbd->data->path, strlen (cbd->data->path));
 
                        if (cbd->check &&
-                                       cbd->data->last_checked != 0 && cbd->stage == map_load_file) {
+                                       cbd->data->last_modified != 0 && cbd->stage == map_load_file) {
                                rspamd_http_date_format (datebuf, sizeof (datebuf),
-                                               cbd->data->last_checked);
+                                               cbd->data->last_modified);
                                rspamd_http_message_add_header (msg, "If-Modified-Since", datebuf);
                        }
                }
@@ -333,13 +336,35 @@ rspamd_map_cache_cb (gint fd, short what, gpointer ud)
 {
        struct rspamd_http_map_cached_cbdata *cache_cbd = ud;
        struct rspamd_map *map;
+       struct timeval tv;
 
        map = cache_cbd->map;
-       g_atomic_int_set (&map->cache->available, 0);
-       MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata");
-       msg_debug_map ("cached data is now expired now for %s", map->name);
-       event_del (&cache_cbd->timeout);
-       g_slice_free1 (sizeof (*cache_cbd), cache_cbd);
+
+       if (cache_cbd->gen != cache_cbd->data->gen) {
+               /* We have another update, so this cache element is obviously expired */
+               /* Important: we do not set cache availability to zero here */
+               MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata");
+               msg_debug_map ("cached data is now expired (gen missmatch) for %s", map->name);
+               event_del (&cache_cbd->timeout);
+               g_slice_free1 (sizeof (*cache_cbd), cache_cbd);
+       }
+       else if (cache_cbd->data->last_checked > cache_cbd->last_checked) {
+               /*
+                * We checked map but we have not found anything more recent,
+                * reschedule cache check
+                */
+               cache_cbd->last_checked = cache_cbd->data->last_checked;
+               msg_debug_map ("cached data is up to date for %s", map->name);
+               double_to_tv (map->poll_timeout * 2, &tv);
+               event_add (&cache_cbd->timeout, &tv);
+       }
+       else {
+               g_atomic_int_set (&map->cache->available, 0);
+               MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata");
+               msg_debug_map ("cached data is now expired for %s", map->name);
+               event_del (&cache_cbd->timeout);
+               g_slice_free1 (sizeof (*cache_cbd), cache_cbd);
+       }
 }
 
 static int
@@ -349,6 +374,8 @@ http_map_finish (struct rspamd_http_connection *conn,
        struct http_callback_data *cbd = conn->ud;
        struct rspamd_map *map;
        struct rspamd_map_backend *bk;
+       struct rspamd_http_map_cached_cbdata *cache_cbd;
+       struct timeval tv;
        guchar *aux_data, *in = NULL;
        gsize inlen = 0, dlen = 0;
 
@@ -368,11 +395,13 @@ http_map_finish (struct rspamd_http_connection *conn,
                }
 
                if (cbd->stage == map_load_file) {
+                       cbd->data->last_checked = msg->date;
+
                        if (msg->last_modified) {
-                               cbd->data->last_checked = msg->last_modified;
+                               cbd->data->last_modified = msg->last_modified;
                        }
                        else {
-                               cbd->data->last_checked = msg->date;
+                               cbd->data->last_modified = msg->date;
                        }
 
                        /* Maybe we need to check signature ? */
@@ -495,29 +524,27 @@ read_data:
                }
 
                MAP_RETAIN (cbd->shmem_data, "shmem_data");
-
+               cbd->data->gen ++;
                /*
                 * We know that a map is in the locked state
                 */
-               if (g_atomic_int_compare_and_exchange (&map->cache->available, 0, 1)) {
-                       /* Store cached data */
-                       struct rspamd_http_map_cached_cbdata *cache_cbd;
-                       struct timeval tv;
-
-                       rspamd_strlcpy (map->cache->shmem_name, cbd->shmem_data->shm_name,
-                                       sizeof (map->cache->shmem_name));
-                       map->cache->len = cbd->data_len;
-                       map->cache->last_checked = cbd->data->last_checked;
-                       cache_cbd = g_slice_alloc0 (sizeof (*cache_cbd));
-                       cache_cbd->shm = cbd->shmem_data;
-                       cache_cbd->map = map;
-                       MAP_RETAIN (cache_cbd->shm, "shmem_data");
-                       event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb,
-                                       cache_cbd);
-                       event_base_set (cbd->ev_base, &cache_cbd->timeout);
-                       double_to_tv (map->poll_timeout, &tv);
-                       event_add (&cache_cbd->timeout, &tv);
-               }
+               g_atomic_int_set (&map->cache->available, 1);
+               /* Store cached data */
+               rspamd_strlcpy (map->cache->shmem_name, cbd->shmem_data->shm_name,
+                               sizeof (map->cache->shmem_name));
+               map->cache->len = cbd->data_len;
+               map->cache->last_modified = cbd->data->last_modified;
+               cache_cbd = g_slice_alloc0 (sizeof (*cache_cbd));
+               cache_cbd->shm = cbd->shmem_data;
+               cache_cbd->map = map;
+               cache_cbd->last_checked = cbd->data->last_checked;
+               cache_cbd->gen = cbd->data->gen;
+               MAP_RETAIN (cache_cbd->shm, "shmem_data");
+               event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb,
+                               cache_cbd);
+               event_base_set (cbd->ev_base, &cache_cbd->timeout);
+               double_to_tv (map->poll_timeout * 2, &tv);
+               event_add (&cache_cbd->timeout, &tv);
 
 
                if (cbd->bk->is_compressed) {
@@ -593,11 +620,13 @@ read_data:
                msg_debug_map ("data is not modified for server %s",
                                cbd->data->host);
 
+               cbd->data->last_checked = msg->date;
+
                if (msg->last_modified) {
-                       cbd->data->last_checked = msg->last_modified;
+                       cbd->data->last_modified = msg->last_modified;
                }
                else {
-                       cbd->data->last_checked = msg->date;
+                       cbd->data->last_modified = msg->date;
                }
 
                cbd->periodic->cur_backend ++;
@@ -854,12 +883,12 @@ rspamd_map_schedule_periodic (struct rspamd_map *map,
        if (initial) {
                timeout = 0.0;
        }
-
-       if (errored) {
-               timeout = map->poll_timeout * error_mult;
-       }
-       else if (locked) {
-               timeout = map->poll_timeout * lock_mult;
+       else {
+               if (errored) {
+                       timeout = map->poll_timeout * error_mult;
+               } else if (locked) {
+                       timeout = map->poll_timeout * lock_mult;
+               }
        }
 
        cbd = g_slice_alloc0 (sizeof (*cbd));
@@ -870,14 +899,8 @@ rspamd_map_schedule_periodic (struct rspamd_map *map,
        cbd->map = map;
        REF_INIT_RETAIN (cbd, rspamd_map_periodic_dtor);
 
-       if (initial) {
-               evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd);
-               event_base_set (map->ev_base, &cbd->ev);
-       }
-       else {
-               evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd);
-               event_base_set (map->ev_base, &cbd->ev);
-       }
+       evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd);
+       event_base_set (map->ev_base, &cbd->ev);
 
        jittered_sec = rspamd_time_jitter (timeout, 0);
        msg_debug_map ("schedule new periodic event %p in %.2f seconds",
@@ -1050,7 +1073,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, struct rspamd_map_backe
        if (g_atomic_int_get (&map->cache->available) == 1) {
                /* Read cached data */
                if (check) {
-                       if (data->last_checked < map->cache->last_checked) {
+                       if (data->last_modified < map->cache->last_modified) {
                                periodic->need_modify = TRUE;
                                /* Reset the whole chain */
                                periodic->cur_backend = 0;
@@ -1067,7 +1090,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, struct rspamd_map_backe
                else if (rspamd_map_read_cached (map, bk, periodic, data->host)) {
                        /* Switch to the next backend */
                        periodic->cur_backend ++;
-                       data->last_checked = map->cache->last_checked;
+                       data->last_modified = map->cache->last_modified;
                        rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic);
 
                        return;

diff --git a/src/libutil/map_private.h b/src/libutil/map_private.h
index abb4ef613337477a33fe3e9bd72d6577895466ad..fc2d8e27945c05ec422e4069f34be88b9d719baf 100644 (file)
--- a/src/libutil/map_private.h
+++ b/src/libutil/map_private.h
@@ -66,7 +66,7 @@ struct rspamd_map_backend {
 struct rspamd_map_cachepoint {
        gint available;
        gsize len;
-       time_t last_checked;
+       time_t last_modified;
        gchar shmem_name[256];
 };
 
@@ -107,8 +107,10 @@ struct http_map_data {
        gchar *path;
        gchar *host;
        gchar *last_signature;
+       time_t last_modified;
        time_t last_checked;
        gboolean request_sent;
+       guint64 gen;
        guint16 port;
 };