From 914afe79b8058a3d4f5463859d693fbd3d8679fc Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 7 Jul 2017 08:36:11 +0100 Subject: [PATCH] [Feature] Improve maps cached data lifetime --- src/libutil/map.c | 117 +++++++++++++++++++++++--------------- src/libutil/map_private.h | 4 +- 2 files changed, 73 insertions(+), 48 deletions(-) diff --git a/src/libutil/map.c b/src/libutil/map.c index 77103ac00..c0953c90f 100644 --- a/src/libutil/map.c +++ b/src/libutil/map.c @@ -61,6 +61,9 @@ struct rspamd_http_map_cached_cbdata { struct event timeout; struct rspamd_storage_shmem *shm; struct rspamd_map *map; + struct http_map_data *data; + guint64 gen; + time_t last_checked; }; /** @@ -97,9 +100,9 @@ write_http_request (struct http_callback_data *cbd) cbd->data->path, strlen (cbd->data->path)); if (cbd->check && - cbd->data->last_checked != 0 && cbd->stage == map_load_file) { + cbd->data->last_modified != 0 && cbd->stage == map_load_file) { rspamd_http_date_format (datebuf, sizeof (datebuf), - cbd->data->last_checked); + cbd->data->last_modified); rspamd_http_message_add_header (msg, "If-Modified-Since", datebuf); } } @@ -333,13 +336,35 @@ rspamd_map_cache_cb (gint fd, short what, gpointer ud) { struct rspamd_http_map_cached_cbdata *cache_cbd = ud; struct rspamd_map *map; + struct timeval tv; map = cache_cbd->map; - g_atomic_int_set (&map->cache->available, 0); - MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata"); - msg_debug_map ("cached data is now expired now for %s", map->name); - event_del (&cache_cbd->timeout); - g_slice_free1 (sizeof (*cache_cbd), cache_cbd); + + if (cache_cbd->gen != cache_cbd->data->gen) { + /* We have another update, so this cache element is obviously expired */ + /* Important: we do not set cache availability to zero here */ + MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata"); + msg_debug_map ("cached data is now expired (gen missmatch) for %s", map->name); + event_del (&cache_cbd->timeout); + g_slice_free1 (sizeof (*cache_cbd), cache_cbd); + } + else if (cache_cbd->data->last_checked > cache_cbd->last_checked) { + /* + * We checked map but we have not found anything more recent, + * reschedule cache check + */ + cache_cbd->last_checked = cache_cbd->data->last_checked; + msg_debug_map ("cached data is up to date for %s", map->name); + double_to_tv (map->poll_timeout * 2, &tv); + event_add (&cache_cbd->timeout, &tv); + } + else { + g_atomic_int_set (&map->cache->available, 0); + MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata"); + msg_debug_map ("cached data is now expired for %s", map->name); + event_del (&cache_cbd->timeout); + g_slice_free1 (sizeof (*cache_cbd), cache_cbd); + } } static int @@ -349,6 +374,8 @@ http_map_finish (struct rspamd_http_connection *conn, struct http_callback_data *cbd = conn->ud; struct rspamd_map *map; struct rspamd_map_backend *bk; + struct rspamd_http_map_cached_cbdata *cache_cbd; + struct timeval tv; guchar *aux_data, *in = NULL; gsize inlen = 0, dlen = 0; @@ -368,11 +395,13 @@ http_map_finish (struct rspamd_http_connection *conn, } if (cbd->stage == map_load_file) { + cbd->data->last_checked = msg->date; + if (msg->last_modified) { - cbd->data->last_checked = msg->last_modified; + cbd->data->last_modified = msg->last_modified; } else { - cbd->data->last_checked = msg->date; + cbd->data->last_modified = msg->date; } /* Maybe we need to check signature ? */ @@ -495,29 +524,27 @@ read_data: } MAP_RETAIN (cbd->shmem_data, "shmem_data"); - + cbd->data->gen ++; /* * We know that a map is in the locked state */ - if (g_atomic_int_compare_and_exchange (&map->cache->available, 0, 1)) { - /* Store cached data */ - struct rspamd_http_map_cached_cbdata *cache_cbd; - struct timeval tv; - - rspamd_strlcpy (map->cache->shmem_name, cbd->shmem_data->shm_name, - sizeof (map->cache->shmem_name)); - map->cache->len = cbd->data_len; - map->cache->last_checked = cbd->data->last_checked; - cache_cbd = g_slice_alloc0 (sizeof (*cache_cbd)); - cache_cbd->shm = cbd->shmem_data; - cache_cbd->map = map; - MAP_RETAIN (cache_cbd->shm, "shmem_data"); - event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb, - cache_cbd); - event_base_set (cbd->ev_base, &cache_cbd->timeout); - double_to_tv (map->poll_timeout, &tv); - event_add (&cache_cbd->timeout, &tv); - } + g_atomic_int_set (&map->cache->available, 1); + /* Store cached data */ + rspamd_strlcpy (map->cache->shmem_name, cbd->shmem_data->shm_name, + sizeof (map->cache->shmem_name)); + map->cache->len = cbd->data_len; + map->cache->last_modified = cbd->data->last_modified; + cache_cbd = g_slice_alloc0 (sizeof (*cache_cbd)); + cache_cbd->shm = cbd->shmem_data; + cache_cbd->map = map; + cache_cbd->last_checked = cbd->data->last_checked; + cache_cbd->gen = cbd->data->gen; + MAP_RETAIN (cache_cbd->shm, "shmem_data"); + event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb, + cache_cbd); + event_base_set (cbd->ev_base, &cache_cbd->timeout); + double_to_tv (map->poll_timeout * 2, &tv); + event_add (&cache_cbd->timeout, &tv); if (cbd->bk->is_compressed) { @@ -593,11 +620,13 @@ read_data: msg_debug_map ("data is not modified for server %s", cbd->data->host); + cbd->data->last_checked = msg->date; + if (msg->last_modified) { - cbd->data->last_checked = msg->last_modified; + cbd->data->last_modified = msg->last_modified; } else { - cbd->data->last_checked = msg->date; + cbd->data->last_modified = msg->date; } cbd->periodic->cur_backend ++; @@ -854,12 +883,12 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, if (initial) { timeout = 0.0; } - - if (errored) { - timeout = map->poll_timeout * error_mult; - } - else if (locked) { - timeout = map->poll_timeout * lock_mult; + else { + if (errored) { + timeout = map->poll_timeout * error_mult; + } else if (locked) { + timeout = map->poll_timeout * lock_mult; + } } cbd = g_slice_alloc0 (sizeof (*cbd)); @@ -870,14 +899,8 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, cbd->map = map; REF_INIT_RETAIN (cbd, rspamd_map_periodic_dtor); - if (initial) { - evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd); - event_base_set (map->ev_base, &cbd->ev); - } - else { - evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd); - event_base_set (map->ev_base, &cbd->ev); - } + evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd); + event_base_set (map->ev_base, &cbd->ev); jittered_sec = rspamd_time_jitter (timeout, 0); msg_debug_map ("schedule new periodic event %p in %.2f seconds", @@ -1050,7 +1073,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, struct rspamd_map_backe if (g_atomic_int_get (&map->cache->available) == 1) { /* Read cached data */ if (check) { - if (data->last_checked < map->cache->last_checked) { + if (data->last_modified < map->cache->last_modified) { periodic->need_modify = TRUE; /* Reset the whole chain */ periodic->cur_backend = 0; @@ -1067,7 +1090,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, struct rspamd_map_backe else if (rspamd_map_read_cached (map, bk, periodic, data->host)) { /* Switch to the next backend */ periodic->cur_backend ++; - data->last_checked = map->cache->last_checked; + data->last_modified = map->cache->last_modified; rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); return; diff --git a/src/libutil/map_private.h b/src/libutil/map_private.h index abb4ef613..fc2d8e279 100644 --- a/src/libutil/map_private.h +++ b/src/libutil/map_private.h @@ -66,7 +66,7 @@ struct rspamd_map_backend { struct rspamd_map_cachepoint { gint available; gsize len; - time_t last_checked; + time_t last_modified; gchar shmem_name[256]; }; @@ -107,8 +107,10 @@ struct http_map_data { gchar *path; gchar *host; gchar *last_signature; + time_t last_modified; time_t last_checked; gboolean request_sent; + guint64 gen; guint16 port; }; -- 2.39.5