[Feature] Improve maps cached data lifetime

2 files changed, 73 insertions, 48 deletions

diff --git a/src/libutil/map.c b/src/libutil/map.c
index 77103ac00..c0953c90f 100644
--- a/src/libutil/map.c
+++ b/src/libutil/map.c
@@ -61,6 +61,9 @@ struct rspamd_http_map_cached_cbdata {
 	struct event timeout;
 	struct rspamd_storage_shmem *shm;
 	struct rspamd_map *map;
+	struct http_map_data *data;
+	guint64 gen;
+	time_t last_checked;
 };
 
 /**
@@ -97,9 +100,9 @@ write_http_request (struct http_callback_data *cbd)
 					cbd->data->path, strlen (cbd->data->path));
 
 			if (cbd->check &&
-					cbd->data->last_checked != 0 && cbd->stage == map_load_file) {
+					cbd->data->last_modified != 0 && cbd->stage == map_load_file) {
 				rspamd_http_date_format (datebuf, sizeof (datebuf),
-						cbd->data->last_checked);
+						cbd->data->last_modified);
 				rspamd_http_message_add_header (msg, "If-Modified-Since", datebuf);
 			}
 		}
@@ -333,13 +336,35 @@ rspamd_map_cache_cb (gint fd, short what, gpointer ud)
 {
 	struct rspamd_http_map_cached_cbdata *cache_cbd = ud;
 	struct rspamd_map *map;
+	struct timeval tv;
 
 	map = cache_cbd->map;
-	g_atomic_int_set (&map->cache->available, 0);
-	MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata");
-	msg_debug_map ("cached data is now expired now for %s", map->name);
-	event_del (&cache_cbd->timeout);
-	g_slice_free1 (sizeof (*cache_cbd), cache_cbd);
+
+	if (cache_cbd->gen != cache_cbd->data->gen) {
+		/* We have another update, so this cache element is obviously expired */
+		/* Important: we do not set cache availability to zero here */
+		MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata");
+		msg_debug_map ("cached data is now expired (gen missmatch) for %s", map->name);
+		event_del (&cache_cbd->timeout);
+		g_slice_free1 (sizeof (*cache_cbd), cache_cbd);
+	}
+	else if (cache_cbd->data->last_checked > cache_cbd->last_checked) {
+		/*
+		 * We checked map but we have not found anything more recent,
+		 * reschedule cache check
+		 */
+		cache_cbd->last_checked = cache_cbd->data->last_checked;
+		msg_debug_map ("cached data is up to date for %s", map->name);
+		double_to_tv (map->poll_timeout * 2, &tv);
+		event_add (&cache_cbd->timeout, &tv);
+	}
+	else {
+		g_atomic_int_set (&map->cache->available, 0);
+		MAP_RELEASE (cache_cbd->shm, "rspamd_http_map_cached_cbdata");
+		msg_debug_map ("cached data is now expired for %s", map->name);
+		event_del (&cache_cbd->timeout);
+		g_slice_free1 (sizeof (*cache_cbd), cache_cbd);
+	}
 }
 
 static int
@@ -349,6 +374,8 @@ http_map_finish (struct rspamd_http_connection *conn,
 	struct http_callback_data *cbd = conn->ud;
 	struct rspamd_map *map;
 	struct rspamd_map_backend *bk;
+	struct rspamd_http_map_cached_cbdata *cache_cbd;
+	struct timeval tv;
 	guchar *aux_data, *in = NULL;
 	gsize inlen = 0, dlen = 0;
 
@@ -368,11 +395,13 @@ http_map_finish (struct rspamd_http_connection *conn,
 		}
 
 		if (cbd->stage == map_load_file) {
+			cbd->data->last_checked = msg->date;
+
 			if (msg->last_modified) {
-				cbd->data->last_checked = msg->last_modified;
+				cbd->data->last_modified = msg->last_modified;
 			}
 			else {
-				cbd->data->last_checked = msg->date;
+				cbd->data->last_modified = msg->date;
 			}
 
 			/* Maybe we need to check signature ? */
@@ -495,29 +524,27 @@ read_data:
 		}
 
 		MAP_RETAIN (cbd->shmem_data, "shmem_data");
-
+		cbd->data->gen ++;
 		/*
 		 * We know that a map is in the locked state
 		 */
-		if (g_atomic_int_compare_and_exchange (&map->cache->available, 0, 1)) {
-			/* Store cached data */
-			struct rspamd_http_map_cached_cbdata *cache_cbd;
-			struct timeval tv;
-
-			rspamd_strlcpy (map->cache->shmem_name, cbd->shmem_data->shm_name,
-					sizeof (map->cache->shmem_name));
-			map->cache->len = cbd->data_len;
-			map->cache->last_checked = cbd->data->last_checked;
-			cache_cbd = g_slice_alloc0 (sizeof (*cache_cbd));
-			cache_cbd->shm = cbd->shmem_data;
-			cache_cbd->map = map;
-			MAP_RETAIN (cache_cbd->shm, "shmem_data");
-			event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb,
-					cache_cbd);
-			event_base_set (cbd->ev_base, &cache_cbd->timeout);
-			double_to_tv (map->poll_timeout, &tv);
-			event_add (&cache_cbd->timeout, &tv);
-		}
+		g_atomic_int_set (&map->cache->available, 1);
+		/* Store cached data */
+		rspamd_strlcpy (map->cache->shmem_name, cbd->shmem_data->shm_name,
+				sizeof (map->cache->shmem_name));
+		map->cache->len = cbd->data_len;
+		map->cache->last_modified = cbd->data->last_modified;
+		cache_cbd = g_slice_alloc0 (sizeof (*cache_cbd));
+		cache_cbd->shm = cbd->shmem_data;
+		cache_cbd->map = map;
+		cache_cbd->last_checked = cbd->data->last_checked;
+		cache_cbd->gen = cbd->data->gen;
+		MAP_RETAIN (cache_cbd->shm, "shmem_data");
+		event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb,
+				cache_cbd);
+		event_base_set (cbd->ev_base, &cache_cbd->timeout);
+		double_to_tv (map->poll_timeout * 2, &tv);
+		event_add (&cache_cbd->timeout, &tv);
 
 
 		if (cbd->bk->is_compressed) {
@@ -593,11 +620,13 @@ read_data:
 		msg_debug_map ("data is not modified for server %s",
 				cbd->data->host);
 
+		cbd->data->last_checked = msg->date;
+
 		if (msg->last_modified) {
-			cbd->data->last_checked = msg->last_modified;
+			cbd->data->last_modified = msg->last_modified;
 		}
 		else {
-			cbd->data->last_checked = msg->date;
+			cbd->data->last_modified = msg->date;
 		}
 
 		cbd->periodic->cur_backend ++;
@@ -854,12 +883,12 @@ rspamd_map_schedule_periodic (struct rspamd_map *map,
 	if (initial) {
 		timeout = 0.0;
 	}
-
-	if (errored) {
-		timeout = map->poll_timeout * error_mult;
-	}
-	else if (locked) {
-		timeout = map->poll_timeout * lock_mult;
+	else {
+		if (errored) {
+			timeout = map->poll_timeout * error_mult;
+		} else if (locked) {
+			timeout = map->poll_timeout * lock_mult;
+		}
 	}
 
 	cbd = g_slice_alloc0 (sizeof (*cbd));
@@ -870,14 +899,8 @@ rspamd_map_schedule_periodic (struct rspamd_map *map,
 	cbd->map = map;
 	REF_INIT_RETAIN (cbd, rspamd_map_periodic_dtor);
 
-	if (initial) {
-		evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd);
-		event_base_set (map->ev_base, &cbd->ev);
-	}
-	else {
-		evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd);
-		event_base_set (map->ev_base, &cbd->ev);
-	}
+	evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd);
+	event_base_set (map->ev_base, &cbd->ev);
 
 	jittered_sec = rspamd_time_jitter (timeout, 0);
 	msg_debug_map ("schedule new periodic event %p in %.2f seconds",
@@ -1050,7 +1073,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, struct rspamd_map_backe
 	if (g_atomic_int_get (&map->cache->available) == 1) {
 		/* Read cached data */
 		if (check) {
-			if (data->last_checked < map->cache->last_checked) {
+			if (data->last_modified < map->cache->last_modified) {
 				periodic->need_modify = TRUE;
 				/* Reset the whole chain */
 				periodic->cur_backend = 0;
@@ -1067,7 +1090,7 @@ rspamd_map_common_http_callback (struct rspamd_map *map, struct rspamd_map_backe
 		else if (rspamd_map_read_cached (map, bk, periodic, data->host)) {
 			/* Switch to the next backend */
 			periodic->cur_backend ++;
-			data->last_checked = map->cache->last_checked;
+			data->last_modified = map->cache->last_modified;
 			rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic);
 
 			return;
diff --git a/src/libutil/map_private.h b/src/libutil/map_private.h
index abb4ef613..fc2d8e279 100644
--- a/src/libutil/map_private.h
+++ b/src/libutil/map_private.h
@@ -66,7 +66,7 @@ struct rspamd_map_backend {
 struct rspamd_map_cachepoint {
 	gint available;
 	gsize len;
-	time_t last_checked;
+	time_t last_modified;
 	gchar shmem_name[256];
 };
 
@@ -107,8 +107,10 @@ struct http_map_data {
 	gchar *path;
 	gchar *host;
 	gchar *last_signature;
+	time_t last_modified;
 	time_t last_checked;
 	gboolean request_sent;
+	guint64 gen;
 	guint16 port;
 };