[Feature] Support Expires header when using HTTP maps

diff --git a/src/libutil/map.c b/src/libutil/map.c
index 2e1eb5287d4feb1e999b971d8c441fc2b49f1331..80ceed93dab9f8df58877058a3e06a1e5b25d1ea 100644 (file)
--- a/src/libutil/map.c
+++ b/src/libutil/map.c
@@ -376,6 +376,7 @@ http_map_finish (struct rspamd_http_connection *conn,
        struct rspamd_map_backend *bk;
        struct rspamd_http_map_cached_cbdata *cache_cbd;
        struct timeval tv;
+       const rspamd_ftok_t *expires_hdr;
        guchar *aux_data, *in = NULL;
        gsize inlen = 0, dlen = 0;
 
@@ -528,6 +529,34 @@ read_data:
                        goto err;
                }
 
+               /* Check for expires */
+               expires_hdr = rspamd_http_message_find_header (msg, "Expires");
+
+               if (expires_hdr) {
+                       time_t hdate;
+
+                       hdate = rspamd_http_parse_date (expires_hdr->begin, expires_hdr->len);
+
+                       if (hdate != (time_t)-1 && hdate > msg->date) {
+                               if (map->next_check) {
+                                       /* If we have multiple backends */
+                                       hdate = MIN (map->next_check, hdate);
+                               }
+
+                               double cached_timeout = map->next_check - msg->date +
+                                       map->poll_timeout * 2;
+
+                               map->next_check = hdate;
+                               double_to_tv (cached_timeout, &tv);
+                       }
+                       else {
+                               double_to_tv (map->poll_timeout * 2, &tv);
+                       }
+               }
+               else {
+                       double_to_tv (map->poll_timeout * 2, &tv);
+               }
+
                MAP_RETAIN (cbd->shmem_data, "shmem_data");
                cbd->data->gen ++;
                /*
@@ -546,10 +575,10 @@ read_data:
                cache_cbd->last_checked = cbd->data->last_checked;
                cache_cbd->gen = cbd->data->gen;
                MAP_RETAIN (cache_cbd->shm, "shmem_data");
+
                event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb,
                                cache_cbd);
                event_base_set (cbd->ev_base, &cache_cbd->timeout);
-               double_to_tv (map->poll_timeout * 2, &tv);
                event_add (&cache_cbd->timeout, &tv);
 
 
@@ -635,6 +664,23 @@ read_data:
                        cbd->data->last_modified = msg->date;
                }
 
+               expires_hdr = rspamd_http_message_find_header (msg, "Expires");
+
+               if (expires_hdr) {
+                       time_t hdate;
+
+                       hdate = rspamd_http_parse_date (expires_hdr->begin, expires_hdr->len);
+
+                       if (hdate != (time_t)-1 && hdate > msg->date) {
+                               if (map->next_check) {
+                                       /* If we have multiple backends */
+                                       hdate = MIN (map->next_check, hdate);
+                               }
+
+                               map->next_check = hdate;
+                       }
+               }
+
                cbd->periodic->cur_backend ++;
                rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
        }
@@ -884,17 +930,43 @@ rspamd_map_schedule_periodic (struct rspamd_map *map,
        gdouble timeout;
        struct map_periodic_cbdata *cbd;
 
-       timeout = map->poll_timeout;
+       if (map->next_check != 0) {
+               timeout = map->next_check - rspamd_get_calendar_ticks ();
+
+               if (timeout < map->poll_timeout) {
+                       timeout = map->poll_timeout;
+
+                       if (errored) {
+                               timeout = map->poll_timeout * error_mult;
+                       }
+                       else if (locked) {
+                               timeout = map->poll_timeout * lock_mult;
+                       }
+
+                       jittered_sec = rspamd_time_jitter (timeout, 0);
+               }
+               else {
+                       jittered_sec = rspamd_time_jitter (timeout, map->poll_timeout);
+               }
 
-       if (initial) {
-               timeout = 0.0;
+               /* Reset till the next usage */
+               map->next_check = 0;
        }
        else {
-               if (errored) {
-                       timeout = map->poll_timeout * error_mult;
-               } else if (locked) {
-                       timeout = map->poll_timeout * lock_mult;
+               timeout = map->poll_timeout;
+
+               if (initial) {
+                       timeout = 0.0;
+               } else {
+                       if (errored) {
+                               timeout = map->poll_timeout * error_mult;
+                       }
+                       else if (locked) {
+                               timeout = map->poll_timeout * lock_mult;
+                       }
                }
+
+               jittered_sec = rspamd_time_jitter (timeout, 0);
        }
 
        cbd = g_slice_alloc0 (sizeof (*cbd));
@@ -908,7 +980,7 @@ rspamd_map_schedule_periodic (struct rspamd_map *map,
        evtimer_set (&cbd->ev, rspamd_map_periodic_callback, cbd);
        event_base_set (map->ev_base, &cbd->ev);
 
-       jittered_sec = rspamd_time_jitter (timeout, 0);
+
        msg_debug_map ("schedule new periodic event %p in %.2f seconds",
                        cbd, jittered_sec);
        double_to_tv (jittered_sec, &map->tv);

diff --git a/src/libutil/map_private.h b/src/libutil/map_private.h
index beb862cc2a8aec69d26b72d184d86e9d97d8431d..79c3164e60538b8dcc9e924d288aeed17ad3199c 100644 (file)
--- a/src/libutil/map_private.h
+++ b/src/libutil/map_private.h
@@ -85,6 +85,7 @@ struct rspamd_map {
        gboolean active_http;
        struct timeval tv;
        gdouble poll_timeout;
+       time_t next_check;
        /* Shared lock for temporary disabling of map reading (e.g. when this map is written by UI) */
        gint *locked;
        /* Shared cache data */