[Feature] Support etag for HTTP maps

src/libutil/map.c

@@ -99,11 +99,17 @@ write_http_request (struct http_callback_data *cbd)
 			msg->url = rspamd_fstring_append (msg->url,
 					cbd->data->path, strlen (cbd->data->path));
 
-			if (cbd->check &&
-					cbd->data->last_modified != 0 && cbd->stage == map_load_file) {
-				rspamd_http_date_format (datebuf, sizeof (datebuf),
-						cbd->data->last_modified);
-				rspamd_http_message_add_header (msg, "If-Modified-Since", datebuf);
+			if (cbd->check && cbd->stage == map_load_file) {
+				if (cbd->data->last_modified != 0) {
+					rspamd_http_date_format (datebuf, sizeof (datebuf),
+							cbd->data->last_modified);
+					rspamd_http_message_add_header (msg, "If-Modified-Since",
+							datebuf);
+				}
+				if (cbd->data->etag) {
+					rspamd_http_message_add_header_len (msg, "If-None-Match",
+							cbd->data->etag->str, cbd->data->etag->len);
+				}
 			}
 		}
 		else if (cbd->stage == map_load_pubkey) {
@@ -431,7 +437,7 @@ http_map_finish (struct rspamd_http_connection *conn,
 	struct rspamd_map_backend *bk;
 	struct rspamd_http_map_cached_cbdata *cache_cbd;
 	struct timeval tv;
-	const rspamd_ftok_t *expires_hdr;
+	const rspamd_ftok_t *expires_hdr, *etag_hdr;
 	char next_check_date[128];
 	guchar *aux_data, *in = NULL;
 	gsize inlen = 0, dlen = 0;
@@ -626,6 +632,25 @@ read_data:
 			double_to_tv (map->poll_timeout * 2, &tv);
 		}
 
+		/* Check for etag */
+		etag_hdr = rspamd_http_message_find_header (msg, "ETag");
+
+		if (etag_hdr) {
+			if (cbd->data->etag) {
+				/* Remove old etag */
+				rspamd_fstring_free (cbd->data->etag);
+				cbd->data->etag = rspamd_fstring_new_init (etag_hdr->begin,
+						etag_hdr->len);
+			}
+		}
+		else {
+			if (cbd->data->etag) {
+				/* Remove and clear old etag */
+				rspamd_fstring_free (cbd->data->etag);
+				cbd->data->etag = NULL;
+			}
+		}
+
 		MAP_RETAIN (cbd->shmem_data, "shmem_data");
 		cbd->data->gen ++;
 		/*
@@ -756,6 +781,17 @@ read_data:
 			}
 		}
 
+		etag_hdr = rspamd_http_message_find_header (msg, "ETag");
+
+		if (etag_hdr) {
+			if (cbd->data->etag) {
+				/* Remove old etag */
+				rspamd_fstring_free (cbd->data->etag);
+				cbd->data->etag = rspamd_fstring_new_init (etag_hdr->begin,
+						etag_hdr->len);
+			}
+		}
+
 		if (map->next_check) {
 			rspamd_http_date_format (next_check_date, sizeof (next_check_date),
 					map->next_check);
@@ -1784,6 +1820,11 @@ rspamd_map_backend_dtor (struct rspamd_map_backend *bk)
 		if (bk->data.hd) {
 			g_free (bk->data.hd->host);
 			g_free (bk->data.hd->path);
+
+			if (bk->data.hd->etag) {
+				rspamd_fstring_free (bk->data.hd->etag);
+			}
+
 			g_free (bk->data.hd);
 		}
 		break;

src/libutil/map_private.h

@@ -49,17 +49,48 @@ enum fetch_proto {
 	MAP_PROTO_STATIC
 };
 
+/**
+ * Data specific to file maps
+ */
+struct file_map_data {
+	gchar *filename;
+	struct stat st;
+};
+
+/**
+ * Data specific to HTTP maps
+ */
+struct http_map_data {
+	gchar *path;
+	gchar *host;
+	gchar *last_signature;
+	rspamd_fstring_t *etag;
+	time_t last_modified;
+	time_t last_checked;
+	gboolean request_sent;
+	guint64 gen;
+	guint16 port;
+};
+
+struct static_map_data {
+	guchar *data;
+	gsize len;
+	gboolean processed;
+};
+
+union rspamd_map_backend_data {
+	struct file_map_data *fd;
+	struct http_map_data *hd;
+	struct static_map_data *sd;
+};
+
 struct rspamd_map_backend {
 	enum fetch_proto protocol;
 	gboolean is_signed;
 	gboolean is_compressed;
 	guint32 id;
 	struct rspamd_cryptobox_pubkey *trusted_pubkey;
-	union {
-		struct file_map_data *fd;
-		struct http_map_data *hd;
-		struct static_map_data *sd;
-	} data;
+	union rspamd_map_backend_data data;
 	gchar *uri;
 	ref_entry_t ref;
 };
@@ -96,34 +127,6 @@ struct rspamd_map {
 	gpointer dtor_data;
 };
 
-/**
- * Data specific to file maps
- */
-struct file_map_data {
-	gchar *filename;
-	struct stat st;
-};
-
-/**
- * Data specific to HTTP maps
- */
-struct http_map_data {
-	gchar *path;
-	gchar *host;
-	gchar *last_signature;
-	time_t last_modified;
-	time_t last_checked;
-	gboolean request_sent;
-	guint64 gen;
-	guint16 port;
-};
-
-struct static_map_data {
-	guchar *data;
-	gsize len;
-	gboolean processed;
-};
-
 enum rspamd_map_http_stage {
 	map_resolve_host2 = 0, /* 2 requests sent */
 	map_resolve_host1, /* 1 requests sent */