From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Tue, 26 Jun 2018 15:36:36 +0000 (+0100)
Subject: [Feature] Preload file and static maps in main process
X-Git-Tag: 1.7.7~19
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=3efd7b744da68245805f0da64661ec5cf7cd5483;p=rspamd.git

[Feature] Preload file and static maps in main process
---

diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h
index 777340e16..a74169b31 100644
--- a/src/libserver/cfg_file.h
+++ b/src/libserver/cfg_file.h
@@ -487,10 +487,15 @@ enum rspamd_post_load_options {
 	RSPAMD_CONFIG_INIT_LIBS = 1 << 1,
 	RSPAMD_CONFIG_INIT_SYMCACHE = 1 << 2,
 	RSPAMD_CONFIG_INIT_VALIDATE = 1 << 3,
-	RSPAMD_CONFIG_INIT_NO_TLD = 1 << 4
+	RSPAMD_CONFIG_INIT_NO_TLD = 1 << 4,
+	RSPAMD_CONFIG_INIT_PRELOAD_MAPS = 1 << 5,
 };
 
-#define RSPAMD_CONFIG_LOAD_ALL (RSPAMD_CONFIG_INIT_URL|RSPAMD_CONFIG_INIT_LIBS|RSPAMD_CONFIG_INIT_SYMCACHE|RSPAMD_CONFIG_INIT_VALIDATE)
+#define RSPAMD_CONFIG_LOAD_ALL (RSPAMD_CONFIG_INIT_URL| \
+		RSPAMD_CONFIG_INIT_LIBS| \
+		RSPAMD_CONFIG_INIT_SYMCACHE| \
+		RSPAMD_CONFIG_INIT_VALIDATE| \
+		RSPAMD_CONFIG_INIT_PRELOAD_MAPS)
 
 /**
  * Do post load actions for config
diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c
index 48c4d5eed..967032777 100644
--- a/src/libserver/cfg_utils.c
+++ b/src/libserver/cfg_utils.c
@@ -850,7 +850,11 @@ rspamd_config_post_load (struct rspamd_config *cfg,
 			ret = FALSE;
 		}
 
-		return rspamd_symbols_cache_validate (cfg->cache, cfg, FALSE) && ret;
+		ret = rspamd_symbols_cache_validate (cfg->cache, cfg, FALSE) && ret;
+	}
+
+	if (opts & RSPAMD_CONFIG_INIT_PRELOAD_MAPS) {
+		rspamd_map_preload (cfg);
 	}
 
 	return ret;
diff --git a/src/libutil/map.c b/src/libutil/map.c
index 7a4d48598..8f36541a2 100644
--- a/src/libutil/map.c
+++ b/src/libutil/map.c
@@ -846,7 +846,8 @@ read_map_file (struct rspamd_map *map, struct file_map_data *data,
 			return FALSE;
 		}
 		else {
-			msg_info_map ("%s: map file is not found",
+			msg_info_map ("%s: map file is not found; "
+						  "it will be read automatically if created",
 					data->filename);
 			return TRUE;
 		}
@@ -1004,6 +1005,9 @@ read_map_file (struct rspamd_map *map, struct file_map_data *data,
 		map->read_callback (NULL, 0, &periodic->cbdata, TRUE);
 	}
 
+	/* Also update at the read time */
+	memcpy (&data->st, &st, sizeof (struct stat));
+
 	return TRUE;
 }
 
@@ -1729,6 +1733,73 @@ rspamd_map_watch (struct rspamd_config *cfg,
 	}
 }
 
+void
+rspamd_map_preload (struct rspamd_config *cfg)
+{
+	GList *cur = cfg->maps;
+	struct rspamd_map *map;
+	struct rspamd_map_backend *bk;
+	guint i;
+	gboolean map_ok;
+
+	/* First of all do synced read of data */
+	while (cur) {
+		map = cur->data;
+		map_ok = TRUE;
+
+		PTR_ARRAY_FOREACH (map->backends, i, bk) {
+			if (!(bk->protocol == MAP_PROTO_FILE ||
+				  bk->protocol == MAP_PROTO_STATIC)) {
+				map_ok = FALSE;
+				break;
+			}
+		}
+
+		if (map_ok) {
+			struct map_periodic_cbdata fake_cbd;
+			gboolean succeed = TRUE;
+
+			memset (&fake_cbd, 0, sizeof (fake_cbd));
+
+			PTR_ARRAY_FOREACH (map->backends, i, bk) {
+				fake_cbd.cbdata.state = 0;
+				fake_cbd.cbdata.prev_data = *map->user_data;
+				fake_cbd.cbdata.cur_data = NULL;
+				fake_cbd.cbdata.map = map;
+				fake_cbd.map = map;
+				fake_cbd.cur_backend = i;
+
+				if (bk->protocol == MAP_PROTO_FILE) {
+					if (!read_map_file (map, bk->data.fd, bk, &fake_cbd)) {
+						succeed = FALSE;
+						break;
+					}
+				}
+				else if (bk->protocol == MAP_PROTO_STATIC) {
+					if (!read_map_static (map, bk->data.sd, bk, &fake_cbd)) {
+						succeed = FALSE;
+						break;
+					}
+				}
+				else {
+					g_assert_not_reached ();
+				}
+			}
+
+			if (succeed) {
+				map->fin_callback (&fake_cbd.cbdata);
+
+				if (fake_cbd.cbdata.cur_data) {
+					*map->user_data = fake_cbd.cbdata.cur_data;
+				}
+			}
+
+		}
+
+		cur = g_list_next (cur);
+	}
+}
+
 void
 rspamd_map_remove_all (struct rspamd_config *cfg)
 {
diff --git a/src/libutil/map.h b/src/libutil/map.h
index 9c7485c58..80aa03825 100644
--- a/src/libutil/map.h
+++ b/src/libutil/map.h
@@ -84,6 +84,12 @@ void rspamd_map_watch (struct rspamd_config *cfg,
 					   struct rspamd_worker *worker,
 					   gboolean active_http);
 
+/**
+ * Preloads maps where all backends are file
+ * @param cfg
+ */
+void rspamd_map_preload (struct rspamd_config *cfg);
+
 /**
  * Remove all maps watched (remove events)
  */
diff --git a/src/rspamd.c b/src/rspamd.c
index cba581419..6d676b968 100644
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -305,6 +305,7 @@ reread_config (struct rspamd_main *rspamd_main)
 		REF_RELEASE (tmp_cfg);
 	}
 	else {
+		rspamd_map_preload (rspamd_main->cfg);
 		msg_info_main ("replacing config");
 		REF_RELEASE (old_cfg);
 		msg_info_main ("config has been reread successfully");