[Feature] Preload file and static maps in main process

diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h
index 777340e16b4eb24a7bc2d53acb1799bbbd95c698..a74169b31bcabf78f76dc62746ec7486b41cf762 100644 (file)
--- a/src/libserver/cfg_file.h
+++ b/src/libserver/cfg_file.h
@@ -487,10 +487,15 @@ enum rspamd_post_load_options {
        RSPAMD_CONFIG_INIT_LIBS = 1 << 1,
        RSPAMD_CONFIG_INIT_SYMCACHE = 1 << 2,
        RSPAMD_CONFIG_INIT_VALIDATE = 1 << 3,
-       RSPAMD_CONFIG_INIT_NO_TLD = 1 << 4
+       RSPAMD_CONFIG_INIT_NO_TLD = 1 << 4,
+       RSPAMD_CONFIG_INIT_PRELOAD_MAPS = 1 << 5,
 };
 
-#define RSPAMD_CONFIG_LOAD_ALL (RSPAMD_CONFIG_INIT_URL|RSPAMD_CONFIG_INIT_LIBS|RSPAMD_CONFIG_INIT_SYMCACHE|RSPAMD_CONFIG_INIT_VALIDATE)
+#define RSPAMD_CONFIG_LOAD_ALL (RSPAMD_CONFIG_INIT_URL| \
+               RSPAMD_CONFIG_INIT_LIBS| \
+               RSPAMD_CONFIG_INIT_SYMCACHE| \
+               RSPAMD_CONFIG_INIT_VALIDATE| \
+               RSPAMD_CONFIG_INIT_PRELOAD_MAPS)
 
 /**
  * Do post load actions for config

diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c
index 48c4d5eed976db876135f84cb28f1722059e8be3..96703277701114d4ab1b2ef8c61dfc5a60507d10 100644 (file)
--- a/src/libserver/cfg_utils.c
+++ b/src/libserver/cfg_utils.c
@@ -850,7 +850,11 @@ rspamd_config_post_load (struct rspamd_config *cfg,
                        ret = FALSE;
                }
 
-               return rspamd_symbols_cache_validate (cfg->cache, cfg, FALSE) && ret;
+               ret = rspamd_symbols_cache_validate (cfg->cache, cfg, FALSE) && ret;
+       }
+
+       if (opts & RSPAMD_CONFIG_INIT_PRELOAD_MAPS) {
+               rspamd_map_preload (cfg);
        }
 
        return ret;

diff --git a/src/libutil/map.c b/src/libutil/map.c
index 7a4d48598f4545a18439842752bbebedd3d168d9..8f36541a2aabb631477dabdca5e44a485bf0e447 100644 (file)
--- a/src/libutil/map.c
+++ b/src/libutil/map.c
@@ -846,7 +846,8 @@ read_map_file (struct rspamd_map *map, struct file_map_data *data,
                        return FALSE;
                }
                else {
-                       msg_info_map ("%s: map file is not found",
+                       msg_info_map ("%s: map file is not found; "
+                                                 "it will be read automatically if created",
                                        data->filename);
                        return TRUE;
                }
@@ -1004,6 +1005,9 @@ read_map_file (struct rspamd_map *map, struct file_map_data *data,
                map->read_callback (NULL, 0, &periodic->cbdata, TRUE);
        }
 
+       /* Also update at the read time */
+       memcpy (&data->st, &st, sizeof (struct stat));
+
        return TRUE;
 }
 
@@ -1729,6 +1733,73 @@ rspamd_map_watch (struct rspamd_config *cfg,
        }
 }
 
+void
+rspamd_map_preload (struct rspamd_config *cfg)
+{
+       GList *cur = cfg->maps;
+       struct rspamd_map *map;
+       struct rspamd_map_backend *bk;
+       guint i;
+       gboolean map_ok;
+
+       /* First of all do synced read of data */
+       while (cur) {
+               map = cur->data;
+               map_ok = TRUE;
+
+               PTR_ARRAY_FOREACH (map->backends, i, bk) {
+                       if (!(bk->protocol == MAP_PROTO_FILE ||
+                                 bk->protocol == MAP_PROTO_STATIC)) {
+                               map_ok = FALSE;
+                               break;
+                       }
+               }
+
+               if (map_ok) {
+                       struct map_periodic_cbdata fake_cbd;
+                       gboolean succeed = TRUE;
+
+                       memset (&fake_cbd, 0, sizeof (fake_cbd));
+
+                       PTR_ARRAY_FOREACH (map->backends, i, bk) {
+                               fake_cbd.cbdata.state = 0;
+                               fake_cbd.cbdata.prev_data = *map->user_data;
+                               fake_cbd.cbdata.cur_data = NULL;
+                               fake_cbd.cbdata.map = map;
+                               fake_cbd.map = map;
+                               fake_cbd.cur_backend = i;
+
+                               if (bk->protocol == MAP_PROTO_FILE) {
+                                       if (!read_map_file (map, bk->data.fd, bk, &fake_cbd)) {
+                                               succeed = FALSE;
+                                               break;
+                                       }
+                               }
+                               else if (bk->protocol == MAP_PROTO_STATIC) {
+                                       if (!read_map_static (map, bk->data.sd, bk, &fake_cbd)) {
+                                               succeed = FALSE;
+                                               break;
+                                       }
+                               }
+                               else {
+                                       g_assert_not_reached ();
+                               }
+                       }
+
+                       if (succeed) {
+                               map->fin_callback (&fake_cbd.cbdata);
+
+                               if (fake_cbd.cbdata.cur_data) {
+                                       *map->user_data = fake_cbd.cbdata.cur_data;
+                               }
+                       }
+
+               }
+
+               cur = g_list_next (cur);
+       }
+}
+
 void
 rspamd_map_remove_all (struct rspamd_config *cfg)
 {

diff --git a/src/libutil/map.h b/src/libutil/map.h
index 9c7485c58a2dc5da38aeeaf5b16f56dd2e98a65e..80aa03825c50cad7054d9b26bc27bcc1a9aa5d33 100644 (file)
--- a/src/libutil/map.h
+++ b/src/libutil/map.h
@@ -84,6 +84,12 @@ void rspamd_map_watch (struct rspamd_config *cfg,
                                           struct rspamd_worker *worker,
                                           gboolean active_http);
 
+/**
+ * Preloads maps where all backends are file
+ * @param cfg
+ */
+void rspamd_map_preload (struct rspamd_config *cfg);
+
 /**
  * Remove all maps watched (remove events)
  */

diff --git a/src/rspamd.c b/src/rspamd.c
index cba581419f63c1496766ba5db501a3f4c1ca90fc..6d676b96810e6cb925da5644f388b0ab0e4375a0 100644 (file)
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -305,6 +305,7 @@ reread_config (struct rspamd_main *rspamd_main)
                REF_RELEASE (tmp_cfg);
        }
        else {
+               rspamd_map_preload (rspamd_main->cfg);
                msg_info_main ("replacing config");
                REF_RELEASE (old_cfg);
                msg_info_main ("config has been reread successfully");