Merge pull request #4818 from rspamd/vstakhov-fuzzy-keymap

Allow dynamic keypairs loading in fuzzy storage

11 files changed, 611 insertions, 151 deletions

diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index d42dffdce..efa53b15b 100644
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Vsevolod Stakhov
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -107,7 +107,38 @@ struct rspamd_leaky_bucket_elt {
 };
 
 static const guint64 rspamd_fuzzy_storage_magic = 0x291a3253eb1b3ea5ULL;
+
+static int64_t
+fuzzy_kp_hash(const unsigned char *p)
+{
+	int64_t res;
+
+	memcpy(&res, p, sizeof(res));
+	return res;
+}
+static bool
+fuzzy_kp_equal(gconstpointer a, gconstpointer b)
+{
+	const guchar *pa = a, *pb = b;
+
+	return (memcmp(pa, pb, RSPAMD_FUZZY_KEYLEN) == 0);
+}
+
 KHASH_SET_INIT_INT(fuzzy_key_ids_set);
+KHASH_INIT(fuzzy_key_flag_stat, int, struct fuzzy_key_stat, 1, kh_int_hash_func,
+		   kh_int_hash_equal);
+struct fuzzy_key {
+	struct rspamd_cryptobox_keypair *key;
+	struct rspamd_cryptobox_pubkey *pk;
+	struct fuzzy_key_stat *stat;
+	khash_t(fuzzy_key_flag_stat) * flags_stat;
+	khash_t(fuzzy_key_ids_set) * forbidden_ids;
+	ref_entry_t ref;
+};
+
+KHASH_INIT(rspamd_fuzzy_keys_hash,
+		   const unsigned char *, struct fuzzy_key *, 1,
+		   fuzzy_kp_hash, fuzzy_kp_equal);
 
 struct rspamd_fuzzy_storage_ctx {
 	guint64 magic;
@@ -133,6 +164,7 @@ struct rspamd_fuzzy_storage_ctx {
 	const ucl_object_t *delay_whitelist_map;
 	const ucl_object_t *blocked_map;
 	const ucl_object_t *ratelimit_whitelist_map;
+	const ucl_object_t *dynamic_keys_map;
 
 	guint keypair_cache_size;
 	ev_timer stat_ev;
@@ -141,7 +173,10 @@ struct rspamd_fuzzy_storage_ctx {
 	/* Local keypair */
 	struct rspamd_cryptobox_keypair *default_keypair; /* Bad clash, need for parse keypair */
 	struct fuzzy_key *default_key;
-	GHashTable *keys;
+	khash_t(rspamd_fuzzy_keys_hash) * keys;
+	/* Those are loaded via map */
+	khash_t(rspamd_fuzzy_keys_hash) * dynamic_keys;
+
 	gboolean encrypted_only;
 	gboolean read_only;
 	gboolean dedicated_update_worker;
@@ -207,16 +242,6 @@ struct fuzzy_peer_request {
 	struct fuzzy_peer_cmd cmd;
 };
 
-KHASH_INIT(fuzzy_key_flag_stat, int, struct fuzzy_key_stat, 1, kh_int_hash_func,
-		   kh_int_hash_equal);
-struct fuzzy_key {
-	struct rspamd_cryptobox_keypair *key;
-	struct rspamd_cryptobox_pubkey *pk;
-	struct fuzzy_key_stat *stat;
-	khash_t(fuzzy_key_flag_stat) * flags_stat;
-	khash_t(fuzzy_key_ids_set) * forbidden_ids;
-};
-
 struct rspamd_updates_cbdata {
 	GArray *updates_pending;
 	struct rspamd_fuzzy_storage_ctx *ctx;
@@ -233,6 +258,151 @@ static gboolean rspamd_fuzzy_check_client(struct rspamd_fuzzy_storage_ctx *ctx,
 static void rspamd_fuzzy_maybe_call_blacklisted(struct rspamd_fuzzy_storage_ctx *ctx,
 												rspamd_inet_addr_t *addr,
 												const gchar *reason);
+static struct fuzzy_key *fuzzy_add_keypair_from_ucl(const ucl_object_t *obj,
+													khash_t(rspamd_fuzzy_keys_hash) * target);
+
+struct fuzzy_keymap_ucl_buf {
+	rspamd_fstring_t *buf;
+	struct rspamd_fuzzy_storage_ctx *ctx;
+};
+
+/* Callbacks for reading json dynamic rules */
+static gchar *
+ucl_keymap_read_cb(gchar *chunk,
+				   gint len,
+				   struct map_cb_data *data,
+				   gboolean final)
+{
+	struct fuzzy_keymap_ucl_buf *jb, *pd;
+
+	pd = data->prev_data;
+
+	g_assert(pd != NULL);
+
+	if (data->cur_data == NULL) {
+		jb = g_malloc0(sizeof(*jb));
+		jb->ctx = pd->ctx;
+		data->cur_data = jb;
+	}
+	else {
+		jb = data->cur_data;
+	}
+
+	if (jb->buf == NULL) {
+		/* Allocate memory for buffer */
+		jb->buf = rspamd_fstring_sized_new(MAX(len, 4096));
+	}
+
+	jb->buf = rspamd_fstring_append(jb->buf, chunk, len);
+
+	return NULL;
+}
+
+static void
+ucl_keymap_fin_cb(struct map_cb_data *data, void **target)
+{
+	struct fuzzy_keymap_ucl_buf *jb;
+	ucl_object_t *top;
+	struct ucl_parser *parser;
+	struct rspamd_config *cfg;
+
+	/* Now parse ucl */
+	if (data->cur_data) {
+		jb = data->cur_data;
+		cfg = jb->ctx->cfg;
+	}
+	else {
+		msg_err("no cur data in the map! might be a bug");
+		return;
+	}
+
+	if (jb->buf->len == 0) {
+		msg_err_config("no data read");
+
+		return;
+	}
+
+	parser = ucl_parser_new(UCL_PARSER_NO_FILEVARS);
+
+	if (!ucl_parser_add_chunk(parser, jb->buf->str, jb->buf->len)) {
+		msg_err_config("cannot load ucl data: parse error %s",
+					   ucl_parser_get_error(parser));
+		ucl_parser_free(parser);
+		return;
+	}
+
+	top = ucl_parser_get_object(parser);
+	ucl_parser_free(parser);
+
+	if (ucl_object_type(top) != UCL_ARRAY) {
+		ucl_object_unref(top);
+		msg_err_config("loaded ucl is not an array");
+		return;
+	}
+
+	if (target) {
+		*target = data->cur_data;
+	}
+
+	if (data->prev_data) {
+		jb = data->prev_data;
+		/* Clean prev data */
+		if (jb->buf) {
+			rspamd_fstring_free(jb->buf);
+		}
+
+		/* Clean the existing keys */
+		struct fuzzy_key *key;
+		kh_foreach_value(jb->ctx->dynamic_keys, key, {
+			REF_RELEASE(key);
+		});
+		kh_clear(rspamd_fuzzy_keys_hash, jb->ctx->dynamic_keys);
+
+		/* Insert new keys */
+		const ucl_object_t *cur;
+		ucl_object_iter_t it = NULL;
+		int success = 0;
+
+		while ((cur = ucl_object_iterate(top, &it, true)) != NULL) {
+			struct fuzzy_key *nk;
+
+			nk = fuzzy_add_keypair_from_ucl(cur, jb->ctx->dynamic_keys);
+
+			if (nk == NULL) {
+				msg_warn_config("cannot add dynamic keypair");
+			}
+			success++;
+		}
+
+		msg_info_config("loaded %d dynamic keypairs", success);
+
+		g_free(jb);
+	}
+
+	ucl_object_unref(top);
+}
+
+static void
+ucl_keymap_dtor_cb(struct map_cb_data *data)
+{
+	struct fuzzy_keymap_ucl_buf *jb;
+
+	if (data->cur_data) {
+		jb = data->cur_data;
+		/* Clean prev data */
+		if (jb->buf) {
+			rspamd_fstring_free(jb->buf);
+		}
+
+		struct fuzzy_key *key;
+		kh_foreach_value(jb->ctx->dynamic_keys, key, {
+			REF_RELEASE(key);
+		});
+		kh_destroy(rspamd_fuzzy_keys_hash, jb->ctx->dynamic_keys);
+
+		g_free(jb);
+	}
+}
 
 static gboolean
 rspamd_fuzzy_check_ratelimit(struct fuzzy_session *session)
@@ -494,6 +664,16 @@ fuzzy_key_dtor(gpointer p)
 }
 
 static void
+fuzzy_hash_table_dtor(khash_t(rspamd_fuzzy_keys_hash) * hash)
+{
+	struct fuzzy_key *key;
+	kh_foreach_value(hash, key, {
+		REF_RELEASE(key);
+	});
+	kh_destroy(rspamd_fuzzy_keys_hash, hash);
+}
+
+static void
 fuzzy_count_callback(guint64 count, void *ud)
 {
 	struct rspamd_fuzzy_storage_ctx *ctx = ud;
@@ -1446,7 +1626,7 @@ rspamd_fuzzy_decrypt_command(struct fuzzy_session *s, guchar *buf, gsize buflen)
 {
 	struct rspamd_fuzzy_encrypted_req_hdr hdr;
 	struct rspamd_cryptobox_pubkey *rk;
-	struct fuzzy_key *key;
+	struct fuzzy_key *key = NULL;
 
 	if (s->ctx->default_key == NULL) {
 		msg_warn("received encrypted request when encryption is not enabled");
@@ -1463,16 +1643,25 @@ rspamd_fuzzy_decrypt_command(struct fuzzy_session *s, guchar *buf, gsize buflen)
 	buflen -= sizeof(hdr);
 
 	/* Try to find the desired key */
-	key = g_hash_table_lookup(s->ctx->keys, hdr.key_id);
+	khiter_t k = kh_get(rspamd_fuzzy_keys_hash, s->ctx->keys, hdr.key_id);
+	if (k == kh_end(s->ctx->keys)) {
 
-	if (key == NULL) {
-		/* Unknown key, assume default one */
 		key = s->ctx->default_key;
-	}
 
-	s->key = key;
+		/* Check dynamic keys */
+		if (s->ctx->dynamic_keys) {
+			k = kh_get(rspamd_fuzzy_keys_hash, s->ctx->dynamic_keys, hdr.key_id);
+
+			if (k != kh_end(s->ctx->keys)) {
+				key = kh_val(s->ctx->dynamic_keys, k);
+			}
+		}
+	}
+	else {
+		key = kh_val(s->ctx->keys, k);
+	}
 
-	/* Now process keypair */
+	/* Now process the remote pubkey */
 	rk = rspamd_pubkey_from_bin(hdr.pubkey, sizeof(hdr.pubkey),
 								RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
 
@@ -1482,6 +1671,7 @@ rspamd_fuzzy_decrypt_command(struct fuzzy_session *s, guchar *buf, gsize buflen)
 		return FALSE;
 	}
 
+	/* Try to get the cached NM */
 	rspamd_keypair_cache_process(s->ctx->keypair_cache, key->key, rk);
 
 	/* Now decrypt request */
@@ -1495,6 +1685,9 @@ rspamd_fuzzy_decrypt_command(struct fuzzy_session *s, guchar *buf, gsize buflen)
 		return FALSE;
 	}
 
+	s->key = key;
+	REF_RETAIN(key);
+
 	memcpy(s->nm, rspamd_pubkey_get_nm(rk, key->key), sizeof(s->nm));
 	rspamd_pubkey_unref(rk);
 
@@ -1750,6 +1943,10 @@ fuzzy_session_destroy(gpointer d)
 		g_free(session->extensions);
 	}
 
+	if (session->key) {
+		REF_RELEASE(session->key);
+	}
+
 	g_free(session);
 }
 
@@ -2088,64 +2285,65 @@ rspamd_fuzzy_storage_stat_key(const struct fuzzy_key_stat *key_stat)
 	return res;
 }
 
-static ucl_object_t *
-rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
+static void
+rspamd_fuzzy_key_stat_iter(const unsigned char *pk_iter, struct fuzzy_key *fuzzy_key, ucl_object_t *keys_obj, gboolean ip_stat)
 {
-	struct fuzzy_key_stat *key_stat;
-	GHashTableIter it;
-	struct fuzzy_key *fuzzy_key;
-	ucl_object_t *obj, *keys_obj, *elt, *ip_elt, *ip_cur;
-	gpointer k, v;
-	gint i;
+	struct fuzzy_key_stat *key_stat = fuzzy_key->stat;
 	gchar keyname[17];
 
-	obj = ucl_object_typed_new(UCL_OBJECT);
+	if (key_stat) {
+		rspamd_snprintf(keyname, sizeof(keyname), "%8bs", pk_iter);
 
-	keys_obj = ucl_object_typed_new(UCL_OBJECT);
-	g_hash_table_iter_init(&it, ctx->keys);
+		ucl_object_t *elt = rspamd_fuzzy_storage_stat_key(key_stat);
 
-	while (g_hash_table_iter_next(&it, &k, &v)) {
-		fuzzy_key = v;
-		key_stat = fuzzy_key->stat;
+		if (key_stat->last_ips && ip_stat) {
+			int i = 0;
+			ucl_object_t *ip_elt = ucl_object_typed_new(UCL_OBJECT);
+			gpointer k, v;
 
-		if (key_stat) {
-			rspamd_snprintf(keyname, sizeof(keyname), "%8bs", k);
+			while ((i = rspamd_lru_hash_foreach(key_stat->last_ips,
+												i, &k, &v)) != -1) {
+				ucl_object_t *ip_cur = rspamd_fuzzy_storage_stat_key(v);
+				ucl_object_insert_key(ip_elt, ip_cur,
+									  rspamd_inet_address_to_string(k), 0, true);
+			}
+			ucl_object_insert_key(elt, ip_elt, "ips", 0, false);
+		}
 
-			elt = rspamd_fuzzy_storage_stat_key(key_stat);
+		int flag;
+		struct fuzzy_key_stat *flag_stat;
+		ucl_object_t *flags_ucl = ucl_object_typed_new(UCL_OBJECT);
 
-			if (key_stat->last_ips && ip_stat) {
-				i = 0;
+		kh_foreach_key_value_ptr(fuzzy_key->flags_stat, flag, flag_stat, {
+			char intbuf[16];
+			rspamd_snprintf(intbuf, sizeof(intbuf), "%d", flag);
+			ucl_object_insert_key(flags_ucl, rspamd_fuzzy_storage_stat_key(flag_stat),
+								  intbuf, 0, true);
+		});
 
-				ip_elt = ucl_object_typed_new(UCL_OBJECT);
+		ucl_object_insert_key(elt, flags_ucl, "flags", 0, false);
 
-				while ((i = rspamd_lru_hash_foreach(key_stat->last_ips,
-													i, &k, &v)) != -1) {
-					ip_cur = rspamd_fuzzy_storage_stat_key(v);
-					ucl_object_insert_key(ip_elt, ip_cur,
-										  rspamd_inet_address_to_string(k), 0, true);
-				}
-				ucl_object_insert_key(elt, ip_elt, "ips", 0, false);
-			}
+		ucl_object_insert_key(elt,
+							  rspamd_keypair_to_ucl(fuzzy_key->key, RSPAMD_KEYPAIR_DUMP_NO_SECRET | RSPAMD_KEYPAIR_DUMP_FLATTENED),
+							  "keypair", 0, false);
+		ucl_object_insert_key(keys_obj, elt, keyname, 0, true);
+	}
+}
 
-			int flag;
-			struct fuzzy_key_stat *flag_stat;
-			ucl_object_t *flags_ucl = ucl_object_typed_new(UCL_OBJECT);
+static ucl_object_t *
+rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
+{
+	struct fuzzy_key *fuzzy_key;
+	ucl_object_t *obj, *keys_obj, *elt, *ip_elt;
+	const unsigned char *pk_iter;
 
-			kh_foreach_key_value_ptr(fuzzy_key->flags_stat, flag, flag_stat, {
-				char intbuf[16];
-				rspamd_snprintf(intbuf, sizeof(intbuf), "%d", flag);
-				ucl_object_insert_key(flags_ucl, rspamd_fuzzy_storage_stat_key(flag_stat),
-									  intbuf, 0, true);
-			});
+	obj = ucl_object_typed_new(UCL_OBJECT);
 
-			ucl_object_insert_key(elt, flags_ucl, "flags", 0, false);
+	keys_obj = ucl_object_typed_new(UCL_OBJECT);
 
-			ucl_object_insert_key(elt,
-								  rspamd_keypair_to_ucl(fuzzy_key->key, RSPAMD_KEYPAIR_DUMP_NO_SECRET | RSPAMD_KEYPAIR_DUMP_FLATTENED),
-								  "keypair", 0, false);
-			ucl_object_insert_key(keys_obj, elt, keyname, 0, true);
-		}
-	}
+	kh_foreach(ctx->keys, pk_iter, fuzzy_key, {
+		rspamd_fuzzy_key_stat_iter(pk_iter, fuzzy_key, keys_obj, ip_stat);
+	});
 
 	ucl_object_insert_key(obj, keys_obj, "keys", 0, false);
 
@@ -2172,8 +2370,8 @@ rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
 						  false);
 
 	if (ctx->errors_ips && ip_stat) {
-		i = 0;
-
+		gpointer k, v;
+		int i = 0;
 		ip_elt = ucl_object_typed_new(UCL_OBJECT);
 
 		while ((i = rspamd_lru_hash_foreach(ctx->errors_ips, i, &k, &v)) != -1) {
@@ -2192,7 +2390,7 @@ rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
 	/* Checked by epoch */
 	elt = ucl_object_typed_new(UCL_ARRAY);
 
-	for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
+	for (int i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
 		ucl_array_append(elt,
 						 ucl_object_fromint(ctx->stat.fuzzy_hashes_checked[i]));
 	}
@@ -2202,7 +2400,7 @@ rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
 	/* Shingles by epoch */
 	elt = ucl_object_typed_new(UCL_ARRAY);
 
-	for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
+	for (int i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
 		ucl_array_append(elt,
 						 ucl_object_fromint(ctx->stat.fuzzy_shingles_checked[i]));
 	}
@@ -2212,7 +2410,7 @@ rspamd_fuzzy_stat_to_ucl(struct rspamd_fuzzy_storage_ctx *ctx, gboolean ip_stat)
 	/* Matched by epoch */
 	elt = ucl_object_typed_new(UCL_ARRAY);
 
-	for (i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
+	for (int i = RSPAMD_FUZZY_EPOCH10; i < RSPAMD_FUZZY_EPOCH_MAX; i++) {
 		ucl_array_append(elt,
 						 ucl_object_fromint(ctx->stat.fuzzy_hashes_found[i]));
 	}
@@ -2558,6 +2756,87 @@ fuzzy_parse_ids(rspamd_mempool_t *pool,
 	return FALSE;
 }
 
+static struct fuzzy_key *
+fuzzy_add_keypair_from_ucl(const ucl_object_t *obj, khash_t(rspamd_fuzzy_keys_hash) * target)
+{
+	struct rspamd_cryptobox_keypair *kp = rspamd_keypair_from_ucl(obj);
+
+	if (kp == NULL) {
+		return NULL;
+	}
+
+	if (rspamd_keypair_alg(kp) != RSPAMD_CRYPTOBOX_MODE_25519 ||
+		rspamd_keypair_type(kp) != RSPAMD_KEYPAIR_KEX) {
+		return FALSE;
+	}
+
+	struct fuzzy_key *key = g_malloc0(sizeof(*key));
+	REF_INIT_RETAIN(key, fuzzy_key_dtor);
+	key->key = kp;
+	struct fuzzy_key_stat *keystat = g_malloc0(sizeof(*keystat));
+	REF_INIT_RETAIN(keystat, fuzzy_key_stat_dtor);
+	/* Hash of ip -> fuzzy_key_stat */
+	keystat->last_ips = rspamd_lru_hash_new_full(1024,
+												 (GDestroyNotify) rspamd_inet_address_free,
+												 fuzzy_key_stat_unref,
+												 rspamd_inet_address_hash, rspamd_inet_address_equal);
+	key->stat = keystat;
+	key->flags_stat = kh_init(fuzzy_key_flag_stat);
+	/* Preallocate some space for flags */
+	kh_resize(fuzzy_key_flag_stat, key->flags_stat, 8);
+	const guchar *pk = rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_PK,
+												NULL);
+	keystat->keypair = rspamd_keypair_ref(kp);
+	/* We map entries by pubkey in binary form for faster lookup */
+	khiter_t k;
+	int r;
+
+	k = kh_put(rspamd_fuzzy_keys_hash, target, pk, &r);
+
+	if (r == 0) {
+		msg_err("duplicate keypair found: pk=%*bs",
+				32, pk);
+		REF_RELEASE(key);
+
+		return FALSE;
+	}
+	else if (r == -1) {
+		msg_err("hash insertion error: pk=%*bs",
+				32, pk);
+		REF_RELEASE(key);
+
+		return FALSE;
+	}
+
+	kh_val(target, k) = key;
+
+	const ucl_object_t *extensions = rspamd_keypair_get_extensions(kp);
+
+	if (extensions) {
+		const ucl_object_t *forbidden_ids = ucl_object_lookup(extensions, "forbidden_ids");
+
+		if (forbidden_ids && ucl_object_type(forbidden_ids) == UCL_ARRAY) {
+			key->forbidden_ids = kh_init(fuzzy_key_ids_set);
+			const ucl_object_t *cur;
+			ucl_object_iter_t it = NULL;
+
+			while ((cur = ucl_object_iterate(forbidden_ids, &it, true)) != NULL) {
+				if (ucl_object_type(cur) == UCL_INT || ucl_object_type(cur) == UCL_FLOAT) {
+					int id = ucl_object_toint(cur);
+					int r;
+
+					kh_put(fuzzy_key_ids_set, key->forbidden_ids, id, &r);
+				}
+			}
+		}
+	}
+
+	msg_debug("loaded keypair %*bs", rspamd_cryptobox_pk_bytes(RSPAMD_CRYPTOBOX_MODE_25519), pk);
+
+	return key;
+}
+
+
 static gboolean
 fuzzy_parse_keypair(rspamd_mempool_t *pool,
 					const ucl_object_t *obj,
@@ -2567,11 +2846,8 @@ fuzzy_parse_keypair(rspamd_mempool_t *pool,
 {
 	struct rspamd_rcl_struct_parser *pd = ud;
 	struct rspamd_fuzzy_storage_ctx *ctx;
-	struct rspamd_cryptobox_keypair *kp;
-	struct fuzzy_key_stat *keystat;
 	struct fuzzy_key *key;
 	const ucl_object_t *cur;
-	const guchar *pk;
 	ucl_object_iter_t it = NULL;
 	gboolean ret;
 
@@ -2588,57 +2864,14 @@ fuzzy_parse_keypair(rspamd_mempool_t *pool,
 			return ret;
 		}
 
-		/* Insert key to the hash table */
-		kp = ctx->default_keypair;
-
-		if (kp == NULL) {
-			return FALSE;
-		}
+		key = fuzzy_add_keypair_from_ucl(obj, ctx->keys);
 
-		if (rspamd_keypair_alg(kp) != RSPAMD_CRYPTOBOX_MODE_25519 ||
-			rspamd_keypair_type(kp) != RSPAMD_KEYPAIR_KEX) {
+		if (key == NULL) {
 			return FALSE;
 		}
 
-		key = g_malloc0(sizeof(*key));
-		key->key = kp;
-		keystat = g_malloc0(sizeof(*keystat));
-		REF_INIT_RETAIN(keystat, fuzzy_key_stat_dtor);
-		/* Hash of ip -> fuzzy_key_stat */
-		keystat->last_ips = rspamd_lru_hash_new_full(1024,
-													 (GDestroyNotify) rspamd_inet_address_free,
-													 fuzzy_key_stat_unref,
-													 rspamd_inet_address_hash, rspamd_inet_address_equal);
-		key->stat = keystat;
-		key->flags_stat = kh_init(fuzzy_key_flag_stat);
-		/* Preallocate some space for flags */
-		kh_resize(fuzzy_key_flag_stat, key->flags_stat, 8);
-		pk = rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_PK,
-									  NULL);
-		keystat->keypair = rspamd_keypair_ref(kp);
-		/* We map entries by pubkey in binary form for faster lookup */
-		g_hash_table_insert(ctx->keys, (gpointer) pk, key);
+		/* Use the last one ? */
 		ctx->default_key = key;
-
-		const ucl_object_t *extensions = rspamd_keypair_get_extensions(kp);
-
-		if (extensions) {
-			const ucl_object_t *forbidden_ids = ucl_object_lookup(extensions, "forbidden_ids");
-
-			if (forbidden_ids && ucl_object_type(forbidden_ids) == UCL_ARRAY) {
-				key->forbidden_ids = kh_init(fuzzy_key_ids_set);
-				while ((cur = ucl_object_iterate(forbidden_ids, &it, true)) != NULL) {
-					if (ucl_object_type(cur) == UCL_INT || ucl_object_type(cur) == UCL_FLOAT) {
-						int id = ucl_object_toint(cur);
-						int r;
-
-						kh_put(fuzzy_key_ids_set, key->forbidden_ids, id, &r);
-					}
-				}
-			}
-		}
-
-		msg_debug_pool_check("loaded keypair %*xs", 8, pk);
 	}
 	else if (ucl_object_type(obj) == UCL_ARRAY) {
 		while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) {
@@ -2651,20 +2884,6 @@ fuzzy_parse_keypair(rspamd_mempool_t *pool,
 	return TRUE;
 }
 
-static guint
-fuzzy_kp_hash(gconstpointer p)
-{
-	return *(guint *) p;
-}
-
-static gboolean
-fuzzy_kp_equal(gconstpointer a, gconstpointer b)
-{
-	const guchar *pa = a, *pb = b;
-
-	return (memcmp(pa, pb, RSPAMD_FUZZY_KEYLEN) == 0);
-}
-
 gpointer
 init_fuzzy(struct rspamd_config *cfg)
 {
@@ -2682,10 +2901,9 @@ init_fuzzy(struct rspamd_config *cfg)
 	ctx->lua_pre_handler_cbref = -1;
 	ctx->lua_post_handler_cbref = -1;
 	ctx->lua_blacklist_cbref = -1;
-	ctx->keys = g_hash_table_new_full(fuzzy_kp_hash, fuzzy_kp_equal,
-									  NULL, fuzzy_key_dtor);
+	ctx->keys = kh_init(rspamd_fuzzy_keys_hash);
 	rspamd_mempool_add_destructor(cfg->cfg_pool,
-								  (rspamd_mempool_destruct_t) g_hash_table_unref, ctx->keys);
+								  (rspamd_mempool_destruct_t) fuzzy_hash_table_dtor, ctx->keys);
 	ctx->errors_ips = rspamd_lru_hash_new_full(1024,
 											   (GDestroyNotify) rspamd_inet_address_free, g_free,
 											   rspamd_inet_address_hash, rspamd_inet_address_equal);
@@ -2770,6 +2988,15 @@ init_fuzzy(struct rspamd_config *cfg)
 
 	rspamd_rcl_register_worker_option(cfg,
 									  type,
+									  "dynamic_keys_map",
+									  rspamd_rcl_parse_struct_ucl,
+									  ctx,
+									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, dynamic_keys_map),
+									  0,
+									  "Dynamic encryption keypairs (can be repeated for different keys)");
+
+	rspamd_rcl_register_worker_option(cfg,
+									  type,
 									  "forbidden_ids",
 									  fuzzy_parse_ids,
 									  ctx,
@@ -3145,6 +3372,31 @@ start_fuzzy(struct rspamd_worker *worker)
 									 worker, "fuzzy ratelimit whitelist");
 	}
 
+	if (ctx->dynamic_keys_map) {
+		struct fuzzy_keymap_ucl_buf *jb, **pjb;
+
+		ctx->dynamic_keys = kh_init(rspamd_fuzzy_keys_hash);
+		/* Now try to add map with ucl data */
+		jb = g_malloc(sizeof(struct fuzzy_keymap_ucl_buf));
+		pjb = g_malloc(sizeof(struct fuzzy_keymap_ucl_buf *));
+		jb->buf = NULL;
+		jb->ctx = ctx;
+		*pjb = jb;
+		rspamd_mempool_add_destructor(ctx->cfg->cfg_pool,
+									  (rspamd_mempool_destruct_t) g_free,
+									  pjb);
+
+		if (!rspamd_map_add_from_ucl(cfg,
+									 ctx->dynamic_keys_map,
+									 "Dynamic fuzzy keys map",
+									 ucl_keymap_read_cb,
+									 ucl_keymap_fin_cb,
+									 ucl_keymap_dtor_cb,
+									 (void **) pjb, worker, RSPAMD_MAP_DEFAULT)) {
+			msg_err("cannot add map for dynamic keys");
+		}
+	}
+
 	if (!isnan(ctx->delay) && ctx->delay_whitelist_map != NULL) {
 		rspamd_config_radix_from_ucl(worker->srv->cfg, ctx->delay_whitelist_map,
 									 "Skip delay from the following ips",
diff --git a/src/libserver/maps/map.c b/src/libserver/maps/map.c
index 7f6a48f8c..3efd5a501 100644
--- a/src/libserver/maps/map.c
+++ b/src/libserver/maps/map.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Vsevolod Stakhov
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -995,7 +995,7 @@ rspamd_map_periodic_dtor(struct map_periodic_cbdata *periodic)
 	struct rspamd_map *map;
 
 	map = periodic->map;
-	msg_debug_map("periodic dtor %p", periodic);
+	msg_debug_map("periodic dtor %p; need_modify=%d", periodic, periodic->need_modify);
 
 	if (periodic->need_modify || periodic->cbdata.errored) {
 		/* Need to notify the real data structure */
@@ -1062,6 +1062,8 @@ rspamd_map_schedule_periodic(struct rspamd_map *map, int how)
 		return;
 	}
 
+	map->seen = true;
+
 	if (map->non_trivial && map->next_check != 0) {
 		timeout = map->next_check - rspamd_get_calendar_ticks();
 		map->next_check = 0;
@@ -1107,7 +1109,7 @@ rspamd_map_schedule_periodic(struct rspamd_map *map, int how)
 		timeout = map->poll_timeout;
 
 		if (how & RSPAMD_MAP_SCHEDULE_INIT) {
-			if (map->active_http) {
+			if (map->non_trivial && map->active_http) {
 				/* Spill maps load to get better chances to hit ssl cache */
 				timeout = rspamd_time_jitter(0.0, 2.0);
 			}
@@ -2189,7 +2191,7 @@ void rspamd_map_watch(struct rspamd_config *cfg,
 
 				data = bk->data.fd;
 
-				if (map->user_data == NULL || *map->user_data == NULL) {
+				if (!map->seen || map->user_data == NULL || *map->user_data == NULL) {
 					/* Map has not been read, init it's reading if possible */
 					struct stat st;
 
@@ -2317,6 +2319,8 @@ void rspamd_map_preload(struct rspamd_config *cfg)
 				if (map->on_load_function) {
 					map->on_load_function(map, map->on_load_ud);
 				}
+
+				map->seen = true;
 			}
 			else {
 				msg_info_map("preload of %s failed", map->name);
diff --git a/src/libserver/maps/map_private.h b/src/libserver/maps/map_private.h
index 60751c0ac..b88a18e3f 100644
--- a/src/libserver/maps/map_private.h
+++ b/src/libserver/maps/map_private.h
@@ -1,11 +1,11 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
+/*
+ * Copyright 2024 Vsevolod Stakhov
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -166,6 +166,7 @@ struct rspamd_map {
 	bool file_only;    /* No HTTP backends found */
 	bool static_only;  /* No need to check */
 	bool no_file_read; /* Do not read files */
+	bool seen;         /* This map has already been watched or pre-loaded */
 	/* Shared lock for temporary disabling of map reading (e.g. when this map is written by UI) */
 	gint *locked;
 	gchar tag[MEMPOOL_UID_LEN];
diff --git a/test/functional/cases/120_fuzzy/encrypted-dyn1.robot b/test/functional/cases/120_fuzzy/encrypted-dyn1.robot
new file mode 100644
index 000000000..dae74879f
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/encrypted-dyn1.robot
@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup     Fuzzy Setup Encrypted Dyn1 Siphash
+Suite Teardown  Rspamd Redis Teardown
+Resource        lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+  Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+  Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+  Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/120_fuzzy/encrypted-dyn2.robot b/test/functional/cases/120_fuzzy/encrypted-dyn2.robot
new file mode 100644
index 000000000..ef10ca3ae
--- /dev/null
+++ b/test/functional/cases/120_fuzzy/encrypted-dyn2.robot
@@ -0,0 +1,14 @@
+*** Settings ***
+Suite Setup     Fuzzy Setup Encrypted Dyn2 Siphash
+Suite Teardown  Rspamd Redis Teardown
+Resource        lib.robot
+
+*** Test Cases ***
+Fuzzy Add
+  Fuzzy Multimessage Add Test
+
+Fuzzy Fuzzy
+  Fuzzy Multimessage Fuzzy Test
+
+Fuzzy Miss
+  Fuzzy Multimessage Miss Test
diff --git a/test/functional/cases/120_fuzzy/lib.robot b/test/functional/cases/120_fuzzy/lib.robot
index 02e0a0af5..fda0af54a 100644
--- a/test/functional/cases/120_fuzzy/lib.robot
+++ b/test/functional/cases/120_fuzzy/lib.robot
@@ -75,6 +75,15 @@ Fuzzy Fuzzy Test
     Expect Symbol  ${FLAG1_SYMBOL}
   END
 
+Fuzzy Encrypted Test
+  [Arguments]  ${message}
+  @{path_info} =  Path Splitter  ${message}
+  @{fuzzy_files} =  List Files In Directory  ${pathinfo}[0]  pattern=${pathinfo}[1].fuzzy*  absolute=1
+  FOR  ${i}  IN  @{fuzzy_files}
+    ${result} =  Run Rspamc  -p  -h  ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_NORMAL}  --key  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${i}
+    Check Rspamc  ${result}  ${FLAG1_SYMBOL}
+  END
+
 Fuzzy Miss Test
   [Arguments]  ${message}
   Scan File  ${message}
@@ -98,15 +107,34 @@ Fuzzy Setup Encrypted
   Set Suite Variable  ${RSPAMD_FUZZY_ALGORITHM}  ${algorithm}
   Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTED_ONLY}  true
   Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
+  Set Suite Variable  ${RSPAMD_FUZZY_CLIENT_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
   Set Suite Variable  ${RSPAMD_FUZZY_INCLUDE}  ${RSPAMD_TESTDIR}/configs/fuzzy-encryption-key.conf
   Rspamd Redis Setup
 
-Fuzzy Setup Encrypted Keyed
+Fuzzy Setup Encrypted Dyn1
   [Arguments]  ${algorithm}
   Set Suite Variable  ${RSPAMD_FUZZY_ALGORITHM}  ${algorithm}
   Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTED_ONLY}  true
-  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1} 
+  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
+  Set Suite Variable  ${RSPAMD_FUZZY_CLIENT_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB2}
+  Set Suite Variable  ${RSPAMD_FUZZY_INCLUDE}  ${RSPAMD_TESTDIR}/configs/fuzzy-encryption-key.conf
+  Rspamd Redis Setup
 
+Fuzzy Setup Encrypted Dyn2
+  [Arguments]  ${algorithm}
+  Set Suite Variable  ${RSPAMD_FUZZY_ALGORITHM}  ${algorithm}
+  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTED_ONLY}  true
+  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
+  Set Suite Variable  ${RSPAMD_FUZZY_CLIENT_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB3}
+  Set Suite Variable  ${RSPAMD_FUZZY_INCLUDE}  ${RSPAMD_TESTDIR}/configs/fuzzy-encryption-key.conf
+  Rspamd Redis Setup
+
+Fuzzy Setup Encrypted Keyed
+  [Arguments]  ${algorithm}
+  Set Suite Variable  ${RSPAMD_FUZZY_ALGORITHM}  ${algorithm}
+  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTED_ONLY}  true
+  Set Suite Variable  ${RSPAMD_FUZZY_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
+  Set Suite Variable  ${RSPAMD_FUZZY_CLIENT_ENCRYPTION_KEY}  ${RSPAMD_KEY_PUB1}
   Set Suite Variable  ${RSPAMD_FUZZY_KEY}  mYN888sydwLTfE32g2hN
   Set Suite Variable  ${RSPAMD_FUZZY_SHINGLES_KEY}  hXUCgul9yYY3Zlk1QIT2
   Rspamd Redis Setup
@@ -150,6 +178,12 @@ Fuzzy Setup Keyed Xxhash
 Fuzzy Setup Encrypted Siphash
   Fuzzy Setup Encrypted  siphash
 
+Fuzzy Setup Encrypted Dyn1 Siphash
+  Fuzzy Setup Encrypted Dyn1  siphash
+
+Fuzzy Setup Encrypted Dyn2 Siphash
+  Fuzzy Setup Encrypted Dyn2  siphash
+
 Fuzzy Skip Hash Test Message
   FOR  ${i}  IN  @{MESSAGES_SKIP}
     Fuzzy Skip Add Test Base  ${i}
@@ -165,6 +199,11 @@ Fuzzy Multimessage Fuzzy Test
     Fuzzy Fuzzy Test  ${i}
   END
 
+Fuzzy Multimessage Fuzzy Encrypted Test
+  FOR  ${i}  IN  @{MESSAGES}
+    Fuzzy Encrypted Test  ${i}
+  END
+
 Fuzzy Multimessage Miss Test
   FOR  ${i}  IN  @{RANDOM_MESSAGES}
     Fuzzy Miss Test  ${i}
diff --git a/test/functional/configs/fuzzy-encryption-key.conf b/test/functional/configs/fuzzy-encryption-key.conf
index 522081b08..3cb29e5af 100644
--- a/test/functional/configs/fuzzy-encryption-key.conf
+++ b/test/functional/configs/fuzzy-encryption-key.conf
@@ -1,2 +1,2 @@
 # Setting this to null does not work out so it's hidden in an include
-encryption_key = {= env.FUZZY_ENCRYPTION_KEY =};
+encryption_key = {= env.FUZZY_CLIENT_ENCRYPTION_KEY =};
diff --git a/test/functional/configs/fuzzy.conf b/test/functional/configs/fuzzy.conf
index 8af1cfa3f..f46faf6d4 100644
--- a/test/functional/configs/fuzzy.conf
+++ b/test/functional/configs/fuzzy.conf
@@ -60,6 +60,7 @@ worker {
 		privkey = "{= env.KEY_PVT1 =}";
 		pubkey = "{= env.KEY_PUB1 =}";
 	}
+	dynamic_keys_map = "{= env.TESTDIR =}/configs/maps/fuzzy_keymap.map";
 }
 
 fuzzy_check {
diff --git a/test/functional/configs/maps/fuzzy_keymap.map b/test/functional/configs/maps/fuzzy_keymap.map
new file mode 100644
index 000000000..df152a9c9
--- /dev/null
+++ b/test/functional/configs/maps/fuzzy_keymap.map
@@ -0,0 +1,16 @@
+[{
+    privkey = "achyfduzs74yc1p95bk9apoknhtzn596pzeai5ybi5tftencoray";
+    id = "xb66rsu7e5i3o95sr7ifd3rxgjruktn8ptsesdxrf4biyc5ckyu6zcye54pkw3cmkhbyoebow85bsqxhryfyy4eep5gai4x1a8s3u5d";
+    pubkey = "mbggdnw3tdx7r3ruakjecpf5hcqr4cb4nmdp1fxynx3drbyujb3y";
+    type = "kex";
+    algorithm = "curve25519";
+    encoding = "base32";
+},
+{
+    privkey = "y1z16mw4n8eaefgwhgneyrntb8rxx911r4q7pgweb7t8sj1q8goy";
+    id = "id8kmo7im37bszdoorpm6cjjg8saazz71bc9ijz974wip3gaockbpymb5e91r8cwsf7kmcbbbygap9bss8r3zkhth5i7pdnyazpkppy";
+    pubkey = "zhypei8sartqrtow84dddgp5exh3gsr65kbw88wj7ppot1bwmuiy";
+    type = "kex";
+    algorithm = "curve25519";
+    encoding = "base32";
+}]
\ No newline at end of file
diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py
index 3976ca949..76132ad5a 100644
--- a/test/functional/lib/rspamd.py
+++ b/test/functional/lib/rspamd.py
@@ -1,3 +1,29 @@
+#  Copyright 2024 Vsevolod Stakhov
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 from urllib.request import urlopen
 import glob
 import grp
@@ -17,6 +43,7 @@ from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 import demjson
 
+
 def Check_JSON(j):
     d = demjson.decode(j, strict=True)
     logger.debug('got json %s' % d)
@@ -24,6 +51,7 @@ def Check_JSON(j):
     assert 'error' not in d
     return d
 
+
 def check_json_log(fn):
     line_count = 0
     f = open(fn, 'r')
@@ -33,9 +61,11 @@ def check_json_log(fn):
         line_count = line_count + 1
     assert line_count > 0
 
+
 def cleanup_temporary_directory(directory):
     shutil.rmtree(directory)
 
+
 def save_run_results(directory, filenames):
     current_directory = os.getcwd()
     suite_name = BuiltIn().get_variable_value("${SUITE_NAME}")
@@ -58,24 +88,29 @@ def save_run_results(directory, filenames):
                 shutil.copy(source_file, "%s/%s" % (destination_directory, file))
                 shutil.copy(source_file, "%s/robot-save/%s.last" % (current_directory, file))
 
+
 def encode_filename(filename):
     return "".join(['%%%0X' % ord(b) for b in filename])
 
+
 def get_test_directory():
     return os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "../../")
 
+
 def get_top_dir():
     if os.environ.get('RSPAMD_TOPDIR'):
         return os.environ['RSPAMD_TOPDIR']
 
     return get_test_directory() + "/../../"
 
+
 def get_install_root():
     if os.environ.get('RSPAMD_INSTALLROOT'):
         return os.path.abspath(os.environ['RSPAMD_INSTALLROOT'])
 
     return os.path.abspath("../install/")
 
+
 def get_rspamd():
     if os.environ.get('RSPAMD'):
         return os.environ['RSPAMD']
@@ -84,6 +119,7 @@ def get_rspamd():
     dname = get_top_dir()
     return dname + "/src/rspamd"
 
+
 def get_rspamc():
     if os.environ.get('RSPAMC'):
         return os.environ['RSPAMC']
@@ -92,6 +128,7 @@ def get_rspamc():
     dname = get_top_dir()
     return dname + "/src/client/rspamc"
 
+
 def get_rspamadm():
     if os.environ.get('RSPAMADM'):
         return os.environ['RSPAMADM']
@@ -100,6 +137,7 @@ def get_rspamadm():
     dname = get_top_dir()
     return dname + "/src/rspamadm/rspamadm"
 
+
 def HTTP(method, host, port, path, data=None, headers={}):
     c = http.client.HTTPConnection("%s:%s" % (host, port))
     c.request(method, path, data, headers)
@@ -109,9 +147,11 @@ def HTTP(method, host, port, path, data=None, headers={}):
     c.close()
     return [s, t]
 
+
 def hard_link(src, dst):
     os.link(src, dst)
 
+
 def make_temporary_directory():
     """Creates and returns a unique temporary directory
 
@@ -128,27 +168,31 @@ def make_temporary_directory():
              stat.S_IXOTH)
     return dirname
 
+
 def make_temporary_file():
     return tempfile.mktemp()
 
+
 def path_splitter(path):
     dirname = os.path.dirname(path)
     basename = os.path.basename(path)
     return [dirname, basename]
 
+
 def rspamc(addr, port, filename):
     mboxgoo = b"From MAILER-DAEMON Fri May 13 19:17:40 2016\r\n"
     goo = open(filename, 'rb').read()
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.connect((addr, port))
     s.send(b"CHECK RSPAMC/1.0\r\nContent-length: ")
-    s.send(str(len(goo+mboxgoo)).encode('utf-8'))
+    s.send(str(len(goo + mboxgoo)).encode('utf-8'))
     s.send(b"\r\n\r\n")
     s.send(mboxgoo)
     s.send(goo)
     r = s.recv(2048)
     return r.decode('utf-8')
 
+
 def Scan_File(filename, **headers):
     addr = BuiltIn().get_variable_value("${RSPAMD_LOCAL_ADDR}")
     port = BuiltIn().get_variable_value("${RSPAMD_PORT_NORMAL}")
@@ -162,16 +206,19 @@ def Scan_File(filename, **headers):
     BuiltIn().set_test_variable("${SCAN_RESULT}", d)
     return
 
+
 def Send_SIGUSR1(pid):
     pid = int(pid)
     os.kill(pid, signal.SIGUSR1)
 
+
 def set_directory_ownership(path, username, groupname):
     if os.getuid() == 0:
-        uid=pwd.getpwnam(username).pw_uid
-        gid=grp.getgrnam(groupname).gr_gid
+        uid = pwd.getpwnam(username).pw_uid
+        gid = grp.getgrnam(groupname).gr_gid
         os.chown(path, uid, gid)
 
+
 def spamc(addr, port, filename):
     goo = open(filename, 'rb').read()
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -184,6 +231,7 @@ def spamc(addr, port, filename):
     r = s.recv(2048)
     return r.decode('utf-8')
 
+
 def TCP_Connect(addr, port):
     """Attempts to open a TCP connection to specified address:port
 
@@ -191,13 +239,15 @@ def TCP_Connect(addr, port):
     | Wait Until Keyword Succeeds | 5s | 10ms | TCP Connect | localhost | 8080 |
     """
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.settimeout(5) # seconds
+    s.settimeout(5)  # seconds
     s.connect((addr, port))
     s.close()
 
+
 def ping_rspamd(addr, port):
     return str(urlopen("http://%s:%s/ping" % (addr, port)).read())
 
+
 def redis_check(addr, port):
     """Attempts to open a TCP connection to specified address:port
 
@@ -205,7 +255,7 @@ def redis_check(addr, port):
     | Wait Until Keyword Succeeds | 5s | 10ms | TCP Connect | localhost | 8080 |
     """
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.settimeout(1.0) # seconds
+    s.settimeout(1.0)  # seconds
     s.connect((addr, port))
     if s.sendall(b"ECHO TEST\n"):
         result = s.recv(128)
@@ -213,6 +263,7 @@ def redis_check(addr, port):
     else:
         return False
 
+
 def update_dictionary(a, b):
     a.update(b)
     return a
@@ -221,6 +272,7 @@ def update_dictionary(a, b):
 TERM_TIMEOUT = 10  # wait after sending a SIGTERM signal
 KILL_WAIT = 20  # additional wait after sending a SIGKILL signal
 
+
 def shutdown_process(process):
     # send SIGTERM
     process.terminate()
@@ -229,7 +281,7 @@ def shutdown_process(process):
         process.wait(TERM_TIMEOUT)
         return
     except psutil.TimeoutExpired:
-        logger.info( "PID {} is not terminated in {} seconds, sending SIGKILL...".format(process.pid, TERM_TIMEOUT))
+        logger.info("PID {} is not terminated in {} seconds, sending SIGKILL...".format(process.pid, TERM_TIMEOUT))
         try:
             # send SIGKILL
             process.kill()
@@ -258,6 +310,7 @@ def shutdown_process_with_children(pid):
             pass
     psutil.wait_procs(children, timeout=KILL_WAIT)
 
+
 def write_to_stdin(process_handle, text):
     if not isinstance(text, bytes):
         text = bytes(text, 'utf-8')
@@ -269,12 +322,14 @@ def write_to_stdin(process_handle, text):
     out = obj.stdout.read(4096)
     return out.decode('utf-8')
 
+
 def get_file_if_exists(file_path):
     if os.path.exists(file_path):
         with open(file_path, 'r') as myfile:
             return myfile.read()
     return None
 
+
 def _merge_luacov_stats(statsfile, coverage):
     """
     Reads a coverage stats file written by luacov and merges coverage data to
@@ -331,7 +386,7 @@ def collect_lua_coverage():
     | Collect Lua Coverage |
     """
     # decided not to do optional coverage so far
-    #if not 'ENABLE_LUA_COVERAGE' in os.environ['HOME']:
+    # if not 'ENABLE_LUA_COVERAGE' in os.environ['HOME']:
     #    logger.info("ENABLE_LUA_COVERAGE is not present in env, will not collect Lua coverage")
     #    return
 
diff --git a/test/functional/lib/vars.py b/test/functional/lib/vars.py
index 0a12a8191..a4bcbadda 100644
--- a/test/functional/lib/vars.py
+++ b/test/functional/lib/vars.py
@@ -1,3 +1,65 @@
+#  Copyright 2024 Vsevolod Stakhov
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 import shutil
 import socket
 
@@ -6,6 +68,8 @@ HAVE_MILTERTEST = shutil.which('miltertest') and True or False
 RSPAMD_EXTERNAL_RELAY_ENABLED = False
 RSPAMD_KEY_PVT1 = 'ekd3x36tfa5gd76t6pa8hqif3ott7n1siuux68exbkk7ukscte9y'
 RSPAMD_KEY_PUB1 = 'm8kneubpcjsb8sbsoj7jy7azj9fdd3xmj63txni86a8ye9ncomny'
+RSPAMD_KEY_PUB2 = 'mbggdnw3tdx7r3ruakjecpf5hcqr4cb4nmdp1fxynx3drbyujb3y'
+RSPAMD_KEY_PUB3 = 'zhypei8sartqrtow84dddgp5exh3gsr65kbw88wj7ppot1bwmuiy'
 RSPAMD_LOCAL_ADDR = '127.0.0.1'
 RSPAMD_MAP_WATCH_INTERVAL = '1min'
 RSPAMD_PORT_CONTROLLER = 56790