[Feature] Add tooling to encrypt strings in Lua

* [Fix] Provide support for OpenSSL 3.0

* [Feature] Provide function to encode header with configured public key

* [Feature] Provide function to decode header with configured public key

* [Test] Add tests for maybe encode/decode header

* [Minor] Fix tests for encode/decode header

* [Minor] Small clean up

* [Minor] Small clean up

* [Minor] Small fix for OpenSSL 3.0 support

* [Minor] Provide logging

* [Minor] Small fix

* [Fix] Fix typo error

* [Fix] Another typo

* [Minor] Little clean up

* [Minor] Little fix

* [Minor] Small fix

* [Minor] Small fix

* [Minor] Rewrite the arguments of secretbox:encrypt/decrypt functions to a more understandable format

* [Fix] Fix problem with nonce was not provided

* [Test] Add test for nonce

* [Minor] Little clean up

* [Minor] Little clean up

* [Test] Test

* [Test] Test

* [Test] Test

* [Minor] Little fix

* [Minor] Small fix

* [Minor] Small fix

* [Test] Small fix

* [Test] Test

* [Test] Test

* [Test] Test

* [Test] Test

* [Minor] Small fix for fips provider

* [Minor] Change provider apply logic

* [Test] Little fix for provider

* [Minor] Provide OpenSSL <3.0 support

* [Test] Possible provider fix

* [Test] Possible provider fix

* [Test] Little fix

* [Minor] Fix provider issue

* [Minor] Small clean up

* [Minor] Change logging errors

* Update lualib/lua_util.lua

---------

Co-authored-by: Vsevolod Stakhov <vsevolod@rspamd.com>

3 files changed, 215 insertions, 28 deletions

diff --git a/lualib/lua_util.lua b/lualib/lua_util.lua
index 470925b95..ffc07842e 100644
--- a/lualib/lua_util.lua
+++ b/lualib/lua_util.lua
@@ -1292,6 +1292,84 @@ exports.maybe_obfuscate_string = function(subject, settings, prefix)
 end
 
 ---[[[
+-- @function lua_util.maybe_encrypt_header(header, settings, prefix)
+-- Encode header with configured public key if enabled in settings.
+-- If header is not set then nil is returned. If pub_key is empty then header is returned.
+-- Supported settings:
+-- * <prefix>_encrypt = false - no need for encryption of a header
+-- * <prefix>_key = 'key' - key that is used encrypt header
+-- * <prefix>_nonce = 'nonce' - nonce to encrypt header(optional)
+-- @return encrypted header
+---]]]
+exports.maybe_encrypt_header = function(header, settings, prefix)
+  local rspamd_secretbox = require "rspamd_cryptobox_secretbox"
+
+  if not header or header == '' then
+    logger.errx(rspamd_config, "Header: %s is empty or nil", header)
+    return nil
+  elseif settings[prefix .. '_encrypt'] then
+    local key = settings[prefix .. '_key']
+    if not key or key == '' then
+      logger.errx(rspamd_config, "Key: %s is empty or nil", key)
+      return header
+    end
+    local cryptobox = rspamd_secretbox.create(key)
+
+    local nonce = settings[prefix .. '_nonce']
+    local encrypted_header = ''
+    if not nonce or nonce == '' then
+      encrypted_header, nonce = cryptobox:encrypt(header)
+    else
+      encrypted_header = cryptobox:encrypt(header, nonce)
+    end
+    return encrypted_header, nonce
+  end
+end
+
+---[[[
+-- @function lua_util.maybe_decrypt_header(header, settings, prefix, nonce)
+-- Decode enoced with configured public_key header if enabled in settings.
+-- If encoded header is not set then nil is returned. If pub_key is empty then encoded header is returned.
+-- Supported settings:
+-- * <prefix>_encrypt = false - no need for decryption of a header
+-- * <prefix>_key = 'key' - key that is used decrypt header
+-- * <prefix>_nonce = 'nonce' - nonce used to encrypt header(optional)
+-- Nonce is an optional argument if <prefix>_nonce is provided, otherwise it is an required argument
+-- and <prefix>_nonce is an optional
+-- @return decrypted header
+---]]]
+exports.maybe_decrypt_header = function(encrypted_header, settings, prefix, nonce)
+  local rspamd_secretbox = require "rspamd_cryptobox_secretbox"
+
+  if not encrypted_header or encrypted_header == '' then
+    logger.errx(rspamd_config, "Encoded header: %s is empty or nil")
+    return nil
+  elseif settings[prefix .. '_encrypt'] then
+    local key = settings[prefix .. '_key']
+    if not key or key == '' then
+      logger.errx(rspamd_config, "Key: %s is empty or nil")
+      return encrypted_header
+    end
+    local cryptobox = rspamd_secretbox.create(key)
+
+    local result = false
+    local header = ''
+    if not nonce then
+      result, header = cryptobox:decrypt(encrypted_header, settings[prefix .. '_nonce'])
+    else
+      result, header = cryptobox:decrypt(encrypted_header, nonce)
+    end
+
+    if not result then
+      logger.infox(rspamd_config, "Decryption is failed with result: %s and decrypted header: %s", result, header)
+      return nil
+    end
+
+    return header
+  end
+end
+
+---[[[
 -- @function lua_util.callback_from_string(str)
 -- Converts a string like `return function(...) end` to lua function and return true and this function
 -- or returns false + error message
diff --git a/src/lua/lua_cryptobox.c b/src/lua/lua_cryptobox.c
index 1b9074f58..fbd44cecd 100644
--- a/src/lua/lua_cryptobox.c
+++ b/src/lua/lua_cryptobox.c
@@ -37,7 +37,9 @@
 
 #include <stdalign.h>
 #include <openssl/hmac.h>
-
+#if OPENSSL_VERSION_MAJOR >= 3
+#include <openssl/provider.h>
+#endif
 
 enum lua_cryptobox_hash_type {
 	LUA_CRYPTOBOX_HASH_BLAKE2 = 0,
@@ -54,7 +56,11 @@ struct rspamd_lua_cryptobox_hash {
 	union {
 		rspamd_cryptobox_hash_state_t *h;
 		EVP_MD_CTX *c;
+#if OPENSSL_VERSION_MAJOR >= 3
+		EVP_MAC_CTX *hmac_c;
+#else
 		HMAC_CTX *hmac_c;
+#endif
 		rspamd_cryptobox_fast_hash_state_t *fh;
 	} content;
 
@@ -899,7 +905,11 @@ rspamd_lua_hash_update(struct rspamd_lua_cryptobox_hash *h,
 			EVP_DigestUpdate(h->content.c, p, len);
 			break;
 		case LUA_CRYPTOBOX_HASH_HMAC:
+#if OPENSSL_VERSION_MAJOR >= 3
+			EVP_MAC_update(h->content.hmac_c, p, len);
+#else
 			HMAC_Update(h->content.hmac_c, p, len);
+#endif
 			break;
 		case LUA_CRYPTOBOX_HASH_XXHASH64:
 		case LUA_CRYPTOBOX_HASH_XXHASH32:
@@ -931,8 +941,12 @@ lua_cryptobox_hash_dtor(struct rspamd_lua_cryptobox_hash *h)
 		HMAC_CTX_cleanup(h->content.hmac_c);
 		g_free(h->content.hmac_c);
 #else
+#if OPENSSL_VERSION_MAJOR >= 3
+		EVP_MAC_CTX_free(h->content.hmac_c);
+#else
 		HMAC_CTX_free(h->content.hmac_c);
 #endif
+#endif
 	}
 	else if (h->type == LUA_CRYPTOBOX_HASH_BLAKE2) {
 		rspamd_explicit_memzero(h->content.h, sizeof(*h->content.h));
@@ -984,25 +998,53 @@ rspamd_lua_ssl_hmac_create(struct rspamd_lua_cryptobox_hash *h, const EVP_MD *ht
 						   bool insecure)
 {
 	h->type = LUA_CRYPTOBOX_HASH_HMAC;
+	OSSL_PROVIDER *dflt = OSSL_PROVIDER_load(NULL, "default");
+
+#if OPENSSL_VERSION_NUMBER > 0x10100000L
+	if (insecure) {
+		/* Should never ever be used for crypto/security purposes! */
+#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+#if OPENSSL_VERSION_MAJOR >= 3
+		OSSL_PROVIDER *fips = OSSL_PROVIDER_load(NULL, "fips");
+#endif
+	}
+#endif
+#endif
 
 #if OPENSSL_VERSION_NUMBER < 0x10100000L || \
 	(defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
 	h->content.hmac_c = g_malloc0(sizeof(*h->content.hmac_c));
 #else
+#if OPENSSL_VERSION_MAJOR >= 3
+	EVP_MAC* mac = EVP_MAC_fetch(NULL, "HMAC", NULL);
+	h->content.hmac_c = EVP_MAC_CTX_new(mac);
+	EVP_MAC_free(mac);
+#else
 	h->content.hmac_c = HMAC_CTX_new();
 #endif
-	h->out_len = EVP_MD_size(htype);
+#endif
 
 #if OPENSSL_VERSION_NUMBER > 0x10100000L
 	if (insecure) {
 		/* Should never ever be used for crypto/security purposes! */
 #ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+#if OPENSSL_VERSION_MAJOR < 3
 		HMAC_CTX_set_flags(h->content.hmac_c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
 #endif
+#endif
 	}
 #endif
 
+	h->out_len = EVP_MD_size(htype);
+#if OPENSSL_VERSION_MAJOR >= 3
+	OSSL_PARAM params[2];
+	params[0] = OSSL_PARAM_construct_utf8_string("digest", EVP_MD_get0_name(htype), 0);
+	params[1] = OSSL_PARAM_construct_end();
+
+	EVP_MAC_init(h->content.hmac_c, key, keylen, params);
+#else
 	HMAC_Init_ex(h->content.hmac_c, key, keylen, htype, NULL);
+#endif
 }
 
 static struct rspamd_lua_cryptobox_hash *
@@ -1385,7 +1427,7 @@ lua_cryptobox_hash_reset(lua_State *L)
 			rspamd_cryptobox_hash_init(h->content.h, NULL, 0);
 			break;
 		case LUA_CRYPTOBOX_HASH_SSL:
-			EVP_DigestInit(h->content.c, EVP_MD_CTX_md(h->content.c));
+			EVP_DigestInit(h->content.c, EVP_MD_CTX_get0_md(h->content.c));
 			break;
 		case LUA_CRYPTOBOX_HASH_HMAC:
 #if OPENSSL_VERSION_NUMBER < 0x10100000L || \
@@ -1393,8 +1435,15 @@ lua_cryptobox_hash_reset(lua_State *L)
 			/* Old openssl is awesome... */
 			HMAC_Init_ex(h->content.hmac_c, NULL, 0, h->content.hmac_c->md, NULL);
 #else
+#if OPENSSL_VERSION_MAJOR >= 3
+			EVP_MAC_CTX_free(h->content.hmac_c);
+			EVP_MAC *mac = EVP_MAC_fetch(NULL, "HMAC", NULL);
+			h->content.hmac_c = EVP_MAC_CTX_new(mac);
+			EVP_MAC_free(mac);
+#else
 			HMAC_CTX_reset(h->content.hmac_c);
 #endif
+#endif
 			break;
 		case LUA_CRYPTOBOX_HASH_XXHASH64:
 			rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
@@ -1452,7 +1501,13 @@ lua_cryptobox_hash_finish(struct rspamd_lua_cryptobox_hash *h)
 		memcpy(h->out, out, ssl_outlen);
 		break;
 	case LUA_CRYPTOBOX_HASH_HMAC:
+#if OPENSSL_VERSION_MAJOR >= 3
+		size_t ssl_outlen_size_t = ssl_outlen;
+		EVP_MAC_final(h->content.hmac_c, out, &ssl_outlen_size_t, sizeof(out));
+		ssl_outlen = ssl_outlen_size_t;
+#else
 		HMAC_Final(h->content.hmac_c, out, &ssl_outlen);
+#endif
 		h->out_len = ssl_outlen;
 		g_assert(ssl_outlen <= sizeof(h->out));
 		memcpy(h->out, out, ssl_outlen);
@@ -2469,31 +2524,31 @@ lua_cryptobox_gen_dkim_keypair(lua_State *L)
 		EVP_PKEY *pk;
 
 		e = BN_new();
-		r = RSA_new();
 		pk = EVP_PKEY_new();
 
 		if (BN_set_word(e, RSA_F4) != 1) {
 			BN_free(e);
-			RSA_free(r);
 			EVP_PKEY_free(pk);
 
 			return luaL_error(L, "BN_set_word failed");
 		}
 
-		if (RSA_generate_key_ex(r, nbits, e, NULL) != 1) {
+		EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
+		if (EVP_PKEY_keygen_init(pctx) != 1) {
 			BN_free(e);
-			RSA_free(r);
 			EVP_PKEY_free(pk);
+			EVP_PKEY_CTX_free(pctx);
 
-			return luaL_error(L, "RSA_generate_key_ex failed");
+			return luaL_error(L, "EVP_PKEY_keygen_init failed");
 		}
-
-		if (EVP_PKEY_set1_RSA(pk, r) != 1) {
+		EVP_PKEY_CTX_set_rsa_keygen_bits(pctx, nbits);
+		EVP_PKEY_CTX_set1_rsa_keygen_pubexp(pctx, e);
+		if (EVP_PKEY_keygen(pctx, &pk) != 1) {
 			BN_free(e);
-			RSA_free(r);
 			EVP_PKEY_free(pk);
+			EVP_PKEY_CTX_free(pctx);
 
-			return luaL_error(L, "EVP_PKEY_set1_RSA failed");
+			return luaL_error(L, "EVP_PKEY_keygen failed");
 		}
 
 		BIO *mbio;
@@ -2505,12 +2560,11 @@ lua_cryptobox_gen_dkim_keypair(lua_State *L)
 		mbio = BIO_new(BIO_s_mem());
 
 		/* Process private key */
-		rc = i2d_RSAPrivateKey_bio(mbio, r);
+		rc = i2d_PrivateKey_bio(mbio, pk);
 
 		if (rc == 0) {
 			BIO_free(mbio);
 			BN_free(e);
-			RSA_free(r);
 			EVP_PKEY_free(pk);
 
 			return luaL_error(L, "i2d_RSAPrivateKey_bio failed");
@@ -2528,12 +2582,11 @@ lua_cryptobox_gen_dkim_keypair(lua_State *L)
 
 		/* Process public key */
 		BIO_reset(mbio);
-		rc = i2d_RSA_PUBKEY_bio(mbio, r);
+		rc = i2d_PUBKEY_bio(mbio, pk);
 
 		if (rc == 0) {
 			BIO_free(mbio);
 			BN_free(e);
-			RSA_free(r);
 			EVP_PKEY_free(pk);
 
 			return luaL_error(L, "i2d_RSA_PUBKEY_bio failed");
@@ -2550,7 +2603,6 @@ lua_cryptobox_gen_dkim_keypair(lua_State *L)
 		pub_out->flags = RSPAMD_TEXT_FLAG_OWN;
 
 		BN_free(e);
-		RSA_free(r);
 		EVP_PKEY_free(pk);
 		BIO_free(mbio);
 	}
@@ -2717,7 +2769,7 @@ lua_cryptobox_secretbox_encrypt(lua_State *L)
 	struct rspamd_lua_text *out;
 
 	if (sbox == NULL) {
-		return luaL_error(L, "invalid arguments");
+		return luaL_error(L, "invalid argument for secretbox state");
 	}
 
 	if (lua_isstring(L, 2)) {
@@ -2727,14 +2779,14 @@ lua_cryptobox_secretbox_encrypt(lua_State *L)
 		struct rspamd_lua_text *t = lua_check_text(L, 2);
 
 		if (!t) {
-			return luaL_error(L, "invalid arguments; userdata is not text");
+			return luaL_error(L, "invalid first argument; userdata is not text");
 		}
 
 		in = t->start;
 		inlen = t->len;
 	}
 	else {
-		return luaL_error(L, "invalid arguments; userdata or string are expected");
+		return luaL_error(L, "invalid first argument; userdata or string are expected");
 	}
 
 	/* Nonce part */
@@ -2746,14 +2798,14 @@ lua_cryptobox_secretbox_encrypt(lua_State *L)
 			struct rspamd_lua_text *t = lua_check_text(L, 3);
 
 			if (!t) {
-				return luaL_error(L, "invalid arguments; userdata is not text");
+				return luaL_error(L, "invalid second argument; userdata is not text");
 			}
 
 			nonce = t->start;
 			nlen = t->len;
 		}
 		else {
-			return luaL_error(L, "invalid arguments; userdata or string are expected");
+			return luaL_error(L, "invalid second argument; userdata or string are expected");
 		}
 
 		if (nlen < 1 || nlen > crypto_secretbox_NONCEBYTES) {
@@ -2791,8 +2843,8 @@ lua_cryptobox_secretbox_encrypt(lua_State *L)
 /***
  * @method rspamd_cryptobox_secretbox:decrypt(input, nonce)
  * Decrypts data using secretbox
- * @param {string/text} nonce nonce used to encrypt
  * @param {string/text} input input to decrypt
+ * @param {string/text} nonce nonce used to encrypt
  * @param {table} params optional parameters - NYI
  * @return {boolean},{rspamd_text} decryption result + decrypted text
  */
@@ -2806,7 +2858,7 @@ lua_cryptobox_secretbox_decrypt(lua_State *L)
 	struct rspamd_lua_text *out;
 
 	if (sbox == NULL) {
-		return luaL_error(L, "invalid arguments");
+		return luaL_error(L, "invalid argument for secretbox state");
 	}
 
 	/* Input argument */
@@ -2817,14 +2869,14 @@ lua_cryptobox_secretbox_decrypt(lua_State *L)
 		struct rspamd_lua_text *t = lua_check_text(L, 2);
 
 		if (!t) {
-			return luaL_error(L, "invalid arguments; userdata is not text");
+			return luaL_error(L, "invalid first argument; userdata is not text");
 		}
 
 		in = t->start;
 		inlen = t->len;
 	}
 	else {
-		return luaL_error(L, "invalid arguments; userdata or string are expected");
+		return luaL_error(L, "invalid first argument; userdata or string are expected");
 	}
 
 	/* Nonce argument */
@@ -2835,14 +2887,14 @@ lua_cryptobox_secretbox_decrypt(lua_State *L)
 		struct rspamd_lua_text *t = lua_check_text(L, 3);
 
 		if (!t) {
-			return luaL_error(L, "invalid arguments; userdata is not text");
+			return luaL_error(L, "invalid second argument; userdata is not text");
 		}
 
 		nonce = t->start;
 		nlen = t->len;
 	}
 	else {
-		return luaL_error(L, "invalid arguments; userdata or string are expected");
+		return luaL_error(L, "invalid second argument; userdata or string are expected");
 	}
 
 
diff --git a/test/lua/unit/lua_util.maybe_encrypt_decrypt_header.lua b/test/lua/unit/lua_util.maybe_encrypt_decrypt_header.lua
new file mode 100644
index 000000000..613101068
--- /dev/null
+++ b/test/lua/unit/lua_util.maybe_encrypt_decrypt_header.lua
@@ -0,0 +1,57 @@
+local util = require 'lua_util'
+
+context("Lua util - maybe encrypt/decrypt header", function()
+    test("Encrypt/Decrypt header with nonce", function()
+        local header = tostring('X-Spamd-Result')
+        local settings = {
+            prefix = 'prefix',
+            prefix_encrypt = true,
+            prefix_key = 'key',
+            prefix_nonce = 'nonce'
+        }
+
+        local encrypted_header = util.maybe_encrypt_header(header, settings, settings.prefix)
+        if encrypted_header == header or encrypted_header == nil then
+            assert_true(false, 'Failed to encrypt header')
+        end
+
+        local decrypted_header = util.maybe_decrypt_header(encrypted_header, settings, settings.prefix)
+        if decrypted_header == encrypted_header or decrypted_header == nil then
+            assert_true(false, 'Failed to decrypt header')
+        end
+
+        if tostring(header) == tostring(decrypted_header) then
+            assert_true(true, 'Succeed to confirm equality of original header and decrypted header')
+        else
+            assert_rspamd_table_eq_sorted({actual = { decrypted_header },
+                                           expect = { header }})
+        end
+    end)
+
+    test("Encrypt/Decrypt header without nonce", function()
+        local header = tostring('X-Spamd-Result')
+        local settings = {
+            prefix = 'prefix',
+            prefix_encrypt = true,
+            prefix_key = 'key'
+        }
+
+        local encrypted_header, nonce = util.maybe_encrypt_header(header, settings, settings.prefix)
+        if encrypted_header == header or encrypted_header == nil then
+            assert_true(false, 'Failed to encrypt header')
+        end
+
+        local decrypted_header = util.maybe_decrypt_header(encrypted_header, settings,
+                settings.prefix, nonce)
+        if decrypted_header == encrypted_header or decrypted_header == nil then
+            assert_true(false, 'Failed to decrypt header')
+        end
+
+        if tostring(header) == tostring(decrypted_header) then
+            assert_true(true, 'Succeed to confirm equality of original header and decrypted header')
+        else
+            assert_rspamd_table_eq_sorted({actual = { decrypted_header },
+                                           expect = { header }})
+        end
+    end)
+end)
\ No newline at end of file