Merge pull request #5154 from rspamd/vstakhov-ssl-fixes

Fixes for OpenSSL

3 files changed, 16 insertions, 24 deletions

diff --git a/.github/workflows/ci_rspamd_build.yml b/.github/workflows/ci_rspamd_build.yml
index aa12c9c6e..9503f1974 100644
--- a/.github/workflows/ci_rspamd_build.yml
+++ b/.github/workflows/ci_rspamd_build.yml
@@ -34,10 +34,12 @@ jobs:
         run: |
           mkdir ${GITHUB_WORKSPACE}/build
           cd ${GITHUB_WORKSPACE}/build
+          if [[ -f /opt/rh/gcc-toolset-10/enable ]] ; then source /opt/rh/gcc-toolset-10/enable ; fi 
           cmake -DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/install -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_COVERAGE=ON -DENABLE_LIBUNWIND=ON  -DENABLE_LUAJIT=ON -DLUA_ROOT=/luajit-build -DENABLE_HYPERSCAN=ON ${{ env.HYPERSCAN_ALTROOT }} ${GITHUB_WORKSPACE}/src
 
       - name: Build rspamd
         run: |
+          if [[ -f /opt/rh/gcc-toolset-10/enable ]] ; then source /opt/rh/gcc-toolset-10/enable ; fi 
           cd ${GITHUB_WORKSPACE}/build
           ncpu=$(getconf _NPROCESSORS_ONLN)
           make -j $ncpu install
diff --git a/src/lua/lua_cryptobox.c b/src/lua/lua_cryptobox.c
index 3fa7d7d4f..9600a4732 100644
--- a/src/lua/lua_cryptobox.c
+++ b/src/lua/lua_cryptobox.c
@@ -1438,7 +1438,11 @@ lua_cryptobox_hash_reset(lua_State *L)
 			rspamd_cryptobox_hash_init(h->content.h, NULL, 0);
 			break;
 		case LUA_CRYPTOBOX_HASH_SSL:
+#if OPENSSL_VERSION_MAJOR >= 3
 			EVP_DigestInit(h->content.c, EVP_MD_CTX_get0_md(h->content.c));
+#else
+			EVP_DigestInit(h->content.c, EVP_MD_CTX_md(h->content.c));
+#endif
 			break;
 		case LUA_CRYPTOBOX_HASH_HMAC:
 #if OPENSSL_VERSION_NUMBER < 0x10100000L || \
@@ -2531,31 +2535,20 @@ lua_cryptobox_gen_dkim_keypair(lua_State *L)
 	}
 
 	if (strcmp(alg_str, "rsa") == 0) {
-		BIGNUM *e;
 		EVP_PKEY *pk;
 
-		e = BN_new();
 		pk = EVP_PKEY_new();
 
-		if (BN_set_word(e, RSA_F4) != 1) {
-			BN_free(e);
-			EVP_PKEY_free(pk);
-
-			return luaL_error(L, "BN_set_word failed");
-		}
 
 		EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
 		if (EVP_PKEY_keygen_init(pctx) != 1) {
-			BN_free(e);
 			EVP_PKEY_free(pk);
 			EVP_PKEY_CTX_free(pctx);
 
 			return luaL_error(L, "EVP_PKEY_keygen_init failed");
 		}
 		EVP_PKEY_CTX_set_rsa_keygen_bits(pctx, nbits);
-		EVP_PKEY_CTX_set1_rsa_keygen_pubexp(pctx, e);
 		if (EVP_PKEY_keygen(pctx, &pk) != 1) {
-			BN_free(e);
 			EVP_PKEY_free(pk);
 			EVP_PKEY_CTX_free(pctx);
 
@@ -2575,7 +2568,6 @@ lua_cryptobox_gen_dkim_keypair(lua_State *L)
 
 		if (rc == 0) {
 			BIO_free(mbio);
-			BN_free(e);
 			EVP_PKEY_free(pk);
 
 			return luaL_error(L, "i2d_RSAPrivateKey_bio failed");
@@ -2597,7 +2589,6 @@ lua_cryptobox_gen_dkim_keypair(lua_State *L)
 
 		if (rc == 0) {
 			BIO_free(mbio);
-			BN_free(e);
 			EVP_PKEY_free(pk);
 
 			return luaL_error(L, "i2d_RSA_PUBKEY_bio failed");
@@ -2613,7 +2604,6 @@ lua_cryptobox_gen_dkim_keypair(lua_State *L)
 		pub_out->len = b64_len;
 		pub_out->flags = RSPAMD_TEXT_FLAG_OWN;
 
-		BN_free(e);
 		EVP_PKEY_free(pk);
 		BIO_free(mbio);
 	}
diff --git a/src/lua/lua_rsa.c b/src/lua/lua_rsa.c
index b7be612b0..4b9aa0354 100644
--- a/src/lua/lua_rsa.c
+++ b/src/lua/lua_rsa.c
@@ -261,6 +261,7 @@ lua_rsa_pubkey_gc(lua_State *L)
 	EVP_PKEY *pkey = lua_check_rsa_pubkey(L, 1);
 
 	if (pkey != NULL) {
+		/* It's actually EVP_PKEY_unref, thanks for that API */
 		EVP_PKEY_free(pkey);
 	}
 
@@ -522,6 +523,7 @@ lua_rsa_privkey_gc(lua_State *L)
 	EVP_PKEY *pkey = lua_check_rsa_privkey(L, 1);
 
 	if (pkey != NULL) {
+		/* It's actually EVP_PKEY_unref, thanks for that API */
 		EVP_PKEY_free(pkey);
 	}
 
@@ -758,7 +760,7 @@ lua_rsa_sign_memory(lua_State *L)
 	data = luaL_checklstring(L, 2, &sz);
 
 	if (pkey != NULL && data != NULL) {
-		signature = rspamd_fstring_sized_new(EVP_PKEY_get_size(pkey));
+		signature = rspamd_fstring_sized_new(EVP_PKEY_size(pkey));
 
 		EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new(pkey, NULL);
 		g_assert(pctx != NULL);
@@ -791,7 +793,6 @@ lua_rsa_sign_memory(lua_State *L)
 static int
 lua_rsa_keypair(lua_State *L)
 {
-	BIGNUM *e;
 	EVP_PKEY *pkey = NULL, *pub_pkey, *priv_pkey, **ppkey;
 	int bits = lua_gettop(L) > 0 ? lua_tointeger(L, 1) : 1024;
 
@@ -799,32 +800,31 @@ lua_rsa_keypair(lua_State *L)
 		return luaL_error(L, "invalid bits count");
 	}
 
-	e = BN_new();
-
-	g_assert(BN_set_word(e, RSA_F4) == 1);
 	EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
 	g_assert(pctx != NULL);
 	g_assert(EVP_PKEY_keygen_init(pctx) == 1);
 
 	g_assert(EVP_PKEY_CTX_set_rsa_keygen_bits(pctx, bits) == 1);
-	g_assert(EVP_PKEY_CTX_set1_rsa_keygen_pubexp(pctx, e) == 1);
-
 	g_assert(EVP_PKEY_keygen(pctx, &pkey) == 1);
 	g_assert(pkey != NULL);
 
-	priv_pkey = EVP_PKEY_dup(pkey);
+	/* Increase refcount and share */
+	g_assert(EVP_PKEY_up_ref(pkey) == 1);
+	priv_pkey = pkey;
+
 	ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *));
 	rspamd_lua_setclass(L, rspamd_rsa_privkey_classname, -1);
 	*ppkey = priv_pkey;
 
-	pub_pkey = EVP_PKEY_dup(pkey);
+	/* Increase refcount and share */
+	g_assert(EVP_PKEY_up_ref(pkey) == 1);
+	pub_pkey = pkey;
 	ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *));
 	rspamd_lua_setclass(L, rspamd_rsa_pubkey_classname, -1);
 	*ppkey = pub_pkey;
 
 	EVP_PKEY_free(pkey);
 	EVP_PKEY_CTX_free(pctx);
-	BN_free(e);
 
 	return 2;
 }