From 1e661a2fc6e31aa0c5be710dde274ec199ff3853 Mon Sep 17 00:00:00 2001 From: Ivan Stakhov <50211739+LeftTry@users.noreply.github.com> Date: Sat, 17 Aug 2024 23:47:01 +0600 Subject: [PATCH] Provide OpenSSL 3.0 support for lua_rsa (#5101) * [Fix] Provide OSSL 3.0 support for lua_rsa (not working) * [Fix] Provide full OpenSSL 3.0 for lua_rsa * [Minor] Little clean up * [Minor] Clean up code from debug * [Test] Replace for fixed size fstring allocation * [Test] Add rsa padding and signature md for pctx * [Test] Little changes * [Fix] Little fix for rsa_sign_memory work correctly * [Minor] Little clean up * [Minor] Small fix * [Minor] Little clean up * [Test] Add signature verification test for rsa2048 --- src/lua/lua_rsa.c | 164 ++++++++++++++++++++++-------------------- test/lua/unit/rsa.lua | 10 +++ 2 files changed, 98 insertions(+), 76 deletions(-) diff --git a/src/lua/lua_rsa.c b/src/lua/lua_rsa.c index 0f67e91d1..0c56b223b 100644 --- a/src/lua/lua_rsa.c +++ b/src/lua/lua_rsa.c @@ -91,22 +91,22 @@ static const struct luaL_reg rsasignlib_m[] = { {"__gc", lua_rsa_signature_gc}, {NULL, NULL}}; -static RSA * +static EVP_PKEY * lua_check_rsa_pubkey(lua_State *L, int pos) { void *ud = rspamd_lua_check_udata(L, pos, rspamd_rsa_pubkey_classname); luaL_argcheck(L, ud != NULL, 1, "'rsa_pubkey' expected"); - return ud ? *((RSA **) ud) : NULL; + return ud ? *((EVP_PKEY **) ud) : NULL; } -static RSA * +static EVP_PKEY * lua_check_rsa_privkey(lua_State *L, int pos) { void *ud = rspamd_lua_check_udata(L, pos, rspamd_rsa_privkey_classname); luaL_argcheck(L, ud != NULL, 1, "'rsa_privkey' expected"); - return ud ? *((RSA **) ud) : NULL; + return ud ? *((EVP_PKEY **) ud) : NULL; } static rspamd_fstring_t * @@ -121,7 +121,7 @@ lua_check_rsa_sign(lua_State *L, int pos) static int lua_rsa_pubkey_load(lua_State *L) { - RSA *rsa = NULL, **prsa; + EVP_PKEY *pkey = NULL, **ppkey; const char *filename; FILE *f; @@ -135,15 +135,15 @@ lua_rsa_pubkey_load(lua_State *L) lua_pushnil(L); } else { - if (!PEM_read_RSA_PUBKEY(f, &rsa, NULL, NULL)) { + if (!PEM_read_PUBKEY(f, &pkey, NULL, NULL)) { msg_err("cannot open pubkey from file: %s, %s", filename, ERR_error_string(ERR_get_error(), NULL)); lua_pushnil(L); } else { - prsa = lua_newuserdata(L, sizeof(RSA *)); + ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *)); rspamd_lua_setclass(L, rspamd_rsa_pubkey_classname, -1); - *prsa = rsa; + *ppkey = pkey; } fclose(f); } @@ -161,15 +161,14 @@ lua_rsa_privkey_save(lua_State *L) const char *type = "pem"; FILE *f; int ret; - - RSA *rsa = lua_check_rsa_privkey(L, 1); + EVP_PKEY *pkey = lua_check_rsa_privkey(L, 1); filename = luaL_checkstring(L, 2); if (lua_gettop(L) > 2) { type = luaL_checkstring(L, 3); } - if (rsa != NULL && filename != NULL) { + if (pkey != NULL && filename != NULL) { if (strcmp(filename, "-") == 0) { f = stdout; } @@ -189,10 +188,10 @@ lua_rsa_privkey_save(lua_State *L) } if (strcmp(type, "der") == 0) { - ret = i2d_RSAPrivateKey_fp(f, rsa); + ret = i2d_PrivateKey_fp(f, pkey); } else { - ret = PEM_write_RSAPrivateKey(f, rsa, NULL, NULL, 0, NULL, NULL); + ret = PEM_write_PrivateKey(f, pkey, NULL, NULL, 0, NULL, NULL); } if (!ret) { @@ -223,7 +222,7 @@ lua_rsa_privkey_save(lua_State *L) static int lua_rsa_pubkey_create(lua_State *L) { - RSA *rsa = NULL, **prsa; + EVP_PKEY *pkey, **ppkey; const char *buf; BIO *bp; @@ -231,15 +230,15 @@ lua_rsa_pubkey_create(lua_State *L) if (buf != NULL) { bp = BIO_new_mem_buf((void *) buf, -1); - if (!PEM_read_bio_RSA_PUBKEY(bp, &rsa, NULL, NULL)) { + if (!PEM_read_bio_PUBKEY(bp, &pkey, NULL, NULL)) { msg_err("cannot parse pubkey: %s", ERR_error_string(ERR_get_error(), NULL)); lua_pushnil(L); } else { - prsa = lua_newuserdata(L, sizeof(RSA *)); + ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *)); rspamd_lua_setclass(L, rspamd_rsa_pubkey_classname, -1); - *prsa = rsa; + *ppkey = pkey; } BIO_free(bp); } @@ -252,10 +251,10 @@ lua_rsa_pubkey_create(lua_State *L) static int lua_rsa_pubkey_gc(lua_State *L) { - RSA *rsa = lua_check_rsa_pubkey(L, 1); + EVP_PKEY *pkey = lua_check_rsa_pubkey(L, 1); - if (rsa != NULL) { - RSA_free(rsa); + if (pkey != NULL) { + EVP_PKEY_free(pkey); } return 0; @@ -264,18 +263,18 @@ lua_rsa_pubkey_gc(lua_State *L) static int lua_rsa_pubkey_tostring(lua_State *L) { - RSA *rsa = lua_check_rsa_pubkey(L, 1); + EVP_PKEY *pkey = lua_check_rsa_pubkey(L, 1); - if (rsa != NULL) { + if (pkey != NULL) { BIO *pubout = BIO_new(BIO_s_mem()); const char *pubdata; gsize publen; - int rc = i2d_RSA_PUBKEY_bio(pubout, rsa); + int rc = i2d_PUBKEY_bio(pubout, pkey); if (rc != 1) { BIO_free(pubout); - return luaL_error(L, "i2d_RSA_PUBKEY_bio failed"); + return luaL_error(L, "i2d_PUBKEY_bio failed"); } publen = BIO_get_mem_data(pubout, &pubdata); @@ -292,7 +291,7 @@ lua_rsa_pubkey_tostring(lua_State *L) static int lua_rsa_privkey_load_file(lua_State *L) { - RSA *rsa = NULL, **prsa; + EVP_PKEY *pkey = NULL, **ppkey; const char *filename; FILE *f; @@ -306,15 +305,15 @@ lua_rsa_privkey_load_file(lua_State *L) lua_pushnil(L); } else { - if (!PEM_read_RSAPrivateKey(f, &rsa, NULL, NULL)) { + if (!PEM_read_PrivateKey(f, &pkey, NULL, NULL)) { msg_err("cannot open private key from file: %s, %s", filename, ERR_error_string(ERR_get_error(), NULL)); lua_pushnil(L); } else { - prsa = lua_newuserdata(L, sizeof(RSA *)); + ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *)); rspamd_lua_setclass(L, rspamd_rsa_privkey_classname, -1); - *prsa = rsa; + *ppkey = pkey; } fclose(f); } @@ -328,7 +327,7 @@ lua_rsa_privkey_load_file(lua_State *L) static int lua_rsa_privkey_load_pem(lua_State *L) { - RSA *rsa = NULL, **prsa; + EVP_PKEY *pkey = NULL, **ppkey; BIO *b; struct rspamd_lua_text *t; const char *data; @@ -351,15 +350,15 @@ lua_rsa_privkey_load_pem(lua_State *L) if (data != NULL) { b = BIO_new_mem_buf(data, len); - if (!PEM_read_bio_RSAPrivateKey(b, &rsa, NULL, NULL)) { + if (!PEM_read_bio_PrivateKey(b, &pkey, NULL, NULL)) { msg_err("cannot open private key from data, %s", ERR_error_string(ERR_get_error(), NULL)); lua_pushnil(L); } else { - prsa = lua_newuserdata(L, sizeof(RSA *)); + ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *)); rspamd_lua_setclass(L, rspamd_rsa_privkey_classname, -1); - *prsa = rsa; + *ppkey = pkey; } BIO_free(b); @@ -374,7 +373,7 @@ lua_rsa_privkey_load_pem(lua_State *L) static int lua_rsa_privkey_load_raw(lua_State *L) { - RSA *rsa = NULL, **prsa; + EVP_PKEY *pkey = NULL, **ppkey; BIO *b; struct rspamd_lua_text *t; const char *data; @@ -396,17 +395,17 @@ lua_rsa_privkey_load_raw(lua_State *L) if (data != NULL) { b = BIO_new_mem_buf(data, len); - rsa = d2i_RSAPrivateKey_bio(b, NULL); + pkey = d2i_PrivateKey_bio(b, NULL); - if (rsa == NULL) { + if (pkey == NULL) { msg_err("cannot open private key from data, %s", ERR_error_string(ERR_get_error(), NULL)); lua_pushnil(L); } else { - prsa = lua_newuserdata(L, sizeof(RSA *)); + ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *)); rspamd_lua_setclass(L, rspamd_rsa_privkey_classname, -1); - *prsa = rsa; + *ppkey = pkey; } BIO_free(b); @@ -421,9 +420,8 @@ lua_rsa_privkey_load_raw(lua_State *L) static int lua_rsa_privkey_load_base64(lua_State *L) { - RSA *rsa = NULL, **prsa; + EVP_PKEY *pkey = NULL, **ppkey; BIO *b; - EVP_PKEY *evp = NULL; struct rspamd_lua_text *t; const char *data; unsigned char *decoded; @@ -454,21 +452,18 @@ lua_rsa_privkey_load_base64(lua_State *L) b = BIO_new_mem_buf(decoded, dec_len); - if (d2i_PrivateKey_bio(b, &evp) != NULL) { - rsa = EVP_PKEY_get1_RSA(evp); - - if (rsa == NULL) { + if (d2i_PrivateKey_bio(b, &pkey) != NULL) { + if (pkey == NULL) { msg_err("cannot open RSA private key from data, %s", ERR_error_string(ERR_get_error(), NULL)); lua_pushnil(L); } else { - prsa = lua_newuserdata(L, sizeof(RSA *)); + ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *)); rspamd_lua_setclass(L, rspamd_rsa_privkey_classname, -1); - *prsa = rsa; + *ppkey = pkey; } - EVP_PKEY_free(evp); } else { msg_err("cannot open EVP private key from data, %s", @@ -489,7 +484,7 @@ lua_rsa_privkey_load_base64(lua_State *L) static int lua_rsa_privkey_create(lua_State *L) { - RSA *rsa = NULL, **prsa; + EVP_PKEY *pkey = NULL, **ppkey; const char *buf; BIO *bp; @@ -497,15 +492,15 @@ lua_rsa_privkey_create(lua_State *L) if (buf != NULL) { bp = BIO_new_mem_buf((void *) buf, -1); - if (!PEM_read_bio_RSAPrivateKey(bp, &rsa, NULL, NULL)) { + if (!PEM_read_bio_PrivateKey(bp, &pkey, NULL, NULL)) { msg_err("cannot parse private key: %s", ERR_error_string(ERR_get_error(), NULL)); lua_pushnil(L); } else { - prsa = lua_newuserdata(L, sizeof(RSA *)); + ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *)); rspamd_lua_setclass(L, rspamd_rsa_privkey_classname, -1); - *prsa = rsa; + *ppkey = pkey; } BIO_free(bp); } @@ -518,10 +513,10 @@ lua_rsa_privkey_create(lua_State *L) static int lua_rsa_privkey_gc(lua_State *L) { - RSA *rsa = lua_check_rsa_privkey(L, 1); + EVP_PKEY *pkey = lua_check_rsa_privkey(L, 1); - if (rsa != NULL) { - RSA_free(rsa); + if (pkey != NULL) { + EVP_PKEY_free(pkey); } return 0; @@ -699,19 +694,22 @@ lua_rsa_signature_base64(lua_State *L) static int lua_rsa_verify_memory(lua_State *L) { - RSA *rsa; + EVP_PKEY *pkey; rspamd_fstring_t *signature; const char *data; gsize sz; int ret; - rsa = lua_check_rsa_pubkey(L, 1); + pkey = lua_check_rsa_pubkey(L, 1); signature = lua_check_rsa_sign(L, 2); data = luaL_checklstring(L, 3, &sz); - if (rsa != NULL && signature != NULL && data != NULL) { - ret = RSA_verify(NID_sha256, data, sz, - signature->str, signature->len, rsa); + if (pkey != NULL && signature != NULL && data != NULL) { + EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new(pkey, NULL); + g_assert(pctx != NULL); + g_assert(EVP_PKEY_verify_init(pctx) == 1); + + ret = EVP_PKEY_verify(pctx, signature->str, signature->len, data, sz); if (ret == 0) { lua_pushboolean(L, FALSE); @@ -722,6 +720,7 @@ lua_rsa_verify_memory(lua_State *L) else { lua_pushboolean(L, TRUE); } + EVP_PKEY_CTX_free(pctx); } else { lua_pushnil(L); @@ -743,22 +742,26 @@ lua_rsa_verify_memory(lua_State *L) static int lua_rsa_sign_memory(lua_State *L) { - RSA *rsa; + EVP_PKEY *pkey; rspamd_fstring_t *signature, **psig; const char *data; gsize sz; int ret; - rsa = lua_check_rsa_privkey(L, 1); + pkey = lua_check_rsa_privkey(L, 1); data = luaL_checklstring(L, 2, &sz); - if (rsa != NULL && data != NULL) { - signature = rspamd_fstring_sized_new(RSA_size(rsa)); + if (pkey != NULL && data != NULL) { + signature = rspamd_fstring_sized_new(EVP_PKEY_get_size(pkey)); + + EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new(pkey, NULL); + g_assert(pctx != NULL); - unsigned int siglen = signature->len; - ret = RSA_sign(NID_sha256, data, sz, - signature->str, &siglen, rsa); + g_assert(EVP_PKEY_sign_init(pctx) == 1); + size_t slen = signature->allocated; + ret = EVP_PKEY_sign(pctx, signature->str, &slen, data, sz); + EVP_PKEY_CTX_free(pctx); if (ret != 1) { rspamd_fstring_free(signature); @@ -766,7 +769,7 @@ lua_rsa_sign_memory(lua_State *L) ERR_error_string(ERR_get_error(), NULL)); } else { - signature->len = siglen; + signature->len = slen; psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *)); rspamd_lua_setclass(L, rspamd_rsa_signature_classname, -1); *psig = signature; @@ -783,7 +786,7 @@ static int lua_rsa_keypair(lua_State *L) { BIGNUM *e; - RSA *rsa, *pub_rsa, *priv_rsa, **prsa; + EVP_PKEY *pkey = NULL, *pub_pkey, *priv_pkey, **ppkey; int bits = lua_gettop(L) > 0 ? lua_tointeger(L, 1) : 1024; if (bits > 4096 || bits < 512) { @@ -791,21 +794,30 @@ lua_rsa_keypair(lua_State *L) } e = BN_new(); - rsa = RSA_new(); + g_assert(BN_set_word(e, RSA_F4) == 1); - g_assert(RSA_generate_key_ex(rsa, bits, e, NULL) == 1); + EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL); + g_assert(pctx != NULL); + g_assert(EVP_PKEY_keygen_init(pctx) == 1); + + g_assert(EVP_PKEY_CTX_set_rsa_keygen_bits(pctx, bits) == 1); + g_assert(EVP_PKEY_CTX_set1_rsa_keygen_pubexp(pctx, e) == 1); + + g_assert(EVP_PKEY_keygen(pctx, &pkey) == 1); + g_assert(pkey != NULL); - priv_rsa = RSAPrivateKey_dup(rsa); - prsa = lua_newuserdata(L, sizeof(RSA *)); + priv_pkey = EVP_PKEY_dup(pkey); + ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *)); rspamd_lua_setclass(L, rspamd_rsa_privkey_classname, -1); - *prsa = priv_rsa; + *ppkey = priv_pkey; - pub_rsa = RSAPublicKey_dup(rsa); - prsa = lua_newuserdata(L, sizeof(RSA *)); + pub_pkey = EVP_PKEY_dup(pkey); + ppkey = lua_newuserdata(L, sizeof(EVP_PKEY *)); rspamd_lua_setclass(L, rspamd_rsa_pubkey_classname, -1); - *prsa = pub_rsa; + *ppkey = pub_pkey; - RSA_free(rsa); + EVP_PKEY_free(pkey); + EVP_PKEY_CTX_free(pctx); BN_free(e); return 2; diff --git a/test/lua/unit/rsa.lua b/test/lua/unit/rsa.lua index c67a36abb..019212df4 100644 --- a/test/lua/unit/rsa.lua +++ b/test/lua/unit/rsa.lua @@ -47,4 +47,14 @@ context("RSA signature verification test", function() sk, pk = rsa.keypair() assert_false(rsa.verify_memory(pk, sig, "test")) end) + + test("RSA-2048 keypair + sign + verify", function() + local sk, pk = rsa.keypair(2048) + local sig = rsa.sign_memory(sk, "test") + assert_true(rsa.verify_memory(pk, sig, "test")) + assert_false(rsa.verify_memory(pk, sig, "test1")) + -- Overwrite + sk, pk = rsa.keypair(2048) + assert_false(rsa.verify_memory(pk, sig, "test")) + end) end) -- 2.39.5