aboutsummaryrefslogtreecommitdiffstats
path: root/common
diff options
context:
space:
mode:
Diffstat (limited to 'common')
-rw-r--r--common/rdr/AESInStream.cxx85
-rw-r--r--common/rdr/AESInStream.h49
-rw-r--r--common/rdr/AESOutStream.cxx103
-rw-r--r--common/rdr/AESOutStream.h53
-rw-r--r--common/rdr/CMakeLists.txt6
-rw-r--r--common/rfb/CMakeLists.txt6
-rw-r--r--common/rfb/CSecurity.h2
-rw-r--r--common/rfb/CSecurityRSAAES.cxx461
-rw-r--r--common/rfb/CSecurityRSAAES.h89
-rw-r--r--common/rfb/CSecurityTLS.h2
-rw-r--r--common/rfb/SSecurityPlain.cxx6
-rw-r--r--common/rfb/SSecurityRSAAES.cxx598
-rw-r--r--common/rfb/SSecurityRSAAES.h98
-rw-r--r--common/rfb/Security.cxx4
-rw-r--r--common/rfb/Security.h7
-rw-r--r--common/rfb/SecurityClient.cxx32
-rw-r--r--common/rfb/SecurityServer.cxx24
17 files changed, 1612 insertions, 13 deletions
diff --git a/common/rdr/AESInStream.cxx b/common/rdr/AESInStream.cxx
new file mode 100644
index 00000000..e6737e64
--- /dev/null
+++ b/common/rdr/AESInStream.cxx
@@ -0,0 +1,85 @@
+/* Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+ * USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <assert.h>
+#include <rdr/AESInStream.h>
+#include <rdr/Exception.h>
+
+#ifdef HAVE_NETTLE
+using namespace rdr;
+
+AESInStream::AESInStream(InStream* _in, const U8* key, int _keySize)
+ : keySize(_keySize), in(_in), counter()
+{
+ if (keySize == 128)
+ EAX_SET_KEY(&eaxCtx128, aes128_set_encrypt_key, aes128_encrypt, key);
+ else if (keySize == 256)
+ EAX_SET_KEY(&eaxCtx256, aes256_set_encrypt_key, aes256_encrypt, key);
+ else
+ assert(!"incorrect key size");
+}
+
+AESInStream::~AESInStream() {}
+
+bool AESInStream::fillBuffer()
+{
+ if (!in->hasData(2))
+ return false;
+ const U8* ptr = in->getptr(2);
+ size_t length = ((int)ptr[0] << 8) | (int)ptr[1];
+ if (!in->hasData(2 + length + 16))
+ return false;
+ ensureSpace(length);
+ ptr = in->getptr(2 + length + 16);
+ const U8* ad = ptr;
+ const U8* data = ptr + 2;
+ const U8* mac = ptr + 2 + length;
+ U8 macComputed[16];
+
+ if (keySize == 128) {
+ EAX_SET_NONCE(&eaxCtx128, aes128_encrypt, 16, counter);
+ EAX_UPDATE(&eaxCtx128, aes128_encrypt, 2, ad);
+ EAX_DECRYPT(&eaxCtx128, aes128_encrypt, length, (rdr::U8*)end, data);
+ EAX_DIGEST(&eaxCtx128, aes128_encrypt, 16, macComputed);
+ } else {
+ EAX_SET_NONCE(&eaxCtx256, aes256_encrypt, 16, counter);
+ EAX_UPDATE(&eaxCtx256, aes256_encrypt, 2, ad);
+ EAX_DECRYPT(&eaxCtx256, aes256_encrypt, length, (rdr::U8*)end, data);
+ EAX_DIGEST(&eaxCtx256, aes256_encrypt, 16, macComputed);
+ }
+ if (memcmp(mac, macComputed, 16) != 0)
+ throw Exception("AESInStream: failed to authenticate message");
+ in->setptr(2 + length + 16);
+ end += length;
+
+ // Update nonce by incrementing the counter as a
+ // 128bit little endian unsigned integer
+ for (int i = 0; i < 16; ++i) {
+ // increment until there is no carry
+ if (++counter[i] != 0) {
+ break;
+ }
+ }
+ return true;
+}
+
+#endif \ No newline at end of file
diff --git a/common/rdr/AESInStream.h b/common/rdr/AESInStream.h
new file mode 100644
index 00000000..3483bd3a
--- /dev/null
+++ b/common/rdr/AESInStream.h
@@ -0,0 +1,49 @@
+/* Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+ * USA.
+ */
+
+#ifndef __RDR_AESINSTREAM_H__
+#define __RDR_AESINSTREAM_H__
+
+#ifdef HAVE_NETTLE
+
+#include <nettle/eax.h>
+#include <nettle/aes.h>
+#include <rdr/BufferedInStream.h>
+
+namespace rdr {
+
+ class AESInStream : public BufferedInStream {
+ public:
+ AESInStream(InStream* in, const U8* key, int keySize);
+ virtual ~AESInStream();
+
+ private:
+ virtual bool fillBuffer();
+
+ int keySize;
+ InStream* in;
+ union {
+ struct EAX_CTX(aes128_ctx) eaxCtx128;
+ struct EAX_CTX(aes256_ctx) eaxCtx256;
+ };
+ U8 counter[16];
+ };
+}
+
+#endif
+#endif
diff --git a/common/rdr/AESOutStream.cxx b/common/rdr/AESOutStream.cxx
new file mode 100644
index 00000000..1559b78a
--- /dev/null
+++ b/common/rdr/AESOutStream.cxx
@@ -0,0 +1,103 @@
+/* Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+ * USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <assert.h>
+#include <rdr/Exception.h>
+#include <rdr/AESOutStream.h>
+
+#ifdef HAVE_NETTLE
+using namespace rdr;
+
+const int MaxMessageSize = 8192;
+
+AESOutStream::AESOutStream(OutStream* _out, const U8* key, int _keySize)
+ : keySize(_keySize), out(_out), counter()
+{
+ msg = new U8[MaxMessageSize + 16 + 2];
+ if (keySize == 128)
+ EAX_SET_KEY(&eaxCtx128, aes128_set_encrypt_key, aes128_encrypt, key);
+ else if (keySize == 256)
+ EAX_SET_KEY(&eaxCtx256, aes256_set_encrypt_key, aes256_encrypt, key);
+ else
+ assert(!"incorrect key size");
+}
+
+AESOutStream::~AESOutStream()
+{
+ delete[] msg;
+}
+
+void AESOutStream::flush()
+{
+ BufferedOutStream::flush();
+ out->flush();
+}
+
+void AESOutStream::cork(bool enable)
+{
+ BufferedOutStream::cork(enable);
+ out->cork(enable);
+}
+
+bool AESOutStream::flushBuffer()
+{
+ while (sentUpTo < ptr) {
+ size_t n = ptr - sentUpTo;
+ if (n > MaxMessageSize)
+ n = MaxMessageSize;
+ writeMessage(sentUpTo, n);
+ sentUpTo += n;
+ }
+ return true;
+}
+
+
+void AESOutStream::writeMessage(const U8* data, size_t length)
+{
+ msg[0] = (length & 0xff00) >> 8;
+ msg[1] = length & 0xff;
+
+ if (keySize == 128) {
+ EAX_SET_NONCE(&eaxCtx128, aes128_encrypt, 16, counter);
+ EAX_UPDATE(&eaxCtx128, aes128_encrypt, 2, msg);
+ EAX_ENCRYPT(&eaxCtx128, aes128_encrypt, length, msg + 2, data);
+ EAX_DIGEST(&eaxCtx128, aes128_encrypt, 16, msg + 2 + length);
+ } else {
+ EAX_SET_NONCE(&eaxCtx256, aes256_encrypt, 16, counter);
+ EAX_UPDATE(&eaxCtx256, aes256_encrypt, 2, msg);
+ EAX_ENCRYPT(&eaxCtx256, aes256_encrypt, length, msg + 2, data);
+ EAX_DIGEST(&eaxCtx256, aes256_encrypt, 16, msg + 2 + length);
+ }
+ out->writeBytes(msg, 2 + length + 16);
+ out->flush();
+
+ // Update nonce by incrementing the counter as a
+ // 128bit little endian unsigned integer
+ for (int i = 0; i < 16; ++i) {
+ // increment until there is no carry
+ if (++counter[i] != 0) {
+ break;
+ }
+ }
+}
+
+#endif
diff --git a/common/rdr/AESOutStream.h b/common/rdr/AESOutStream.h
new file mode 100644
index 00000000..e5d6aa7a
--- /dev/null
+++ b/common/rdr/AESOutStream.h
@@ -0,0 +1,53 @@
+/* Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+ * USA.
+ */
+
+#ifndef __RDR_AESOUTSTREAM_H__
+#define __RDR_AESOUTSTREAM_H__
+
+#ifdef HAVE_NETTLE
+#include <nettle/eax.h>
+#include <nettle/aes.h>
+#include <rdr/BufferedOutStream.h>
+
+namespace rdr {
+
+ class AESOutStream : public BufferedOutStream {
+ public:
+ AESOutStream(OutStream* out, const U8* key, int keySize);
+ virtual ~AESOutStream();
+
+ virtual void flush();
+ virtual void cork(bool enable);
+
+ private:
+ virtual bool flushBuffer();
+ void writeMessage(const U8* data, size_t length);
+
+ int keySize;
+ OutStream* out;
+ U8* msg;
+ union {
+ struct EAX_CTX(aes128_ctx) eaxCtx128;
+ struct EAX_CTX(aes256_ctx) eaxCtx256;
+ };
+ U8 counter[16];
+ };
+};
+
+#endif
+#endif
diff --git a/common/rdr/CMakeLists.txt b/common/rdr/CMakeLists.txt
index 08c93d8b..96ffe163 100644
--- a/common/rdr/CMakeLists.txt
+++ b/common/rdr/CMakeLists.txt
@@ -2,6 +2,8 @@ include_directories(${CMAKE_SOURCE_DIR}/common)
include_directories(${ZLIB_INCLUDE_DIRS})
add_library(rdr STATIC
+ AESInStream.cxx
+ AESOutStream.cxx
BufferedInStream.cxx
BufferedOutStream.cxx
Exception.cxx
@@ -23,6 +25,10 @@ if(GNUTLS_FOUND)
include_directories(${GNUTLS_INCLUDE_DIR})
target_link_libraries(rdr ${GNUTLS_LIBRARIES})
endif()
+if (NETTLE_FOUND)
+ include_directories(${NETTLE_INCLUDE_DIRS})
+ target_link_libraries(rdr ${NETTLE_LINK_LIBRARIES})
+endif()
if(WIN32)
target_link_libraries(rdr ws2_32)
endif()
diff --git a/common/rfb/CMakeLists.txt b/common/rfb/CMakeLists.txt
index 06966410..914dd894 100644
--- a/common/rfb/CMakeLists.txt
+++ b/common/rfb/CMakeLists.txt
@@ -102,6 +102,12 @@ if(GNUTLS_FOUND)
target_link_libraries(rfb ${GNUTLS_LIBRARIES})
endif()
+if (NETTLE_FOUND)
+ target_sources(rfb PRIVATE CSecurityRSAAES.cxx SSecurityRSAAES.cxx)
+ include_directories(${NETTLE_INCLUDE_DIRS} ${GMP_INCLUDE_DIRS})
+ target_link_libraries(rfb ${HOGWEED_LINK_LIBRARIES} ${NETTLE_LINK_LIBRARIES} ${GMP_LINK_LIBRARIES})
+endif()
+
if(UNIX)
libtool_create_control_file(rfb)
endif()
diff --git a/common/rfb/CSecurity.h b/common/rfb/CSecurity.h
index 476f8770..549db794 100644
--- a/common/rfb/CSecurity.h
+++ b/common/rfb/CSecurity.h
@@ -39,6 +39,7 @@
#define __RFB_CSECURITY_H__
#include <rfb/UserPasswdGetter.h>
+#include <rfb/UserMsgBox.h>
namespace rfb {
class CConnection;
@@ -55,6 +56,7 @@ namespace rfb {
* It MUST be set by viewer.
*/
static UserPasswdGetter *upg;
+ static UserMsgBox *msg;
protected:
CConnection* cc;
diff --git a/common/rfb/CSecurityRSAAES.cxx b/common/rfb/CSecurityRSAAES.cxx
new file mode 100644
index 00000000..db6e43ad
--- /dev/null
+++ b/common/rfb/CSecurityRSAAES.cxx
@@ -0,0 +1,461 @@
+/*
+ * Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+ * USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifndef HAVE_NETTLE
+#error "This header should not be compiled without HAVE_NETTLE defined"
+#endif
+
+#include <stdlib.h>
+#ifndef WIN32
+#include <unistd.h>
+#endif
+#include <assert.h>
+
+#include <nettle/bignum.h>
+#include <nettle/sha1.h>
+#include <nettle/sha2.h>
+#include <rfb/CSecurityRSAAES.h>
+#include <rfb/CConnection.h>
+#include <rfb/LogWriter.h>
+#include <rfb/Exception.h>
+#include <rfb/UserMsgBox.h>
+#include <rdr/AESInStream.h>
+#include <rdr/AESOutStream.h>
+#include <os/os.h>
+
+enum {
+ ReadPublicKey,
+ ReadRandom,
+ ReadHash,
+ ReadSubtype,
+};
+
+const int MinKeyLength = 1024;
+const int MaxKeyLength = 8192;
+
+using namespace rfb;
+
+CSecurityRSAAES::CSecurityRSAAES(CConnection* cc, rdr::U32 _secType,
+ int _keySize, bool _isAllEncrypted)
+ : CSecurity(cc), state(ReadPublicKey),
+ keySize(_keySize), isAllEncrypted(_isAllEncrypted), secType(_secType),
+ clientKey(), clientPublicKey(), serverKey(),
+ serverKeyN(NULL), serverKeyE(NULL),
+ clientKeyN(NULL), clientKeyE(NULL),
+ rais(NULL), raos(NULL), rawis(NULL), rawos(NULL)
+{
+ assert(keySize == 128 || keySize == 256);
+}
+
+CSecurityRSAAES::~CSecurityRSAAES()
+{
+ cleanup();
+}
+
+void CSecurityRSAAES::cleanup()
+{
+ if (serverKeyN)
+ delete[] serverKeyN;
+ if (serverKeyE)
+ delete[] serverKeyE;
+ if (clientKeyN)
+ delete[] clientKeyN;
+ if (clientKeyE)
+ delete[] clientKeyE;
+ if (clientKey.size)
+ rsa_private_key_clear(&clientKey);
+ if (clientPublicKey.size)
+ rsa_public_key_clear(&clientPublicKey);
+ if (serverKey.size)
+ rsa_public_key_clear(&serverKey);
+ if (isAllEncrypted && rawis && rawos)
+ cc->setStreams(rawis, rawos);
+ if (rais)
+ delete rais;
+ if (raos)
+ delete raos;
+}
+
+bool CSecurityRSAAES::processMsg()
+{
+ switch (state) {
+ case ReadPublicKey:
+ if (readPublicKey()) {
+ verifyServer();
+ writePublicKey();
+ writeRandom();
+ state = ReadRandom;
+ }
+ return false;
+ case ReadRandom:
+ if (readRandom()) {
+ setCipher();
+ writeHash();
+ state = ReadHash;
+ }
+ return false;
+ case ReadHash:
+ if (readHash()) {
+ clearSecrets();
+ state = ReadSubtype;
+ }
+ case ReadSubtype:
+ if (readSubtype()) {
+ writeCredentials();
+ return true;
+ }
+ return false;
+ }
+ assert(!"unreachable");
+ return false;
+}
+
+static void random_func(void* ctx, size_t length, uint8_t* dst)
+{
+ rdr::RandomStream* rs = (rdr::RandomStream*)ctx;
+ if (!rs->hasData(length))
+ throw ConnFailedException("failed to generate random");
+ rs->readBytes(dst, length);
+}
+
+void CSecurityRSAAES::writePublicKey()
+{
+ rdr::OutStream* os = cc->getOutStream();
+ // generate client key
+ rsa_public_key_init(&clientPublicKey);
+ rsa_private_key_init(&clientKey);
+ // match the server key size
+ clientKeyLength = serverKeyLength;
+ int rsaKeySize = (clientKeyLength + 7) / 8;
+ // set key size to non-zero to allow clearing the keys when cleanup
+ clientPublicKey.size = rsaKeySize;
+ clientKey.size = rsaKeySize;
+ // set e = 65537
+ mpz_set_ui(clientPublicKey.e, 65537);
+ if (!rsa_generate_keypair(&clientPublicKey, &clientKey,
+ &rs, random_func, NULL, NULL, clientKeyLength, 0))
+ throw AuthFailureException("failed to generate key");
+ clientKeyN = new rdr::U8[rsaKeySize];
+ clientKeyE = new rdr::U8[rsaKeySize];
+ nettle_mpz_get_str_256(rsaKeySize, clientKeyN, clientPublicKey.n);
+ nettle_mpz_get_str_256(rsaKeySize, clientKeyE, clientPublicKey.e);
+ os->writeU32(clientKeyLength);
+ os->writeBytes(clientKeyN, rsaKeySize);
+ os->writeBytes(clientKeyE, rsaKeySize);
+ os->flush();
+}
+
+bool CSecurityRSAAES::readPublicKey()
+{
+ rdr::InStream* is = cc->getInStream();
+ if (!is->hasData(4))
+ return false;
+ is->setRestorePoint();
+ serverKeyLength = is->readU32();
+ if (serverKeyLength < MinKeyLength)
+ throw AuthFailureException("server key is too short");
+ if (serverKeyLength > MaxKeyLength)
+ throw AuthFailureException("server key is too long");
+ size_t size = (serverKeyLength + 7) / 8;
+ if (!is->hasDataOrRestore(size * 2))
+ return false;
+ is->clearRestorePoint();
+ serverKeyE = new rdr::U8[size];
+ serverKeyN = new rdr::U8[size];
+ is->readBytes(serverKeyN, size);
+ is->readBytes(serverKeyE, size);
+ rsa_public_key_init(&serverKey);
+ nettle_mpz_set_str_256_u(serverKey.n, size, serverKeyN);
+ nettle_mpz_set_str_256_u(serverKey.e, size, serverKeyE);
+ if (!rsa_public_key_prepare(&serverKey))
+ throw AuthFailureException("server key is invalid");
+ return true;
+}
+
+void CSecurityRSAAES::verifyServer()
+{
+ rdr::U8 lenServerKey[4] = {
+ (rdr::U8)((serverKeyLength & 0xff000000) >> 24),
+ (rdr::U8)((serverKeyLength & 0xff0000) >> 16),
+ (rdr::U8)((serverKeyLength & 0xff00) >> 8),
+ (rdr::U8)(serverKeyLength & 0xff)
+ };
+ rdr::U8 f[8];
+ struct sha1_ctx ctx;
+ sha1_init(&ctx);
+ sha1_update(&ctx, 4, lenServerKey);
+ sha1_update(&ctx, serverKey.size, serverKeyN);
+ sha1_update(&ctx, serverKey.size, serverKeyE);
+ sha1_digest(&ctx, sizeof(f), f);
+ const char *title = "Server key fingerprint";
+ CharArray text;
+ text.format(
+ "The server has provided the following identifying information:\n"
+ "Fingerprint: %02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x\n"
+ "Please verify that the information is correct and press \"Yes\". "
+ "Otherwise press \"No\"", f[0], f[1], f[2], f[3], f[4], f[5], f[6], f[7]);
+ if (!msg->showMsgBox(UserMsgBox::M_YESNO, title, text.buf))
+ throw AuthFailureException("server key mismatch");
+}
+
+void CSecurityRSAAES::writeRandom()
+{
+ rdr::OutStream* os = cc->getOutStream();
+ if (!rs.hasData(keySize / 8))
+ throw ConnFailedException("failed to generate random");
+ rs.readBytes(clientRandom, keySize / 8);
+ mpz_t x;
+ mpz_init(x);
+ int res;
+ try {
+ res = rsa_encrypt(&serverKey, &rs, random_func, keySize / 8,
+ clientRandom, x);
+ } catch (...) {
+ mpz_clear(x);
+ throw;
+ }
+ if (!res) {
+ mpz_clear(x);
+ throw AuthFailureException("failed to encrypt random");
+ }
+ rdr::U8* buffer = new rdr::U8[serverKey.size];
+ nettle_mpz_get_str_256(serverKey.size, buffer, x);
+ mpz_clear(x);
+ os->writeU16(serverKey.size);
+ os->writeBytes(buffer, serverKey.size);
+ os->flush();
+ delete[] buffer;
+}
+
+bool CSecurityRSAAES::readRandom()
+{
+ rdr::InStream* is = cc->getInStream();
+ if (!is->hasData(2))
+ return false;
+ is->setRestorePoint();
+ size_t size = is->readU16();
+ if (size != clientKey.size)
+ throw AuthFailureException("client key length doesn't match");
+ if (!is->hasDataOrRestore(size))
+ return false;
+ is->clearRestorePoint();
+ rdr::U8* buffer = new rdr::U8[size];
+ is->readBytes(buffer, size);
+ size_t randomSize = keySize / 8;
+ mpz_t x;
+ nettle_mpz_init_set_str_256_u(x, size, buffer);
+ delete[] buffer;
+ if (!rsa_decrypt(&clientKey, &randomSize, serverRandom, x) ||
+ randomSize != (size_t)keySize / 8) {
+ mpz_clear(x);
+ throw AuthFailureException("failed to decrypt server random");
+ }
+ mpz_clear(x);
+ return true;
+}
+
+void CSecurityRSAAES::setCipher()
+{
+ rawis = cc->getInStream();
+ rawos = cc->getOutStream();
+ rdr::U8 key[32];
+ if (keySize == 128) {
+ struct sha1_ctx ctx;
+ sha1_init(&ctx);
+ sha1_update(&ctx, 16, clientRandom);
+ sha1_update(&ctx, 16, serverRandom);
+ sha1_digest(&ctx, 16, key);
+ rais = new rdr::AESInStream(rawis, key, 128);
+ sha1_init(&ctx);
+ sha1_update(&ctx, 16, serverRandom);
+ sha1_update(&ctx, 16, clientRandom);
+ sha1_digest(&ctx, 16, key);
+ raos = new rdr::AESOutStream(rawos, key, 128);
+ } else {
+ struct sha256_ctx ctx;
+ sha256_init(&ctx);
+ sha256_update(&ctx, 32, clientRandom);
+ sha256_update(&ctx, 32, serverRandom);
+ sha256_digest(&ctx, 32, key);
+ rais = new rdr::AESInStream(rawis, key, 256);
+ sha256_init(&ctx);
+ sha256_update(&ctx, 32, serverRandom);
+ sha256_update(&ctx, 32, clientRandom);
+ sha256_digest(&ctx, 32, key);
+ raos = new rdr::AESOutStream(rawos, key, 256);
+ }
+ if (isAllEncrypted)
+ cc->setStreams(rais, raos);
+}
+
+void CSecurityRSAAES::writeHash()
+{
+ rdr::U8 hash[32];
+ size_t len = serverKeyLength;
+ rdr::U8 lenServerKey[4] = {
+ (rdr::U8)((len & 0xff000000) >> 24),
+ (rdr::U8)((len & 0xff0000) >> 16),
+ (rdr::U8)((len & 0xff00) >> 8),
+ (rdr::U8)(len & 0xff)
+ };
+ len = clientKeyLength;
+ rdr::U8 lenClientKey[4] = {
+ (rdr::U8)((len & 0xff000000) >> 24),
+ (rdr::U8)((len & 0xff0000) >> 16),
+ (rdr::U8)((len & 0xff00) >> 8),
+ (rdr::U8)(len & 0xff)
+ };
+ int hashSize;
+ if (keySize == 128) {
+ hashSize = 20;
+ struct sha1_ctx ctx;
+ sha1_init(&ctx);
+ sha1_update(&ctx, 4, lenClientKey);
+ sha1_update(&ctx, clientKey.size, clientKeyN);
+ sha1_update(&ctx, clientKey.size, clientKeyE);
+ sha1_update(&ctx, 4, lenServerKey);
+ sha1_update(&ctx, serverKey.size, serverKeyN);
+ sha1_update(&ctx, serverKey.size, serverKeyE);
+ sha1_digest(&ctx, hashSize, hash);
+ } else {
+ hashSize = 32;
+ struct sha256_ctx ctx;
+ sha256_init(&ctx);
+ sha256_update(&ctx, 4, lenClientKey);
+ sha256_update(&ctx, clientKey.size, clientKeyN);
+ sha256_update(&ctx, clientKey.size, clientKeyE);
+ sha256_update(&ctx, 4, lenServerKey);
+ sha256_update(&ctx, serverKey.size, serverKeyN);
+ sha256_update(&ctx, serverKey.size, serverKeyE);
+ sha256_digest(&ctx, hashSize, hash);
+ }
+ raos->writeBytes(hash, hashSize);
+ raos->flush();
+}
+
+bool CSecurityRSAAES::readHash()
+{
+ rdr::U8 hash[32];
+ rdr::U8 realHash[32];
+ int hashSize = keySize == 128 ? 20 : 32;
+ if (!rais->hasData(hashSize))
+ return false;
+ rais->readBytes(hash, hashSize);
+ size_t len = serverKeyLength;
+ rdr::U8 lenServerKey[4] = {
+ (rdr::U8)((len & 0xff000000) >> 24),
+ (rdr::U8)((len & 0xff0000) >> 16),
+ (rdr::U8)((len & 0xff00) >> 8),
+ (rdr::U8)(len & 0xff)
+ };
+ len = clientKeyLength;
+ rdr::U8 lenClientKey[4] = {
+ (rdr::U8)((len & 0xff000000) >> 24),
+ (rdr::U8)((len & 0xff0000) >> 16),
+ (rdr::U8)((len & 0xff00) >> 8),
+ (rdr::U8)(len & 0xff)
+ };
+ if (keySize == 128) {
+ struct sha1_ctx ctx;
+ sha1_init(&ctx);
+ sha1_update(&ctx, 4, lenServerKey);
+ sha1_update(&ctx, serverKey.size, serverKeyN);
+ sha1_update(&ctx, serverKey.size, serverKeyE);
+ sha1_update(&ctx, 4, lenClientKey);
+ sha1_update(&ctx, clientKey.size, clientKeyN);
+ sha1_update(&ctx, clientKey.size, clientKeyE);
+ sha1_digest(&ctx, hashSize, realHash);
+ } else {
+ struct sha256_ctx ctx;
+ sha256_init(&ctx);
+ sha256_update(&ctx, 4, lenServerKey);
+ sha256_update(&ctx, serverKey.size, serverKeyN);
+ sha256_update(&ctx, serverKey.size, serverKeyE);
+ sha256_update(&ctx, 4, lenClientKey);
+ sha256_update(&ctx, clientKey.size, clientKeyN);
+ sha256_update(&ctx, clientKey.size, clientKeyE);
+ sha256_digest(&ctx, hashSize, realHash);
+ }
+ if (memcmp(hash, realHash, hashSize) != 0)
+ throw AuthFailureException("hash doesn't match");
+ return true;
+}
+
+void CSecurityRSAAES::clearSecrets()
+{
+ rsa_private_key_clear(&clientKey);
+ rsa_public_key_clear(&clientPublicKey);
+ rsa_public_key_clear(&serverKey);
+ clientKey.size = 0;
+ clientPublicKey.size = 0;
+ serverKey.size = 0;
+ delete[] serverKeyN;
+ delete[] serverKeyE;
+ delete[] clientKeyN;
+ delete[] clientKeyE;
+ serverKeyN = NULL;
+ serverKeyE = NULL;
+ clientKeyN = NULL;
+ clientKeyE = NULL;
+ memset(serverRandom, 0, sizeof(serverRandom));
+ memset(clientRandom, 0, sizeof(clientRandom));
+}
+
+bool CSecurityRSAAES::readSubtype()
+{
+ if (!rais->hasData(1))
+ return false;
+ subtype = rais->readU8();
+ if (subtype != secTypeRA2UserPass && subtype != secTypeRA2Pass)
+ throw AuthFailureException("unknown RSA-AES subtype");
+ return true;
+}
+
+void CSecurityRSAAES::writeCredentials()
+{
+ CharArray username;
+ CharArray password;
+
+ (CSecurity::upg)->getUserPasswd(
+ isSecure(),
+ subtype == secTypeRA2UserPass ? &username.buf : NULL, &password.buf
+ );
+ size_t len;
+ if (username.buf) {
+ len = strlen(username.buf);
+ if (len > 255)
+ throw AuthFailureException("username is too long");
+ raos->writeU8(len);
+ if (len)
+ raos->writeBytes(username.buf, len);
+ } else {
+ raos->writeU8(0);
+ }
+ len = strlen(password.buf);
+ if (len > 255)
+ throw AuthFailureException("password is too long");
+ raos->writeU8(len);
+ if (len)
+ raos->writeBytes(password.buf, len);
+ raos->flush();
+} \ No newline at end of file
diff --git a/common/rfb/CSecurityRSAAES.h b/common/rfb/CSecurityRSAAES.h
new file mode 100644
index 00000000..f58bb87f
--- /dev/null
+++ b/common/rfb/CSecurityRSAAES.h
@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+ * USA.
+ */
+
+#ifndef __C_SECURITY_RSAAES_H__
+#define __C_SECURITY_RSAAES_H__
+
+#ifndef HAVE_NETTLE
+#error "This header should not be compiled without HAVE_NETTLE defined"
+#endif
+
+#include <nettle/rsa.h>
+#include <rfb/CSecurity.h>
+#include <rfb/Security.h>
+#include <rfb/UserMsgBox.h>
+#include <rdr/InStream.h>
+#include <rdr/OutStream.h>
+#include <rdr/RandomStream.h>
+
+namespace rfb {
+ class UserMsgBox;
+ class CSecurityRSAAES : public CSecurity {
+ public:
+ CSecurityRSAAES(CConnection* cc, rdr::U32 secType,
+ int keySize, bool isAllEncrypted);
+ virtual ~CSecurityRSAAES();
+ virtual bool processMsg();
+ virtual int getType() const { return secType; }
+ virtual bool isSecure() const { return secType == secTypeRA256; }
+
+ static IntParameter RSAKeyLength;
+
+ private:
+ void cleanup();
+ void writePublicKey();
+ bool readPublicKey();
+ void verifyServer();
+ void writeRandom();
+ bool readRandom();
+ void setCipher();
+ void writeHash();
+ bool readHash();
+ void clearSecrets();
+ bool readSubtype();
+ void writeCredentials();
+
+ int state;
+ int keySize;
+ bool isAllEncrypted;
+ rdr::U32 secType;
+ rdr::U8 subtype;
+ struct rsa_private_key clientKey;
+ struct rsa_public_key clientPublicKey;
+ struct rsa_public_key serverKey;
+ rdr::U32 serverKeyLength;
+ rdr::U8* serverKeyN;
+ rdr::U8* serverKeyE;
+ rdr::U32 clientKeyLength;
+ rdr::U8* clientKeyN;
+ rdr::U8* clientKeyE;
+ rdr::U8 serverRandom[32];
+ rdr::U8 clientRandom[32];
+
+ rdr::InStream* rais;
+ rdr::OutStream* raos;
+
+ rdr::InStream* rawis;
+ rdr::OutStream* rawos;
+
+ rdr::RandomStream rs;
+ };
+}
+
+#endif
diff --git a/common/rfb/CSecurityTLS.h b/common/rfb/CSecurityTLS.h
index b2d68b6f..fb317748 100644
--- a/common/rfb/CSecurityTLS.h
+++ b/common/rfb/CSecurityTLS.h
@@ -34,7 +34,6 @@
#include <gnutls/gnutls.h>
namespace rfb {
- class UserMsgBox;
class CSecurityTLS : public CSecurity {
public:
CSecurityTLS(CConnection* cc, bool _anon);
@@ -45,7 +44,6 @@ namespace rfb {
static StringParameter X509CA;
static StringParameter X509CRL;
- static UserMsgBox *msg;
protected:
void shutdown();
diff --git a/common/rfb/SSecurityPlain.cxx b/common/rfb/SSecurityPlain.cxx
index 6ae19557..6f65e87a 100644
--- a/common/rfb/SSecurityPlain.cxx
+++ b/common/rfb/SSecurityPlain.cxx
@@ -36,7 +36,11 @@ using namespace rfb;
StringParameter PasswordValidator::plainUsers
("PlainUsers",
- "Users permitted to access via Plain security type (including TLSPlain, X509Plain etc.)",
+ "Users permitted to access via Plain security type (including TLSPlain, X509Plain etc.)"
+#ifdef HAVE_NETTLE
+ " or RSA-AES security types (RA2, RA2ne, RA2_256, RA2ne_256)"
+#endif
+ ,
"");
bool PasswordValidator::validUser(const char* username)
diff --git a/common/rfb/SSecurityRSAAES.cxx b/common/rfb/SSecurityRSAAES.cxx
new file mode 100644
index 00000000..15d2e97b
--- /dev/null
+++ b/common/rfb/SSecurityRSAAES.cxx
@@ -0,0 +1,598 @@
+/* Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+ * USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifndef HAVE_NETTLE
+#error "This source should not be compiled without HAVE_NETTLE defined"
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include <nettle/bignum.h>
+#include <nettle/sha1.h>
+#include <nettle/sha2.h>
+#include <nettle/base64.h>
+#include <nettle/asn1.h>
+#include <rfb/SSecurityRSAAES.h>
+#include <rfb/SConnection.h>
+#include <rfb/LogWriter.h>
+#include <rfb/Exception.h>
+#include <rdr/AESInStream.h>
+#include <rdr/AESOutStream.h>
+#if !defined(WIN32) && !defined(__APPLE__)
+#include <rfb/UnixPasswordValidator.h>
+#endif
+#ifdef WIN32
+#include <rfb/WinPasswdValidator.h>
+#endif
+#include <rfb/SSecurityVncAuth.h>
+
+enum {
+ SendPublicKey,
+ ReadPublicKey,
+ ReadRandom,
+ ReadHash,
+ ReadCredentials,
+};
+
+const int MinKeyLength = 1024;
+const int MaxKeyLength = 8192;
+const size_t MaxKeyFileSize = 32 * 1024;
+
+using namespace rfb;
+
+StringParameter SSecurityRSAAES::keyFile
+("RSAKey", "Path to the RSA key for the RSA-AES security types in "
+ "PEM format", "", ConfServer);
+BoolParameter SSecurityRSAAES::requireUsername
+("RequireUsername", "Require username for the RSA-AES security types",
+ false, ConfServer);
+
+SSecurityRSAAES::SSecurityRSAAES(SConnection* sc, rdr::U32 _secType,
+ int _keySize, bool _isAllEncrypted)
+ : SSecurity(sc), state(SendPublicKey),
+ keySize(_keySize), isAllEncrypted(_isAllEncrypted), secType(_secType),
+ serverKey(), clientKey(),
+ serverKeyN(NULL), serverKeyE(NULL), clientKeyN(NULL), clientKeyE(NULL),
+ accessRights(SConnection::AccessDefault),
+ rais(NULL), raos(NULL), rawis(NULL), rawos(NULL)
+{
+ assert(keySize == 128 || keySize == 256);
+}
+
+SSecurityRSAAES::~SSecurityRSAAES()
+{
+ cleanup();
+}
+
+void SSecurityRSAAES::cleanup()
+{
+ if (serverKeyN)
+ delete[] serverKeyN;
+ if (serverKeyE)
+ delete[] serverKeyE;
+ if (clientKeyN)
+ delete[] clientKeyN;
+ if (clientKeyE)
+ delete[] clientKeyE;
+ if (serverKey.size)
+ rsa_private_key_clear(&serverKey);
+ if (clientKey.size)
+ rsa_public_key_clear(&clientKey);
+ if (isAllEncrypted && rawis && rawos)
+ sc->setStreams(rawis, rawos);
+ if (rais)
+ delete rais;
+ if (raos)
+ delete raos;
+}
+
+static inline ssize_t findSubstr(rdr::U8* data, size_t size, const char *pattern)
+{
+ size_t patternLength = strlen(pattern);
+ for (size_t i = 0; i + patternLength < size; ++i) {
+ for (size_t j = 0; j < patternLength; ++j)
+ if (data[i + j] != pattern[j])
+ goto next;
+ return i;
+next:
+ continue;
+ }
+ return -1;
+}
+
+static bool loadPEM(rdr::U8* data, size_t size, const char *begin,
+ const char *end, rdr::U8** der, size_t *derSize)
+{
+ ssize_t pos1 = findSubstr(data, size, begin);
+ if (pos1 == -1)
+ return false;
+ pos1 += strlen(begin);
+ ssize_t base64Size = findSubstr(data + pos1, size - pos1, end);
+ if (base64Size == -1)
+ return false;
+ char *derBase64 = (char *)data + pos1;
+ if (!base64Size)
+ return false;
+ *der = new rdr::U8[BASE64_DECODE_LENGTH(base64Size)];
+ struct base64_decode_ctx ctx;
+ base64_decode_init(&ctx);
+ if (!base64_decode_update(&ctx, derSize, *der, base64Size, derBase64))
+ return false;
+ if (!base64_decode_final(&ctx))
+ return false;
+ return true;
+}
+
+void SSecurityRSAAES::loadPrivateKey()
+{
+ FILE* file = fopen(keyFile.getData(), "rb");
+ if (!file)
+ throw ConnFailedException("failed to open key file");
+ fseek(file, 0, SEEK_END);
+ size_t size = ftell(file);
+ if (size == 0 || size > MaxKeyFileSize) {
+ fclose(file);
+ throw ConnFailedException("size of key file is zero or too big");
+ }
+ fseek(file, 0, SEEK_SET);
+ rdr::U8Array data(size);
+ if (fread(data.buf, 1, size, file) != size) {
+ fclose(file);
+ throw ConnFailedException("failed to read key");
+ }
+ fclose(file);
+
+ rdr::U8Array der;
+ size_t derSize;
+ if (loadPEM(data.buf, size, "-----BEGIN RSA PRIVATE KEY-----\n",
+ "-----END RSA PRIVATE KEY-----", &der.buf, &derSize)) {
+ loadPKCS1Key(der.buf, derSize);
+ return;
+ }
+ if (der.buf)
+ delete[] der.takeBuf();
+ if (loadPEM(data.buf, size, "-----BEGIN PRIVATE KEY-----\n",
+ "-----END PRIVATE KEY-----", &der.buf, &derSize)) {
+ loadPKCS8Key(der.buf, derSize);
+ return;
+ }
+ throw ConnFailedException("failed to import key");
+}
+
+void SSecurityRSAAES::loadPKCS1Key(const rdr::U8* data, size_t size)
+{
+ struct rsa_public_key pub;
+ rsa_private_key_init(&serverKey);
+ rsa_public_key_init(&pub);
+ if (!rsa_keypair_from_der(&pub, &serverKey, 0, size, data)) {
+ rsa_private_key_clear(&serverKey);
+ rsa_public_key_clear(&pub);
+ throw ConnFailedException("failed to import key");
+ }
+ serverKeyLength = serverKey.size * 8;
+ serverKeyN = new rdr::U8[serverKey.size];
+ serverKeyE = new rdr::U8[serverKey.size];
+ nettle_mpz_get_str_256(serverKey.size, serverKeyN, pub.n);
+ nettle_mpz_get_str_256(serverKey.size, serverKeyE, pub.e);
+ rsa_public_key_clear(&pub);
+}
+
+void SSecurityRSAAES::loadPKCS8Key(const rdr::U8* data, size_t size)
+{
+ struct asn1_der_iterator i, j;
+ uint32_t version;
+ const char* rsaIdentifier = "\x2a\x86\x48\x86\xf7\x0d\x01\x01\x01";
+ const size_t rsaIdentifierLength = 9;
+ enum asn1_iterator_result res = asn1_der_iterator_first(&i, size, data);
+ if (res != ASN1_ITERATOR_CONSTRUCTED)
+ goto failed;
+ if (i.type != ASN1_SEQUENCE)
+ goto failed;
+ if (asn1_der_decode_constructed_last(&i) != ASN1_ITERATOR_PRIMITIVE)
+ goto failed;
+ if (!(i.type == ASN1_INTEGER &&
+ asn1_der_get_uint32(&i, &version) &&
+ version == 0))
+ goto failed;
+ if (!(asn1_der_iterator_next(&i) == ASN1_ITERATOR_CONSTRUCTED &&
+ i.type == ASN1_SEQUENCE &&
+ asn1_der_decode_constructed(&i, &j) == ASN1_ITERATOR_PRIMITIVE &&
+ j.type == ASN1_IDENTIFIER &&
+ j.length == rsaIdentifierLength &&
+ memcmp(j.data, rsaIdentifier, rsaIdentifierLength) == 0))
+ goto failed;
+ if (!(asn1_der_iterator_next(&i) == ASN1_ITERATOR_PRIMITIVE &&
+ i.type == ASN1_OCTETSTRING && i.length))
+ goto failed;
+ loadPKCS1Key(i.data, i.length);
+ return;
+failed:
+ throw ConnFailedException("failed to import key");
+}
+
+bool SSecurityRSAAES::processMsg()
+{
+ switch (state) {
+ case SendPublicKey:
+ loadPrivateKey();
+ writePublicKey();
+ state = ReadPublicKey;
+ // fall through
+ case ReadPublicKey:
+ if (readPublicKey()) {
+ writeRandom();
+ state = ReadRandom;
+ }
+ return false;
+ case ReadRandom:
+ if (readRandom()) {
+ setCipher();
+ writeHash();
+ state = ReadHash;
+ }
+ return false;
+ case ReadHash:
+ if (readHash()) {
+ clearSecrets();
+ writeSubtype();
+ state = ReadCredentials;
+ }
+ return false;
+ case ReadCredentials:
+ if (readCredentials()) {
+ if (requireUsername)
+ verifyUserPass();
+ else
+ verifyPass();
+ return true;
+ }
+ return false;
+ }
+ assert(!"unreachable");
+ return false;
+}
+
+void SSecurityRSAAES::writePublicKey()
+{
+ rdr::OutStream* os = sc->getOutStream();
+ os->writeU32(serverKeyLength);
+ os->writeBytes(serverKeyN, serverKey.size);
+ os->writeBytes(serverKeyE, serverKey.size);
+ os->flush();
+}
+
+bool SSecurityRSAAES::readPublicKey()
+{
+ rdr::InStream* is = sc->getInStream();
+ if (!is->hasData(4))
+ return false;
+ is->setRestorePoint();
+ clientKeyLength = is->readU32();
+ if (clientKeyLength < MinKeyLength)
+ throw ConnFailedException("client key is too short");
+ if (clientKeyLength > MaxKeyLength)
+ throw ConnFailedException("client key is too long");
+ size_t size = (clientKeyLength + 7) / 8;
+ if (!is->hasDataOrRestore(size * 2))
+ return false;
+ is->clearRestorePoint();
+ clientKeyE = new rdr::U8[size];
+ clientKeyN = new rdr::U8[size];
+ is->readBytes(clientKeyN, size);
+ is->readBytes(clientKeyE, size);
+ rsa_public_key_init(&clientKey);
+ nettle_mpz_set_str_256_u(clientKey.n, size, clientKeyN);
+ nettle_mpz_set_str_256_u(clientKey.e, size, clientKeyE);
+ if (!rsa_public_key_prepare(&clientKey))
+ throw ConnFailedException("client key is invalid");
+ return true;
+}
+
+static void random_func(void* ctx, size_t length, uint8_t* dst)
+{
+ rdr::RandomStream* rs = (rdr::RandomStream*)ctx;
+ if (!rs->hasData(length))
+ throw ConnFailedException("failed to encrypt random");
+ rs->readBytes(dst, length);
+}
+
+void SSecurityRSAAES::writeRandom()
+{
+ rdr::OutStream* os = sc->getOutStream();
+ if (!rs.hasData(keySize / 8))
+ throw ConnFailedException("failed to generate random");
+ rs.readBytes(serverRandom, keySize / 8);
+ mpz_t x;
+ mpz_init(x);
+ int res;
+ try {
+ res = rsa_encrypt(&clientKey, &rs, random_func, keySize / 8,
+ serverRandom, x);
+ } catch (...) {
+ mpz_clear(x);
+ throw;
+ }
+ if (!res) {
+ mpz_clear(x);
+ throw ConnFailedException("failed to encrypt random");
+ }
+ rdr::U8* buffer = new rdr::U8[clientKey.size];
+ nettle_mpz_get_str_256(clientKey.size, buffer, x);
+ mpz_clear(x);
+ os->writeU16(clientKey.size);
+ os->writeBytes(buffer, clientKey.size);
+ os->flush();
+ delete[] buffer;
+}
+
+bool SSecurityRSAAES::readRandom()
+{
+ rdr::InStream* is = sc->getInStream();
+ if (!is->hasData(2))
+ return false;
+ is->setRestorePoint();
+ size_t size = is->readU16();
+ if (size != serverKey.size)
+ throw ConnFailedException("server key length doesn't match");
+ if (!is->hasDataOrRestore(size))
+ return false;
+ is->clearRestorePoint();
+ rdr::U8* buffer = new rdr::U8[size];
+ is->readBytes(buffer, size);
+ size_t randomSize = keySize / 8;
+ mpz_t x;
+ nettle_mpz_init_set_str_256_u(x, size, buffer);
+ delete[] buffer;
+ if (!rsa_decrypt(&serverKey, &randomSize, clientRandom, x) ||
+ randomSize != (size_t)keySize / 8) {
+ mpz_clear(x);
+ throw ConnFailedException("failed to decrypt client random");
+ }
+ mpz_clear(x);
+ return true;
+}
+
+void SSecurityRSAAES::setCipher()
+{
+ rawis = sc->getInStream();
+ rawos = sc->getOutStream();
+ rdr::U8 key[32];
+ if (keySize == 128) {
+ struct sha1_ctx ctx;
+ sha1_init(&ctx);
+ sha1_update(&ctx, 16, serverRandom);
+ sha1_update(&ctx, 16, clientRandom);
+ sha1_digest(&ctx, 16, key);
+ rais = new rdr::AESInStream(rawis, key, 128);
+ sha1_init(&ctx);
+ sha1_update(&ctx, 16, clientRandom);
+ sha1_update(&ctx, 16, serverRandom);
+ sha1_digest(&ctx, 16, key);
+ raos = new rdr::AESOutStream(rawos, key, 128);
+ } else {
+ struct sha256_ctx ctx;
+ sha256_init(&ctx);
+ sha256_update(&ctx, 32, serverRandom);
+ sha256_update(&ctx, 32, clientRandom);
+ sha256_digest(&ctx, 32, key);
+ rais = new rdr::AESInStream(rawis, key, 256);
+ sha256_init(&ctx);
+ sha256_update(&ctx, 32, clientRandom);
+ sha256_update(&ctx, 32, serverRandom);
+ sha256_digest(&ctx, 32, key);
+ raos = new rdr::AESOutStream(rawos, key, 256);
+ }
+ if (isAllEncrypted)
+ sc->setStreams(rais, raos);
+}
+
+void SSecurityRSAAES::writeHash()
+{
+ rdr::U8 hash[32];
+ size_t len = serverKeyLength;
+ rdr::U8 lenServerKey[4] = {
+ (rdr::U8)((len & 0xff000000) >> 24),
+ (rdr::U8)((len & 0xff0000) >> 16),
+ (rdr::U8)((len & 0xff00) >> 8),
+ (rdr::U8)(len & 0xff)
+ };
+ len = clientKeyLength;
+ rdr::U8 lenClientKey[4] = {
+ (rdr::U8)((len & 0xff000000) >> 24),
+ (rdr::U8)((len & 0xff0000) >> 16),
+ (rdr::U8)((len & 0xff00) >> 8),
+ (rdr::U8)(len & 0xff)
+ };
+ int hashSize;
+ if (keySize == 128) {
+ hashSize = 20;
+ struct sha1_ctx ctx;
+ sha1_init(&ctx);
+ sha1_update(&ctx, 4, lenServerKey);
+ sha1_update(&ctx, serverKey.size, serverKeyN);
+ sha1_update(&ctx, serverKey.size, serverKeyE);
+ sha1_update(&ctx, 4, lenClientKey);
+ sha1_update(&ctx, clientKey.size, clientKeyN);
+ sha1_update(&ctx, clientKey.size, clientKeyE);
+ sha1_digest(&ctx, hashSize, hash);
+ } else {
+ hashSize = 32;
+ struct sha256_ctx ctx;
+ sha256_init(&ctx);
+ sha256_update(&ctx, 4, lenServerKey);
+ sha256_update(&ctx, serverKey.size, serverKeyN);
+ sha256_update(&ctx, serverKey.size, serverKeyE);
+ sha256_update(&ctx, 4, lenClientKey);
+ sha256_update(&ctx, clientKey.size, clientKeyN);
+ sha256_update(&ctx, clientKey.size, clientKeyE);
+ sha256_digest(&ctx, hashSize, hash);
+ }
+ raos->writeBytes(hash, hashSize);
+ raos->flush();
+}
+
+bool SSecurityRSAAES::readHash()
+{
+ rdr::U8 hash[32];
+ rdr::U8 realHash[32];
+ int hashSize = keySize == 128 ? 20 : 32;
+ if (!rais->hasData(hashSize))
+ return false;
+ rais->readBytes(hash, hashSize);
+ size_t len = serverKeyLength;
+ rdr::U8 lenServerKey[4] = {
+ (rdr::U8)((len & 0xff000000) >> 24),
+ (rdr::U8)((len & 0xff0000) >> 16),
+ (rdr::U8)((len & 0xff00) >> 8),
+ (rdr::U8)(len & 0xff)
+ };
+ len = clientKeyLength;
+ rdr::U8 lenClientKey[4] = {
+ (rdr::U8)((len & 0xff000000) >> 24),
+ (rdr::U8)((len & 0xff0000) >> 16),
+ (rdr::U8)((len & 0xff00) >> 8),
+ (rdr::U8)(len & 0xff)
+ };
+ if (keySize == 128) {
+ struct sha1_ctx ctx;
+ sha1_init(&ctx);
+ sha1_update(&ctx, 4, lenClientKey);
+ sha1_update(&ctx, clientKey.size, clientKeyN);
+ sha1_update(&ctx, clientKey.size, clientKeyE);
+ sha1_update(&ctx, 4, lenServerKey);
+ sha1_update(&ctx, serverKey.size, serverKeyN);
+ sha1_update(&ctx, serverKey.size, serverKeyE);
+ sha1_digest(&ctx, hashSize, realHash);
+ } else {
+ struct sha256_ctx ctx;
+ sha256_init(&ctx);
+ sha256_update(&ctx, 4, lenClientKey);
+ sha256_update(&ctx, clientKey.size, clientKeyN);
+ sha256_update(&ctx, clientKey.size, clientKeyE);
+ sha256_update(&ctx, 4, lenServerKey);
+ sha256_update(&ctx, serverKey.size, serverKeyN);
+ sha256_update(&ctx, serverKey.size, serverKeyE);
+ sha256_digest(&ctx, hashSize, realHash);
+ }
+ if (memcmp(hash, realHash, hashSize) != 0)
+ throw ConnFailedException("hash doesn't match");
+ return true;
+}
+
+void SSecurityRSAAES::clearSecrets()
+{
+ rsa_private_key_clear(&serverKey);
+ rsa_public_key_clear(&clientKey);
+ serverKey.size = 0;
+ clientKey.size = 0;
+ delete[] serverKeyN;
+ delete[] serverKeyE;
+ delete[] clientKeyN;
+ delete[] clientKeyE;
+ serverKeyN = NULL;
+ serverKeyE = NULL;
+ clientKeyN = NULL;
+ clientKeyE = NULL;
+ memset(serverRandom, 0, sizeof(serverRandom));
+ memset(clientRandom, 0, sizeof(clientRandom));
+}
+
+void SSecurityRSAAES::writeSubtype()
+{
+ if (requireUsername)
+ raos->writeU8(secTypeRA2UserPass);
+ else
+ raos->writeU8(secTypeRA2Pass);
+ raos->flush();
+}
+
+bool SSecurityRSAAES::readCredentials()
+{
+ rais->setRestorePoint();
+ if (!rais->hasData(1))
+ return false;
+ rdr::U8 lenUsername = rais->readU8();
+ if (!rais->hasDataOrRestore(lenUsername + 1))
+ return false;
+ if (!username.buf) {
+ username.replaceBuf(new char[lenUsername + 1]);
+ rais->readBytes(username.buf, lenUsername);
+ username.buf[lenUsername] = 0;
+ } else {
+ rais->skip(lenUsername);
+ }
+ rdr::U8 lenPassword = rais->readU8();
+ if (!rais->hasDataOrRestore(lenPassword))
+ return false;
+ password.replaceBuf(new char[lenPassword + 1]);
+ rais->readBytes(password.buf, lenPassword);
+ password.buf[lenPassword] = 0;
+ rais->clearRestorePoint();
+ return true;
+}
+
+void SSecurityRSAAES::verifyUserPass()
+{
+#ifndef __APPLE__
+#ifdef WIN32
+ WinPasswdValidator* valid = new WinPasswdValidator();
+#elif !defined(__APPLE__)
+ UnixPasswordValidator *valid = new UnixPasswordValidator();
+#endif
+ if (!valid->validate(sc, username.buf, password.buf)) {
+ delete valid;
+ throw AuthFailureException("invalid password or username");
+ }
+ delete valid;
+#else
+ throw AuthFailureException("No password validator configured");
+#endif
+}
+
+void SSecurityRSAAES::verifyPass()
+{
+ VncAuthPasswdGetter* pg = &SSecurityVncAuth::vncAuthPasswd;
+ PlainPasswd passwd, passwdReadOnly;
+ pg->getVncAuthPasswd(&passwd, &passwdReadOnly);
+
+ if (!passwd.buf)
+ throw AuthFailureException("No password configured for VNC Auth");
+
+ if (strcmp(password.buf, passwd.buf) == 0) {
+ accessRights = SConnection::AccessDefault;
+ return;
+ }
+
+ if (passwdReadOnly.buf && strcmp(password.buf, passwdReadOnly.buf) == 0) {
+ accessRights = SConnection::AccessView;
+ return;
+ }
+
+ throw AuthFailureException();
+}
+
+const char* SSecurityRSAAES::getUserName() const
+{
+ return username.buf;
+}
diff --git a/common/rfb/SSecurityRSAAES.h b/common/rfb/SSecurityRSAAES.h
new file mode 100644
index 00000000..17e0d407
--- /dev/null
+++ b/common/rfb/SSecurityRSAAES.h
@@ -0,0 +1,98 @@
+/* Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+ * USA.
+ */
+
+#ifndef __S_SECURITY_RSAAES_H__
+#define __S_SECURITY_RSAAES_H__
+
+#ifndef HAVE_NETTLE
+#error "This header should not be included without HAVE_NETTLE defined"
+#endif
+
+#include <nettle/rsa.h>
+#include <rfb/SSecurity.h>
+#include <rdr/InStream.h>
+#include <rdr/OutStream.h>
+#include <rdr/RandomStream.h>
+
+namespace rfb {
+
+ class SSecurityRSAAES : public SSecurity {
+ public:
+ SSecurityRSAAES(SConnection* sc, rdr::U32 secType,
+ int keySize, bool isAllEncrypted);
+ virtual ~SSecurityRSAAES();
+ virtual bool processMsg();
+ virtual const char* getUserName() const;
+ virtual int getType() const { return secType; }
+ virtual SConnection::AccessRights getAccessRights() const
+ {
+ return accessRights;
+ }
+
+ static StringParameter keyFile;
+ static BoolParameter requireUsername;
+
+ private:
+ void cleanup();
+ void loadPrivateKey();
+ void loadPKCS1Key(const rdr::U8* data, size_t size);
+ void loadPKCS8Key(const rdr::U8* data, size_t size);
+ void writePublicKey();
+ bool readPublicKey();
+ void writeRandom();
+ bool readRandom();
+ void setCipher();
+ void writeHash();
+ bool readHash();
+ void clearSecrets();
+ void writeSubtype();
+ bool readCredentials();
+ void verifyUserPass();
+ void verifyPass();
+
+ int state;
+ int keySize;
+ bool isAllEncrypted;
+ rdr::U32 secType;
+ struct rsa_private_key serverKey;
+ struct rsa_public_key clientKey;
+ rdr::U32 serverKeyLength;
+ rdr::U8* serverKeyN;
+ rdr::U8* serverKeyE;
+ rdr::U32 clientKeyLength;
+ rdr::U8* clientKeyN;
+ rdr::U8* clientKeyE;
+ rdr::U8 serverRandom[32];
+ rdr::U8 clientRandom[32];
+
+ CharArray username;
+ CharArray password;
+ SConnection::AccessRights accessRights;
+
+ rdr::InStream* rais;
+ rdr::OutStream* raos;
+
+ rdr::InStream* rawis;
+ rdr::OutStream* rawos;
+
+ rdr::RandomStream rs;
+ };
+
+}
+
+#endif
diff --git a/common/rfb/Security.cxx b/common/rfb/Security.cxx
index 59deb78d..3ae4e093 100644
--- a/common/rfb/Security.cxx
+++ b/common/rfb/Security.cxx
@@ -160,6 +160,8 @@ rdr::U32 rfb::secTypeNum(const char* name)
if (strcasecmp(name, "Tight") == 0) return secTypeTight;
if (strcasecmp(name, "RA2") == 0) return secTypeRA2;
if (strcasecmp(name, "RA2ne") == 0) return secTypeRA2ne;
+ if (strcasecmp(name, "RA2_256") == 0) return secTypeRA256;
+ if (strcasecmp(name, "RA2ne_256") == 0) return secTypeRAne256;
if (strcasecmp(name, "SSPI") == 0) return secTypeSSPI;
if (strcasecmp(name, "SSPIne") == 0) return secTypeSSPIne;
if (strcasecmp(name, "VeNCrypt") == 0) return secTypeVeNCrypt;
@@ -184,6 +186,8 @@ const char* rfb::secTypeName(rdr::U32 num)
case secTypeTight: return "Tight";
case secTypeRA2: return "RA2";
case secTypeRA2ne: return "RA2ne";
+ case secTypeRA256: return "RA2_256";
+ case secTypeRAne256: return "RA2ne_256";
case secTypeSSPI: return "SSPI";
case secTypeSSPIne: return "SSPIne";
case secTypeVeNCrypt: return "VeNCrypt";
diff --git a/common/rfb/Security.h b/common/rfb/Security.h
index a4987f5f..03a2218f 100644
--- a/common/rfb/Security.h
+++ b/common/rfb/Security.h
@@ -43,6 +43,9 @@ namespace rfb {
const rdr::U8 secTypeTLS = 18;
const rdr::U8 secTypeVeNCrypt= 19;
+ const rdr::U8 secTypeRA256 = 129;
+ const rdr::U8 secTypeRAne256 = 130;
+
/* VeNCrypt subtypes */
const int secTypePlain = 256;
const int secTypeTLSNone = 257;
@@ -52,6 +55,10 @@ namespace rfb {
const int secTypeX509Vnc = 261;
const int secTypeX509Plain = 262;
+ /* RSA-AES subtypes */
+ const int secTypeRA2UserPass = 1;
+ const int secTypeRA2Pass = 2;
+
// result types
const rdr::U32 secResultOK = 0;
diff --git a/common/rfb/SecurityClient.cxx b/common/rfb/SecurityClient.cxx
index dcd28bae..b2e974f8 100644
--- a/common/rfb/SecurityClient.cxx
+++ b/common/rfb/SecurityClient.cxx
@@ -32,13 +32,16 @@
#ifdef HAVE_GNUTLS
#include <rfb/CSecurityTLS.h>
#endif
+#ifdef HAVE_NETTLE
+#include <rfb/CSecurityRSAAES.h>
+#endif
using namespace rdr;
using namespace rfb;
UserPasswdGetter *CSecurity::upg = NULL;
-#ifdef HAVE_GNUTLS
-UserMsgBox *CSecurityTLS::msg = NULL;
+#if defined(HAVE_GNUTLS) || defined(HAVE_NETTLE)
+UserMsgBox *CSecurity::msg = NULL;
#endif
StringParameter SecurityClient::secTypes
@@ -47,19 +50,24 @@ StringParameter SecurityClient::secTypes
#ifdef HAVE_GNUTLS
", TLSNone, TLSVnc, TLSPlain, X509None, X509Vnc, X509Plain"
#endif
+#ifdef HAVE_NETTLE
+ ", RA2, RA2ne, RA2_256, RA2ne_256"
+#endif
")",
#ifdef HAVE_GNUTLS
- "X509Plain,TLSPlain,X509Vnc,TLSVnc,X509None,TLSNone,VncAuth,None",
-#else
- "VncAuth,None",
+ "X509Plain,TLSPlain,X509Vnc,TLSVnc,X509None,TLSNone,"
#endif
+#ifdef HAVE_NETTLE
+ "RA2,RA2_256,RA2ne,RA2ne_256,"
+#endif
+ "VncAuth,None",
ConfViewer);
CSecurity* SecurityClient::GetCSecurity(CConnection* cc, U32 secType)
{
assert (CSecurity::upg != NULL); /* (upg == NULL) means bug in the viewer */
-#ifdef HAVE_GNUTLS
- assert (CSecurityTLS::msg != NULL);
+#if defined(HAVE_GNUTLS) || defined(HAVE_NETTLE)
+ assert (CSecurity::msg != NULL);
#endif
if (!IsSupported(secType))
@@ -94,6 +102,16 @@ CSecurity* SecurityClient::GetCSecurity(CConnection* cc, U32 secType)
new CSecurityTLS(cc, false),
new CSecurityPlain(cc));
#endif
+#ifdef HAVE_NETTLE
+ case secTypeRA2:
+ return new CSecurityRSAAES(cc, secTypeRA2, 128, true);
+ case secTypeRA2ne:
+ return new CSecurityRSAAES(cc, secTypeRA2ne, 128, false);
+ case secTypeRA256:
+ return new CSecurityRSAAES(cc, secTypeRA256, 256, true);
+ case secTypeRAne256:
+ return new CSecurityRSAAES(cc, secTypeRAne256, 256, false);
+#endif
}
bail:
diff --git a/common/rfb/SecurityServer.cxx b/common/rfb/SecurityServer.cxx
index 97b133c7..04022a90 100644
--- a/common/rfb/SecurityServer.cxx
+++ b/common/rfb/SecurityServer.cxx
@@ -31,6 +31,9 @@
#ifdef HAVE_GNUTLS
#include <rfb/SSecurityTLS.h>
#endif
+#ifdef HAVE_NETTLE
+#include <rfb/SSecurityRSAAES.h>
+#endif
using namespace rdr;
using namespace rfb;
@@ -41,12 +44,17 @@ StringParameter SecurityServer::secTypes
#ifdef HAVE_GNUTLS
", TLSNone, TLSVnc, TLSPlain, X509None, X509Vnc, X509Plain"
#endif
+#ifdef HAVE_NETTLE
+ ", RA2, RA2ne, RA2_256, RA2ne_256"
+#endif
")",
#ifdef HAVE_GNUTLS
- "TLSVnc,VncAuth",
-#else
- "VncAuth",
+ "TLSVnc,"
#endif
+#ifdef HAVE_NETTLE
+ "RA2_256,RA2,RA2ne_256,RA2ne,"
+#endif
+ "VncAuth",
ConfServer);
SSecurity* SecurityServer::GetSSecurity(SConnection* sc, U32 secType)
@@ -73,6 +81,16 @@ SSecurity* SecurityServer::GetSSecurity(SConnection* sc, U32 secType)
case secTypeX509Plain:
return new SSecurityStack(sc, secTypeX509Plain, new SSecurityTLS(sc, false), new SSecurityPlain(sc));
#endif
+#ifdef HAVE_NETTLE
+ case secTypeRA2:
+ return new SSecurityRSAAES(sc, secTypeRA2, 128, true);
+ case secTypeRA2ne:
+ return new SSecurityRSAAES(sc, secTypeRA2ne, 128, false);
+ case secTypeRA256:
+ return new SSecurityRSAAES(sc, secTypeRA256, 256, true);
+ case secTypeRAne256:
+ return new SSecurityRSAAES(sc, secTypeRAne256, 256, false);
+#endif
}
bail:
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-23 06:49:45 +0000