Add support for RSA-AES security types

1 files changed, 89 insertions, 0 deletions

diff --git a/common/rfb/CSecurityRSAAES.h b/common/rfb/CSecurityRSAAES.h
new file mode 100644
index 00000000..f58bb87f
--- /dev/null
+++ b/common/rfb/CSecurityRSAAES.h
@@ -0,0 +1,89 @@
+/* 
+ * Copyright (C) 2022 Dinglan Peng
+ *    
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#ifndef __C_SECURITY_RSAAES_H__
+#define __C_SECURITY_RSAAES_H__
+
+#ifndef HAVE_NETTLE
+#error "This header should not be compiled without HAVE_NETTLE defined"
+#endif
+
+#include <nettle/rsa.h>
+#include <rfb/CSecurity.h>
+#include <rfb/Security.h>
+#include <rfb/UserMsgBox.h>
+#include <rdr/InStream.h>
+#include <rdr/OutStream.h>
+#include <rdr/RandomStream.h>
+
+namespace rfb {
+  class UserMsgBox;
+  class CSecurityRSAAES : public CSecurity {
+  public:
+    CSecurityRSAAES(CConnection* cc, rdr::U32 secType,
+                    int keySize, bool isAllEncrypted);
+    virtual ~CSecurityRSAAES();
+    virtual bool processMsg();
+    virtual int getType() const { return secType; }
+    virtual bool isSecure() const { return secType == secTypeRA256; }
+
+    static IntParameter RSAKeyLength;
+
+  private:
+    void cleanup();
+    void writePublicKey();
+    bool readPublicKey();
+    void verifyServer();
+    void writeRandom();
+    bool readRandom();
+    void setCipher();
+    void writeHash();
+    bool readHash();
+    void clearSecrets();
+    bool readSubtype();
+    void writeCredentials();
+
+    int state;
+    int keySize;
+    bool isAllEncrypted;
+    rdr::U32 secType;
+    rdr::U8 subtype;
+    struct rsa_private_key clientKey;
+    struct rsa_public_key clientPublicKey;
+    struct rsa_public_key serverKey;
+    rdr::U32 serverKeyLength;
+    rdr::U8* serverKeyN;
+    rdr::U8* serverKeyE;
+    rdr::U32 clientKeyLength;
+    rdr::U8* clientKeyN;
+    rdr::U8* clientKeyE;
+    rdr::U8 serverRandom[32];
+    rdr::U8 clientRandom[32];
+
+    rdr::InStream* rais;
+    rdr::OutStream* raos;
+
+    rdr::InStream* rawis;
+    rdr::OutStream* rawos;
+
+    rdr::RandomStream rs;
+  };
+}
+
+#endif