Merge pull request #1525 from pdlan/add-ra2-java

Add support for RSA-AES security types (Java version)

java/com/tigervnc/rdr/AESEAXCipher.java

@@ -0,0 +1,150 @@
+/* Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+ * USA.
+ */
+
+package com.tigervnc.rdr;
+
+import java.util.Arrays;
+
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.security.InvalidAlgorithmParameterException;
+
+import javax.crypto.Cipher;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+
+public class AESEAXCipher {
+    
+  private static final byte[] zeroBlock = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+  private static final byte[] prefixBlock0 = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+  private static final byte[] prefixBlock1 = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1};
+  private static final byte[] prefixBlock2 = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2};
+  private static final int[] lut = {0x0,0x87,0x0e,0x89};
+
+  public AESEAXCipher(byte[] key)
+  {
+    try {
+      Cipher blockCipher = Cipher.getInstance("AES");
+      cbcCipher = Cipher.getInstance("AES/CBC/NOPADDING");
+      ctrCipher = Cipher.getInstance("AES/CTR/NOPADDING");
+      keySpec = new SecretKeySpec(key, "AES");
+      blockCipher.init(Cipher.ENCRYPT_MODE, keySpec);
+      cbcCipher.init(Cipher.ENCRYPT_MODE, keySpec,
+                     new IvParameterSpec(zeroBlock));
+      subKey1 = Arrays.copyOfRange(blockCipher.doFinal(zeroBlock), 0, 16);
+      subKey2 = new byte[16];
+      int v = (subKey1[0] & 0xff) >>> 6;
+      for (int i = 0; i < 15; i++) {
+        subKey2[i] = (byte)(((subKey1[i + 1] & 0xff) >>> 6) |
+                            ((subKey1[i] & 0xff) << 2));
+        subKey1[i] = (byte)(((subKey1[i + 1] & 0xff) >>> 7) |
+                            ((subKey1[i] & 0xff) << 1));
+      }
+      subKey2[14] ^= v >>> 1;
+      subKey2[15] = (byte)(((subKey1[15] & 0xff) << 2) ^ lut[v]);
+      subKey1[15] = (byte)(((subKey1[15] & 0xff) << 1) ^ lut[v >>> 1]);
+    } catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
+      throw new Exception("AESEAXCipher: AES algorithm is not supported");
+    } catch (IllegalBlockSizeException | BadPaddingException |
+             InvalidKeyException | InvalidAlgorithmParameterException e) {
+      throw new Exception("AESEAXCipher: invalid key");
+    }
+  }
+
+  private void encryptCTR(byte[] input, int inputOffset, int inputLength,
+                          byte[] output, int outputOffset, byte[] iv)
+  {
+    try {
+      ctrCipher.init(Cipher.ENCRYPT_MODE, keySpec, new IvParameterSpec(iv));
+      ctrCipher.doFinal(input, inputOffset, inputLength, output, outputOffset);
+    } catch (java.lang.Exception e) {
+      throw new Exception("AESEAXCipher: " + e.toString());
+    }
+  }
+
+  private byte[] computeCMAC(byte[] input, int offset,
+                             int length, byte[] prefix)
+  {
+    int n = length / 16;
+    int m = (length + 15) / 16;
+    int r = length - n * 16;
+    byte[] cbcData = new byte[(m + 1) * 16];
+    System.arraycopy(prefix, 0, cbcData, 0, 16);
+    System.arraycopy(input, offset, cbcData, 16, length);
+    
+    if (r == 0) {
+      for (int i = 0; i < 16; i++) {
+        cbcData[n * 16 + i] ^= subKey1[i] & 0xff;
+      }
+    } else {
+      cbcData[(n + 1) * 16 + r] = (byte)0x80;
+      for (int i = 0; i < 16; i++) {
+        cbcData[(n + 1) * 16 + i] ^= subKey2[i] & 0xff;
+      }
+    }
+    try {
+      byte[] encrypted = cbcCipher.doFinal(cbcData);
+      return Arrays.copyOfRange(encrypted, encrypted.length - 16, encrypted.length - 0);
+    } catch (java.lang.Exception e) {
+      throw new Exception("AESEAXCipher: " + e.getMessage());
+    }
+  }
+
+  public void encrypt(byte[] input, int inputOffset, int inputLength,
+                      byte[] ad,  int adOffset, int adLength,
+                      byte[] nonce,
+                      byte[] output, int outputOffset,
+                      byte[] mac, int macOffset)
+  {
+    byte[] nCMAC = computeCMAC(nonce, 0, nonce.length, prefixBlock0);
+    encryptCTR(input, inputOffset, inputLength, output, outputOffset, nCMAC);
+    byte[] adCMAC = computeCMAC(ad, adOffset, adLength, prefixBlock1);
+    byte[] m = computeCMAC(output, outputOffset, inputLength, prefixBlock2);
+    for (int i = 0; i < 16; i++) {
+      mac[macOffset + i] = (byte)((m[i] & 0xff) ^
+                                  (nCMAC[i] & 0xff) ^
+                                  (adCMAC[i] & 0xff));
+    }
+  }
+
+  public void decrypt(byte[] input, int inputOffset, int inputLength,
+                      byte[] ad,  int adOffset, int adLength,
+                      byte[] nonce,
+                      byte[] output, int outputOffset,
+                      byte[] mac, int macOffset)
+  {
+    byte[] nCMAC = computeCMAC(nonce, 0, nonce.length, prefixBlock0);
+    byte[] adCMAC = computeCMAC(ad, adOffset, adLength, prefixBlock1);
+    byte[] m = computeCMAC(input, inputOffset, inputLength, prefixBlock2);
+    for (int i = 0; i < 16; i++) {
+      byte x = (byte)((m[i] & 0xff) ^ (nCMAC[i] & 0xff) ^ (adCMAC[i] & 0xff));
+      if (x != mac[macOffset + i])
+        throw new Exception("AESEAXCipher: failed to authenticate message"); 
+    }
+    encryptCTR(input, inputOffset, inputLength, output, outputOffset, nCMAC);
+  }
+
+  private SecretKeySpec keySpec;
+  private Cipher ctrCipher;
+  private Cipher cbcCipher;
+  private byte[] subKey1;
+  private byte[] subKey2;
+}

java/com/tigervnc/rdr/AESInStream.java

@@ -0,0 +1,169 @@
+/* Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+ * USA.
+ */
+
+package com.tigervnc.rdr;
+
+import java.nio.ByteBuffer;
+
+public class AESInStream extends InStream {
+
+  private static final int maxMessageSize = 65536;
+
+  public AESInStream(InStream _in, byte[] key)
+  {
+    in = _in;
+    offset = 0;
+    bufSize = maxMessageSize;
+    b = new byte[bufSize];
+    ptr = end = start = 0;
+    cipher = new AESEAXCipher(key);
+    messageSize = 0;
+    messageOffset = 0;
+    message = new byte[maxMessageSize + 16];
+    decryptedMessageOffset = 0;
+    decryptedMessage = new byte[maxMessageSize];
+    counter = new byte[16];
+    state = 0;
+  }
+
+  public final int pos()
+  {
+    return offset + ptr - start;
+  }
+
+  protected final int overrun(int itemSize, int nItems, boolean wait)
+  {
+    if (itemSize > bufSize)
+      throw new Exception("AESInStream overrun: max itemSize exceeded");
+
+    if (end - ptr != 0)
+      System.arraycopy(b, ptr, b, 0, end - ptr);
+
+    offset += ptr - start;
+    end -= ptr - start;
+    ptr = start;
+
+    while ((end - start) < itemSize) {
+      int n = readMessage(b, end, start + bufSize - end, wait);
+      if (!wait && n == 0)
+        return 0;
+      end += n;
+    }
+
+    int nAvail;
+    nAvail = (end - ptr) / itemSize;
+    if (nAvail < nItems)
+      return nAvail;
+
+    return nItems;
+  }
+
+  private int readMessage(byte[] buf, int bufPtr, int len, boolean wait)
+  {
+    if (state == 0 || state == 1) {
+      if (!fillDecryptedMessageBuffer(wait) && !wait)
+        return 0;
+    }
+    if (state == 2) {
+      int readSize = messageSize - decryptedMessageOffset;
+      if (readSize > len)
+        readSize = len;
+      System.arraycopy(decryptedMessage, decryptedMessageOffset,
+                       buf, bufPtr, readSize);
+      decryptedMessageOffset += readSize;
+      if (decryptedMessageOffset == messageSize)
+        state = 0;
+      return readSize;
+    }
+    return 0;
+  }
+
+  private boolean fillDecryptedMessageBuffer(boolean wait)
+  {
+    if (state == 0) {
+      while (true) {
+        if (in.check(2, 1, wait) != 0) {
+          messageSize = in.readU16();
+          messageOffset = 0;
+          state = 1;
+          break;
+        } else if (!wait) {
+          return false;
+        }
+      }
+    }
+    if (state == 1) {
+      if (wait) {
+        in.readBytes(ByteBuffer.wrap(message, messageOffset,
+                                     messageSize + 16 - messageOffset),
+                     messageSize + 16 - messageOffset);
+      } else {
+        while (true) {
+          int readSize = messageSize + 16 - messageOffset;
+          if (in.check(1, readSize, false) != 0) {
+            int availSize = in.getend() - in.getptr();
+            if (readSize > availSize)
+              readSize = availSize;
+            in.readBytes(ByteBuffer.wrap(message, messageOffset, readSize),
+                         readSize);
+            messageOffset += readSize;
+            if (messageSize + 16 == messageOffset) {
+              break;
+            }
+          } else {
+            return false;
+          }
+        }
+      }
+    }
+    byte[] ad = new byte[] {
+      (byte)((messageSize & 0xff00) >> 8),
+      (byte)(messageSize & 0xff)
+    };
+    cipher.decrypt(message, 0, messageSize,
+                   ad, 0, 2,
+                   counter,
+                   decryptedMessage, 0,
+                   message, messageSize);
+    // Update nonce by incrementing the counter as a
+    // 128bit little endian unsigned integer
+    for (int i = 0; i < 16; ++i) {
+      // increment until there is no carry
+      if (++counter[i] != 0) {
+        break;
+      }
+    }
+    decryptedMessageOffset = 0;
+    state = 2;
+    return true;
+  }
+
+
+  private AESEAXCipher cipher;
+  private int offset;
+  private int start;
+  private int bufSize;
+  private int state;
+  private int messageSize;
+  private int messageOffset;
+  private byte[] message;
+  private int decryptedMessageOffset;
+  private byte[] decryptedMessage;
+  private byte[] counter;
+  private InStream in;
+}

java/com/tigervnc/rdr/AESOutStream.java

@@ -0,0 +1,103 @@
+/* Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+ * USA.
+ */
+
+package com.tigervnc.rdr;
+
+import java.nio.ByteBuffer;
+
+public class AESOutStream extends OutStream {
+
+  static final int maxMessageSize = 8192;
+
+  public AESOutStream(OutStream _out, byte[] key)
+  {
+    out = _out;
+    bufSize = maxMessageSize;
+    b = new byte[bufSize];
+    buffer = new byte[bufSize + 16 + 2];
+    ptr = offset = start = 0;
+    end = start + bufSize;
+    cipher = new AESEAXCipher(key);
+    counter = new byte[16];
+  }
+
+  public int length()
+  {
+    return offset + ptr - start;
+  }
+
+  public void flush()
+  {
+    int sentUpTo = start;
+    while (sentUpTo < ptr) {
+      int n = writeMessage(b, sentUpTo, ptr - sentUpTo);
+      sentUpTo += n;
+      offset += n;
+    }
+
+    ptr = start;
+  }
+
+  protected int overrun(int itemSize, int nItems)
+  {
+    if (itemSize > bufSize)
+      throw new Exception("AESOutStream overrun: max itemSize exceeded");
+
+    flush();
+
+    int nAvail;
+    nAvail = (end - ptr) / itemSize;
+    if (nAvail < nItems)
+      return nAvail;
+
+    return nItems;
+  }
+
+  protected int writeMessage(byte[] data, int dataPtr, int length)
+  {
+    if (length == 0)
+      return 0;
+    buffer[0] = (byte)((length & 0xff00) >> 8);
+    buffer[1] = (byte)(length & 0xff);
+    cipher.encrypt(data, dataPtr, length,
+                   buffer, 0, 2,
+                   counter,
+                   buffer, 2,
+                   buffer, 2 + length);
+    out.writeBytes(ByteBuffer.wrap(buffer, 0, length + 16 + 2),
+                   length + 16 + 2);
+    out.flush();
+    // Update nonce by incrementing the counter as a
+    // 128bit little endian unsigned integer
+    for (int i = 0; i < 16; ++i) {
+      // increment until there is no carry
+      if (++counter[i] != 0) {
+        break;
+      }
+    }
+    return length;
+  }
+
+  private AESEAXCipher cipher;
+  private int start;
+  private int offset;
+  private int bufSize;
+  private byte[] buffer;
+  private byte[] counter;
+  private OutStream out;
+}

java/com/tigervnc/rfb/CConnection.java

@@ -197,7 +197,7 @@ abstract public class CConnection extends CMsgHandler {
 
     verStr.clear();
     verStr.put(String.format("RFB %03d.%03d\n",
-               majorVersion, minorVersion).getBytes()).flip();
+               server.majorVersion, server.minorVersion).getBytes()).flip();
     os.writeBytes(verStr.array(), 0, 12);
     os.flush();
 

java/com/tigervnc/rfb/CSecurity.java

@@ -44,4 +44,5 @@ abstract public class CSecurity {
    * It MUST be set by viewer.
    */
   public static UserPasswdGetter upg;
+  public static UserMsgBox msg;
 }

java/com/tigervnc/rfb/CSecurityRSAAES.java

@@ -0,0 +1,364 @@
+/* Copyright (C) 2022 Dinglan Peng
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+ * USA.
+ */
+
+package com.tigervnc.rfb;
+
+import java.io.UnsupportedEncodingException;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.security.InvalidKeyException;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.SecureRandom;
+import java.security.interfaces.RSAPublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.RSAPublicKeySpec;
+import java.util.Arrays;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.swing.JOptionPane;
+
+import com.tigervnc.rdr.*;
+import com.tigervnc.vncviewer.*;
+
+public class CSecurityRSAAES extends CSecurity {
+
+  private static final int MinKeyLength = 1024;
+  private static final int MaxKeyLength = 8192;
+
+  private static final byte[] bigIntToBytes(BigInteger n, int bytes) {
+    int bits = n.bitCount();
+    byte[] arr = n.toByteArray();
+    //int len = bits % 8 == 0 ? arr.length - 1 : arr.length;
+    //System.out.printf("%d %d %d\n", bits, len, bytes);
+    //if (len > bytes)
+    //  return null;
+    int len = arr.length < bytes ? arr.length : bytes;
+    byte[] res = new byte[bytes];
+    System.arraycopy(arr, arr.length - len, res, bytes - len, len);
+    return res;
+  }
+
+  public CSecurityRSAAES(int secType, int keySize, boolean isAllEncrypted) {
+    this.secType = secType;
+    this.keySize = keySize;
+    this.isAllEncrypted = isAllEncrypted;
+  }
+
+  public boolean processMsg(CConnection cc) {
+    readPubclicKey(cc);
+    verifyServer();
+    writePublicKey(cc);
+    writeRandom(cc);
+    readRandom(cc);
+    setCipher(cc);
+    writeHash(cc);
+    readHash();
+    readSubtype();
+    writeCredentials();
+    return true;
+  }
+
+  private void readPubclicKey(CConnection cc) {
+    InStream is = cc.getInStream();
+    serverKeyLength = is.readU32();
+    if (serverKeyLength < MinKeyLength)
+      throw new AuthFailureException("server key is too short");
+    if (serverKeyLength > MaxKeyLength)
+      throw new AuthFailureException("server key is too long");
+    int size = (serverKeyLength + 7) / 8;
+    serverKeyN = new byte[size];
+    serverKeyE = new byte[size];
+    is.readBytes(ByteBuffer.wrap(serverKeyN), size);
+    is.readBytes(ByteBuffer.wrap(serverKeyE), size);
+    BigInteger modulus = new BigInteger(1, serverKeyN);
+    BigInteger publicExponent = new BigInteger(1, serverKeyE);
+    RSAPublicKeySpec spec = new RSAPublicKeySpec(modulus, publicExponent);
+    try {
+      KeyFactory factory = KeyFactory.getInstance("RSA");
+      serverKey = factory.generatePublic(spec);
+    } catch (NoSuchAlgorithmException e) {
+      throw new AuthFailureException("RSA algorithm is not supported");
+    } catch (InvalidKeySpecException e) {
+      throw new AuthFailureException("server key is invalid");
+    }
+  }
+
+  private void verifyServer() {
+    MessageDigest digest;
+    try {
+      digest = MessageDigest.getInstance("SHA-1");
+    } catch (NoSuchAlgorithmException e) {
+      throw new AuthFailureException("SHA-1 algorithm is not supported");
+    }
+    byte[] length = new byte[4];
+    length[0] = (byte) ((serverKeyLength & 0xff000000) >> 24);
+    length[1] = (byte) ((serverKeyLength & 0xff0000) >> 16);
+    length[2] = (byte) ((serverKeyLength & 0xff00) >> 8);
+    length[3] = (byte) (serverKeyLength & 0xff);
+    digest.update(length);
+    digest.update(serverKeyN);
+    digest.update(serverKeyE);
+    byte[] f = digest.digest();
+    String title = "Server key fingerprint";
+    String text = String.format(
+      "The server has provided the following identifying information:\n" +
+      "Fingerprint: %02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x\n" +
+      "Please verify that the information is correct and press \"Yes\". " +
+      "Otherwise press \"No\"", f[0], f[1], f[2], f[3], f[4], f[5], f[6], f[7]);
+    if (!msg.showMsgBox(JOptionPane.YES_NO_OPTION, title, text))
+      throw new AuthFailureException("server key mismatch");
+  }
+
+  private void writePublicKey(CConnection cc) {
+    OutStream os = cc.getOutStream();
+    clientKeyLength = serverKeyLength;
+    KeyPairGenerator kpg;
+    try {
+      kpg = KeyPairGenerator.getInstance("RSA");
+    } catch (NoSuchAlgorithmException e) {
+      throw new AuthFailureException("RSA algorithm is not supported");
+    }
+    kpg.initialize(clientKeyLength);
+    KeyPair kp = kpg.generateKeyPair();
+    clientKey = kp.getPrivate();
+    clientPublicKey = kp.getPublic();
+    RSAPublicKey rsaKey = (RSAPublicKey) clientPublicKey;
+    BigInteger modulus = rsaKey.getModulus();
+    BigInteger publicExponent = rsaKey.getPublicExponent();
+
+    clientKeyN = bigIntToBytes(modulus, (clientKeyLength + 7) / 8);
+    clientKeyE = bigIntToBytes(publicExponent, (clientKeyLength + 7) / 8);
+    if (clientKeyN == null || clientKeyN == null) {
+      throw new AuthFailureException("failed to generate RSA keys");
+    }
+    os.writeU32(clientKeyLength);
+    os.writeBytes(clientKeyN, 0, clientKeyN.length);
+    os.writeBytes(clientKeyE, 0, clientKeyE.length);
+    os.flush();
+  }
+
+  private void writeRandom(CConnection cc) {
+    OutStream os = cc.getOutStream();
+    SecureRandom sr = new SecureRandom();
+    clientRandom = new byte[keySize / 8];
+    sr.nextBytes(clientRandom);
+    byte[] encrypted;
+    try {
+      Cipher cipher = Cipher.getInstance("RSA/ECB/PKCS1Padding");
+      cipher.init(Cipher.ENCRYPT_MODE, serverKey);
+      encrypted = cipher.doFinal(clientRandom);
+    } catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
+      throw new AuthFailureException("RSA algorithm is not supported");
+    } catch (InvalidKeyException | IllegalBlockSizeException |
+             BadPaddingException e) {
+      throw new AuthFailureException("failed to encrypt random");
+    }
+    os.writeU16(encrypted.length);
+    os.writeBytes(encrypted, 0, encrypted.length);
+    os.flush();
+  }
+
+  private void readRandom(CConnection cc) {
+    InStream is = cc.getInStream();
+    int size = is.readU16();
+    if (size != clientKeyN.length)
+      throw new AuthFailureException("client key length doesn't match");
+    byte[] buffer = new byte[size];
+    is.readBytes(ByteBuffer.wrap(buffer), size);
+    try {
+      Cipher cipher = Cipher.getInstance("RSA/ECB/PKCS1Padding");
+      cipher.init(Cipher.DECRYPT_MODE, clientKey);
+      serverRandom = cipher.doFinal(buffer);
+    } catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
+      throw new AuthFailureException("RSA algorithm is not supported");
+    } catch (InvalidKeyException | IllegalBlockSizeException |
+             BadPaddingException e) {
+      System.out.println(e.getMessage());
+      throw new AuthFailureException("failed to decrypt server random");
+    }
+    if (serverRandom.length != keySize / 8)
+      throw new AuthFailureException("server random length doesn't match");
+  }
+
+  private void setCipher(CConnection cc) {
+    rawis = cc.getInStream();
+    rawos = cc.getOutStream();
+    MessageDigest digest;
+    try {
+      digest = MessageDigest.getInstance(keySize == 128 ? "SHA-1" : "SHA-256");
+    } catch (NoSuchAlgorithmException e) {
+      throw new AuthFailureException("hash algorithm is not supported");
+    }
+    digest.update(clientRandom);
+    digest.update(serverRandom);
+    byte[] key = Arrays.copyOfRange(digest.digest(), 0, keySize / 8);
+    rais = new AESInStream(rawis, key);
+    digest.reset();
+    digest.update(serverRandom);
+    digest.update(clientRandom);
+    key = Arrays.copyOfRange(digest.digest(), 0, keySize / 8);
+    raos = new AESOutStream(rawos, key);
+    if (isAllEncrypted)
+      cc.setStreams(rais, raos);
+  }
+
+  private void writeHash(CConnection cc) {
+    MessageDigest digest;
+    try {
+      digest = MessageDigest.getInstance(keySize == 128 ? "SHA-1" : "SHA-256");
+    } catch (NoSuchAlgorithmException e) {
+      throw new AuthFailureException("hash algorithm is not supported");
+    }
+    int len = serverKeyLength;
+    byte[] lenServerKey = new byte[] {
+        (byte) ((len & 0xff000000) >> 24),
+        (byte) ((len & 0xff0000) >> 16),
+        (byte) ((len & 0xff00) >> 8),
+        (byte) (len & 0xff)
+    };
+    len = clientKeyLength;
+    byte[] lenClientKey = new byte[] {
+        (byte) ((len & 0xff000000) >> 24),
+        (byte) ((len & 0xff0000) >> 16),
+        (byte) ((len & 0xff00) >> 8),
+        (byte) (len & 0xff)
+    };
+    digest.update(lenClientKey);
+    digest.update(clientKeyN);
+    digest.update(clientKeyE);
+    digest.update(lenServerKey);
+    digest.update(serverKeyN);
+    digest.update(serverKeyE);
+    byte[] hash = digest.digest();
+    raos.writeBytes(hash, 0, hash.length);
+    raos.flush();
+  }
+
+  void readHash() {
+    MessageDigest digest;
+    try {
+      digest = MessageDigest.getInstance(keySize == 128 ? "SHA-1" : "SHA-256");
+    } catch (NoSuchAlgorithmException e) {
+      throw new AuthFailureException("hash algorithm is not supported");
+    }
+    int len = serverKeyLength;
+    byte[] lenServerKey = new byte[] {
+        (byte)((len & 0xff000000) >> 24),
+        (byte)((len & 0xff0000) >> 16),
+        (byte)((len & 0xff00) >> 8),
+        (byte)(len & 0xff)
+    };
+    len = clientKeyLength;
+    byte[] lenClientKey = new byte[] {
+        (byte)((len & 0xff000000) >> 24),
+        (byte)((len & 0xff0000) >> 16),
+        (byte)((len & 0xff00) >> 8),
+        (byte)(len & 0xff)
+    };
+    digest.update(lenServerKey);
+    digest.update(serverKeyN);
+    digest.update(serverKeyE);
+    digest.update(lenClientKey);
+    digest.update(clientKeyN);
+    digest.update(clientKeyE);
+    byte[] realHash = digest.digest();
+    ByteBuffer hash = ByteBuffer.allocate(realHash.length);
+    rais.readBytes(hash, realHash.length);
+    if (!Arrays.equals(hash.array(), realHash)) {
+      throw new AuthFailureException("hash doesn't match");
+    }
+  }
+
+  private void readSubtype() {
+    subtype = rais.readU8();
+    if (subtype != Security.secTypeRA2UserPass &&
+        subtype != Security.secTypeRA2Pass)
+      throw new AuthFailureException("unknown RSA-AES subtype");
+  }
+
+  private void writeCredentials() {
+    StringBuffer username = new StringBuffer();
+    StringBuffer password = new StringBuffer();
+    CSecurity.upg.getUserPasswd(secType == Security.secTypeRA256,
+                                subtype == Security.secTypeRA2UserPass ?
+                                  username : null, 
+                                password);
+    if (username.length() > 255)
+      throw new AuthFailureException("username is too long");
+    byte[] usernameBytes;
+    try {
+      usernameBytes = username.toString().getBytes("UTF-8");
+    } catch (UnsupportedEncodingException e) {
+      throw new AuthFailureException("UTF-8 is not supported");
+    }
+    raos.writeU8(usernameBytes.length);
+    if (usernameBytes.length != 0)
+      raos.writeBytes(usernameBytes, 0, usernameBytes.length);
+    if (password.length() > 255)
+      throw new AuthFailureException("password is too long");
+    byte[] passwordBytes;
+    try {
+      passwordBytes = password.toString().getBytes("UTF-8");
+    } catch (UnsupportedEncodingException e) {
+      throw new AuthFailureException("UTF-8 is not supported");
+    }
+    raos.writeU8(passwordBytes.length);
+    if (passwordBytes.length != 0)
+      raos.writeBytes(passwordBytes, 0, passwordBytes.length);
+    raos.flush();
+  }
+
+  public int getType() {
+    return secType;
+  }
+
+  public String description() {
+    return "RSA-ASE security types";
+  }
+
+  private int secType;
+  private int subtype;
+  private int keySize;
+  private boolean isAllEncrypted;
+
+  private PrivateKey clientKey;
+  private PublicKey clientPublicKey;
+  private PublicKey serverKey;
+  private int serverKeyLength;
+  private byte[] serverKeyN;
+  private byte[] serverKeyE;
+  private int clientKeyLength;
+  private byte[] clientKeyN;
+  private byte[] clientKeyE;
+  private byte[] serverRandom;
+  private byte[] clientRandom;
+
+  private AESInStream rais;
+  private AESOutStream raos;
+  private InStream rawis;
+  private OutStream rawos;
+}

java/com/tigervnc/rfb/CSecurityTLS.java

@@ -66,7 +66,6 @@ public class CSecurityTLS extends CSecurity {
   public static StringParameter X509CRL
   = new StringParameter("X509CRL",
                         "X509 CRL file", "", Configuration.ConfigurationObject.ConfViewer);
-  public static UserMsgBox msg;
 
   private void initGlobal()
   {

java/com/tigervnc/rfb/Security.java

@@ -42,6 +42,9 @@ public class Security {
   public static final int secTypeTLS       = 18;
   public static final int secTypeVeNCrypt  = 19;
 
+  public static final int secTypeRA256     = 129;
+  public static final int secTypeRAne256   = 130;
+
   /* VeNCrypt subtypes */
   public static final int secTypePlain     = 256;
   public static final int secTypeTLSNone   = 257;
@@ -54,6 +57,9 @@ public class Security {
   public static final int secTypeTLSIdent  = 266;
   public static final int secTypeX509Ident = 267;
 
+  public static final int secTypeRA2UserPass = 1;
+  public static final int secTypeRA2Pass     = 2;
+
   // result types
 
   public static final int secResultOK = 0;
@@ -166,6 +172,8 @@ public class Security {
     if (name.equalsIgnoreCase("Tight"))	    return secTypeTight;
     if (name.equalsIgnoreCase("RA2"))       return secTypeRA2;
     if (name.equalsIgnoreCase("RA2ne"))	    return secTypeRA2ne;
+    if (name.equalsIgnoreCase("RA2_256"))   return secTypeRA256;
+    if (name.equalsIgnoreCase("RA2ne_256"))	return secTypeRAne256;
     if (name.equalsIgnoreCase("SSPI"))      return secTypeSSPI;
     if (name.equalsIgnoreCase("SSPIne"))	  return secTypeSSPIne;
     //if (name.equalsIgnoreCase("ultra"))	    return secTypeUltra;
@@ -194,6 +202,8 @@ public class Security {
     case secTypeTight:      return "Tight";
     case secTypeRA2:        return "RA2";
     case secTypeRA2ne:      return "RA2ne";
+    case secTypeRA256:      return "RA2_256";
+    case secTypeRAne256:    return "RA2ne_256";
     case secTypeSSPI:       return "SSPI";
     case secTypeSSPIne:     return "SSPIne";
     //case secTypeUltra:      return "Ultra";

java/com/tigervnc/rfb/SecurityClient.java

@@ -62,6 +62,14 @@ public class SecurityClient extends Security {
     case Security.secTypeX509Ident:
       return (new CSecurityStack(secTypeX509Ident, "X509 with username only",
   			      new CSecurityTLS(false), new CSecurityIdent()));
+    case Security.secTypeRA2:
+      return (new CSecurityRSAAES(secType, 128, true));
+    case Security.secTypeRA2ne:
+      return (new CSecurityRSAAES(secType, 128, false));
+    case Security.secTypeRA256:
+      return (new CSecurityRSAAES(secType, 256, true));
+    case Security.secTypeRAne256:
+      return (new CSecurityRSAAES(secType, 256, false));
     default:
       throw new Exception("Security type not supported");
     }
@@ -75,7 +83,7 @@ public class SecurityClient extends Security {
 
   public static StringParameter secTypes
   = new StringParameter("SecurityTypes",
-                        "Specify which security scheme to use (None, VncAuth, Plain, Ident, TLSNone, TLSVnc, TLSPlain, TLSIdent, X509None, X509Vnc, X509Plain, X509Ident)",
-                        "X509Ident,X509Plain,TLSIdent,TLSPlain,X509Vnc,TLSVnc,X509None,TLSNone,Ident,VncAuth,None", Configuration.ConfigurationObject.ConfViewer);
+                        "Specify which security scheme to use (None, VncAuth, Plain, Ident, TLSNone, TLSVnc, TLSPlain, TLSIdent, X509None, X509Vnc, X509Plain, X509Ident, RA2, RA2ne, RA2_256, RA2ne_256)",
+                        "X509Ident,X509Plain,TLSIdent,TLSPlain,X509Vnc,TLSVnc,X509None,TLSNone,Ident,RA2_256,RA2,RA2ne_256,RA2ne,VncAuth,None", Configuration.ConfigurationObject.ConfViewer);
 
 }

java/com/tigervnc/vncviewer/OptionsDialog.java

@@ -119,6 +119,7 @@ class OptionsDialog extends Dialog {
   JCheckBox encNoneCheckbox;
   JCheckBox encTLSCheckbox;
   JCheckBox encX509Checkbox;
+  JCheckBox encRSAAESCheckbox;
   JTextField caInput;
   JTextField crlInput;
   JButton caChooser;
@@ -323,6 +324,7 @@ class OptionsDialog extends Dialog {
     encNoneCheckbox.setSelected(false);
     encTLSCheckbox.setSelected(false);
     encX509Checkbox.setSelected(false);
+    encRSAAESCheckbox.setSelected(false);
 
     authNoneCheckbox.setSelected(false);
     authVncCheckbox.setSelected(false);
@@ -387,6 +389,14 @@ class OptionsDialog extends Dialog {
         encX509Checkbox.setSelected(true);
         authIdentCheckbox.setSelected(true);
         break;
+      case Security.secTypeRA2:
+      case Security.secTypeRA256:
+        encRSAAESCheckbox.setSelected(true);
+      case Security.secTypeRA2ne:
+      case Security.secTypeRAne256:
+        authVncCheckbox.setSelected(true);
+        authPlainCheckbox.setSelected(true);
+        break;
       }
     }
 
@@ -516,10 +526,16 @@ class OptionsDialog extends Dialog {
     if (encNoneCheckbox.isSelected()) {
       if (authNoneCheckbox.isSelected())
         security.EnableSecType(Security.secTypeNone);
-      if (authVncCheckbox.isSelected())
+      if (authVncCheckbox.isSelected()) {
         security.EnableSecType(Security.secTypeVncAuth);
-      if (authPlainCheckbox.isSelected())
+        security.EnableSecType(Security.secTypeRA2ne);
+        security.EnableSecType(Security.secTypeRAne256);
+      }
+      if (authPlainCheckbox.isSelected()) {
         security.EnableSecType(Security.secTypePlain);
+        security.EnableSecType(Security.secTypeRA2ne);
+        security.EnableSecType(Security.secTypeRAne256);
+      }
       if (authIdentCheckbox.isSelected())
         security.EnableSecType(Security.secTypeIdent);
     }
@@ -548,6 +564,11 @@ class OptionsDialog extends Dialog {
         security.EnableSecType(Security.secTypeX509Ident);
     }
 
+    if (encRSAAESCheckbox.isSelected()) {
+      security.EnableSecType(Security.secTypeRA2);
+      security.EnableSecType(Security.secTypeRA256);
+    }
+
     if (authIdentCheckbox.isSelected() ||
         authPlainCheckbox.isSelected()) {
       sendLocalUsername.setParam(sendLocalUsernameCheckbox.isSelected());
@@ -809,6 +830,12 @@ class OptionsDialog extends Dialog {
           crlInput.setText(f.getAbsolutePath());
       }
     });
+    encRSAAESCheckbox = new JCheckBox("RSA-AES");
+    encRSAAESCheckbox.addItemListener(new ItemListener() {
+      public void itemStateChanged(ItemEvent e) {
+        handleRSAAES();
+      }
+    });
     encrPanel.add(encNoneCheckbox,
                   new GridBagConstraints(0, 0,
                                          REMAINDER, 1,
@@ -873,6 +900,13 @@ class OptionsDialog extends Dialog {
                                          LINE_START, VERTICAL,
                                          new Insets(0, 5, 0, 0),
                                          0, 0));
+    encrPanel.add(encRSAAESCheckbox,
+                  new GridBagConstraints(0, 5,
+                                         REMAINDER, 1,
+                                         HEAVY, LIGHT,
+                                         LINE_START, NONE,
+                                         new Insets(0, 0, 4, 0),
+                                         NONE, NONE));
 
     JPanel authPanel = new JPanel(new GridBagLayout());
     authPanel.setBorder(BorderFactory.createTitledBorder("Authentication"));
@@ -1502,6 +1536,12 @@ class OptionsDialog extends Dialog {
     crlChooser.setEnabled(encX509Checkbox.isSelected());
   }
 
+  private void handleRSAAES()
+  {
+    authVncCheckbox.setSelected(true);
+    authPlainCheckbox.setSelected(true);
+  }
+
   private void handleSendLocalUsername()
   {
     boolean value = authIdentCheckbox.isSelected() ||

java/com/tigervnc/vncviewer/VncViewer.java

@@ -387,7 +387,7 @@ public class VncViewer implements Runnable {
     cc = null;
     UserDialog dlg = new UserDialog();
     CSecurity.upg = dlg;
-    CSecurityTLS.msg = dlg;
+    CSecurity.msg = dlg;
     Socket sock = null;
 
     /* Specifying -via and -listen together is nonsense */

vncviewer/OptionsDialog.cxx

@@ -395,6 +395,7 @@ void OptionsDialog::storeOptions(void)
     if (authVncCheckbox->value()) {
       security.EnableSecType(secTypeVncAuth);
 #ifdef HAVE_NETTLE
+      security.EnableSecType(secTypeRA2ne);
       security.EnableSecType(secTypeRAne256);
 #endif
     }