From 8a1e99d5bcf96748fc76f3a28797eb7415cd6cb9 Mon Sep 17 00:00:00 2001
From: Brian Hinz <bphinz@users.sourceforge.net>
Date: Sun, 26 Aug 2012 18:24:15 +0000
Subject: [PATCH] Refactor the handshake code from first principles to remove
 code that was borrowed from Netty, which is released under an incompatible
 license (Apache v2.)  Since our SSLEngineManager class was loosely inspired
 by, but no code was actually borrowed from, the Oracle JSSE examples, it is
 my opinion that it was incorrect to attribute copyright to Oracle, so that
 message has been removed (their examples are released under a BSD-style
 license, anyhow, so it's a non-issue.)

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4962 3789f03b-4d11-0410-bbf8-ca57d06f2519
---
 .../tigervnc/network/SSLEngineManager.java    | 83 +++++++------------
 1 file changed, 29 insertions(+), 54 deletions(-)

diff --git a/java/com/tigervnc/network/SSLEngineManager.java b/java/com/tigervnc/network/SSLEngineManager.java
index 1a48543a..caf89ac4 100644
--- a/java/com/tigervnc/network/SSLEngineManager.java
+++ b/java/com/tigervnc/network/SSLEngineManager.java
@@ -1,6 +1,5 @@
-/* Copyright (c) 2004, 2006, Oracle and/or its affiliates. All rights reserved.
- * Copyright (C) 2008  Trustin Heuiseung Lee
- * Copyright (C) 2012  Brian P. Hinz
+/* Copyright (C) 2012 Brian P. Hinz
+ * Copyright (C) 2012 D. R. Commander.  All Rights Reserved.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -213,7 +212,6 @@ public class SSLEngineManager {
   }
 
   private int pull(ByteBuffer dst) throws IOException {
-    int packetLength = 0;
     inStream.checkNoWait(5);
     //if (!inStream.checkNoWait(5)) {
     //  return 0;
@@ -222,65 +220,42 @@ public class SSLEngineManager {
     byte[] header = new byte[5];
     inStream.readBytes(header, 0, 5);
 
-    boolean tls;
-    int h = header[0] & 0xFF;
-    switch (header[0] & 0xFF) {
-    case 20:  // change_cipher_spec
-    case 21:  // alert
-    case 22:  // handshake
-    case 23:  // application_data
-      tls = true;
-      break;
-    default:
-      //  SSLv2 bad data
-      tls = false;
-    }
-
-    if (tls) {
-      int majorVersion = (int)(header[1] & 0xFF);
-      if (majorVersion >= 3 && majorVersion < 10) {
-        // SSLv3 or TLS
-        packetLength = (int)(((header[3] << 8) | (header[4] & 0xFF)) & 0xFFFF) + 5;
-        if (packetLength <= 5) {
-          // Neither SSLv2 or TLSv1
-          tls = false;
-        }
+    // Reference: http://publib.boulder.ibm.com/infocenter/tpfhelp/current/index.jsp?topic=%2Fcom.ibm.ztpf-ztpfdf.doc_put.cur%2Fgtps5%2Fs5rcd.html
+    int sslRecordType = header[0] & 0xFF;
+    int sslVersion = header[1] & 0xFF;
+    int sslDataLength = (int)((header[3] << 8) | (header[4] & 0xFF));
+
+    if (sslRecordType < 20 || sslRecordType > 23 || sslVersion != 3 ||
+        sslDataLength == 0) {
+      // Not SSL v3 or TLS.  Could be SSL v2 or bad data
+
+      // Reference: http://www.homeport.org/~adam/ssl.html
+      // and the SSL v2 protocol specification
+      int headerBytes;
+      if ((header[0] & 0x80) != 0x80) {
+        headerBytes = 2;
+        sslDataLength = (int)(((header[0] & 0x7f) << 8) | header[1]);
       } else {
-        // Neither SSLv2 or TLSv1
-        tls = false;
+        headerBytes = 3;
+        sslDataLength = (int)(((header[0] & 0x3f) << 8) | header[1]);
       }
-    }
 
-    if (!tls) {
-      boolean sslv2 = true;
-      int headerLength = (int)((header[0] & 0xFF) & 0x80) != 0 ? 2 : 3;
-      int majorVersion = (int)(header[headerLength + 1] & 0xFF);
-      if (majorVersion >= 2 && majorVersion < 10) {
-        // SSLv2
-        if (headerLength == 2) {
-          packetLength = (int)(((header[0] << 8) | (header[1] & 0xFF)) & 0x7FFF) + 2;
-        } else {
-          packetLength = (int)(((header[0] << 8) | (header[1] & 0xFF)) & 0x3FFF) + 3;
-        }
-        if (packetLength <= headerLength) {
-          sslv2 = false;
-        }
-      } else {
-        sslv2 = false;
-      }
-
-      if (!sslv2) {
+      // In SSL v2, the version is part of the handshake
+      sslVersion = header[headerBytes + 1] & 0xFF;
+      if (sslVersion < 2 || sslVersion > 3 || sslDataLength == 0)
         throw new IOException("not an SSL/TLS record");
-      }
+
+      // The number of bytes left to read
+      sslDataLength -= (5 - headerBytes);
     }
 
-    assert packetLength > 0;
+    assert sslDataLength > 0;
 
-    byte[] buf = new byte[packetLength - 5];
-    inStream.readBytes(buf, 0, packetLength - 5);
+    byte[] buf = new byte[sslDataLength];
+    inStream.readBytes(buf, 0, sslDataLength);
     dst.put(header);
     dst.put(buf);
-    return packetLength;
+    return sslDataLength;
   }
 
   public SSLSession getSession() {
-- 
2.39.5