From b40235af946e6854e7fcf1f8f7a00c165e71008e Mon Sep 17 00:00:00 2001 From: "Brian P. Hinz" Date: Mon, 1 Jan 2018 17:24:20 -0500 Subject: [PATCH] Fixes for erros in java hextile/zrle decoders Various errors exposed when connecting to RealVNC servers on alternative platforms (ARM, SPARC). SSLEngineManager was also cleaned up but most of the changes are cosmetic. --- java/com/tigervnc/network/FileDescriptor.java | 5 +- .../tigervnc/network/SSLEngineManager.java | 88 ++++---------- .../tigervnc/network/SocketDescriptor.java | 82 ++----------- java/com/tigervnc/rdr/FdInStream.java | 25 ++-- java/com/tigervnc/rdr/FdOutStream.java | 101 ++++++++-------- java/com/tigervnc/rdr/InStream.java | 63 ++-------- java/com/tigervnc/rdr/OutStream.java | 14 ++- java/com/tigervnc/rdr/TLSInStream.java | 6 +- java/com/tigervnc/rdr/TLSOutStream.java | 2 +- java/com/tigervnc/rdr/ZlibInStream.java | 52 ++++---- java/com/tigervnc/rfb/CMsgReader.java | 33 ++--- java/com/tigervnc/rfb/CSecurityVncAuth.java | 10 +- java/com/tigervnc/rfb/HextileDecoder.java | 36 +++--- java/com/tigervnc/rfb/PixelFormat.java | 7 +- java/com/tigervnc/rfb/TightDecoder.java | 22 ++-- java/com/tigervnc/rfb/ZRLEDecoder.java | 114 +++++++++--------- 16 files changed, 274 insertions(+), 386 deletions(-) diff --git a/java/com/tigervnc/network/FileDescriptor.java b/java/com/tigervnc/network/FileDescriptor.java index 8a4f693d..d8a91735 100644 --- a/java/com/tigervnc/network/FileDescriptor.java +++ b/java/com/tigervnc/network/FileDescriptor.java @@ -18,12 +18,13 @@ package com.tigervnc.network; import java.io.IOException; +import java.nio.ByteBuffer; import com.tigervnc.rdr.Exception; public interface FileDescriptor { - public int read(byte[] buf, int bufPtr, int length) throws Exception; - public int write(byte[] buf, int bufPtr, int length) throws Exception; + public int read(ByteBuffer buf, int length) throws Exception; + public int write(ByteBuffer buf, int length) throws Exception; public int select(int interestOps, Integer timeout) throws Exception; public void close() throws IOException; diff --git a/java/com/tigervnc/network/SSLEngineManager.java b/java/com/tigervnc/network/SSLEngineManager.java index e6abe3e9..27a9a3ea 100644 --- a/java/com/tigervnc/network/SSLEngineManager.java +++ b/java/com/tigervnc/network/SSLEngineManager.java @@ -32,12 +32,7 @@ public class SSLEngineManager { private SSLEngine engine = null; - private int appBufSize; - private int pktBufSize; - - private ByteBuffer myAppData; private ByteBuffer myNetData; - private ByteBuffer peerAppData; private ByteBuffer peerNetData; private Executor executor; @@ -53,13 +48,8 @@ public class SSLEngineManager { executor = Executors.newSingleThreadExecutor(); - pktBufSize = engine.getSession().getPacketBufferSize(); - appBufSize = engine.getSession().getApplicationBufferSize(); - - myAppData = - ByteBuffer.allocate(Math.max(appBufSize, os.getBufSize())); + int pktBufSize = engine.getSession().getPacketBufferSize(); myNetData = ByteBuffer.allocate(pktBufSize); - peerAppData = ByteBuffer.allocate(appBufSize); peerNetData = ByteBuffer.allocate(pktBufSize); } @@ -70,6 +60,10 @@ public class SSLEngineManager { SSLEngineResult.HandshakeStatus hs = engine.getHandshakeStatus(); // Process handshaking message + SSLEngineResult res = null; + int appBufSize = engine.getSession().getApplicationBufferSize(); + ByteBuffer peerAppData = ByteBuffer.allocate(appBufSize); + ByteBuffer myAppData = ByteBuffer.allocate(appBufSize); while (hs != SSLEngineResult.HandshakeStatus.FINISHED && hs != SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING) { @@ -78,36 +72,26 @@ public class SSLEngineManager { case NEED_UNWRAP: // Receive handshaking data from peer peerNetData.flip(); - SSLEngineResult res = engine.unwrap(peerNetData, peerAppData); + res = engine.unwrap(peerNetData, peerAppData); peerNetData.compact(); hs = res.getHandshakeStatus(); + // Check status switch (res.getStatus()) { case BUFFER_UNDERFLOW: - int max = Math.min(peerNetData.remaining(), in.getBufSize()); - int m = in.check(1, max, true); - int pos = peerNetData.position(); - in.readBytes(peerNetData.array(), pos, m); - peerNetData.position(pos+m); - peerNetData.flip(); - peerNetData.compact(); + int avail = in.check(1, peerNetData.remaining(), false); + in.readBytes(peerNetData, avail); break; - case OK: // Process incoming handshaking data break; - case CLOSED: engine.closeInbound(); break; - } break; case NEED_WRAP: - // Empty the local network packet buffer. - myNetData.clear(); - // Generate handshaking data res = engine.wrap(myAppData, myNetData); hs = res.getHandshakeStatus(); @@ -115,21 +99,14 @@ public class SSLEngineManager { // Check status switch (res.getStatus()) { case OK: - myAppData.compact(); myNetData.flip(); - os.writeBytes(myNetData.array(), 0, myNetData.remaining()); + os.writeBytes(myNetData, myNetData.remaining()); os.flush(); - myNetData.clear(); + myNetData.compact(); break; - - case BUFFER_OVERFLOW: - // FIXME: How much larger should the buffer be? - break; - case CLOSED: engine.closeOutbound(); break; - } break; @@ -149,30 +126,19 @@ public class SSLEngineManager { } } - public int read(byte[] data, int dataPtr, int length) throws IOException { + public int read(ByteBuffer data, int length) throws IOException { // Read SSL/TLS encoded data from peer - int bytesRead = 0; peerNetData.flip(); - SSLEngineResult res = engine.unwrap(peerNetData, peerAppData); + SSLEngineResult res = engine.unwrap(peerNetData, data); peerNetData.compact(); switch (res.getStatus()) { case OK : - bytesRead = Math.min(length, res.bytesProduced()); - peerAppData.flip(); - peerAppData.get(data, dataPtr, bytesRead); - peerAppData.compact(); - break; + return res.bytesProduced(); case BUFFER_UNDERFLOW: - // need more net data - int pos = peerNetData.position(); // attempt to drain the underlying buffer first int need = peerNetData.remaining(); - int avail = in.check(1, in.getBufSize(), false); - if (avail < need) - avail = in.check(1, Math.min(need, in.getBufSize()), true); - in.readBytes(peerNetData.array(), pos, Math.min(need, avail)); - peerNetData.position(pos+Math.min(need, avail)); + in.readBytes(peerNetData, in.check(1, need, true)); break; case CLOSED: @@ -180,26 +146,27 @@ public class SSLEngineManager { break; } - return bytesRead; + return 0; } - public int write(byte[] data, int dataPtr, int length) throws IOException { + public int write(ByteBuffer data, int length) throws IOException { int n = 0; - myAppData.put(data, dataPtr, length); - myAppData.flip(); - while (myAppData.hasRemaining()) { - SSLEngineResult res = engine.wrap(myAppData, myNetData); + while (data.hasRemaining()) { + SSLEngineResult res = engine.wrap(data, myNetData); n += res.bytesConsumed(); switch (res.getStatus()) { case OK: + myNetData.flip(); + os.writeBytes(myNetData, myNetData.remaining()); + os.flush(); + myNetData.compact(); break; case BUFFER_OVERFLOW: // Make room in the buffer by flushing the outstream myNetData.flip(); - os.writeBytes(myNetData.array(), 0, myNetData.remaining()); - os.flush(); - myNetData.clear(); + os.writeBytes(myNetData, myNetData.remaining()); + myNetData.compact(); break; case CLOSED: @@ -207,11 +174,6 @@ public class SSLEngineManager { break; } } - myAppData.clear(); - myNetData.flip(); - os.writeBytes(myNetData.array(), 0, myNetData.remaining()); - os.flush(); - myNetData.clear(); return n; } diff --git a/java/com/tigervnc/network/SocketDescriptor.java b/java/com/tigervnc/network/SocketDescriptor.java index 3cbe5ab2..b4c99c74 100644 --- a/java/com/tigervnc/network/SocketDescriptor.java +++ b/java/com/tigervnc/network/SocketDescriptor.java @@ -74,37 +74,6 @@ public class SocketDescriptor implements FileDescriptor { return SelectorProvider.provider(); } - synchronized public int read(byte[] buf, int bufPtr, int length) throws Exception { - int n; - ByteBuffer b = ByteBuffer.allocate(length); - try { - n = channel.read(b); - } catch (java.io.IOException e) { - throw new Exception(e.getMessage()); - } - if (n <= 0) - return (n == 0) ? -1 : 0; - b.flip(); - b.get(buf, bufPtr, n); - b.clear(); - return n; - - } - - synchronized public int write(byte[] buf, int bufPtr, int length) throws Exception { - int n; - ByteBuffer b = ByteBuffer.allocate(length); - b.put(buf, bufPtr, length); - b.flip(); - try { - n = channel.write(b); - } catch (java.io.IOException e) { - throw new Exception(e.getMessage()); - } - b.clear(); - return n; - } - synchronized public int select(int interestOps, Integer timeout) throws Exception { int n; Selector selector; @@ -134,48 +103,24 @@ public class SocketDescriptor implements FileDescriptor { return n; } - public int write(ByteBuffer buf) throws Exception { - int n = 0; - try { - n = channel.write(buf); - } catch (java.io.IOException e) { - throw new Exception(e.getMessage()); - } - return n; - } - - public long write(ByteBuffer[] buf, int offset, int length) - throws IOException - { - long n = 0; + public int write(ByteBuffer buf, int len) throws Exception { try { - n = channel.write(buf, offset, length); + int n = channel.write((ByteBuffer)buf.slice().limit(len)); + buf.position(buf.position()+n); + return n; } catch (java.io.IOException e) { throw new Exception(e.getMessage()); } - return n; } - public int read(ByteBuffer buf) throws IOException { - int n = 0; + public int read(ByteBuffer buf, int len) throws Exception { try { - n = channel.read(buf); - } catch (java.io.IOException e) { + int n = channel.read((ByteBuffer)buf.slice().limit(len)); + buf.position(buf.position()+n); + return (n < 0) ? 0 : n; + } catch (java.lang.Exception e) { throw new Exception(e.getMessage()); } - return n; - } - - public long read(ByteBuffer[] buf, int offset, int length) - throws IOException - { - long n = 0; - try { - n = channel.read(buf, offset, length); - } catch (java.io.IOException e) { - throw new Exception(e.getMessage()); - } - return n; } public java.net.Socket socket() { @@ -210,15 +155,6 @@ public class SocketDescriptor implements FileDescriptor { return channel.isConnected(); } - protected void implConfigureBlocking(boolean block) throws IOException { - channel.configureBlocking(block); - } - - protected synchronized void implCloseSelectableChannel() throws IOException { - channel.close(); - notifyAll(); - } - protected void setChannel(SocketChannel channel_) { try { if (channel != null) diff --git a/java/com/tigervnc/rdr/FdInStream.java b/java/com/tigervnc/rdr/FdInStream.java index b782f74a..8b3c7e2c 100644 --- a/java/com/tigervnc/rdr/FdInStream.java +++ b/java/com/tigervnc/rdr/FdInStream.java @@ -19,12 +19,14 @@ package com.tigervnc.rdr; -import com.tigervnc.network.*; +import java.nio.*; import java.nio.channels.Selector; import java.nio.channels.SelectionKey; import java.util.Set; import java.util.Iterator; +import com.tigervnc.network.*; + public class FdInStream extends InStream { static final int DEFAULT_BUF_SIZE = 16384; @@ -58,22 +60,24 @@ public class FdInStream extends InStream { this(fd_, blockCallback_, 0); } - public final void readBytes(byte[] data, int dataPtr, int length) { + public final void readBytes(ByteBuffer data, int length) { if (length < minBulkSize) { - super.readBytes(data, dataPtr, length); + super.readBytes(data, length); return; } + int dataPtr = data.position(); + int n = end - ptr; if (n > length) n = length; - System.arraycopy(b, ptr, data, dataPtr, n); + data.put(b, ptr, n); dataPtr += n; length -= n; ptr += n; while (length > 0) { - n = readWithTimeoutOrCallback(data, dataPtr, length); + n = readWithTimeoutOrCallback(data, length); dataPtr += n; length -= n; offset += n; @@ -139,7 +143,8 @@ public class FdInStream extends InStream { // bytes is ineffecient. bytes_to_read = Math.min(bytes_to_read, Math.max(itemSize*nItems, 8)); } - int n = readWithTimeoutOrCallback(b, end, bytes_to_read, wait); + Buffer buf = ByteBuffer.wrap(b).position(end); + int n = readWithTimeoutOrCallback((ByteBuffer)buf, bytes_to_read, wait); if (n == 0) return 0; end += n; } @@ -150,7 +155,7 @@ public class FdInStream extends InStream { return nItems; } - protected int readWithTimeoutOrCallback(byte[] buf, int bufPtr, int len, boolean wait) { + protected int readWithTimeoutOrCallback(ByteBuffer buf, int len, boolean wait) { long before = 0; if (timing) before = System.nanoTime(); @@ -184,7 +189,7 @@ public class FdInStream extends InStream { } try { - n = fd.read(buf, bufPtr, len); + n = fd.read(buf, len); } catch (Exception e) { throw new SystemException("read:"+e.toString()); } @@ -211,8 +216,8 @@ public class FdInStream extends InStream { return n; } - private int readWithTimeoutOrCallback(byte[] buf, int bufPtr, int len) { - return readWithTimeoutOrCallback(buf, bufPtr, len, true); + private int readWithTimeoutOrCallback(ByteBuffer buf, int len) { + return readWithTimeoutOrCallback(buf, len, true); } public FileDescriptor getFd() { diff --git a/java/com/tigervnc/rdr/FdOutStream.java b/java/com/tigervnc/rdr/FdOutStream.java index a9dea781..acb03a29 100644 --- a/java/com/tigervnc/rdr/FdOutStream.java +++ b/java/com/tigervnc/rdr/FdOutStream.java @@ -20,9 +20,11 @@ package com.tigervnc.rdr; -import com.tigervnc.network.*; +import java.nio.ByteBuffer; import java.nio.channels.SelectionKey; +import com.tigervnc.network.*; + public class FdOutStream extends OutStream { static final int DEFAULT_BUF_SIZE = 16384; @@ -36,6 +38,8 @@ public class FdOutStream extends OutStream { offset = 0; ptr = sentUpTo = start = 0; end = start + bufSize; + + lastWrite = System.currentTimeMillis(); } public FdOutStream(FileDescriptor fd_) { this(fd_, true, -1, 0); } @@ -53,17 +57,22 @@ public class FdOutStream extends OutStream { return offset + ptr - sentUpTo; } - public void flush() + int bufferUsage() { - int timeoutms_; + return ptr - sentUpTo; + } - if (blocking) - timeoutms_ = timeoutms; - else - timeoutms_ = 0; + long getIdleTime() + { + return System.currentTimeMillis()-lastWrite; + } + public void flush() + { while (sentUpTo < ptr) { - int n = writeWithTimeout(b, sentUpTo, ptr - sentUpTo, timeoutms_); + int n = writeWithTimeout(b, sentUpTo, + ptr - sentUpTo, + blocking ? timeoutms : 0); // Timeout? if (n == 0) { @@ -71,13 +80,6 @@ public class FdOutStream extends OutStream { if (!blocking) break; - // Otherwise try blocking (with possible timeout) - if ((timeoutms_ == 0) && (timeoutms != 0)) { - timeoutms_ = timeoutms; - break; - } - - // Proper timeout throw new TimedOut(); } @@ -90,39 +92,6 @@ public class FdOutStream extends OutStream { ptr = sentUpTo = start; } - private int writeWithTimeout(byte[] data, int dataPtr, int length, int timeoutms) - { - int n; - - do { - - Integer tv; - if (timeoutms != -1) { - tv = new Integer(timeoutms); - } else { - tv = null; - } - - try { - n = fd.select(SelectionKey.OP_WRITE, tv); - } catch (java.lang.Exception e) { - System.out.println(e.toString()); - throw new Exception(e.getMessage()); - } - - } while (n < 0); - - if (n == 0) return 0; - - try { - n = fd.write(data, dataPtr, length); - } catch (java.lang.Exception e) { - throw new Exception(e.getMessage()); - } - - return n; - } - protected int overrun(int itemSize, int nItems) { if (itemSize > bufSize) @@ -159,6 +128,41 @@ public class FdOutStream extends OutStream { return nItems; } + private int writeWithTimeout(byte[] data, int dataPtr, int length, int timeoutms) + { + int n; + + do { + + Integer tv; + if (timeoutms != -1) { + tv = new Integer(timeoutms); + } else { + tv = null; + } + + try { + n = fd.select(SelectionKey.OP_WRITE, tv); + } catch (java.lang.Exception e) { + System.out.println(e.toString()); + throw new Exception(e.getMessage()); + } + + } while (n < 0); + + if (n == 0) return 0; + + try { + n = fd.write(ByteBuffer.wrap(data, dataPtr, length), length); + } catch (java.lang.Exception e) { + throw new Exception(e.getMessage()); + } + + lastWrite = System.currentTimeMillis(); + + return n; + } + public FileDescriptor getFd() { return fd; } @@ -178,4 +182,5 @@ public class FdOutStream extends OutStream { protected int sentUpTo; protected int offset; protected int bufSize; + private long lastWrite; } diff --git a/java/com/tigervnc/rdr/InStream.java b/java/com/tigervnc/rdr/InStream.java index 0906d326..58afcfb6 100644 --- a/java/com/tigervnc/rdr/InStream.java +++ b/java/com/tigervnc/rdr/InStream.java @@ -74,11 +74,11 @@ abstract public class InStream { if (len > maxStringLength) throw new Exception("InStream max string length exceeded"); - byte[] str = new byte[len]; - readBytes(str, 0, len); + ByteBuffer str = ByteBuffer.allocate(len); + readBytes(str, len); String utf8string = new String(); try { - utf8string = new String(str,"UTF8"); + utf8string = new String(str.array(),"UTF8"); } catch(java.io.UnsupportedEncodingException e) { e.printStackTrace(); } @@ -98,26 +98,16 @@ abstract public class InStream { } } - // readBytes() reads an exact number of bytes into an array at an offset. + // readBytes() reads an exact number of bytes public void readBytes(ByteBuffer data, int length) { - int dataEnd = data.mark().position() + length; - while (data.position() < dataEnd) { - int n = check(1, dataEnd - data.position()); - data.put(b, ptr, n); + ByteBuffer dataPtr = data; + int dataEnd = dataPtr.position() + length; + while (dataPtr.position() < dataEnd) { + int n = check(1, dataEnd - dataPtr.position()); + dataPtr.put(b, ptr, n); ptr += n; } - data.reset(); - } - - public void readBytes(byte[] data, int dataPtr, int length) { - int dataEnd = dataPtr + length; - while (dataPtr < dataEnd) { - int n = check(1, dataEnd - dataPtr); - System.arraycopy(b, ptr, data, dataPtr, n); - ptr += n; - dataPtr += n; - } } // readOpaqueN() reads a quantity "without byte-swapping". Because java has @@ -126,38 +116,6 @@ abstract public class InStream { public final int readOpaque8() { return readU8(); } public final int readOpaque16() { return readU16(); } public final int readOpaque32() { return readU32(); } - public final int readOpaque24A() { check(3); int b0 = b[ptr++]; - int b1 = b[ptr++]; int b2 = b[ptr++]; - return b0 << 24 | b1 << 16 | b2 << 8; } - public final int readOpaque24B() { check(3); int b0 = b[ptr++]; - int b1 = b[ptr++]; int b2 = b[ptr++]; - return b0 << 16 | b1 << 8 | b2; } - - public final int readPixel(int bytesPerPixel, boolean bigEndian) { - byte[] pix = new byte[4]; - readBytes(pix, 0, bytesPerPixel); - - if (bigEndian) { - return 0xff000000 | (pix[0] & 0xff)<<16 | (pix[1] & 0xff)<<8 | (pix[2] & 0xff); - } else { - return 0xff000000 | (pix[2] & 0xff)<<16 | (pix[1] & 0xff)<<8 | (pix[0] & 0xff); - } - } - - public final void readPixels(int[] buf, int length, int bytesPerPixel, boolean bigEndian) { - int npixels = length*bytesPerPixel; - byte[] pixels = new byte[npixels]; - readBytes(pixels, 0, npixels); - for (int i = 0; i < length; i++) { - byte[] pix = new byte[4]; - System.arraycopy(pixels, i*bytesPerPixel, pix, 0, bytesPerPixel); - if (bigEndian) { - buf[i] = 0xff000000 | (pix[0] & 0xff)<<16 | (pix[1] & 0xff)<<8 | (pix[2] & 0xff); - } else { - buf[i] = 0xff000000 | (pix[2] & 0xff)<<16 | (pix[1] & 0xff)<<8 | (pix[0] & 0xff); - } - } - } // pos() returns the position in the stream. @@ -184,6 +142,9 @@ abstract public class InStream { // is supposed to be "small" (a few bytes). abstract protected int overrun(int itemSize, int nItems, boolean wait); + protected int overrun(int itemSize, int nItems) { + return overrun(itemSize, nItems, true); + } protected InStream() {} protected byte[] b; diff --git a/java/com/tigervnc/rdr/OutStream.java b/java/com/tigervnc/rdr/OutStream.java index a3b1a6c0..0e201339 100644 --- a/java/com/tigervnc/rdr/OutStream.java +++ b/java/com/tigervnc/rdr/OutStream.java @@ -23,6 +23,8 @@ package com.tigervnc.rdr; +import java.nio.*; + import com.tigervnc.network.*; abstract public class OutStream { @@ -116,12 +118,22 @@ abstract public class OutStream { } } + public void writeBytes(ByteBuffer data, int length) { + ByteBuffer dataPtr = data; + int dataEnd = dataPtr.position() + length; + while (dataPtr.position() < dataEnd) { + int n = check(1, dataEnd - dataPtr.position()); + dataPtr.get(b, ptr, n); + ptr += n; + } + } + // copyBytes() efficiently transfers data between streams public void copyBytes(InStream is, int length) { while (length > 0) { int n = check(1, length); - is.readBytes(b, ptr, n); + is.readBytes(ByteBuffer.wrap(b, ptr, n), n); ptr += n; length -= n; } diff --git a/java/com/tigervnc/rdr/TLSInStream.java b/java/com/tigervnc/rdr/TLSInStream.java index 0cdc4d9d..c0e57a61 100644 --- a/java/com/tigervnc/rdr/TLSInStream.java +++ b/java/com/tigervnc/rdr/TLSInStream.java @@ -89,12 +89,8 @@ public class TLSInStream extends InStream { { int n = -1; - //n = in.check(1, 1, wait); - //if (n == 0) - // return 0; - try { - n = manager.read(buf, bufPtr, len); + n = manager.read(ByteBuffer.wrap(buf, bufPtr, len), len); } catch (java.io.IOException e) { e.printStackTrace(); } diff --git a/java/com/tigervnc/rdr/TLSOutStream.java b/java/com/tigervnc/rdr/TLSOutStream.java index 80514b27..f5685d5b 100644 --- a/java/com/tigervnc/rdr/TLSOutStream.java +++ b/java/com/tigervnc/rdr/TLSOutStream.java @@ -77,7 +77,7 @@ public class TLSOutStream extends OutStream { int n = 0; try { - n = manager.write(data, dataPtr, length); + n = manager.write(ByteBuffer.wrap(data, dataPtr, length), length); } catch (java.io.IOException e) { throw new Exception(e.getMessage()); } diff --git a/java/com/tigervnc/rdr/ZlibInStream.java b/java/com/tigervnc/rdr/ZlibInStream.java index 151633ec..12de0e8b 100644 --- a/java/com/tigervnc/rdr/ZlibInStream.java +++ b/java/com/tigervnc/rdr/ZlibInStream.java @@ -33,28 +33,12 @@ public class ZlibInStream extends InStream { bufSize = bufSize_; b = new byte[bufSize]; bytesIn = offset = 0; - zs = new ZStream(); - zs.next_in = null; - zs.next_in_index = 0; - zs.avail_in = 0; - if (zs.inflateInit() != JZlib.Z_OK) { - zs = null; - throw new Exception("ZlinInStream: inflateInit failed"); - } ptr = end = start = 0; + init(); } public ZlibInStream() { this(defaultBufSize); } - protected void finalize() throws Throwable { - try { - b = null; - zs.inflateEnd(); - } finally { - super.finalize(); - } - } - public void setUnderlying(InStream is, int bytesIn_) { underlying = is; @@ -62,6 +46,11 @@ public class ZlibInStream extends InStream { ptr = end = start; } + public int pos() + { + return offset + ptr - start; + } + public void removeUnderlying() { ptr = end = start; @@ -74,21 +63,32 @@ public class ZlibInStream extends InStream { underlying = null; } - public int pos() + public void reset() { - return offset + ptr - start; + deinit(); + init(); } - public void reset() + public void init() { - ptr = end = start; - if (underlying == null) return; + assert(zs == null); - while (bytesIn > 0) { - decompress(true); - end = start; // throw away any data + zs = new ZStream(); + zs.next_in = null; + zs.next_in_index = 0; + zs.avail_in = 0; + if (zs.inflateInit() != JZlib.Z_OK) { + zs = null; + throw new Exception("ZlinInStream: inflateInit failed"); } - underlying = null; + } + + public void deinit() + { + assert(zs != null); + removeUnderlying(); + zs.inflateEnd(); + zs = null; } protected int overrun(int itemSize, int nItems, boolean wait) diff --git a/java/com/tigervnc/rfb/CMsgReader.java b/java/com/tigervnc/rfb/CMsgReader.java index a79cf842..e65db8c6 100644 --- a/java/com/tigervnc/rfb/CMsgReader.java +++ b/java/com/tigervnc/rfb/CMsgReader.java @@ -153,10 +153,10 @@ public class CMsgReader { vlog.error("cut text too long ("+len+" bytes) - ignoring"); return; } - byte[] buf = new byte[len]; - is.readBytes(buf, 0, len); + ByteBuffer buf = ByteBuffer.allocate(len); + is.readBytes(buf, len); Charset latin1 = Charset.forName("ISO-8859-1"); - CharBuffer chars = latin1.decode(ByteBuffer.wrap(buf)); + CharBuffer chars = latin1.decode(buf.compact()); handler.serverCutText(chars.toString(), len); } @@ -164,22 +164,22 @@ public class CMsgReader { { int flags; int len; - byte[] data = new byte[64]; + ByteBuffer data = ByteBuffer.allocate(64); is.skip(3); flags = is.readU32(); len = is.readU8(); - if (len > data.length) { + if (len > data.capacity()) { System.out.println("Ignoring fence with too large payload\n"); is.skip(len); return; } - is.readBytes(data, 0, len); + is.readBytes(data, len); - handler.fence(flags, len, data); + handler.fence(flags, len, data.array()); } protected void readEndOfContinuousUpdates() @@ -243,6 +243,12 @@ public class CMsgReader { int byte_ = y * maskBytesPerRow + x / 8; int bit = 7 - x % 8; + // NOTE: BufferedImage needs ARGB, rather than RGBA + if ((mask.get(byte_) & (1 << bit)) > 0) + out.put(out.position(), (byte)255); + else + out.put(out.position(), (byte)0); + if ((data.get(byte_) & (1 << bit)) > 0) { out.put(out.position() + 1, pr); out.put(out.position() + 2, pg); @@ -253,11 +259,6 @@ public class CMsgReader { out.put(out.position() + 3, sb); } - if ((mask.get(byte_) & (1 << bit)) > 0) - out.put(out.position() + 0, (byte)255); - else - out.put(out.position() + 0, (byte)0); - out.position(out.position() + 4); } } @@ -281,8 +282,8 @@ public class CMsgReader { is.readBytes(mask, mask_len); int maskBytesPerRow = (width+7)/8; - in = (ByteBuffer)data.duplicate().mark(); - out = (ByteBuffer)ByteBuffer.wrap(buf).mark(); + in = ByteBuffer.wrap(data.array()); + out = ByteBuffer.wrap(buf); for (y = 0;y < height;y++) { for (x = 0;x < width;x++) { int byte_ = y * maskBytesPerRow + x / 8; @@ -294,10 +295,10 @@ public class CMsgReader { else out.put((byte)0); - handler.cp.pf().rgbFromBuffer(out, in.duplicate(), 1); + handler.cp.pf().rgbFromBuffer(out.duplicate(), in.duplicate(), 1); in.position(in.position() + handler.cp.pf().bpp/8); - out.position(out.reset().position() + 4).mark(); + out.position(out.position() + 3); } } diff --git a/java/com/tigervnc/rfb/CSecurityVncAuth.java b/java/com/tigervnc/rfb/CSecurityVncAuth.java index 06154953..b1859f11 100644 --- a/java/com/tigervnc/rfb/CSecurityVncAuth.java +++ b/java/com/tigervnc/rfb/CSecurityVncAuth.java @@ -18,6 +18,8 @@ package com.tigervnc.rfb; +import java.nio.*; + import com.tigervnc.rdr.*; import com.tigervnc.vncviewer.*; @@ -33,8 +35,8 @@ public class CSecurityVncAuth extends CSecurity { OutStream os = cc.getOutStream(); // Read the challenge & obtain the user's password - byte[] challenge = new byte[vncAuthChallengeSize]; - is.readBytes(challenge, 0, vncAuthChallengeSize); + ByteBuffer challenge = ByteBuffer.allocate(vncAuthChallengeSize); + is.readBytes(challenge, vncAuthChallengeSize); StringBuffer passwd = new StringBuffer(); upg.getUserPasswd(cc.isSecure(), null, passwd); @@ -51,10 +53,10 @@ public class CSecurityVncAuth extends CSecurity { key[i] = i> 4) & 15) + 1; int h = (wh & 15) + 1; + if (x + w > 16 || y + h > 16) { + throw new Exception("HEXTILE_DECODE: Hextile out of bounds"); + } ptr = buf.duplicate(); ptr.position((y * t.width() + x)*pf.bpp/8); int rowAdd = (t.width() - w)*pf.bpp/8; while (h-- > 0) { len = w; while (len-- > 0) ptr.put(fg.array()); - ptr.position(ptr.position()+Math.min(rowAdd,ptr.remaining())); + if (h > 0) ptr.position(ptr.position()+rowAdd); } } } diff --git a/java/com/tigervnc/rfb/PixelFormat.java b/java/com/tigervnc/rfb/PixelFormat.java index 9a269992..593b7804 100644 --- a/java/com/tigervnc/rfb/PixelFormat.java +++ b/java/com/tigervnc/rfb/PixelFormat.java @@ -204,11 +204,13 @@ public class PixelFormat { return 0; } + // This method should be invoked with duplicates of dst/src Buffers public void bufferFromRGB(ByteBuffer dst, ByteBuffer src, int pixels) { bufferFromRGB(dst, src, pixels, pixels, 1); } + // This method should be invoked with duplicates of dst/src Buffers public void bufferFromRGB(ByteBuffer dst, ByteBuffer src, int w, int stride, int h) { @@ -269,11 +271,13 @@ public class PixelFormat { } } + // This method should be invoked with duplicates of dst/src Buffers public void rgbFromBuffer(ByteBuffer dst, ByteBuffer src, int pixels) { rgbFromBuffer(dst, src, pixels, pixels, 1); } + // This method should be invoked with duplicates of dst/src Buffers public void rgbFromBuffer(ByteBuffer dst, ByteBuffer src, int w, int stride, int h) { @@ -326,7 +330,7 @@ public class PixelFormat { dst.put(b); src.position(src.position() + bpp/8); } - src.reset().position(src.position() + srcPad).mark(); + src.position(src.position() + srcPad); } } } @@ -345,6 +349,7 @@ public class PixelFormat { } } + // This method should be invoked with a duplicates of buffer public int pixelFromBuffer(ByteBuffer buffer) { int p; diff --git a/java/com/tigervnc/rfb/TightDecoder.java b/java/com/tigervnc/rfb/TightDecoder.java index b180707d..262fcc34 100644 --- a/java/com/tigervnc/rfb/TightDecoder.java +++ b/java/com/tigervnc/rfb/TightDecoder.java @@ -211,7 +211,7 @@ public class TightDecoder extends Decoder { assert(buflen >= 3); - pf.bufferFromRGB(pix, bufptr, 1); + pf.bufferFromRGB(pix.duplicate(), bufptr, 1); pb.fillRect(pf, r, pix.array()); } else { assert(buflen >= pf.bpp/8); @@ -296,7 +296,7 @@ public class TightDecoder extends Decoder { // Determine if the data should be decompressed or just copied. int rowSize, dataSize; - byte[] netbuf; + ByteBuffer netbuf; if (palSize != 0) { if (palSize <= 2) @@ -330,14 +330,14 @@ public class TightDecoder extends Decoder { zis[streamId].setUnderlying(ms, len); // Allocate netbuf and read in data - netbuf = new byte[dataSize]; + netbuf = ByteBuffer.allocate(dataSize); - zis[streamId].readBytes(netbuf, 0, dataSize); + zis[streamId].readBytes(netbuf, dataSize); zis[streamId].removeUnderlying(); ms = null; - bufptr = ByteBuffer.wrap(netbuf); + bufptr = (ByteBuffer)netbuf.flip(); buflen = dataSize; } @@ -426,7 +426,7 @@ public class TightDecoder extends Decoder { pix.put(c, (byte)(inbuf.get(y*rectWidth*3+c) + prevRow[c])); thisRow[c] = pix.get(c); } - pf.bufferFromRGB((ByteBuffer)outbuf.position(y*stride), pix, 1); + pf.bufferFromRGB((ByteBuffer)outbuf.duplicate().position(y*stride), pix, 1); /* Remaining pixels of a row */ for (x = 1; x < rectWidth; x++) { @@ -440,7 +440,7 @@ public class TightDecoder extends Decoder { pix.put(c, (byte)(inbuf.get((y*rectWidth+x)*3+c) + est[c])); thisRow[x*3+c] = pix.get(c); } - pf.bufferFromRGB((ByteBuffer)outbuf.position(y*stride+x), pix, 1); + pf.bufferFromRGB((ByteBuffer)outbuf.duplicate().position(y*stride+x), pix, 1); } System.arraycopy(thisRow, 0, prevRow, 0, prevRow.length); @@ -463,13 +463,13 @@ public class TightDecoder extends Decoder { for (y = 0; y < rectHeight; y++) { /* First pixel in a row */ - pf.rgbFromBuffer(pix, (ByteBuffer)inbuf.position(y*rectWidth), 1); + pf.rgbFromBuffer(pix.duplicate(), (ByteBuffer)inbuf.position(y*rectWidth), 1); for (c = 0; c < 3; c++) pix.put(c, (byte)(pix.get(c) + prevRow[c])); System.arraycopy(pix.array(), 0, thisRow, 0, pix.capacity()); - pf.bufferFromRGB((ByteBuffer)outbuf.position(y*stride), pix, 1); + pf.bufferFromRGB((ByteBuffer)outbuf.duplicate().position(y*stride), pix, 1); /* Remaining pixels of a row */ for (x = 1; x < rectWidth; x++) { @@ -482,13 +482,13 @@ public class TightDecoder extends Decoder { } } - pf.rgbFromBuffer(pix, (ByteBuffer)inbuf.position(y*rectWidth+x), 1); + pf.rgbFromBuffer(pix.duplicate(), (ByteBuffer)inbuf.position(y*rectWidth+x), 1); for (c = 0; c < 3; c++) pix.put(c, (byte)(pix.get(c) + est[c])); System.arraycopy(pix.array(), 0, thisRow, x*3, pix.capacity()); - pf.bufferFromRGB((ByteBuffer)outbuf.position(y*stride+x), pix, 1); + pf.bufferFromRGB((ByteBuffer)outbuf.duplicate().position(y*stride+x), pix, 1); } System.arraycopy(thisRow, 0, prevRow, 0, prevRow.length); diff --git a/java/com/tigervnc/rfb/ZRLEDecoder.java b/java/com/tigervnc/rfb/ZRLEDecoder.java index c1f908ab..03692b96 100644 --- a/java/com/tigervnc/rfb/ZRLEDecoder.java +++ b/java/com/tigervnc/rfb/ZRLEDecoder.java @@ -26,24 +26,24 @@ import com.tigervnc.rdr.*; public class ZRLEDecoder extends Decoder { - private static int readOpaque24A(InStream is) + private static ByteBuffer readOpaque24A(InStream is) { is.check(3); ByteBuffer r = ByteBuffer.allocate(4); r.put(0, (byte)is.readU8()); r.put(1, (byte)is.readU8()); r.put(2, (byte)is.readU8()); - return ((ByteBuffer)r.rewind()).getInt(); + return r; } - private static int readOpaque24B(InStream is) + private static ByteBuffer readOpaque24B(InStream is) { is.check(3); ByteBuffer r = ByteBuffer.allocate(4); - r.put(2, (byte)is.readU8()); r.put(1, (byte)is.readU8()); - r.put(0, (byte)is.readU8()); - return ((ByteBuffer)r.rewind()).getInt(); + r.put(2, (byte)is.readU8()); + r.put(3, (byte)is.readU8()); + return r; } public ZRLEDecoder() { @@ -67,97 +67,98 @@ public class ZRLEDecoder extends Decoder { { MemInStream is = new MemInStream((byte[])buffer, 0, buflen); PixelFormat pf = cp.pf(); - ByteBuffer buf = ByteBuffer.allocate(64 * 64 * 4); switch (pf.bpp) { - case 8: zrleDecode8(r, is, zis, buf, pf, pb); break; - case 16: zrleDecode16(r, is, zis, buf, pf, pb); break; + case 8: zrleDecode8(r, is, zis, pf, pb); break; + case 16: zrleDecode16(r, is, zis, pf, pb); break; case 32: - int maxPixel = pf.pixelFromRGB(-1, -1, -1, pf.getColorModel()); - boolean fitsInLS3Bytes = maxPixel < (1<<24); - boolean fitsInMS3Bytes = (maxPixel & 0xff) == 0; - - if ((fitsInLS3Bytes && pf.isLittleEndian()) || - (fitsInMS3Bytes && pf.isBigEndian())) - { - zrleDecode24A(r, is, zis, buf, pf, pb); - } - else if ((fitsInLS3Bytes && pf.isBigEndian()) || - (fitsInMS3Bytes && pf.isLittleEndian())) - { - zrleDecode24B(r, is, zis, buf, pf, pb); - } - else - { - zrleDecode32(r, is, zis, buf, pf, pb); + { + if (pf.depth <= 24) { + int maxPixel = pf.pixelFromRGB(-1, -1, -1, pf.getColorModel()); + boolean fitsInLS3Bytes = maxPixel < (1<<24); + boolean fitsInMS3Bytes = (maxPixel & 0xff) == 0; + + if ((fitsInLS3Bytes && pf.isLittleEndian()) || + (fitsInMS3Bytes && pf.isBigEndian())) + { + zrleDecode24A(r, is, zis, pf, pb); + break; + } + + if ((fitsInLS3Bytes && pf.isBigEndian()) || + (fitsInMS3Bytes && pf.isLittleEndian())) + { + zrleDecode24B(r, is, zis, pf, pb); + break; + } } + + zrleDecode32(r, is, zis, pf, pb); break; + } } } private static enum PIXEL_T { U8, U16, U24A, U24B, U32 }; private static ByteBuffer READ_PIXEL(InStream is, PIXEL_T type) { - ByteBuffer b = ByteBuffer.allocate(4); switch (type) { case U8: - b.putInt(is.readOpaque8()); - return (ByteBuffer)ByteBuffer.allocate(1).put(b.get(3)).rewind(); + return ByteBuffer.allocate(1).put(0, (byte)is.readOpaque8()); case U16: - b.putInt(is.readOpaque16()); - return (ByteBuffer)ByteBuffer.allocate(2).put(b.array(), 2, 2).rewind(); + return ByteBuffer.allocate(2).putShort(0, (short)is.readOpaque16()); case U24A: - return (ByteBuffer)b.putInt(readOpaque24A(is)).rewind(); + return readOpaque24A(is); case U24B: - return (ByteBuffer)b.putInt(readOpaque24B(is)).rewind(); - case U32: + return readOpaque24B(is); default: - return (ByteBuffer)b.putInt(is.readOpaque32()).rewind(); + return ByteBuffer.allocate(4).putInt(0, is.readOpaque32()); } } private void zrleDecode8(Rect r, InStream is, - ZlibInStream zis, ByteBuffer buf, + ZlibInStream zis, PixelFormat pf, ModifiablePixelBuffer pb) { - ZRLE_DECODE(r, is, zis, buf, pf, pb, PIXEL_T.U8); + ZRLE_DECODE(r, is, zis, pf, pb, PIXEL_T.U8); } private void zrleDecode16(Rect r, InStream is, - ZlibInStream zis, ByteBuffer buf, + ZlibInStream zis, PixelFormat pf, ModifiablePixelBuffer pb) { - ZRLE_DECODE(r, is, zis, buf, pf, pb, PIXEL_T.U16); + ZRLE_DECODE(r, is, zis, pf, pb, PIXEL_T.U16); } private void zrleDecode24A(Rect r, InStream is, - ZlibInStream zis, ByteBuffer buf, + ZlibInStream zis, PixelFormat pf, ModifiablePixelBuffer pb) { - ZRLE_DECODE(r, is, zis, buf, pf, pb, PIXEL_T.U24A); + ZRLE_DECODE(r, is, zis, pf, pb, PIXEL_T.U24A); } private void zrleDecode24B(Rect r, InStream is, - ZlibInStream zis, ByteBuffer buf, + ZlibInStream zis, PixelFormat pf, ModifiablePixelBuffer pb) { - ZRLE_DECODE(r, is, zis, buf, pf, pb, PIXEL_T.U24B); + ZRLE_DECODE(r, is, zis, pf, pb, PIXEL_T.U24B); } private void zrleDecode32(Rect r, InStream is, - ZlibInStream zis, ByteBuffer buf, + ZlibInStream zis, PixelFormat pf, ModifiablePixelBuffer pb) { - ZRLE_DECODE(r, is, zis, buf, pf, pb, PIXEL_T.U32); + ZRLE_DECODE(r, is, zis, pf, pb, PIXEL_T.U32); } private void ZRLE_DECODE(Rect r, InStream is, - ZlibInStream zis, ByteBuffer buf, + ZlibInStream zis, PixelFormat pf, ModifiablePixelBuffer pb, PIXEL_T pix_t) { int length = is.readU32(); zis.setUnderlying(is, length); Rect t = new Rect(); + ByteBuffer buf = ByteBuffer.allocate(64 * 64 * pf.bpp/8); for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 64) { @@ -175,11 +176,12 @@ public class ZRLEDecoder extends Decoder { for (int i = 0; i < palSize; i++) { palette.put(READ_PIXEL(zis, pix_t)); } + palette.flip(); if (palSize == 1) { - ByteBuffer pix = - ByteBuffer.allocate(pf.bpp/8).put(palette.array(), 0, pf.bpp/8); - pb.fillRect(pf, t, pix.array()); + byte[] pix = new byte[pf.bpp/8]; + palette.get(pix); + pb.fillRect(pf, t, pix); continue; } @@ -191,12 +193,12 @@ public class ZRLEDecoder extends Decoder { case U24A: case U24B: ByteBuffer ptr = buf.duplicate(); - for (int iptr=0; iptr < t.area(); iptr++) { + for (int i=0; i < t.area(); i++) { ptr.put(READ_PIXEL(zis, pix_t)); } break; default: - zis.readBytes(buf, t.area() * (pf.bpp/8)); + zis.readBytes(buf.duplicate(), t.area() * (pf.bpp/8)); } } else { @@ -241,12 +243,12 @@ public class ZRLEDecoder extends Decoder { len += b; } while (b == 255); - if (end - ptr.position() < len*(pf.bpp/8)) { + if (end - ptr.position() < len*pf.bpp/8) { System.err.println("ZRLE decode error\n"); throw new Exception("ZRLE decode error"); } - while (len-- > 0) ptr.put(pix); + while (len-- > 0) ptr.put(pix.array()); } } else { @@ -265,7 +267,7 @@ public class ZRLEDecoder extends Decoder { len += b; } while (b == 255); - if (end - ptr.position() < len*(pf.bpp/8)) { + if (end - ptr.position() < len*pf.bpp/8) { System.err.println("ZRLE decode error\n"); throw new Exception("ZRLE decode error"); } @@ -273,8 +275,10 @@ public class ZRLEDecoder extends Decoder { index &= 127; - while (len-- > 0) ptr.put(palette.array(), index*pf.bpp/8, pf.bpp/8); + ByteBuffer pix = ByteBuffer.allocate(pf.bpp/8); + pix.put(palette.array(), index*pf.bpp/8, pf.bpp/8); + while (len-- > 0) ptr.put(pix.array()); } } } -- 2.39.5