aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPierre Ossman <ossman@cendio.se>2019-12-20 07:39:06 +0100
committerPierre Ossman <ossman@cendio.se>2019-12-20 07:39:06 +0100
commitd461f7fdb8b01f655260ea2f495ece700f3c9898 (patch)
tree5113b5a2844de4ce1469864d56c9786b3c4d63b0
parentc59f195d0ef2922bc163ac7cabfc94b9f1b25860 (diff)
parent05e28490873a861379c943bf616614b78b558b89 (diff)
downloadtigervnc-d461f7fdb8b01f655260ea2f495ece700f3c9898.tar.gz
tigervnc-d461f7fdb8b01f655260ea2f495ece700f3c9898.zip
Merge branch 'secfix' of https://github.com/CendioOssman/tigervnc
-rw-r--r--common/rdr/FdInStream.cxx26
-rw-r--r--common/rdr/FdInStream.h17
-rw-r--r--common/rdr/FdOutStream.cxx25
-rw-r--r--common/rdr/FdOutStream.h12
-rw-r--r--common/rdr/FileInStream.cxx14
-rw-r--r--common/rdr/FileInStream.h4
-rw-r--r--common/rdr/FixedMemOutStream.h52
-rw-r--r--common/rdr/HexInStream.cxx26
-rw-r--r--common/rdr/HexInStream.h12
-rw-r--r--common/rdr/HexOutStream.cxx24
-rw-r--r--common/rdr/HexOutStream.h12
-rw-r--r--common/rdr/InStream.h38
-rw-r--r--common/rdr/MemInStream.h8
-rw-r--r--common/rdr/MemOutStream.h16
-rw-r--r--common/rdr/OutStream.h42
-rw-r--r--common/rdr/RandomStream.cxx18
-rw-r--r--common/rdr/RandomStream.h6
-rw-r--r--common/rdr/TLSInStream.cxx18
-rw-r--r--common/rdr/TLSInStream.h10
-rw-r--r--common/rdr/TLSOutStream.cxx14
-rw-r--r--common/rdr/TLSOutStream.h10
-rw-r--r--common/rdr/ZlibInStream.cxx33
-rw-r--r--common/rdr/ZlibInStream.h16
-rw-r--r--common/rdr/ZlibOutStream.cxx14
-rw-r--r--common/rdr/ZlibOutStream.h10
-rw-r--r--common/rfb/CMsgReader.cxx3
-rw-r--r--common/rfb/Configuration.cxx6
-rw-r--r--common/rfb/Configuration.h13
-rw-r--r--common/rfb/Cursor.cxx3
-rw-r--r--common/rfb/EncodeManager.cxx11
-rw-r--r--common/rfb/EncodeManager.h3
-rw-r--r--common/rfb/Password.cxx6
-rw-r--r--common/rfb/Password.h6
-rw-r--r--common/rfb/PixelBuffer.cxx125
-rw-r--r--common/rfb/PixelBuffer.h17
-rw-r--r--common/rfb/PixelFormat.cxx18
-rw-r--r--common/rfb/SMsgReader.cxx3
-rw-r--r--common/rfb/TightDecoder.cxx3
-rw-r--r--common/rfb/tightDecode.h37
-rw-r--r--common/rfb/util.h2
-rw-r--r--common/rfb/zrleDecode.h3
-rw-r--r--tests/perf/encperf.cxx10
-rw-r--r--tests/unit/CMakeLists.txt3
-rw-r--r--tests/unit/pixelformat.cxx188
-rw-r--r--unix/x0vncserver/XPixelBuffer.cxx9
-rw-r--r--unix/xserver/hw/vnc/XserverDesktop.cc24
-rw-r--r--unix/xserver/hw/vnc/XserverDesktop.h2
-rw-r--r--vncviewer/PlatformPixelBuffer.cxx9
-rw-r--r--win/rfb_win32/DIBSectionBuffer.cxx41
-rw-r--r--win/rfb_win32/Registry.cxx6
-rw-r--r--win/rfb_win32/Registry.h6
51 files changed, 633 insertions, 401 deletions
diff --git a/common/rdr/FdInStream.cxx b/common/rdr/FdInStream.cxx
index 1b9a322a..1730d6d1 100644
--- a/common/rdr/FdInStream.cxx
+++ b/common/rdr/FdInStream.cxx
@@ -56,7 +56,7 @@ using namespace rdr;
enum { DEFAULT_BUF_SIZE = 8192,
MIN_BULK_SIZE = 1024 };
-FdInStream::FdInStream(int fd_, int timeoutms_, int bufSize_,
+FdInStream::FdInStream(int fd_, int timeoutms_, size_t bufSize_,
bool closeWhenDone_)
: fd(fd_), closeWhenDone(closeWhenDone_),
timeoutms(timeoutms_), blockCallback(0),
@@ -67,7 +67,7 @@ FdInStream::FdInStream(int fd_, int timeoutms_, int bufSize_,
}
FdInStream::FdInStream(int fd_, FdInStreamBlockCallback* blockCallback_,
- int bufSize_)
+ size_t bufSize_)
: fd(fd_), timeoutms(0), blockCallback(blockCallback_),
timing(false), timeWaitedIn100us(5), timedKbits(0),
bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_SIZE), offset(0)
@@ -92,12 +92,12 @@ void FdInStream::setBlockCallback(FdInStreamBlockCallback* blockCallback_)
timeoutms = 0;
}
-int FdInStream::pos()
+size_t FdInStream::pos()
{
return offset + ptr - start;
}
-void FdInStream::readBytes(void* data, int length)
+void FdInStream::readBytes(void* data, size_t length)
{
if (length < MIN_BULK_SIZE) {
InStream::readBytes(data, length);
@@ -106,7 +106,7 @@ void FdInStream::readBytes(void* data, int length)
U8* dataPtr = (U8*)data;
- int n = end - ptr;
+ size_t n = end - ptr;
if (n > length) n = length;
memcpy(dataPtr, ptr, n);
@@ -123,7 +123,7 @@ void FdInStream::readBytes(void* data, int length)
}
-int FdInStream::overrun(int itemSize, int nItems, bool wait)
+size_t FdInStream::overrun(size_t itemSize, size_t nItems, bool wait)
{
if (itemSize > bufSize)
throw Exception("FdInStream overrun: max itemSize exceeded");
@@ -135,8 +135,8 @@ int FdInStream::overrun(int itemSize, int nItems, bool wait)
end -= ptr - start;
ptr = start;
- int bytes_to_read;
- while (end < start + itemSize) {
+ size_t bytes_to_read;
+ while ((size_t)(end - start) < itemSize) {
bytes_to_read = start + bufSize - end;
if (!timing) {
// When not timing, we must be careful not to read too much
@@ -147,13 +147,15 @@ int FdInStream::overrun(int itemSize, int nItems, bool wait)
// bytes is ineffecient.
bytes_to_read = vncmin(bytes_to_read, vncmax(itemSize*nItems, 8));
}
- int n = readWithTimeoutOrCallback((U8*)end, bytes_to_read, wait);
+ size_t n = readWithTimeoutOrCallback((U8*)end, bytes_to_read, wait);
if (n == 0) return 0;
end += n;
}
- if (itemSize * nItems > end - ptr)
- nItems = (end - ptr) / itemSize;
+ size_t nAvail;
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
return nItems;
}
@@ -171,7 +173,7 @@ int FdInStream::overrun(int itemSize, int nItems, bool wait)
// returning EINTR.
//
-int FdInStream::readWithTimeoutOrCallback(void* buf, int len, bool wait)
+size_t FdInStream::readWithTimeoutOrCallback(void* buf, size_t len, bool wait)
{
struct timeval before, after;
if (timing)
diff --git a/common/rdr/FdInStream.h b/common/rdr/FdInStream.h
index b4c87653..d99ad3ce 100644
--- a/common/rdr/FdInStream.h
+++ b/common/rdr/FdInStream.h
@@ -37,16 +37,17 @@ namespace rdr {
public:
- FdInStream(int fd, int timeoutms=-1, int bufSize=0,
+ FdInStream(int fd, int timeoutms=-1, size_t bufSize=0,
bool closeWhenDone_=false);
- FdInStream(int fd, FdInStreamBlockCallback* blockCallback, int bufSize=0);
+ FdInStream(int fd, FdInStreamBlockCallback* blockCallback,
+ size_t bufSize=0);
virtual ~FdInStream();
void setTimeout(int timeoutms);
void setBlockCallback(FdInStreamBlockCallback* blockCallback);
int getFd() { return fd; }
- int pos();
- void readBytes(void* data, int length);
+ size_t pos();
+ void readBytes(void* data, size_t length);
void startTiming();
void stopTiming();
@@ -54,10 +55,10 @@ namespace rdr {
unsigned int timeWaited() { return timeWaitedIn100us; }
protected:
- int overrun(int itemSize, int nItems, bool wait);
+ size_t overrun(size_t itemSize, size_t nItems, bool wait);
private:
- int readWithTimeoutOrCallback(void* buf, int len, bool wait=true);
+ size_t readWithTimeoutOrCallback(void* buf, size_t len, bool wait=true);
int fd;
bool closeWhenDone;
@@ -68,8 +69,8 @@ namespace rdr {
unsigned int timeWaitedIn100us;
unsigned int timedKbits;
- int bufSize;
- int offset;
+ size_t bufSize;
+ size_t offset;
U8* start;
};
diff --git a/common/rdr/FdOutStream.cxx b/common/rdr/FdOutStream.cxx
index cf857f85..f5d07e4b 100644
--- a/common/rdr/FdOutStream.cxx
+++ b/common/rdr/FdOutStream.cxx
@@ -51,7 +51,7 @@ using namespace rdr;
enum { DEFAULT_BUF_SIZE = 16384 };
-FdOutStream::FdOutStream(int fd_, bool blocking_, int timeoutms_, int bufSize_)
+FdOutStream::FdOutStream(int fd_, bool blocking_, int timeoutms_, size_t bufSize_)
: fd(fd_), blocking(blocking_), timeoutms(timeoutms_),
bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_SIZE), offset(0)
{
@@ -79,7 +79,7 @@ void FdOutStream::setBlocking(bool blocking_) {
blocking = blocking_;
}
-int FdOutStream::length()
+size_t FdOutStream::length()
{
return offset + ptr - sentUpTo;
}
@@ -97,9 +97,9 @@ unsigned FdOutStream::getIdleTime()
void FdOutStream::flush()
{
while (sentUpTo < ptr) {
- int n = writeWithTimeout((const void*) sentUpTo,
- ptr - sentUpTo,
- blocking? timeoutms : 0);
+ size_t n = writeWithTimeout((const void*) sentUpTo,
+ ptr - sentUpTo,
+ blocking? timeoutms : 0);
// Timeout?
if (n == 0) {
@@ -120,7 +120,7 @@ void FdOutStream::flush()
}
-int FdOutStream::overrun(int itemSize, int nItems)
+size_t FdOutStream::overrun(size_t itemSize, size_t nItems)
{
if (itemSize > bufSize)
throw Exception("FdOutStream overrun: max itemSize exceeded");
@@ -129,10 +129,10 @@ int FdOutStream::overrun(int itemSize, int nItems)
flush();
// Still not enough space?
- if (itemSize > end - ptr) {
+ if (itemSize > (size_t)(end - ptr)) {
// Can we shuffle things around?
// (don't do this if it gains us less than 25%)
- if ((sentUpTo - start > bufSize / 4) &&
+ if (((size_t)(sentUpTo - start) > bufSize / 4) &&
(itemSize < bufSize - (ptr - sentUpTo))) {
memmove(start, sentUpTo, ptr - sentUpTo);
ptr = start + (ptr - sentUpTo);
@@ -149,9 +149,10 @@ int FdOutStream::overrun(int itemSize, int nItems)
}
}
- // Can we fit all the items asked for?
- if (itemSize * nItems > end - ptr)
- nItems = (end - ptr) / itemSize;
+ size_t nAvail;
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
return nItems;
}
@@ -166,7 +167,7 @@ int FdOutStream::overrun(int itemSize, int nItems)
// select() and send() returning EINTR.
//
-int FdOutStream::writeWithTimeout(const void* data, int length, int timeoutms)
+size_t FdOutStream::writeWithTimeout(const void* data, size_t length, int timeoutms)
{
int n;
diff --git a/common/rdr/FdOutStream.h b/common/rdr/FdOutStream.h
index b7f6cb01..ed84fdb5 100644
--- a/common/rdr/FdOutStream.h
+++ b/common/rdr/FdOutStream.h
@@ -34,7 +34,7 @@ namespace rdr {
public:
- FdOutStream(int fd, bool blocking=true, int timeoutms=-1, int bufSize=0);
+ FdOutStream(int fd, bool blocking=true, int timeoutms=-1, size_t bufSize=0);
virtual ~FdOutStream();
void setTimeout(int timeoutms);
@@ -42,20 +42,20 @@ namespace rdr {
int getFd() { return fd; }
void flush();
- int length();
+ size_t length();
int bufferUsage();
unsigned getIdleTime();
private:
- int overrun(int itemSize, int nItems);
- int writeWithTimeout(const void* data, int length, int timeoutms);
+ size_t overrun(size_t itemSize, size_t nItems);
+ size_t writeWithTimeout(const void* data, size_t length, int timeoutms);
int fd;
bool blocking;
int timeoutms;
- int bufSize;
- int offset;
+ size_t bufSize;
+ size_t offset;
U8* start;
U8* sentUpTo;
struct timeval lastWrite;
diff --git a/common/rdr/FileInStream.cxx b/common/rdr/FileInStream.cxx
index 3acdfd45..bdb05a3a 100644
--- a/common/rdr/FileInStream.cxx
+++ b/common/rdr/FileInStream.cxx
@@ -48,7 +48,7 @@ void FileInStream::reset(void) {
ptr = end = b;
}
-int FileInStream::pos()
+size_t FileInStream::pos()
{
if (!file)
throw Exception("File is not open");
@@ -56,9 +56,9 @@ int FileInStream::pos()
return ftell(file) + ptr - b;
}
-int FileInStream::overrun(int itemSize, int nItems, bool wait)
+size_t FileInStream::overrun(size_t itemSize, size_t nItems, bool wait)
{
- if (itemSize > (int)sizeof(b))
+ if (itemSize > sizeof(b))
throw Exception("FileInStream overrun: max itemSize exceeded");
if (end - ptr != 0)
@@ -68,7 +68,7 @@ int FileInStream::overrun(int itemSize, int nItems, bool wait)
ptr = b;
- while (end < b + itemSize) {
+ while ((size_t)(end - b) < itemSize) {
size_t n = fread((U8 *)end, b + sizeof(b) - end, 1, file);
if (n == 0) {
if (ferror(file))
@@ -80,8 +80,10 @@ int FileInStream::overrun(int itemSize, int nItems, bool wait)
end += b + sizeof(b) - end;
}
- if (itemSize * nItems > end - ptr)
- nItems = (end - ptr) / itemSize;
+ size_t nAvail;
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
return nItems;
}
diff --git a/common/rdr/FileInStream.h b/common/rdr/FileInStream.h
index ace04f37..a33c765e 100644
--- a/common/rdr/FileInStream.h
+++ b/common/rdr/FileInStream.h
@@ -35,10 +35,10 @@ namespace rdr {
void reset(void);
- int pos();
+ size_t pos();
protected:
- int overrun(int itemSize, int nItems, bool wait = true);
+ size_t overrun(size_t itemSize, size_t nItems, bool wait = true);
private:
U8 b[131072];
diff --git a/common/rdr/FixedMemOutStream.h b/common/rdr/FixedMemOutStream.h
deleted file mode 100644
index e4ec52cb..00000000
--- a/common/rdr/FixedMemOutStream.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/* Copyright (C) 2002-2005 RealVNC Ltd. All Rights Reserved.
- *
- * This is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this software; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
- * USA.
- */
-
-//
-// A FixedMemOutStream writes to a buffer of a fixed length.
-//
-
-#ifndef __RDR_FIXEDMEMOUTSTREAM_H__
-#define __RDR_FIXEDMEMOUTSTREAM_H__
-
-#include <rdr/OutStream.h>
-#include <rdr/Exception.h>
-
-namespace rdr {
-
- class FixedMemOutStream : public OutStream {
-
- public:
-
- FixedMemOutStream(void* buf, int len) {
- ptr = start = (U8*)buf;
- end = start + len;
- }
-
- int length() { return ptr - start; }
- void reposition(int pos) { ptr = start + pos; }
- const void* data() { return (const void*)start; }
-
- private:
-
- int overrun(int itemSize, int nItems) { throw EndOfStream(); }
- U8* start;
- };
-
-}
-
-#endif
diff --git a/common/rdr/HexInStream.cxx b/common/rdr/HexInStream.cxx
index 80f8a796..a6bc92cd 100644
--- a/common/rdr/HexInStream.cxx
+++ b/common/rdr/HexInStream.cxx
@@ -28,7 +28,7 @@ const int DEFAULT_BUF_LEN = 16384;
static inline int min(int a, int b) {return a<b ? a : b;}
-HexInStream::HexInStream(InStream& is, int bufSize_)
+HexInStream::HexInStream(InStream& is, size_t bufSize_)
: bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_LEN), offset(0), in_stream(is)
{
ptr = end = start = new U8[bufSize];
@@ -50,8 +50,8 @@ bool HexInStream::readHexAndShift(char c, int* v) {
return true;
}
-bool HexInStream::hexStrToBin(const char* s, char** data, int* length) {
- int l=strlen(s);
+bool HexInStream::hexStrToBin(const char* s, char** data, size_t* length) {
+ size_t l=strlen(s);
if ((l % 2) == 0) {
delete [] *data;
*data = 0; *length = 0;
@@ -59,7 +59,7 @@ bool HexInStream::hexStrToBin(const char* s, char** data, int* length) {
return true;
*data = new char[l/2];
*length = l/2;
- for(int i=0;i<l;i+=2) {
+ for(size_t i=0;i<l;i+=2) {
int byte = 0;
if (!readHexAndShift(s[i], &byte) ||
!readHexAndShift(s[i+1], &byte))
@@ -76,11 +76,11 @@ decodeError:
}
-int HexInStream::pos() {
+size_t HexInStream::pos() {
return offset + ptr - start;
}
-int HexInStream::overrun(int itemSize, int nItems, bool wait) {
+size_t HexInStream::overrun(size_t itemSize, size_t nItems, bool wait) {
if (itemSize > bufSize)
throw Exception("HexInStream overrun: max itemSize exceeded");
@@ -91,15 +91,15 @@ int HexInStream::overrun(int itemSize, int nItems, bool wait) {
offset += ptr - start;
ptr = start;
- while (end < ptr + itemSize) {
- int n = in_stream.check(2, 1, wait);
+ while ((size_t)(end - ptr) < itemSize) {
+ size_t n = in_stream.check(2, 1, wait);
if (n == 0) return 0;
const U8* iptr = in_stream.getptr();
const U8* eptr = in_stream.getend();
- int length = min((eptr - iptr)/2, start + bufSize - end);
+ size_t length = min((eptr - iptr)/2, start + bufSize - end);
U8* optr = (U8*) end;
- for (int i=0; i<length; i++) {
+ for (size_t i=0; i<length; i++) {
int v = 0;
readHexAndShift(iptr[i*2], &v);
readHexAndShift(iptr[i*2+1], &v);
@@ -110,8 +110,10 @@ int HexInStream::overrun(int itemSize, int nItems, bool wait) {
end += length;
}
- if (itemSize * nItems > end - ptr)
- nItems = (end - ptr) / itemSize;
+ size_t nAvail;
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
return nItems;
}
diff --git a/common/rdr/HexInStream.h b/common/rdr/HexInStream.h
index 6bfb8433..8e495fbe 100644
--- a/common/rdr/HexInStream.h
+++ b/common/rdr/HexInStream.h
@@ -26,21 +26,21 @@ namespace rdr {
class HexInStream : public InStream {
public:
- HexInStream(InStream& is, int bufSize=0);
+ HexInStream(InStream& is, size_t bufSize=0);
virtual ~HexInStream();
- int pos();
+ size_t pos();
static bool readHexAndShift(char c, int* v);
- static bool hexStrToBin(const char* s, char** data, int* length);
+ static bool hexStrToBin(const char* s, char** data, size_t* length);
protected:
- int overrun(int itemSize, int nItems, bool wait);
+ size_t overrun(size_t itemSize, size_t nItems, bool wait);
private:
- int bufSize;
+ size_t bufSize;
U8* start;
- int offset;
+ size_t offset;
InStream& in_stream;
};
diff --git a/common/rdr/HexOutStream.cxx b/common/rdr/HexOutStream.cxx
index 9b0b6c4d..eac2eff8 100644
--- a/common/rdr/HexOutStream.cxx
+++ b/common/rdr/HexOutStream.cxx
@@ -23,9 +23,9 @@ using namespace rdr;
const int DEFAULT_BUF_LEN = 16384;
-static inline int min(int a, int b) {return a<b ? a : b;}
+static inline size_t min(size_t a, size_t b) {return a<b ? a : b;}
-HexOutStream::HexOutStream(OutStream& os, int buflen)
+HexOutStream::HexOutStream(OutStream& os, size_t buflen)
: out_stream(os), offset(0), bufSize(buflen ? buflen : DEFAULT_BUF_LEN)
{
if (bufSize % 2)
@@ -48,9 +48,9 @@ char HexOutStream::intToHex(int i) {
throw rdr::Exception("intToHex failed");
}
-char* HexOutStream::binToHexStr(const char* data, int length) {
+char* HexOutStream::binToHexStr(const char* data, size_t length) {
char* buffer = new char[length*2+1];
- for (int i=0; i<length; i++) {
+ for (size_t i=0; i<length; i++) {
buffer[i*2] = intToHex((data[i] >> 4) & 15);
buffer[i*2+1] = intToHex((data[i] & 15));
if (!buffer[i*2] || !buffer[i*2+1]) {
@@ -70,9 +70,9 @@ HexOutStream::writeBuffer() {
out_stream.check(2);
U8* optr = out_stream.getptr();
U8* oend = out_stream.getend();
- int length = min(ptr-pos, (oend-optr)/2);
+ size_t length = min(ptr-pos, (oend-optr)/2);
- for (int i=0; i<length; i++) {
+ for (size_t i=0; i<length; i++) {
optr[i*2] = intToHex((pos[i] >> 4) & 0xf);
optr[i*2+1] = intToHex(pos[i] & 0xf);
}
@@ -84,7 +84,7 @@ HexOutStream::writeBuffer() {
ptr = start;
}
-int HexOutStream::length()
+size_t HexOutStream::length()
{
return offset + ptr - start;
}
@@ -95,15 +95,17 @@ HexOutStream::flush() {
out_stream.flush();
}
-int
-HexOutStream::overrun(int itemSize, int nItems) {
+size_t
+HexOutStream::overrun(size_t itemSize, size_t nItems) {
if (itemSize > bufSize)
throw Exception("HexOutStream overrun: max itemSize exceeded");
writeBuffer();
- if (itemSize * nItems > end - ptr)
- nItems = (end - ptr) / itemSize;
+ size_t nAvail;
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
return nItems;
}
diff --git a/common/rdr/HexOutStream.h b/common/rdr/HexOutStream.h
index 10247e68..92442a72 100644
--- a/common/rdr/HexOutStream.h
+++ b/common/rdr/HexOutStream.h
@@ -26,24 +26,24 @@ namespace rdr {
class HexOutStream : public OutStream {
public:
- HexOutStream(OutStream& os, int buflen=0);
+ HexOutStream(OutStream& os, size_t buflen=0);
virtual ~HexOutStream();
void flush();
- int length();
+ size_t length();
static char intToHex(int i);
- static char* binToHexStr(const char* data, int length);
+ static char* binToHexStr(const char* data, size_t length);
private:
void writeBuffer();
- int overrun(int itemSize, int nItems);
+ size_t overrun(size_t itemSize, size_t nItems);
OutStream& out_stream;
U8* start;
- int offset;
- int bufSize;
+ size_t offset;
+ size_t bufSize;
};
}
diff --git a/common/rdr/InStream.h b/common/rdr/InStream.h
index 212a2ec9..f71a4d9e 100644
--- a/common/rdr/InStream.h
+++ b/common/rdr/InStream.h
@@ -41,14 +41,17 @@ namespace rdr {
// for the bytes, zero is returned if the bytes are not immediately
// available.
- inline int check(int itemSize, int nItems=1, bool wait=true)
+ inline size_t check(size_t itemSize, size_t nItems=1, bool wait=true)
{
- if (ptr + itemSize * nItems > end) {
- if (ptr + itemSize > end)
- return overrun(itemSize, nItems, wait);
+ size_t nAvail;
+
+ if (itemSize > (size_t)(end - ptr))
+ return overrun(itemSize, nItems, wait);
+
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
- nItems = (end - ptr) / itemSize;
- }
return nItems;
}
@@ -56,7 +59,7 @@ namespace rdr {
// be read without blocking. It returns true if this is the case, false
// otherwise. The length must be "small" (less than the buffer size).
- inline bool checkNoWait(int length) { return check(length, 1, false)!=0; }
+ inline bool checkNoWait(size_t length) { return check(length, 1, false)!=0; }
// readU/SN() methods read unsigned and signed N-bit integers.
@@ -82,9 +85,9 @@ namespace rdr {
static U32 maxStringLength;
- inline void skip(int bytes) {
+ inline void skip(size_t bytes) {
while (bytes > 0) {
- int n = check(1, bytes);
+ size_t n = check(1, bytes);
ptr += n;
bytes -= n;
}
@@ -92,14 +95,13 @@ namespace rdr {
// readBytes() reads an exact number of bytes.
- void readBytes(void* data, int length) {
- U8* dataPtr = (U8*)data;
- U8* dataEnd = dataPtr + length;
- while (dataPtr < dataEnd) {
- int n = check(1, dataEnd - dataPtr);
- memcpy(dataPtr, ptr, n);
+ void readBytes(void* data, size_t length) {
+ while (length > 0) {
+ size_t n = check(1, length);
+ memcpy(data, ptr, n);
ptr += n;
- dataPtr += n;
+ data = (U8*)data + n;
+ length -= n;
}
}
@@ -114,7 +116,7 @@ namespace rdr {
// pos() returns the position in the stream.
- virtual int pos() = 0;
+ virtual size_t pos() = 0;
// getptr(), getend() and setptr() are "dirty" methods which allow you to
// manipulate the buffer directly. This is useful for a stream which is a
@@ -133,7 +135,7 @@ namespace rdr {
// instead of blocking to wait for the bytes, zero is returned if the bytes
// are not immediately available.
- virtual int overrun(int itemSize, int nItems, bool wait=true) = 0;
+ virtual size_t overrun(size_t itemSize, size_t nItems, bool wait=true) = 0;
protected:
diff --git a/common/rdr/MemInStream.h b/common/rdr/MemInStream.h
index 1a6a7982..3e9e77bc 100644
--- a/common/rdr/MemInStream.h
+++ b/common/rdr/MemInStream.h
@@ -36,7 +36,7 @@ namespace rdr {
public:
- MemInStream(const void* data, int len, bool deleteWhenDone_=false)
+ MemInStream(const void* data, size_t len, bool deleteWhenDone_=false)
: start((const U8*)data), deleteWhenDone(deleteWhenDone_)
{
ptr = start;
@@ -48,12 +48,12 @@ namespace rdr {
delete [] start;
}
- int pos() { return ptr - start; }
- void reposition(int pos) { ptr = start + pos; }
+ size_t pos() { return ptr - start; }
+ void reposition(size_t pos) { ptr = start + pos; }
private:
- int overrun(int itemSize, int nItems, bool wait) { throw EndOfStream(); }
+ size_t overrun(size_t itemSize, size_t nItems, bool wait) { throw EndOfStream(); }
const U8* start;
bool deleteWhenDone;
};
diff --git a/common/rdr/MemOutStream.h b/common/rdr/MemOutStream.h
index 3b17e555..b56bac3a 100644
--- a/common/rdr/MemOutStream.h
+++ b/common/rdr/MemOutStream.h
@@ -23,6 +23,7 @@
#ifndef __RDR_MEMOUTSTREAM_H__
#define __RDR_MEMOUTSTREAM_H__
+#include <rdr/Exception.h>
#include <rdr/OutStream.h>
namespace rdr {
@@ -40,16 +41,16 @@ namespace rdr {
delete [] start;
}
- void writeBytes(const void* data, int length) {
+ void writeBytes(const void* data, size_t length) {
check(length);
memcpy(ptr, data, length);
ptr += length;
}
- int length() { return ptr - start; }
+ size_t length() { return ptr - start; }
void clear() { ptr = start; };
void clearAndZero() { memset(start, 0, ptr-start); clear(); }
- void reposition(int pos) { ptr = start + pos; }
+ void reposition(size_t pos) { ptr = start + pos; }
// data() returns a pointer to the buffer.
@@ -60,11 +61,14 @@ namespace rdr {
// overrun() either doubles the buffer or adds enough space for nItems of
// size itemSize bytes.
- int overrun(int itemSize, int nItems) {
- int len = ptr - start + itemSize * nItems;
- if (len < (end - start) * 2)
+ size_t overrun(size_t itemSize, size_t nItems) {
+ size_t len = ptr - start + itemSize * nItems;
+ if (len < (size_t)(end - start) * 2)
len = (end - start) * 2;
+ if (len < (size_t)(end - start))
+ throw Exception("Overflow in MemOutStream::overrun()");
+
U8* newStart = new U8[len];
memcpy(newStart, start, ptr - start);
ptr = newStart + (ptr - start);
diff --git a/common/rdr/OutStream.h b/common/rdr/OutStream.h
index a749a208..0f60ccc1 100644
--- a/common/rdr/OutStream.h
+++ b/common/rdr/OutStream.h
@@ -44,14 +44,17 @@ namespace rdr {
// itemSize bytes. Returns the number of items which fit (up to a maximum
// of nItems).
- inline int check(int itemSize, int nItems=1)
+ inline size_t check(size_t itemSize, size_t nItems=1)
{
- if (ptr + itemSize * nItems > end) {
- if (ptr + itemSize > end)
- return overrun(itemSize, nItems);
+ size_t nAvail;
+
+ if (itemSize > (size_t)(end - ptr))
+ return overrun(itemSize, nItems);
+
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
- nItems = (end - ptr) / itemSize;
- }
return nItems;
}
@@ -76,13 +79,13 @@ namespace rdr {
writeBytes(str, len);
}
- inline void pad(int bytes) {
+ inline void pad(size_t bytes) {
while (bytes-- > 0) writeU8(0);
}
- inline void skip(int bytes) {
+ inline void skip(size_t bytes) {
while (bytes > 0) {
- int n = check(1, bytes);
+ size_t n = check(1, bytes);
ptr += n;
bytes -= n;
}
@@ -90,22 +93,21 @@ namespace rdr {
// writeBytes() writes an exact number of bytes.
- void writeBytes(const void* data, int length) {
- const U8* dataPtr = (const U8*)data;
- const U8* dataEnd = dataPtr + length;
- while (dataPtr < dataEnd) {
- int n = check(1, dataEnd - dataPtr);
- memcpy(ptr, dataPtr, n);
+ void writeBytes(const void* data, size_t length) {
+ while (length > 0) {
+ size_t n = check(1, length);
+ memcpy(ptr, data, n);
ptr += n;
- dataPtr += n;
+ data = (U8*)data + n;
+ length -= n;
}
}
// copyBytes() efficiently transfers data between streams
- void copyBytes(InStream* is, int length) {
+ void copyBytes(InStream* is, size_t length) {
while (length > 0) {
- int n = check(1, length);
+ size_t n = check(1, length);
is->readBytes(ptr, n);
ptr += n;
length -= n;
@@ -124,7 +126,7 @@ namespace rdr {
// length() returns the length of the stream.
- virtual int length() = 0;
+ virtual size_t length() = 0;
// flush() requests that the stream be flushed.
@@ -145,7 +147,7 @@ namespace rdr {
// the number of items which fit (up to a maximum of nItems). itemSize is
// supposed to be "small" (a few bytes).
- virtual int overrun(int itemSize, int nItems) = 0;
+ virtual size_t overrun(size_t itemSize, size_t nItems) = 0;
protected:
diff --git a/common/rdr/RandomStream.cxx b/common/rdr/RandomStream.cxx
index e22da3d0..1be9b251 100644
--- a/common/rdr/RandomStream.cxx
+++ b/common/rdr/RandomStream.cxx
@@ -35,7 +35,7 @@ static rfb::LogWriter vlog("RandomStream");
using namespace rdr;
-const int DEFAULT_BUF_LEN = 256;
+const size_t DEFAULT_BUF_LEN = 256;
unsigned int RandomStream::seed;
@@ -86,11 +86,11 @@ RandomStream::~RandomStream() {
#endif
}
-int RandomStream::pos() {
+size_t RandomStream::pos() {
return offset + ptr - start;
}
-int RandomStream::overrun(int itemSize, int nItems, bool wait) {
+size_t RandomStream::overrun(size_t itemSize, size_t nItems, bool wait) {
if (itemSize > DEFAULT_BUF_LEN)
throw Exception("RandomStream overrun: max itemSize exceeded");
@@ -101,7 +101,7 @@ int RandomStream::overrun(int itemSize, int nItems, bool wait) {
offset += ptr - start;
ptr = start;
- int length = start + DEFAULT_BUF_LEN - end;
+ size_t length = start + DEFAULT_BUF_LEN - end;
#ifdef RFB_HAVE_WINCRYPT
if (provider) {
@@ -112,7 +112,7 @@ int RandomStream::overrun(int itemSize, int nItems, bool wait) {
#else
#ifndef WIN32
if (fp) {
- int n = fread((U8*)end, length, 1, fp);
+ size_t n = fread((U8*)end, length, 1, fp);
if (n != 1)
throw rdr::SystemException("reading /dev/urandom or /dev/random failed",
errno);
@@ -122,12 +122,14 @@ int RandomStream::overrun(int itemSize, int nItems, bool wait) {
{
#endif
#endif
- for (int i=0; i<length; i++)
+ for (size_t i=0; i<length; i++)
*(U8*)end++ = (int) (256.0*rand()/(RAND_MAX+1.0));
}
- if (itemSize * nItems > end - ptr)
- nItems = (end - ptr) / itemSize;
+ size_t nAvail;
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
return nItems;
}
diff --git a/common/rdr/RandomStream.h b/common/rdr/RandomStream.h
index c33360d7..80b389b2 100644
--- a/common/rdr/RandomStream.h
+++ b/common/rdr/RandomStream.h
@@ -39,14 +39,14 @@ namespace rdr {
RandomStream();
virtual ~RandomStream();
- int pos();
+ size_t pos();
protected:
- int overrun(int itemSize, int nItems, bool wait);
+ size_t overrun(size_t itemSize, size_t nItems, bool wait);
private:
U8* start;
- int offset;
+ size_t offset;
static unsigned int seed;
#ifdef RFB_HAVE_WINCRYPT
diff --git a/common/rdr/TLSInStream.cxx b/common/rdr/TLSInStream.cxx
index 8cd07b6c..cd81f224 100644
--- a/common/rdr/TLSInStream.cxx
+++ b/common/rdr/TLSInStream.cxx
@@ -43,7 +43,7 @@ ssize_t TLSInStream::pull(gnutls_transport_ptr_t str, void* data, size_t size)
return -1;
}
- if (in->getend() - in->getptr() < (ptrdiff_t)size)
+ if ((size_t)(in->getend() - in->getptr()) < size)
size = in->getend() - in->getptr();
in->readBytes(data, size);
@@ -75,12 +75,12 @@ TLSInStream::~TLSInStream()
delete[] start;
}
-int TLSInStream::pos()
+size_t TLSInStream::pos()
{
return offset + ptr - start;
}
-int TLSInStream::overrun(int itemSize, int nItems, bool wait)
+size_t TLSInStream::overrun(size_t itemSize, size_t nItems, bool wait)
{
if (itemSize > bufSize)
throw Exception("TLSInStream overrun: max itemSize exceeded");
@@ -92,20 +92,22 @@ int TLSInStream::overrun(int itemSize, int nItems, bool wait)
end -= ptr - start;
ptr = start;
- while (end < start + itemSize) {
- int n = readTLS((U8*) end, start + bufSize - end, wait);
+ while ((size_t)(end - start) < itemSize) {
+ size_t n = readTLS((U8*) end, start + bufSize - end, wait);
if (!wait && n == 0)
return 0;
end += n;
}
- if (itemSize * nItems > end - ptr)
- nItems = (end - ptr) / itemSize;
+ size_t nAvail;
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
return nItems;
}
-int TLSInStream::readTLS(U8* buf, int len, bool wait)
+size_t TLSInStream::readTLS(U8* buf, size_t len, bool wait)
{
int n;
diff --git a/common/rdr/TLSInStream.h b/common/rdr/TLSInStream.h
index b16d9f5a..5f9dee7f 100644
--- a/common/rdr/TLSInStream.h
+++ b/common/rdr/TLSInStream.h
@@ -36,17 +36,17 @@ namespace rdr {
TLSInStream(InStream* in, gnutls_session_t session);
virtual ~TLSInStream();
- int pos();
+ size_t pos();
private:
- int overrun(int itemSize, int nItems, bool wait);
- int readTLS(U8* buf, int len, bool wait);
+ size_t overrun(size_t itemSize, size_t nItems, bool wait);
+ size_t readTLS(U8* buf, size_t len, bool wait);
static ssize_t pull(gnutls_transport_ptr_t str, void* data, size_t size);
gnutls_session_t session;
InStream* in;
- int bufSize;
- int offset;
+ size_t bufSize;
+ size_t offset;
U8* start;
};
};
diff --git a/common/rdr/TLSOutStream.cxx b/common/rdr/TLSOutStream.cxx
index 44d2d9f3..7d7c3b56 100644
--- a/common/rdr/TLSOutStream.cxx
+++ b/common/rdr/TLSOutStream.cxx
@@ -75,7 +75,7 @@ TLSOutStream::~TLSOutStream()
delete [] start;
}
-int TLSOutStream::length()
+size_t TLSOutStream::length()
{
return offset + ptr - start;
}
@@ -84,7 +84,7 @@ void TLSOutStream::flush()
{
U8* sentUpTo = start;
while (sentUpTo < ptr) {
- int n = writeTLS(sentUpTo, ptr - sentUpTo);
+ size_t n = writeTLS(sentUpTo, ptr - sentUpTo);
sentUpTo += n;
offset += n;
}
@@ -93,20 +93,22 @@ void TLSOutStream::flush()
out->flush();
}
-int TLSOutStream::overrun(int itemSize, int nItems)
+size_t TLSOutStream::overrun(size_t itemSize, size_t nItems)
{
if (itemSize > bufSize)
throw Exception("TLSOutStream overrun: max itemSize exceeded");
flush();
- if (itemSize * nItems > end - ptr)
- nItems = (end - ptr) / itemSize;
+ size_t nAvail;
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
return nItems;
}
-int TLSOutStream::writeTLS(const U8* data, int length)
+size_t TLSOutStream::writeTLS(const U8* data, size_t length)
{
int n;
diff --git a/common/rdr/TLSOutStream.h b/common/rdr/TLSOutStream.h
index 81dd237a..71a7f3bf 100644
--- a/common/rdr/TLSOutStream.h
+++ b/common/rdr/TLSOutStream.h
@@ -36,20 +36,20 @@ namespace rdr {
virtual ~TLSOutStream();
void flush();
- int length();
+ size_t length();
protected:
- int overrun(int itemSize, int nItems);
+ size_t overrun(size_t itemSize, size_t nItems);
private:
- int writeTLS(const U8* data, int length);
+ size_t writeTLS(const U8* data, size_t length);
static ssize_t push(gnutls_transport_ptr_t str, const void* data, size_t size);
gnutls_session_t session;
OutStream* out;
- int bufSize;
+ size_t bufSize;
U8* start;
- int offset;
+ size_t offset;
};
};
diff --git a/common/rdr/ZlibInStream.cxx b/common/rdr/ZlibInStream.cxx
index 4053bd19..9fcfaf6b 100644
--- a/common/rdr/ZlibInStream.cxx
+++ b/common/rdr/ZlibInStream.cxx
@@ -26,7 +26,7 @@ using namespace rdr;
enum { DEFAULT_BUF_SIZE = 16384 };
-ZlibInStream::ZlibInStream(int bufSize_)
+ZlibInStream::ZlibInStream(size_t bufSize_)
: underlying(0), bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_SIZE), offset(0),
zs(NULL), bytesIn(0)
{
@@ -40,28 +40,28 @@ ZlibInStream::~ZlibInStream()
delete [] start;
}
-void ZlibInStream::setUnderlying(InStream* is, int bytesIn_)
+void ZlibInStream::setUnderlying(InStream* is, size_t bytesIn_)
{
underlying = is;
bytesIn = bytesIn_;
ptr = end = start;
}
-int ZlibInStream::pos()
+size_t ZlibInStream::pos()
{
return offset + ptr - start;
}
-void ZlibInStream::removeUnderlying()
+void ZlibInStream::flushUnderlying()
{
ptr = end = start;
- if (!underlying) return;
while (bytesIn > 0) {
decompress(true);
end = start; // throw away any data
}
- underlying = 0;
+
+ setUnderlying(NULL, 0);
}
void ZlibInStream::reset()
@@ -90,18 +90,16 @@ void ZlibInStream::init()
void ZlibInStream::deinit()
{
assert(zs != NULL);
- removeUnderlying();
+ setUnderlying(NULL, 0);
inflateEnd(zs);
delete zs;
zs = NULL;
}
-int ZlibInStream::overrun(int itemSize, int nItems, bool wait)
+size_t ZlibInStream::overrun(size_t itemSize, size_t nItems, bool wait)
{
if (itemSize > bufSize)
throw Exception("ZlibInStream overrun: max itemSize exceeded");
- if (!underlying)
- throw Exception("ZlibInStream overrun: no underlying stream");
if (end - ptr != 0)
memmove(start, ptr, end - ptr);
@@ -110,13 +108,15 @@ int ZlibInStream::overrun(int itemSize, int nItems, bool wait)
end -= ptr - start;
ptr = start;
- while (end - ptr < itemSize) {
+ while ((size_t)(end - ptr) < itemSize) {
if (!decompress(wait))
return 0;
}
- if (itemSize * nItems > end - ptr)
- nItems = (end - ptr) / itemSize;
+ size_t nAvail;
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
return nItems;
}
@@ -127,14 +127,17 @@ int ZlibInStream::overrun(int itemSize, int nItems, bool wait)
bool ZlibInStream::decompress(bool wait)
{
+ if (!underlying)
+ throw Exception("ZlibInStream overrun: no underlying stream");
+
zs->next_out = (U8*)end;
zs->avail_out = start + bufSize - end;
- int n = underlying->check(1, 1, wait);
+ size_t n = underlying->check(1, 1, wait);
if (n == 0) return false;
zs->next_in = (U8*)underlying->getptr();
zs->avail_in = underlying->getend() - underlying->getptr();
- if ((int)zs->avail_in > bytesIn)
+ if (zs->avail_in > bytesIn)
zs->avail_in = bytesIn;
int rc = inflate(zs, Z_SYNC_FLUSH);
diff --git a/common/rdr/ZlibInStream.h b/common/rdr/ZlibInStream.h
index 6bd4da4c..08784b0f 100644
--- a/common/rdr/ZlibInStream.h
+++ b/common/rdr/ZlibInStream.h
@@ -34,12 +34,12 @@ namespace rdr {
public:
- ZlibInStream(int bufSize=0);
+ ZlibInStream(size_t bufSize=0);
virtual ~ZlibInStream();
- void setUnderlying(InStream* is, int bytesIn);
- void removeUnderlying();
- int pos();
+ void setUnderlying(InStream* is, size_t bytesIn);
+ void flushUnderlying();
+ size_t pos();
void reset();
private:
@@ -47,14 +47,14 @@ namespace rdr {
void init();
void deinit();
- int overrun(int itemSize, int nItems, bool wait);
+ size_t overrun(size_t itemSize, size_t nItems, bool wait);
bool decompress(bool wait);
InStream* underlying;
- int bufSize;
- int offset;
+ size_t bufSize;
+ size_t offset;
z_stream_s* zs;
- int bytesIn;
+ size_t bytesIn;
U8* start;
};
diff --git a/common/rdr/ZlibOutStream.cxx b/common/rdr/ZlibOutStream.cxx
index dd3c2367..7a0d692c 100644
--- a/common/rdr/ZlibOutStream.cxx
+++ b/common/rdr/ZlibOutStream.cxx
@@ -33,7 +33,7 @@ using namespace rdr;
enum { DEFAULT_BUF_SIZE = 16384 };
-ZlibOutStream::ZlibOutStream(OutStream* os, int bufSize_, int compressLevel)
+ZlibOutStream::ZlibOutStream(OutStream* os, size_t bufSize_, int compressLevel)
: underlying(os), compressionLevel(compressLevel), newLevel(compressLevel),
bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_SIZE), offset(0)
{
@@ -75,7 +75,7 @@ void ZlibOutStream::setCompressionLevel(int level)
newLevel = level;
}
-int ZlibOutStream::length()
+size_t ZlibOutStream::length()
{
return offset + ptr - start;
}
@@ -98,7 +98,7 @@ void ZlibOutStream::flush()
ptr = start;
}
-int ZlibOutStream::overrun(int itemSize, int nItems)
+size_t ZlibOutStream::overrun(size_t itemSize, size_t nItems)
{
#ifdef ZLIBOUT_DEBUG
vlog.debug("overrun");
@@ -109,7 +109,7 @@ int ZlibOutStream::overrun(int itemSize, int nItems)
checkCompressionLevel();
- while (end - ptr < itemSize) {
+ while ((size_t)(end - ptr) < itemSize) {
zs->next_in = start;
zs->avail_in = ptr - start;
@@ -130,8 +130,10 @@ int ZlibOutStream::overrun(int itemSize, int nItems)
}
}
- if (itemSize * nItems > end - ptr)
- nItems = (end - ptr) / itemSize;
+ size_t nAvail;
+ nAvail = (end - ptr) / itemSize;
+ if (nAvail < nItems)
+ return nAvail;
return nItems;
}
diff --git a/common/rdr/ZlibOutStream.h b/common/rdr/ZlibOutStream.h
index 2d82a13b..11bb0468 100644
--- a/common/rdr/ZlibOutStream.h
+++ b/common/rdr/ZlibOutStream.h
@@ -35,25 +35,25 @@ namespace rdr {
public:
- ZlibOutStream(OutStream* os=0, int bufSize=0, int compressionLevel=-1);
+ ZlibOutStream(OutStream* os=0, size_t bufSize=0, int compressionLevel=-1);
virtual ~ZlibOutStream();
void setUnderlying(OutStream* os);
void setCompressionLevel(int level=-1);
void flush();
- int length();
+ size_t length();
private:
- int overrun(int itemSize, int nItems);
+ size_t overrun(size_t itemSize, size_t nItems);
void deflate(int flush);
void checkCompressionLevel();
OutStream* underlying;
int compressionLevel;
int newLevel;
- int bufSize;
- int offset;
+ size_t bufSize;
+ size_t offset;
z_stream_s* zs;
U8* start;
};
diff --git a/common/rfb/CMsgReader.cxx b/common/rfb/CMsgReader.cxx
index a9e12d70..52d40ce7 100644
--- a/common/rfb/CMsgReader.cxx
+++ b/common/rfb/CMsgReader.cxx
@@ -242,7 +242,8 @@ void CMsgReader::readExtendedClipboard(rdr::S32 len)
num++;
}
- zis.removeUnderlying();
+ zis.flushUnderlying();
+ zis.setUnderlying(NULL, 0);
handler->handleClipboardProvide(flags, lengths, buffers);
diff --git a/common/rfb/Configuration.cxx b/common/rfb/Configuration.cxx
index ea000600..ceaefccb 100644
--- a/common/rfb/Configuration.cxx
+++ b/common/rfb/Configuration.cxx
@@ -421,7 +421,7 @@ StringParameter::operator const char *() const {
// -=- BinaryParameter
BinaryParameter::BinaryParameter(const char* name_, const char* desc_,
- const void* v, int l, ConfigurationObject co)
+ const void* v, size_t l, ConfigurationObject co)
: VoidParameter(name_, desc_, co), value(0), length(0), def_value((char*)v), def_length(l) {
if (l) {
value = new char[l];
@@ -441,7 +441,7 @@ bool BinaryParameter::setParam(const char* v) {
return rdr::HexInStream::hexStrToBin(v, &value, &length);
}
-void BinaryParameter::setParam(const void* v, int len) {
+void BinaryParameter::setParam(const void* v, size_t len) {
LOCK_CONFIG;
if (immutable) return;
vlog.debug("set %s(Binary)", getName());
@@ -462,7 +462,7 @@ char* BinaryParameter::getValueStr() const {
return rdr::HexOutStream::binToHexStr(value, length);
}
-void BinaryParameter::getData(void** data_, int* length_) const {
+void BinaryParameter::getData(void** data_, size_t* length_) const {
LOCK_CONFIG;
if (length_) *length_ = length;
if (data_) {
diff --git a/common/rfb/Configuration.h b/common/rfb/Configuration.h
index 6197317b..e23e8a51 100644
--- a/common/rfb/Configuration.h
+++ b/common/rfb/Configuration.h
@@ -256,24 +256,25 @@ namespace rfb {
class BinaryParameter : public VoidParameter {
public:
- BinaryParameter(const char* name_, const char* desc_, const void* v, int l,
- ConfigurationObject co=ConfGlobal);
+ BinaryParameter(const char* name_, const char* desc_,
+ const void* v, size_t l,
+ ConfigurationObject co=ConfGlobal);
using VoidParameter::setParam;
virtual ~BinaryParameter();
virtual bool setParam(const char* value);
- virtual void setParam(const void* v, int l);
+ virtual void setParam(const void* v, size_t l);
virtual char* getDefaultStr() const;
virtual char* getValueStr() const;
// getData() will return length zero if there is no data
// NB: data may be set to zero, OR set to a zero-length buffer
- void getData(void** data, int* length) const;
+ void getData(void** data, size_t* length) const;
protected:
char* value;
- int length;
+ size_t length;
char* def_value;
- int def_length;
+ size_t def_length;
};
// -=- ParameterIterator
diff --git a/common/rfb/Cursor.cxx b/common/rfb/Cursor.cxx
index d7b536de..3ca69f7c 100644
--- a/common/rfb/Cursor.cxx
+++ b/common/rfb/Cursor.cxx
@@ -272,8 +272,7 @@ void RenderedCursor::update(PixelBuffer* framebuffer,
assert(cursor);
format = framebuffer->getPF();
- width_ = framebuffer->width();
- height_ = framebuffer->height();
+ setSize(framebuffer->width(), framebuffer->height());
rawOffset = pos.subtract(cursor->hotspot());
clippedRect = Rect(0, 0, cursor->width(), cursor->height())
diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx
index 57020fee..a29502ae 100644
--- a/common/rfb/EncodeManager.cxx
+++ b/common/rfb/EncodeManager.cxx
@@ -28,6 +28,7 @@
#include <rfb/SMsgWriter.h>
#include <rfb/UpdateTracker.h>
#include <rfb/LogWriter.h>
+#include <rfb/Exception.h>
#include <rfb/RawEncoder.h>
#include <rfb/RREEncoder.h>
@@ -1051,11 +1052,13 @@ void EncodeManager::OffsetPixelBuffer::update(const PixelFormat& pf,
int stride_)
{
format = pf;
- width_ = width;
- height_ = height;
// Forced cast. We never write anything though, so it should be safe.
- data = (rdr::U8*)data_;
- stride = stride_;
+ setBuffer(width, height, (rdr::U8*)data_, stride_);
+}
+
+rdr::U8* EncodeManager::OffsetPixelBuffer::getBufferRW(const Rect& r, int* stride)
+{
+ throw rfb::Exception("Invalid write attempt to OffsetPixelBuffer");
}
// Preprocessor generated, optimised methods
diff --git a/common/rfb/EncodeManager.h b/common/rfb/EncodeManager.h
index bdae9063..f8201c34 100644
--- a/common/rfb/EncodeManager.h
+++ b/common/rfb/EncodeManager.h
@@ -148,6 +148,9 @@ namespace rfb {
void update(const PixelFormat& pf, int width, int height,
const rdr::U8* data_, int stride);
+
+ private:
+ virtual rdr::U8* getBufferRW(const Rect& r, int* stride);
};
OffsetPixelBuffer offsetPixelBuffer;
diff --git a/common/rfb/Password.cxx b/common/rfb/Password.cxx
index 240c9d4f..e4a508c8 100644
--- a/common/rfb/Password.cxx
+++ b/common/rfb/Password.cxx
@@ -38,7 +38,7 @@ PlainPasswd::PlainPasswd() {}
PlainPasswd::PlainPasswd(char* pwd) : CharArray(pwd) {
}
-PlainPasswd::PlainPasswd(int len) : CharArray(len) {
+PlainPasswd::PlainPasswd(size_t len) : CharArray(len) {
}
PlainPasswd::PlainPasswd(const ObfuscatedPasswd& obfPwd) : CharArray(9) {
@@ -63,11 +63,11 @@ void PlainPasswd::replaceBuf(char* b) {
ObfuscatedPasswd::ObfuscatedPasswd() : length(0) {
}
-ObfuscatedPasswd::ObfuscatedPasswd(int len) : CharArray(len), length(len) {
+ObfuscatedPasswd::ObfuscatedPasswd(size_t len) : CharArray(len), length(len) {
}
ObfuscatedPasswd::ObfuscatedPasswd(const PlainPasswd& plainPwd) : CharArray(8), length(8) {
- int l = strlen(plainPwd.buf), i;
+ size_t l = strlen(plainPwd.buf), i;
for (i=0; i<8; i++)
buf[i] = i<l ? plainPwd.buf[i] : 0;
deskey(d3desObfuscationKey, EN0);
diff --git a/common/rfb/Password.h b/common/rfb/Password.h
index e5196ee0..712bc813 100644
--- a/common/rfb/Password.h
+++ b/common/rfb/Password.h
@@ -28,7 +28,7 @@ namespace rfb {
public:
PlainPasswd();
PlainPasswd(char* pwd);
- PlainPasswd(int len);
+ PlainPasswd(size_t len);
PlainPasswd(const ObfuscatedPasswd& obfPwd);
~PlainPasswd();
void replaceBuf(char* b);
@@ -37,10 +37,10 @@ namespace rfb {
class ObfuscatedPasswd : public CharArray {
public:
ObfuscatedPasswd();
- ObfuscatedPasswd(int l);
+ ObfuscatedPasswd(size_t l);
ObfuscatedPasswd(const PlainPasswd& plainPwd);
~ObfuscatedPasswd();
- int length;
+ size_t length;
};
}
diff --git a/common/rfb/PixelBuffer.cxx b/common/rfb/PixelBuffer.cxx
index 7f4c1ad3..fe406b96 100644
--- a/common/rfb/PixelBuffer.cxx
+++ b/common/rfb/PixelBuffer.cxx
@@ -31,12 +31,26 @@ using namespace rdr;
static LogWriter vlog("PixelBuffer");
+// We do a lot of byte offset calculations that assume the result fits
+// inside a signed 32 bit integer. Limit the maximum size of pixel
+// buffers so that these calculations never overflow.
+
+const int maxPixelBufferWidth = 16384;
+const int maxPixelBufferHeight = 16384;
+const int maxPixelBufferStride = 16384;
+
// -=- Generic pixel buffer class
PixelBuffer::PixelBuffer(const PixelFormat& pf, int w, int h)
- : format(pf), width_(w), height_(h) {}
-PixelBuffer::PixelBuffer() : width_(0), height_(0) {}
+ : format(pf), width_(0), height_(0)
+{
+ setSize(w, h);
+}
+
+PixelBuffer::PixelBuffer() : width_(0), height_(0)
+{
+}
PixelBuffer::~PixelBuffer() {}
@@ -53,7 +67,7 @@ PixelBuffer::getImage(void* imageBuf, const Rect& r, int outStride) const
if (!r.enclosed_by(getRect()))
throw rfb::Exception("Source rect %dx%d at %d,%d exceeds framebuffer %dx%d",
r.width(), r.height(),
- r.tl.x, r.tl.y, width_, height_);
+ r.tl.x, r.tl.y, width(), height());
data = getBuffer(r, &inStride);
@@ -89,7 +103,7 @@ void PixelBuffer::getImage(const PixelFormat& pf, void* imageBuf,
if (!r.enclosed_by(getRect()))
throw rfb::Exception("Source rect %dx%d at %d,%d exceeds framebuffer %dx%d",
r.width(), r.height(),
- r.tl.x, r.tl.y, width_, height_);
+ r.tl.x, r.tl.y, width(), height());
if (stride == 0)
stride = r.width();
@@ -100,6 +114,17 @@ void PixelBuffer::getImage(const PixelFormat& pf, void* imageBuf,
stride, srcStride);
}
+void PixelBuffer::setSize(int width, int height)
+{
+ if ((width < 0) || (width > maxPixelBufferWidth))
+ throw rfb::Exception("Invalid PixelBuffer width of %d pixels requested", width);
+ if ((height < 0) || (height > maxPixelBufferHeight))
+ throw rfb::Exception("Invalid PixelBuffer height of %d pixels requested", height);
+
+ width_ = width;
+ height_ = height;
+}
+
// -=- Modifiable generic pixel buffer class
ModifiablePixelBuffer::ModifiablePixelBuffer(const PixelFormat& pf,
@@ -124,7 +149,7 @@ void ModifiablePixelBuffer::fillRect(const Rect& r, const void* pix)
if (!r.enclosed_by(getRect()))
throw rfb::Exception("Destination rect %dx%d at %d,%d exceeds framebuffer %dx%d",
- r.width(), r.height(), r.tl.x, r.tl.y, width_, height_);
+ r.width(), r.height(), r.tl.x, r.tl.y, width(), height());
w = r.width();
h = r.height();
@@ -175,7 +200,7 @@ void ModifiablePixelBuffer::imageRect(const Rect& r,
if (!r.enclosed_by(getRect()))
throw rfb::Exception("Destination rect %dx%d at %d,%d exceeds framebuffer %dx%d",
r.width(), r.height(),
- r.tl.x, r.tl.y, width_, height_);
+ r.tl.x, r.tl.y, width(), height());
bytesPerPixel = getPF().bpp/8;
@@ -214,13 +239,13 @@ void ModifiablePixelBuffer::copyRect(const Rect &rect,
if (!drect.enclosed_by(getRect()))
throw rfb::Exception("Destination rect %dx%d at %d,%d exceeds framebuffer %dx%d",
drect.width(), drect.height(),
- drect.tl.x, drect.tl.y, width_, height_);
+ drect.tl.x, drect.tl.y, width(), height());
srect = drect.translate(move_by_delta.negate());
if (!srect.enclosed_by(getRect()))
throw rfb::Exception("Source rect %dx%d at %d,%d exceeds framebuffer %dx%d",
srect.width(), srect.height(),
- srect.tl.x, srect.tl.y, width_, height_);
+ srect.tl.x, srect.tl.y, width(), height());
bytesPerPixel = format.bpp/8;
@@ -275,7 +300,7 @@ void ModifiablePixelBuffer::imageRect(const PixelFormat& pf, const Rect &dest,
if (!dest.enclosed_by(getRect()))
throw rfb::Exception("Destination rect %dx%d at %d,%d exceeds framebuffer %dx%d",
dest.width(), dest.height(),
- dest.tl.x, dest.tl.y, width_, height_);
+ dest.tl.x, dest.tl.y, width(), height());
if (stride == 0)
stride = dest.width();
@@ -304,7 +329,7 @@ rdr::U8* FullFramePixelBuffer::getBufferRW(const Rect& r, int* stride_)
if (!r.enclosed_by(getRect()))
throw rfb::Exception("Pixel buffer request %dx%d at %d,%d exceeds framebuffer %dx%d",
r.width(), r.height(),
- r.tl.x, r.tl.y, width_, height_);
+ r.tl.x, r.tl.y, width(), height());
*stride_ = stride;
return &data[(r.tl.x + (r.tl.y * stride)) * (format.bpp/8)];
@@ -319,55 +344,77 @@ const rdr::U8* FullFramePixelBuffer::getBuffer(const Rect& r, int* stride_) cons
if (!r.enclosed_by(getRect()))
throw rfb::Exception("Pixel buffer request %dx%d at %d,%d exceeds framebuffer %dx%d",
r.width(), r.height(),
- r.tl.x, r.tl.y, width_, height_);
+ r.tl.x, r.tl.y, width(), height());
*stride_ = stride;
return &data[(r.tl.x + (r.tl.y * stride)) * (format.bpp/8)];
}
+void FullFramePixelBuffer::setBuffer(int width, int height,
+ rdr::U8* data_, int stride_)
+{
+ if ((width < 0) || (width > maxPixelBufferWidth))
+ throw rfb::Exception("Invalid PixelBuffer width of %d pixels requested", width);
+ if ((height < 0) || (height > maxPixelBufferHeight))
+ throw rfb::Exception("Invalid PixelBuffer height of %d pixels requested", height);
+ if ((stride_ < 0) || (stride_ > maxPixelBufferStride) || (stride_ < width))
+ throw rfb::Exception("Invalid PixelBuffer stride of %d pixels requested", stride_);
+ if ((width != 0) && (height != 0) && (data_ == NULL))
+ throw rfb::Exception("PixelBuffer requested without a valid memory area");
+
+ ModifiablePixelBuffer::setSize(width, height);
+ stride = stride_;
+ data = data_;
+}
+
+void FullFramePixelBuffer::setSize(int w, int h)
+{
+ // setBuffer() should be used
+ throw rfb::Exception("Invalid call to FullFramePixelBuffer::setSize()");
+}
+
// -=- Managed pixel buffer class
// Automatically allocates enough space for the specified format & area
ManagedPixelBuffer::ManagedPixelBuffer()
- : datasize(0)
+ : data_(NULL), datasize(0)
{
- checkDataSize();
-};
+}
ManagedPixelBuffer::ManagedPixelBuffer(const PixelFormat& pf, int w, int h)
- : FullFramePixelBuffer(pf, w, h, NULL, w), datasize(0)
+ : FullFramePixelBuffer(pf, 0, 0, NULL, 0), data_(NULL), datasize(0)
{
- checkDataSize();
-};
-
-ManagedPixelBuffer::~ManagedPixelBuffer() {
- if (data) delete [] data;
-};
+ setSize(w, h);
+}
+ManagedPixelBuffer::~ManagedPixelBuffer()
+{
+ if (data_)
+ delete [] data_;
+}
-void
-ManagedPixelBuffer::setPF(const PixelFormat &pf) {
- format = pf; checkDataSize();
-};
-void
-ManagedPixelBuffer::setSize(int w, int h) {
- width_ = w; height_ = h; stride = w; checkDataSize();
-};
+void ManagedPixelBuffer::setPF(const PixelFormat &pf)
+{
+ format = pf;
+ setSize(width(), height());
+}
+void ManagedPixelBuffer::setSize(int w, int h)
+{
+ unsigned long new_datasize = w * h * (format.bpp/8);
-inline void
-ManagedPixelBuffer::checkDataSize() {
- unsigned long new_datasize = width_ * height_ * (format.bpp/8);
+ new_datasize = w * h * (format.bpp/8);
if (datasize < new_datasize) {
- if (data) {
- delete [] data;
- datasize = 0; data = 0;
+ if (data_) {
+ delete [] data_;
+ data_ = NULL;
+ datasize = 0;
}
if (new_datasize) {
- data = new U8[new_datasize];
- if (!data)
- throw Exception("rfb::ManagedPixelBuffer unable to allocate buffer");
+ data_ = new U8[new_datasize];
datasize = new_datasize;
}
}
-};
+
+ setBuffer(w, h, data_, w);
+}
diff --git a/common/rfb/PixelBuffer.h b/common/rfb/PixelBuffer.h
index d89793f5..3e4018f9 100644
--- a/common/rfb/PixelBuffer.h
+++ b/common/rfb/PixelBuffer.h
@@ -90,7 +90,12 @@ namespace rfb {
protected:
PixelBuffer();
+ virtual void setSize(int width, int height);
+
+ protected:
PixelFormat format;
+
+ private:
int width_, height_;
};
@@ -154,7 +159,12 @@ namespace rfb {
protected:
FullFramePixelBuffer();
+ virtual void setBuffer(int width, int height, rdr::U8* data, int stride);
+ private:
+ virtual void setSize(int w, int h);
+
+ private:
rdr::U8* data;
int stride;
};
@@ -172,12 +182,9 @@ namespace rfb {
virtual void setPF(const PixelFormat &pf);
virtual void setSize(int w, int h);
- // Return the total number of bytes of pixel data in the buffer
- int dataLen() const { return width_ * height_ * (format.bpp/8); }
-
- protected:
+ private:
+ rdr::U8* data_; // Mirrors FullFramePixelBuffer::data
unsigned long datasize;
- void checkDataSize();
};
};
diff --git a/common/rfb/PixelFormat.cxx b/common/rfb/PixelFormat.cxx
index 883b0410..1b4ab1ba 100644
--- a/common/rfb/PixelFormat.cxx
+++ b/common/rfb/PixelFormat.cxx
@@ -81,7 +81,8 @@ PixelFormat::PixelFormat(int b, int d, bool e, bool t,
redMax(rm), greenMax(gm), blueMax(bm),
redShift(rs), greenShift(gs), blueShift(bs)
{
- assert(isSane());
+ if (!isSane())
+ throw Exception("invalid pixel format");
updateState();
}
@@ -205,6 +206,12 @@ bool PixelFormat::is888(void) const
return false;
if (blueMax != 255)
return false;
+ if ((redShift & 0x7) != 0)
+ return false;
+ if ((greenShift & 0x7) != 0)
+ return false;
+ if ((blueShift & 0x7) != 0)
+ return false;
return true;
}
@@ -678,7 +685,14 @@ bool PixelFormat::isSane(void)
return false;
totalBits = bits(redMax) + bits(greenMax) + bits(blueMax);
- if (totalBits > bpp)
+ if (totalBits > depth)
+ return false;
+
+ if ((bits(redMax) + redShift) > bpp)
+ return false;
+ if ((bits(greenMax) + greenShift) > bpp)
+ return false;
+ if ((bits(blueMax) + blueShift) > bpp)
return false;
if (((redMax << redShift) & (greenMax << greenShift)) != 0)
diff --git a/common/rfb/SMsgReader.cxx b/common/rfb/SMsgReader.cxx
index ab42e59a..dc7ddea6 100644
--- a/common/rfb/SMsgReader.cxx
+++ b/common/rfb/SMsgReader.cxx
@@ -293,7 +293,8 @@ void SMsgReader::readExtendedClipboard(rdr::S32 len)
num++;
}
- zis.removeUnderlying();
+ zis.flushUnderlying();
+ zis.setUnderlying(NULL, 0);
handler->handleClipboardProvide(flags, lengths, buffers);
diff --git a/common/rfb/TightDecoder.cxx b/common/rfb/TightDecoder.cxx
index 5b7c553d..ebc98b06 100644
--- a/common/rfb/TightDecoder.cxx
+++ b/common/rfb/TightDecoder.cxx
@@ -341,7 +341,8 @@ void TightDecoder::decodeRect(const Rect& r, const void* buffer,
zis[streamId].readBytes(netbuf, dataSize);
- zis[streamId].removeUnderlying();
+ zis[streamId].flushUnderlying();
+ zis[streamId].setUnderlying(NULL, 0);
delete ms;
bufptr = netbuf;
diff --git a/common/rfb/tightDecode.h b/common/rfb/tightDecode.h
index b6e86ed5..8f77aebd 100644
--- a/common/rfb/tightDecode.h
+++ b/common/rfb/tightDecode.h
@@ -56,15 +56,17 @@ TightDecoder::FilterGradient24(const rdr::U8 *inbuf,
int rectWidth = r.width();
for (y = 0; y < rectHeight; y++) {
- /* First pixel in a row */
- for (c = 0; c < 3; c++) {
- pix[c] = inbuf[y*rectWidth*3+c] + prevRow[c];
- thisRow[c] = pix[c];
- }
- pf.bufferFromRGB((rdr::U8*)&outbuf[y*stride], pix, 1);
+ for (x = 0; x < rectWidth; x++) {
+ /* First pixel in a row */
+ if (x == 0) {
+ for (c = 0; c < 3; c++) {
+ pix[c] = inbuf[y*rectWidth*3+c] + prevRow[c];
+ thisRow[c] = pix[c];
+ }
+ pf.bufferFromRGB((rdr::U8*)&outbuf[y*stride], pix, 1);
+ continue;
+ }
- /* Remaining pixels of a row */
- for (x = 1; x < rectWidth; x++) {
for (c = 0; c < 3; c++) {
est[c] = prevRow[x*3+c] + pix[c] - prevRow[(x-1)*3+c];
if (est[c] > 0xff) {
@@ -103,17 +105,20 @@ void TightDecoder::FilterGradient(const rdr::U8* inbuf,
int rectWidth = r.width();
for (y = 0; y < rectHeight; y++) {
- /* First pixel in a row */
- pf.rgbFromBuffer(pix, &inbuf[y*rectWidth], 1);
- for (c = 0; c < 3; c++)
- pix[c] += prevRow[c];
+ for (x = 0; x < rectWidth; x++) {
+ /* First pixel in a row */
+ if (x == 0) {
+ pf.rgbFromBuffer(pix, &inbuf[y*rectWidth], 1);
+ for (c = 0; c < 3; c++)
+ pix[c] += prevRow[c];
- memcpy(thisRow, pix, sizeof(pix));
+ memcpy(thisRow, pix, sizeof(pix));
- pf.bufferFromRGB((rdr::U8*)&outbuf[y*stride], pix, 1);
+ pf.bufferFromRGB((rdr::U8*)&outbuf[y*stride], pix, 1);
+
+ continue;
+ }
- /* Remaining pixels of a row */
- for (x = 1; x < rectWidth; x++) {
for (c = 0; c < 3; c++) {
est[c] = prevRow[x*3+c] + pix[c] - prevRow[(x-1)*3+c];
if (est[c] > 255) {
diff --git a/common/rfb/util.h b/common/rfb/util.h
index 8503519d..3100f90f 100644
--- a/common/rfb/util.h
+++ b/common/rfb/util.h
@@ -50,7 +50,7 @@ namespace rfb {
public:
CharArray() : buf(0) {}
CharArray(char* str) : buf(str) {} // note: assumes ownership
- CharArray(int len) {
+ CharArray(size_t len) {
buf = new char[len]();
}
~CharArray() {
diff --git a/common/rfb/zrleDecode.h b/common/rfb/zrleDecode.h
index 32b5c92b..f4325385 100644
--- a/common/rfb/zrleDecode.h
+++ b/common/rfb/zrleDecode.h
@@ -174,7 +174,8 @@ void ZRLE_DECODE (const Rect& r, rdr::InStream* is,
}
}
- zis->removeUnderlying();
+ zis->flushUnderlying();
+ zis->setUnderlying(NULL, 0);
}
#undef ZRLE_DECODE
diff --git a/tests/perf/encperf.cxx b/tests/perf/encperf.cxx
index e461197e..171564d1 100644
--- a/tests/perf/encperf.cxx
+++ b/tests/perf/encperf.cxx
@@ -71,11 +71,11 @@ class DummyOutStream : public rdr::OutStream {
public:
DummyOutStream();
- virtual int length();
+ virtual size_t length();
virtual void flush();
private:
- virtual int overrun(int itemSize, int nItems);
+ virtual size_t overrun(size_t itemSize, size_t nItems);
int offset;
rdr::U8 buf[131072];
@@ -141,7 +141,7 @@ DummyOutStream::DummyOutStream()
end = buf + sizeof(buf);
}
-int DummyOutStream::length()
+size_t DummyOutStream::length()
{
flush();
return offset;
@@ -153,10 +153,10 @@ void DummyOutStream::flush()
ptr = buf;
}
-int DummyOutStream::overrun(int itemSize, int nItems)
+size_t DummyOutStream::overrun(size_t itemSize, size_t nItems)
{
flush();
- if (itemSize * nItems > end - ptr)
+ if (itemSize * nItems > (size_t)(end - ptr))
nItems = (end - ptr) / itemSize;
return nItems;
}
diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index c847238d..acc3adcd 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -8,3 +8,6 @@ target_link_libraries(convertlf rfb)
add_executable(hostport hostport.cxx)
target_link_libraries(hostport rfb)
+
+add_executable(pixelformat pixelformat.cxx)
+target_link_libraries(pixelformat rfb)
diff --git a/tests/unit/pixelformat.cxx b/tests/unit/pixelformat.cxx
new file mode 100644
index 00000000..2e0c0bbb
--- /dev/null
+++ b/tests/unit/pixelformat.cxx
@@ -0,0 +1,188 @@
+/* Copyright 2019 Pierre Ossman <ossman@cendio.se> for Cendio AB
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+ * USA.
+ */
+
+#include <stdio.h>
+
+#include <rfb/PixelFormat.h>
+#include <rfb/Exception.h>
+
+static void doTest(bool should_fail, int b, int d, bool e, bool t,
+ int rm, int gm, int bm, int rs, int gs, int bs)
+{
+ rfb::PixelFormat* pf;
+
+ printf("PixelFormat(%d, %d, %s, %s, %d, %d, %d, %d, %d, %d): ",
+ b, d, e ? "true" : "false", t ? "true": "false",
+ rm, gm, bm, rs, gs, bs);
+
+ try {
+ pf = new rfb::PixelFormat(b, d, e, t, rm, gm, bm, rs, gs, bs);
+ } catch(rfb::Exception &e) {
+ if (should_fail)
+ printf("OK");
+ else
+ printf("FAILED");
+ printf("\n");
+ fflush(stdout);
+ return;
+ }
+
+ delete pf;
+
+ if (should_fail)
+ printf("FAILED");
+ else
+ printf("OK");
+ printf("\n");
+ fflush(stdout);
+}
+
+static void do888Test(bool expected, int b, int d, bool e, bool t,
+ int rm, int gm, int bm, int rs, int gs, int bs)
+{
+ rfb::PixelFormat* pf;
+
+ printf("PixelFormat(%d, %d, %s, %s, %d, %d, %d, %d, %d, %d): ",
+ b, d, e ? "true" : "false", t ? "true": "false",
+ rm, gm, bm, rs, gs, bs);
+
+ pf = new rfb::PixelFormat(b, d, e, t, rm, gm, bm, rs, gs, bs);
+
+ if (pf->is888() == expected)
+ printf("OK");
+ else
+ printf("FAILED");
+ printf("\n");
+ fflush(stdout);
+
+ delete pf;
+}
+
+static void sanityTests()
+{
+ printf("Sanity checks:\n\n");
+
+ /* Normal true color formats */
+
+ doTest(false, 32, 24, false, true, 255, 255, 255, 0, 8, 16);
+ doTest(false, 32, 24, false, true, 255, 255, 255, 24, 16, 8);
+
+ doTest(false, 16, 16, false, true, 15, 31, 15, 0, 5, 11);
+
+ doTest(false, 8, 8, false, true, 3, 7, 3, 0, 2, 5);
+
+ /* Excessive bpp */
+
+ doTest(false, 32, 16, false, true, 15, 31, 15, 0, 5, 11);
+
+ doTest(false, 16, 16, false, true, 15, 31, 15, 0, 5, 11);
+
+ doTest(false, 32, 8, false, true, 3, 7, 3, 0, 2, 5);
+
+ doTest(false, 16, 8, false, true, 3, 7, 3, 0, 2, 5);
+
+ /* Colour map */
+
+ doTest(false, 8, 8, false, false, 0, 0, 0, 0, 0, 0);
+
+ /* Invalid bpp */
+
+ doTest(true, 64, 24, false, true, 255, 255, 255, 0, 8, 16);
+
+ doTest(true, 18, 16, false, true, 15, 31, 15, 0, 5, 11);
+
+ doTest(true, 3, 3, false, true, 1, 1, 1, 0, 1, 2);
+
+ /* Invalid depth */
+
+ doTest(true, 16, 24, false, true, 15, 31, 15, 0, 5, 11);
+
+ doTest(true, 8, 24, false, true, 3, 7, 3, 0, 2, 5);
+ doTest(true, 8, 16, false, true, 3, 7, 3, 0, 2, 5);
+
+ doTest(true, 32, 24, false, false, 0, 0, 0, 0, 0, 0);
+
+ /* Invalid max values */
+
+ doTest(true, 32, 24, false, true, 254, 255, 255, 0, 8, 16);
+ doTest(true, 32, 24, false, true, 255, 253, 255, 0, 8, 16);
+ doTest(true, 32, 24, false, true, 255, 255, 252, 0, 8, 16);
+
+ doTest(true, 32, 24, false, true, 511, 127, 127, 0, 16, 20);
+ doTest(true, 32, 24, false, true, 127, 511, 127, 0, 4, 20);
+ doTest(true, 32, 24, false, true, 127, 127, 511, 0, 4, 8);
+
+ /* Insufficient depth */
+
+ doTest(true, 32, 16, false, true, 255, 255, 255, 0, 8, 16);
+
+ /* Invalid shift values */
+
+ doTest(true, 32, 24, false, true, 255, 255, 255, 25, 8, 16);
+ doTest(true, 32, 24, false, true, 255, 255, 255, 0, 25, 16);
+ doTest(true, 32, 24, false, true, 255, 255, 255, 0, 8, 25);
+
+ /* Overlapping channels */
+
+ doTest(true, 32, 24, false, true, 255, 255, 255, 0, 7, 16);
+ doTest(true, 32, 24, false, true, 255, 255, 255, 0, 8, 15);
+ doTest(true, 32, 24, false, true, 255, 255, 255, 0, 16, 7);
+
+ printf("\n");
+}
+
+void is888Tests()
+{
+ printf("Simple format detection:\n\n");
+
+ /* Positive cases */
+
+ do888Test(true, 32, 24, false, true, 255, 255, 255, 0, 8, 16);
+ do888Test(true, 32, 24, false, true, 255, 255, 255, 24, 16, 8);
+ do888Test(true, 32, 24, false, true, 255, 255, 255, 24, 8, 0);
+
+ /* Low depth */
+
+ do888Test(false, 32, 16, false, true, 15, 31, 15, 0, 8, 16);
+ do888Test(false, 32, 8, false, true, 3, 7, 3, 0, 8, 16);
+
+ /* Low bpp and depth */
+
+ do888Test(false, 16, 16, false, true, 15, 31, 15, 0, 5, 11);
+ do888Test(false, 8, 8, false, true, 3, 7, 3, 0, 2, 5);
+
+ /* Colour map */
+
+ do888Test(false, 8, 8, false, false, 0, 0, 0, 0, 0, 0);
+
+ /* Odd shifts */
+
+ do888Test(false, 32, 24, false, true, 255, 255, 255, 0, 8, 18);
+ do888Test(false, 32, 24, false, true, 255, 255, 255, 0, 11, 24);
+ do888Test(false, 32, 24, false, true, 255, 255, 255, 4, 16, 24);
+
+ printf("\n");
+}
+
+int main(int argc, char** argv)
+{
+ sanityTests();
+ is888Tests();
+
+ return 0;
+}
diff --git a/unix/x0vncserver/XPixelBuffer.cxx b/unix/x0vncserver/XPixelBuffer.cxx
index 4769b651..f0b06967 100644
--- a/unix/x0vncserver/XPixelBuffer.cxx
+++ b/unix/x0vncserver/XPixelBuffer.cxx
@@ -50,13 +50,8 @@ XPixelBuffer::XPixelBuffer(Display *dpy, ImageFactory &factory,
ffs(m_image->xim->blue_mask) - 1);
// Set up the remaining data of the parent class.
- width_ = rect.width();
- height_ = rect.height();
- data = (rdr::U8 *)m_image->xim->data;
-
- // Calculate the distance in pixels between two subsequent scan
- // lines of the framebuffer. This may differ from image width.
- stride = m_image->xim->bytes_per_line * 8 / m_image->xim->bits_per_pixel;
+ setBuffer(rect.width(), rect.height(), (rdr::U8 *)m_image->xim->data,
+ m_image->xim->bytes_per_line * 8 / m_image->xim->bits_per_pixel);
// Get initial screen image from the X display.
m_image->get(DefaultRootWindow(m_dpy), m_offsetLeft, m_offsetTop);
diff --git a/unix/xserver/hw/vnc/XserverDesktop.cc b/unix/xserver/hw/vnc/XserverDesktop.cc
index 4edffec7..5c8b4cef 100644
--- a/unix/xserver/hw/vnc/XserverDesktop.cc
+++ b/unix/xserver/hw/vnc/XserverDesktop.cc
@@ -75,7 +75,7 @@ XserverDesktop::XserverDesktop(int screenIndex_,
void* fbptr, int stride)
: screenIndex(screenIndex_),
server(0), listeners(listeners_),
- directFbptr(true),
+ shadowFramebuffer(NULL),
queryConnectId(0), queryConnectTimer(this)
{
format = pf;
@@ -97,8 +97,8 @@ XserverDesktop::~XserverDesktop()
delete listeners.back();
listeners.pop_back();
}
- if (!directFbptr)
- delete [] data;
+ if (shadowFramebuffer)
+ delete [] shadowFramebuffer;
delete server;
}
@@ -116,22 +116,18 @@ void XserverDesktop::setFramebuffer(int w, int h, void* fbptr, int stride_)
{
ScreenSet layout;
- width_ = w;
- height_ = h;
-
- if (!directFbptr) {
- delete [] data;
- directFbptr = true;
+ if (shadowFramebuffer) {
+ delete [] shadowFramebuffer;
+ shadowFramebuffer = NULL;
}
if (!fbptr) {
- fbptr = new rdr::U8[w * h * (format.bpp/8)];
+ shadowFramebuffer = new rdr::U8[w * h * (format.bpp/8)];
+ fbptr = shadowFramebuffer;
stride_ = w;
- directFbptr = false;
}
- data = (rdr::U8*)fbptr;
- stride = stride_;
+ setBuffer(w, h, (rdr::U8*)fbptr, stride_);
vncSetGlueContext(screenIndex);
layout = ::computeScreenLayout(&outputIdMap);
@@ -492,7 +488,7 @@ void XserverDesktop::handleClipboardData(const char* data_)
void XserverDesktop::grabRegion(const rfb::Region& region)
{
- if (directFbptr)
+ if (shadowFramebuffer == NULL)
return;
std::vector<rfb::Rect> rects;
diff --git a/unix/xserver/hw/vnc/XserverDesktop.h b/unix/xserver/hw/vnc/XserverDesktop.h
index 6c670689..cc50f9e9 100644
--- a/unix/xserver/hw/vnc/XserverDesktop.h
+++ b/unix/xserver/hw/vnc/XserverDesktop.h
@@ -118,7 +118,7 @@ private:
int screenIndex;
rfb::VNCServer* server;
std::list<network::SocketListener*> listeners;
- bool directFbptr;
+ rdr::U8* shadowFramebuffer;
uint32_t queryConnectId;
network::Socket* queryConnectSocket;
diff --git a/vncviewer/PlatformPixelBuffer.cxx b/vncviewer/PlatformPixelBuffer.cxx
index ff1935e7..61f7b743 100644
--- a/vncviewer/PlatformPixelBuffer.cxx
+++ b/vncviewer/PlatformPixelBuffer.cxx
@@ -36,7 +36,7 @@ static rfb::LogWriter vlog("PlatformPixelBuffer");
PlatformPixelBuffer::PlatformPixelBuffer(int width, int height) :
FullFramePixelBuffer(rfb::PixelFormat(32, 24, false, true,
255, 255, 255, 16, 8, 0),
- width, height, NULL, 0),
+ 0, 0, NULL, 0),
Surface(width, height)
#if !defined(WIN32) && !defined(__APPLE__)
, shminfo(NULL), xim(NULL)
@@ -56,14 +56,13 @@ PlatformPixelBuffer::PlatformPixelBuffer(int width, int height) :
vlog.debug("Using standard XImage");
}
- data = (rdr::U8*)xim->data;
- stride = xim->bytes_per_line / (getPF().bpp/8);
+ setBuffer(width, height, (rdr::U8*)xim->data,
+ xim->bytes_per_line / (getPF().bpp/8));
// On X11, the Pixmap backing this Surface is uninitialized.
clear(0, 0, 0);
#else
- FullFramePixelBuffer::data = (rdr::U8*)Surface::data;
- stride = width;
+ setBuffer(width, height, (rdr::U8*)Surface::data, width);
#endif
}
diff --git a/win/rfb_win32/DIBSectionBuffer.cxx b/win/rfb_win32/DIBSectionBuffer.cxx
index e2b0d641..e00cf233 100644
--- a/win/rfb_win32/DIBSectionBuffer.cxx
+++ b/win/rfb_win32/DIBSectionBuffer.cxx
@@ -52,39 +52,28 @@ void DIBSectionBuffer::setPF(const PixelFormat& pf) {
if (!pf.trueColour)
throw rfb::Exception("palette format not supported");
format = pf;
- recreateBuffer();
+ setSize(width(), height());
}
-void DIBSectionBuffer::setSize(int w, int h) {
- if (width_ == w && height_ == h) {
- vlog.debug("size unchanged by setSize()");
- return;
- }
- width_ = w;
- height_ = h;
- recreateBuffer();
-}
-
-
inline void initMaxAndShift(DWORD mask, int* max, int* shift) {
for ((*shift) = 0; (mask & 1) == 0; (*shift)++) mask >>= 1;
(*max) = (rdr::U16)mask;
}
-void DIBSectionBuffer::recreateBuffer() {
+void DIBSectionBuffer::setSize(int w, int h) {
HBITMAP new_bitmap = 0;
rdr::U8* new_data = 0;
- if (width_ && height_ && (format.depth != 0)) {
+ if (w && h && (format.depth != 0)) {
BitmapInfo bi;
memset(&bi, 0, sizeof(bi));
UINT iUsage = DIB_RGB_COLORS;
bi.bmiHeader.biSize = sizeof(BITMAPINFOHEADER);
bi.bmiHeader.biBitCount = format.bpp;
- bi.bmiHeader.biSizeImage = (format.bpp / 8) * width_ * height_;
+ bi.bmiHeader.biSizeImage = (format.bpp / 8) * w * h;
bi.bmiHeader.biPlanes = 1;
- bi.bmiHeader.biWidth = width_;
- bi.bmiHeader.biHeight = -height_;
+ bi.bmiHeader.biWidth = w;
+ bi.bmiHeader.biHeight = -h;
bi.bmiHeader.biCompression = (format.bpp > 8) ? BI_BITFIELDS : BI_RGB;
bi.mask.red = format.pixelFromRGB((rdr::U16)~0, 0, 0);
bi.mask.green = format.pixelFromRGB(0, (rdr::U16)~0, 0);
@@ -115,12 +104,12 @@ void DIBSectionBuffer::recreateBuffer() {
if (device) {
BitmapDC src_dev(device, bitmap);
BitmapDC dest_dev(device, new_bitmap);
- BitBlt(dest_dev, 0, 0, width_, height_, src_dev, 0, 0, SRCCOPY);
+ BitBlt(dest_dev, 0, 0, w, h, src_dev, 0, 0, SRCCOPY);
} else {
WindowDC wndDC(window);
BitmapDC src_dev(wndDC, bitmap);
BitmapDC dest_dev(wndDC, new_bitmap);
- BitBlt(dest_dev, 0, 0, width_, height_, src_dev, 0, 0, SRCCOPY);
+ BitBlt(dest_dev, 0, 0, w, h, src_dev, 0, 0, SRCCOPY);
}
}
@@ -128,17 +117,17 @@ void DIBSectionBuffer::recreateBuffer() {
// Delete the old bitmap
DeleteObject(bitmap);
bitmap = 0;
- data = 0;
+ setBuffer(0, 0, NULL, 0);
}
if (new_bitmap) {
int bpp, depth;
int redMax, greenMax, blueMax;
int redShift, greenShift, blueShift;
+ int new_stride;
// Set up the new bitmap
bitmap = new_bitmap;
- data = new_data;
// Determine the *actual* DIBSection format
DIBSECTION ds;
@@ -147,14 +136,16 @@ void DIBSectionBuffer::recreateBuffer() {
// Correct the "stride" of the DIB
// *** This code DWORD aligns each row - is that right???
- stride = width_;
- int bytesPerRow = stride * format.bpp/8;
+ new_stride = w;
+ int bytesPerRow = new_stride * format.bpp/8;
if (bytesPerRow % 4) {
bytesPerRow += 4 - (bytesPerRow % 4);
- stride = (bytesPerRow * 8) / format.bpp;
- vlog.info("adjusting DIB stride: %d to %d", width_, stride);
+ new_stride = (bytesPerRow * 8) / format.bpp;
+ vlog.info("adjusting DIB stride: %d to %d", w, new_stride);
}
+ setBuffer(w, h, new_data, new_stride);
+
// Calculate the PixelFormat for the DIB
bpp = depth = ds.dsBm.bmBitsPixel;
diff --git a/win/rfb_win32/Registry.cxx b/win/rfb_win32/Registry.cxx
index 9cd50184..5d78c4c4 100644
--- a/win/rfb_win32/Registry.cxx
+++ b/win/rfb_win32/Registry.cxx
@@ -146,7 +146,7 @@ void RegKey::setString(const TCHAR* valname, const TCHAR* value) const {
if (result != ERROR_SUCCESS) throw rdr::SystemException("setString", result);
}
-void RegKey::setBinary(const TCHAR* valname, const void* value, int length) const {
+void RegKey::setBinary(const TCHAR* valname, const void* value, size_t length) const {
LONG result = RegSetValueEx(key, valname, 0, REG_BINARY, (const BYTE*)value, length);
if (result != ERROR_SUCCESS) throw rdr::SystemException("setBinary", result);
}
@@ -169,12 +169,12 @@ TCHAR* RegKey::getString(const TCHAR* valname, const TCHAR* def) const {
}
}
-void RegKey::getBinary(const TCHAR* valname, void** data, int* length) const {
+void RegKey::getBinary(const TCHAR* valname, void** data, size_t* length) const {
TCharArray hex(getRepresentation(valname));
if (!rdr::HexInStream::hexStrToBin(CStr(hex.buf), (char**)data, length))
throw rdr::Exception("getBinary failed");
}
-void RegKey::getBinary(const TCHAR* valname, void** data, int* length, void* def, int deflen) const {
+void RegKey::getBinary(const TCHAR* valname, void** data, size_t* length, void* def, size_t deflen) const {
try {
getBinary(valname, data, length);
} catch(rdr::Exception&) {
diff --git a/win/rfb_win32/Registry.h b/win/rfb_win32/Registry.h
index 68d535cd..2bb16911 100644
--- a/win/rfb_win32/Registry.h
+++ b/win/rfb_win32/Registry.h
@@ -71,15 +71,15 @@ namespace rfb {
void setExpandString(const TCHAR* valname, const TCHAR* s) const;
void setString(const TCHAR* valname, const TCHAR* s) const;
- void setBinary(const TCHAR* valname, const void* data, int length) const;
+ void setBinary(const TCHAR* valname, const void* data, size_t length) const;
void setInt(const TCHAR* valname, int i) const;
void setBool(const TCHAR* valname, bool b) const;
TCHAR* getString(const TCHAR* valname) const;
TCHAR* getString(const TCHAR* valname, const TCHAR* def) const;
- void getBinary(const TCHAR* valname, void** data, int* length) const;
- void getBinary(const TCHAR* valname, void** data, int* length, void* def, int deflength) const;
+ void getBinary(const TCHAR* valname, void** data, size_t* length) const;
+ void getBinary(const TCHAR* valname, void** data, size_t* length, void* def, size_t deflength) const;
int getInt(const TCHAR* valname) const;
int getInt(const TCHAR* valname, int def) const;