diff options
author | Pierre Ossman <ossman@cendio.se> | 2019-12-20 07:39:06 +0100 |
---|---|---|
committer | Pierre Ossman <ossman@cendio.se> | 2019-12-20 07:39:06 +0100 |
commit | d461f7fdb8b01f655260ea2f495ece700f3c9898 (patch) | |
tree | 5113b5a2844de4ce1469864d56c9786b3c4d63b0 | |
parent | c59f195d0ef2922bc163ac7cabfc94b9f1b25860 (diff) | |
parent | 05e28490873a861379c943bf616614b78b558b89 (diff) | |
download | tigervnc-d461f7fdb8b01f655260ea2f495ece700f3c9898.tar.gz tigervnc-d461f7fdb8b01f655260ea2f495ece700f3c9898.zip |
Merge branch 'secfix' of https://github.com/CendioOssman/tigervnc
51 files changed, 633 insertions, 401 deletions
diff --git a/common/rdr/FdInStream.cxx b/common/rdr/FdInStream.cxx index 1b9a322a..1730d6d1 100644 --- a/common/rdr/FdInStream.cxx +++ b/common/rdr/FdInStream.cxx @@ -56,7 +56,7 @@ using namespace rdr; enum { DEFAULT_BUF_SIZE = 8192, MIN_BULK_SIZE = 1024 }; -FdInStream::FdInStream(int fd_, int timeoutms_, int bufSize_, +FdInStream::FdInStream(int fd_, int timeoutms_, size_t bufSize_, bool closeWhenDone_) : fd(fd_), closeWhenDone(closeWhenDone_), timeoutms(timeoutms_), blockCallback(0), @@ -67,7 +67,7 @@ FdInStream::FdInStream(int fd_, int timeoutms_, int bufSize_, } FdInStream::FdInStream(int fd_, FdInStreamBlockCallback* blockCallback_, - int bufSize_) + size_t bufSize_) : fd(fd_), timeoutms(0), blockCallback(blockCallback_), timing(false), timeWaitedIn100us(5), timedKbits(0), bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_SIZE), offset(0) @@ -92,12 +92,12 @@ void FdInStream::setBlockCallback(FdInStreamBlockCallback* blockCallback_) timeoutms = 0; } -int FdInStream::pos() +size_t FdInStream::pos() { return offset + ptr - start; } -void FdInStream::readBytes(void* data, int length) +void FdInStream::readBytes(void* data, size_t length) { if (length < MIN_BULK_SIZE) { InStream::readBytes(data, length); @@ -106,7 +106,7 @@ void FdInStream::readBytes(void* data, int length) U8* dataPtr = (U8*)data; - int n = end - ptr; + size_t n = end - ptr; if (n > length) n = length; memcpy(dataPtr, ptr, n); @@ -123,7 +123,7 @@ void FdInStream::readBytes(void* data, int length) } -int FdInStream::overrun(int itemSize, int nItems, bool wait) +size_t FdInStream::overrun(size_t itemSize, size_t nItems, bool wait) { if (itemSize > bufSize) throw Exception("FdInStream overrun: max itemSize exceeded"); @@ -135,8 +135,8 @@ int FdInStream::overrun(int itemSize, int nItems, bool wait) end -= ptr - start; ptr = start; - int bytes_to_read; - while (end < start + itemSize) { + size_t bytes_to_read; + while ((size_t)(end - start) < itemSize) { bytes_to_read = start + bufSize - end; if (!timing) { // When not timing, we must be careful not to read too much @@ -147,13 +147,15 @@ int FdInStream::overrun(int itemSize, int nItems, bool wait) // bytes is ineffecient. bytes_to_read = vncmin(bytes_to_read, vncmax(itemSize*nItems, 8)); } - int n = readWithTimeoutOrCallback((U8*)end, bytes_to_read, wait); + size_t n = readWithTimeoutOrCallback((U8*)end, bytes_to_read, wait); if (n == 0) return 0; end += n; } - if (itemSize * nItems > end - ptr) - nItems = (end - ptr) / itemSize; + size_t nAvail; + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; return nItems; } @@ -171,7 +173,7 @@ int FdInStream::overrun(int itemSize, int nItems, bool wait) // returning EINTR. // -int FdInStream::readWithTimeoutOrCallback(void* buf, int len, bool wait) +size_t FdInStream::readWithTimeoutOrCallback(void* buf, size_t len, bool wait) { struct timeval before, after; if (timing) diff --git a/common/rdr/FdInStream.h b/common/rdr/FdInStream.h index b4c87653..d99ad3ce 100644 --- a/common/rdr/FdInStream.h +++ b/common/rdr/FdInStream.h @@ -37,16 +37,17 @@ namespace rdr { public: - FdInStream(int fd, int timeoutms=-1, int bufSize=0, + FdInStream(int fd, int timeoutms=-1, size_t bufSize=0, bool closeWhenDone_=false); - FdInStream(int fd, FdInStreamBlockCallback* blockCallback, int bufSize=0); + FdInStream(int fd, FdInStreamBlockCallback* blockCallback, + size_t bufSize=0); virtual ~FdInStream(); void setTimeout(int timeoutms); void setBlockCallback(FdInStreamBlockCallback* blockCallback); int getFd() { return fd; } - int pos(); - void readBytes(void* data, int length); + size_t pos(); + void readBytes(void* data, size_t length); void startTiming(); void stopTiming(); @@ -54,10 +55,10 @@ namespace rdr { unsigned int timeWaited() { return timeWaitedIn100us; } protected: - int overrun(int itemSize, int nItems, bool wait); + size_t overrun(size_t itemSize, size_t nItems, bool wait); private: - int readWithTimeoutOrCallback(void* buf, int len, bool wait=true); + size_t readWithTimeoutOrCallback(void* buf, size_t len, bool wait=true); int fd; bool closeWhenDone; @@ -68,8 +69,8 @@ namespace rdr { unsigned int timeWaitedIn100us; unsigned int timedKbits; - int bufSize; - int offset; + size_t bufSize; + size_t offset; U8* start; }; diff --git a/common/rdr/FdOutStream.cxx b/common/rdr/FdOutStream.cxx index cf857f85..f5d07e4b 100644 --- a/common/rdr/FdOutStream.cxx +++ b/common/rdr/FdOutStream.cxx @@ -51,7 +51,7 @@ using namespace rdr; enum { DEFAULT_BUF_SIZE = 16384 }; -FdOutStream::FdOutStream(int fd_, bool blocking_, int timeoutms_, int bufSize_) +FdOutStream::FdOutStream(int fd_, bool blocking_, int timeoutms_, size_t bufSize_) : fd(fd_), blocking(blocking_), timeoutms(timeoutms_), bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_SIZE), offset(0) { @@ -79,7 +79,7 @@ void FdOutStream::setBlocking(bool blocking_) { blocking = blocking_; } -int FdOutStream::length() +size_t FdOutStream::length() { return offset + ptr - sentUpTo; } @@ -97,9 +97,9 @@ unsigned FdOutStream::getIdleTime() void FdOutStream::flush() { while (sentUpTo < ptr) { - int n = writeWithTimeout((const void*) sentUpTo, - ptr - sentUpTo, - blocking? timeoutms : 0); + size_t n = writeWithTimeout((const void*) sentUpTo, + ptr - sentUpTo, + blocking? timeoutms : 0); // Timeout? if (n == 0) { @@ -120,7 +120,7 @@ void FdOutStream::flush() } -int FdOutStream::overrun(int itemSize, int nItems) +size_t FdOutStream::overrun(size_t itemSize, size_t nItems) { if (itemSize > bufSize) throw Exception("FdOutStream overrun: max itemSize exceeded"); @@ -129,10 +129,10 @@ int FdOutStream::overrun(int itemSize, int nItems) flush(); // Still not enough space? - if (itemSize > end - ptr) { + if (itemSize > (size_t)(end - ptr)) { // Can we shuffle things around? // (don't do this if it gains us less than 25%) - if ((sentUpTo - start > bufSize / 4) && + if (((size_t)(sentUpTo - start) > bufSize / 4) && (itemSize < bufSize - (ptr - sentUpTo))) { memmove(start, sentUpTo, ptr - sentUpTo); ptr = start + (ptr - sentUpTo); @@ -149,9 +149,10 @@ int FdOutStream::overrun(int itemSize, int nItems) } } - // Can we fit all the items asked for? - if (itemSize * nItems > end - ptr) - nItems = (end - ptr) / itemSize; + size_t nAvail; + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; return nItems; } @@ -166,7 +167,7 @@ int FdOutStream::overrun(int itemSize, int nItems) // select() and send() returning EINTR. // -int FdOutStream::writeWithTimeout(const void* data, int length, int timeoutms) +size_t FdOutStream::writeWithTimeout(const void* data, size_t length, int timeoutms) { int n; diff --git a/common/rdr/FdOutStream.h b/common/rdr/FdOutStream.h index b7f6cb01..ed84fdb5 100644 --- a/common/rdr/FdOutStream.h +++ b/common/rdr/FdOutStream.h @@ -34,7 +34,7 @@ namespace rdr { public: - FdOutStream(int fd, bool blocking=true, int timeoutms=-1, int bufSize=0); + FdOutStream(int fd, bool blocking=true, int timeoutms=-1, size_t bufSize=0); virtual ~FdOutStream(); void setTimeout(int timeoutms); @@ -42,20 +42,20 @@ namespace rdr { int getFd() { return fd; } void flush(); - int length(); + size_t length(); int bufferUsage(); unsigned getIdleTime(); private: - int overrun(int itemSize, int nItems); - int writeWithTimeout(const void* data, int length, int timeoutms); + size_t overrun(size_t itemSize, size_t nItems); + size_t writeWithTimeout(const void* data, size_t length, int timeoutms); int fd; bool blocking; int timeoutms; - int bufSize; - int offset; + size_t bufSize; + size_t offset; U8* start; U8* sentUpTo; struct timeval lastWrite; diff --git a/common/rdr/FileInStream.cxx b/common/rdr/FileInStream.cxx index 3acdfd45..bdb05a3a 100644 --- a/common/rdr/FileInStream.cxx +++ b/common/rdr/FileInStream.cxx @@ -48,7 +48,7 @@ void FileInStream::reset(void) { ptr = end = b; } -int FileInStream::pos() +size_t FileInStream::pos() { if (!file) throw Exception("File is not open"); @@ -56,9 +56,9 @@ int FileInStream::pos() return ftell(file) + ptr - b; } -int FileInStream::overrun(int itemSize, int nItems, bool wait) +size_t FileInStream::overrun(size_t itemSize, size_t nItems, bool wait) { - if (itemSize > (int)sizeof(b)) + if (itemSize > sizeof(b)) throw Exception("FileInStream overrun: max itemSize exceeded"); if (end - ptr != 0) @@ -68,7 +68,7 @@ int FileInStream::overrun(int itemSize, int nItems, bool wait) ptr = b; - while (end < b + itemSize) { + while ((size_t)(end - b) < itemSize) { size_t n = fread((U8 *)end, b + sizeof(b) - end, 1, file); if (n == 0) { if (ferror(file)) @@ -80,8 +80,10 @@ int FileInStream::overrun(int itemSize, int nItems, bool wait) end += b + sizeof(b) - end; } - if (itemSize * nItems > end - ptr) - nItems = (end - ptr) / itemSize; + size_t nAvail; + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; return nItems; } diff --git a/common/rdr/FileInStream.h b/common/rdr/FileInStream.h index ace04f37..a33c765e 100644 --- a/common/rdr/FileInStream.h +++ b/common/rdr/FileInStream.h @@ -35,10 +35,10 @@ namespace rdr { void reset(void); - int pos(); + size_t pos(); protected: - int overrun(int itemSize, int nItems, bool wait = true); + size_t overrun(size_t itemSize, size_t nItems, bool wait = true); private: U8 b[131072]; diff --git a/common/rdr/FixedMemOutStream.h b/common/rdr/FixedMemOutStream.h deleted file mode 100644 index e4ec52cb..00000000 --- a/common/rdr/FixedMemOutStream.h +++ /dev/null @@ -1,52 +0,0 @@ -/* Copyright (C) 2002-2005 RealVNC Ltd. All Rights Reserved. - * - * This is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This software is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this software; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, - * USA. - */ - -// -// A FixedMemOutStream writes to a buffer of a fixed length. -// - -#ifndef __RDR_FIXEDMEMOUTSTREAM_H__ -#define __RDR_FIXEDMEMOUTSTREAM_H__ - -#include <rdr/OutStream.h> -#include <rdr/Exception.h> - -namespace rdr { - - class FixedMemOutStream : public OutStream { - - public: - - FixedMemOutStream(void* buf, int len) { - ptr = start = (U8*)buf; - end = start + len; - } - - int length() { return ptr - start; } - void reposition(int pos) { ptr = start + pos; } - const void* data() { return (const void*)start; } - - private: - - int overrun(int itemSize, int nItems) { throw EndOfStream(); } - U8* start; - }; - -} - -#endif diff --git a/common/rdr/HexInStream.cxx b/common/rdr/HexInStream.cxx index 80f8a796..a6bc92cd 100644 --- a/common/rdr/HexInStream.cxx +++ b/common/rdr/HexInStream.cxx @@ -28,7 +28,7 @@ const int DEFAULT_BUF_LEN = 16384; static inline int min(int a, int b) {return a<b ? a : b;} -HexInStream::HexInStream(InStream& is, int bufSize_) +HexInStream::HexInStream(InStream& is, size_t bufSize_) : bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_LEN), offset(0), in_stream(is) { ptr = end = start = new U8[bufSize]; @@ -50,8 +50,8 @@ bool HexInStream::readHexAndShift(char c, int* v) { return true; } -bool HexInStream::hexStrToBin(const char* s, char** data, int* length) { - int l=strlen(s); +bool HexInStream::hexStrToBin(const char* s, char** data, size_t* length) { + size_t l=strlen(s); if ((l % 2) == 0) { delete [] *data; *data = 0; *length = 0; @@ -59,7 +59,7 @@ bool HexInStream::hexStrToBin(const char* s, char** data, int* length) { return true; *data = new char[l/2]; *length = l/2; - for(int i=0;i<l;i+=2) { + for(size_t i=0;i<l;i+=2) { int byte = 0; if (!readHexAndShift(s[i], &byte) || !readHexAndShift(s[i+1], &byte)) @@ -76,11 +76,11 @@ decodeError: } -int HexInStream::pos() { +size_t HexInStream::pos() { return offset + ptr - start; } -int HexInStream::overrun(int itemSize, int nItems, bool wait) { +size_t HexInStream::overrun(size_t itemSize, size_t nItems, bool wait) { if (itemSize > bufSize) throw Exception("HexInStream overrun: max itemSize exceeded"); @@ -91,15 +91,15 @@ int HexInStream::overrun(int itemSize, int nItems, bool wait) { offset += ptr - start; ptr = start; - while (end < ptr + itemSize) { - int n = in_stream.check(2, 1, wait); + while ((size_t)(end - ptr) < itemSize) { + size_t n = in_stream.check(2, 1, wait); if (n == 0) return 0; const U8* iptr = in_stream.getptr(); const U8* eptr = in_stream.getend(); - int length = min((eptr - iptr)/2, start + bufSize - end); + size_t length = min((eptr - iptr)/2, start + bufSize - end); U8* optr = (U8*) end; - for (int i=0; i<length; i++) { + for (size_t i=0; i<length; i++) { int v = 0; readHexAndShift(iptr[i*2], &v); readHexAndShift(iptr[i*2+1], &v); @@ -110,8 +110,10 @@ int HexInStream::overrun(int itemSize, int nItems, bool wait) { end += length; } - if (itemSize * nItems > end - ptr) - nItems = (end - ptr) / itemSize; + size_t nAvail; + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; return nItems; } diff --git a/common/rdr/HexInStream.h b/common/rdr/HexInStream.h index 6bfb8433..8e495fbe 100644 --- a/common/rdr/HexInStream.h +++ b/common/rdr/HexInStream.h @@ -26,21 +26,21 @@ namespace rdr { class HexInStream : public InStream { public: - HexInStream(InStream& is, int bufSize=0); + HexInStream(InStream& is, size_t bufSize=0); virtual ~HexInStream(); - int pos(); + size_t pos(); static bool readHexAndShift(char c, int* v); - static bool hexStrToBin(const char* s, char** data, int* length); + static bool hexStrToBin(const char* s, char** data, size_t* length); protected: - int overrun(int itemSize, int nItems, bool wait); + size_t overrun(size_t itemSize, size_t nItems, bool wait); private: - int bufSize; + size_t bufSize; U8* start; - int offset; + size_t offset; InStream& in_stream; }; diff --git a/common/rdr/HexOutStream.cxx b/common/rdr/HexOutStream.cxx index 9b0b6c4d..eac2eff8 100644 --- a/common/rdr/HexOutStream.cxx +++ b/common/rdr/HexOutStream.cxx @@ -23,9 +23,9 @@ using namespace rdr; const int DEFAULT_BUF_LEN = 16384; -static inline int min(int a, int b) {return a<b ? a : b;} +static inline size_t min(size_t a, size_t b) {return a<b ? a : b;} -HexOutStream::HexOutStream(OutStream& os, int buflen) +HexOutStream::HexOutStream(OutStream& os, size_t buflen) : out_stream(os), offset(0), bufSize(buflen ? buflen : DEFAULT_BUF_LEN) { if (bufSize % 2) @@ -48,9 +48,9 @@ char HexOutStream::intToHex(int i) { throw rdr::Exception("intToHex failed"); } -char* HexOutStream::binToHexStr(const char* data, int length) { +char* HexOutStream::binToHexStr(const char* data, size_t length) { char* buffer = new char[length*2+1]; - for (int i=0; i<length; i++) { + for (size_t i=0; i<length; i++) { buffer[i*2] = intToHex((data[i] >> 4) & 15); buffer[i*2+1] = intToHex((data[i] & 15)); if (!buffer[i*2] || !buffer[i*2+1]) { @@ -70,9 +70,9 @@ HexOutStream::writeBuffer() { out_stream.check(2); U8* optr = out_stream.getptr(); U8* oend = out_stream.getend(); - int length = min(ptr-pos, (oend-optr)/2); + size_t length = min(ptr-pos, (oend-optr)/2); - for (int i=0; i<length; i++) { + for (size_t i=0; i<length; i++) { optr[i*2] = intToHex((pos[i] >> 4) & 0xf); optr[i*2+1] = intToHex(pos[i] & 0xf); } @@ -84,7 +84,7 @@ HexOutStream::writeBuffer() { ptr = start; } -int HexOutStream::length() +size_t HexOutStream::length() { return offset + ptr - start; } @@ -95,15 +95,17 @@ HexOutStream::flush() { out_stream.flush(); } -int -HexOutStream::overrun(int itemSize, int nItems) { +size_t +HexOutStream::overrun(size_t itemSize, size_t nItems) { if (itemSize > bufSize) throw Exception("HexOutStream overrun: max itemSize exceeded"); writeBuffer(); - if (itemSize * nItems > end - ptr) - nItems = (end - ptr) / itemSize; + size_t nAvail; + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; return nItems; } diff --git a/common/rdr/HexOutStream.h b/common/rdr/HexOutStream.h index 10247e68..92442a72 100644 --- a/common/rdr/HexOutStream.h +++ b/common/rdr/HexOutStream.h @@ -26,24 +26,24 @@ namespace rdr { class HexOutStream : public OutStream { public: - HexOutStream(OutStream& os, int buflen=0); + HexOutStream(OutStream& os, size_t buflen=0); virtual ~HexOutStream(); void flush(); - int length(); + size_t length(); static char intToHex(int i); - static char* binToHexStr(const char* data, int length); + static char* binToHexStr(const char* data, size_t length); private: void writeBuffer(); - int overrun(int itemSize, int nItems); + size_t overrun(size_t itemSize, size_t nItems); OutStream& out_stream; U8* start; - int offset; - int bufSize; + size_t offset; + size_t bufSize; }; } diff --git a/common/rdr/InStream.h b/common/rdr/InStream.h index 212a2ec9..f71a4d9e 100644 --- a/common/rdr/InStream.h +++ b/common/rdr/InStream.h @@ -41,14 +41,17 @@ namespace rdr { // for the bytes, zero is returned if the bytes are not immediately // available. - inline int check(int itemSize, int nItems=1, bool wait=true) + inline size_t check(size_t itemSize, size_t nItems=1, bool wait=true) { - if (ptr + itemSize * nItems > end) { - if (ptr + itemSize > end) - return overrun(itemSize, nItems, wait); + size_t nAvail; + + if (itemSize > (size_t)(end - ptr)) + return overrun(itemSize, nItems, wait); + + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; - nItems = (end - ptr) / itemSize; - } return nItems; } @@ -56,7 +59,7 @@ namespace rdr { // be read without blocking. It returns true if this is the case, false // otherwise. The length must be "small" (less than the buffer size). - inline bool checkNoWait(int length) { return check(length, 1, false)!=0; } + inline bool checkNoWait(size_t length) { return check(length, 1, false)!=0; } // readU/SN() methods read unsigned and signed N-bit integers. @@ -82,9 +85,9 @@ namespace rdr { static U32 maxStringLength; - inline void skip(int bytes) { + inline void skip(size_t bytes) { while (bytes > 0) { - int n = check(1, bytes); + size_t n = check(1, bytes); ptr += n; bytes -= n; } @@ -92,14 +95,13 @@ namespace rdr { // readBytes() reads an exact number of bytes. - void readBytes(void* data, int length) { - U8* dataPtr = (U8*)data; - U8* dataEnd = dataPtr + length; - while (dataPtr < dataEnd) { - int n = check(1, dataEnd - dataPtr); - memcpy(dataPtr, ptr, n); + void readBytes(void* data, size_t length) { + while (length > 0) { + size_t n = check(1, length); + memcpy(data, ptr, n); ptr += n; - dataPtr += n; + data = (U8*)data + n; + length -= n; } } @@ -114,7 +116,7 @@ namespace rdr { // pos() returns the position in the stream. - virtual int pos() = 0; + virtual size_t pos() = 0; // getptr(), getend() and setptr() are "dirty" methods which allow you to // manipulate the buffer directly. This is useful for a stream which is a @@ -133,7 +135,7 @@ namespace rdr { // instead of blocking to wait for the bytes, zero is returned if the bytes // are not immediately available. - virtual int overrun(int itemSize, int nItems, bool wait=true) = 0; + virtual size_t overrun(size_t itemSize, size_t nItems, bool wait=true) = 0; protected: diff --git a/common/rdr/MemInStream.h b/common/rdr/MemInStream.h index 1a6a7982..3e9e77bc 100644 --- a/common/rdr/MemInStream.h +++ b/common/rdr/MemInStream.h @@ -36,7 +36,7 @@ namespace rdr { public: - MemInStream(const void* data, int len, bool deleteWhenDone_=false) + MemInStream(const void* data, size_t len, bool deleteWhenDone_=false) : start((const U8*)data), deleteWhenDone(deleteWhenDone_) { ptr = start; @@ -48,12 +48,12 @@ namespace rdr { delete [] start; } - int pos() { return ptr - start; } - void reposition(int pos) { ptr = start + pos; } + size_t pos() { return ptr - start; } + void reposition(size_t pos) { ptr = start + pos; } private: - int overrun(int itemSize, int nItems, bool wait) { throw EndOfStream(); } + size_t overrun(size_t itemSize, size_t nItems, bool wait) { throw EndOfStream(); } const U8* start; bool deleteWhenDone; }; diff --git a/common/rdr/MemOutStream.h b/common/rdr/MemOutStream.h index 3b17e555..b56bac3a 100644 --- a/common/rdr/MemOutStream.h +++ b/common/rdr/MemOutStream.h @@ -23,6 +23,7 @@ #ifndef __RDR_MEMOUTSTREAM_H__ #define __RDR_MEMOUTSTREAM_H__ +#include <rdr/Exception.h> #include <rdr/OutStream.h> namespace rdr { @@ -40,16 +41,16 @@ namespace rdr { delete [] start; } - void writeBytes(const void* data, int length) { + void writeBytes(const void* data, size_t length) { check(length); memcpy(ptr, data, length); ptr += length; } - int length() { return ptr - start; } + size_t length() { return ptr - start; } void clear() { ptr = start; }; void clearAndZero() { memset(start, 0, ptr-start); clear(); } - void reposition(int pos) { ptr = start + pos; } + void reposition(size_t pos) { ptr = start + pos; } // data() returns a pointer to the buffer. @@ -60,11 +61,14 @@ namespace rdr { // overrun() either doubles the buffer or adds enough space for nItems of // size itemSize bytes. - int overrun(int itemSize, int nItems) { - int len = ptr - start + itemSize * nItems; - if (len < (end - start) * 2) + size_t overrun(size_t itemSize, size_t nItems) { + size_t len = ptr - start + itemSize * nItems; + if (len < (size_t)(end - start) * 2) len = (end - start) * 2; + if (len < (size_t)(end - start)) + throw Exception("Overflow in MemOutStream::overrun()"); + U8* newStart = new U8[len]; memcpy(newStart, start, ptr - start); ptr = newStart + (ptr - start); diff --git a/common/rdr/OutStream.h b/common/rdr/OutStream.h index a749a208..0f60ccc1 100644 --- a/common/rdr/OutStream.h +++ b/common/rdr/OutStream.h @@ -44,14 +44,17 @@ namespace rdr { // itemSize bytes. Returns the number of items which fit (up to a maximum // of nItems). - inline int check(int itemSize, int nItems=1) + inline size_t check(size_t itemSize, size_t nItems=1) { - if (ptr + itemSize * nItems > end) { - if (ptr + itemSize > end) - return overrun(itemSize, nItems); + size_t nAvail; + + if (itemSize > (size_t)(end - ptr)) + return overrun(itemSize, nItems); + + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; - nItems = (end - ptr) / itemSize; - } return nItems; } @@ -76,13 +79,13 @@ namespace rdr { writeBytes(str, len); } - inline void pad(int bytes) { + inline void pad(size_t bytes) { while (bytes-- > 0) writeU8(0); } - inline void skip(int bytes) { + inline void skip(size_t bytes) { while (bytes > 0) { - int n = check(1, bytes); + size_t n = check(1, bytes); ptr += n; bytes -= n; } @@ -90,22 +93,21 @@ namespace rdr { // writeBytes() writes an exact number of bytes. - void writeBytes(const void* data, int length) { - const U8* dataPtr = (const U8*)data; - const U8* dataEnd = dataPtr + length; - while (dataPtr < dataEnd) { - int n = check(1, dataEnd - dataPtr); - memcpy(ptr, dataPtr, n); + void writeBytes(const void* data, size_t length) { + while (length > 0) { + size_t n = check(1, length); + memcpy(ptr, data, n); ptr += n; - dataPtr += n; + data = (U8*)data + n; + length -= n; } } // copyBytes() efficiently transfers data between streams - void copyBytes(InStream* is, int length) { + void copyBytes(InStream* is, size_t length) { while (length > 0) { - int n = check(1, length); + size_t n = check(1, length); is->readBytes(ptr, n); ptr += n; length -= n; @@ -124,7 +126,7 @@ namespace rdr { // length() returns the length of the stream. - virtual int length() = 0; + virtual size_t length() = 0; // flush() requests that the stream be flushed. @@ -145,7 +147,7 @@ namespace rdr { // the number of items which fit (up to a maximum of nItems). itemSize is // supposed to be "small" (a few bytes). - virtual int overrun(int itemSize, int nItems) = 0; + virtual size_t overrun(size_t itemSize, size_t nItems) = 0; protected: diff --git a/common/rdr/RandomStream.cxx b/common/rdr/RandomStream.cxx index e22da3d0..1be9b251 100644 --- a/common/rdr/RandomStream.cxx +++ b/common/rdr/RandomStream.cxx @@ -35,7 +35,7 @@ static rfb::LogWriter vlog("RandomStream"); using namespace rdr; -const int DEFAULT_BUF_LEN = 256; +const size_t DEFAULT_BUF_LEN = 256; unsigned int RandomStream::seed; @@ -86,11 +86,11 @@ RandomStream::~RandomStream() { #endif } -int RandomStream::pos() { +size_t RandomStream::pos() { return offset + ptr - start; } -int RandomStream::overrun(int itemSize, int nItems, bool wait) { +size_t RandomStream::overrun(size_t itemSize, size_t nItems, bool wait) { if (itemSize > DEFAULT_BUF_LEN) throw Exception("RandomStream overrun: max itemSize exceeded"); @@ -101,7 +101,7 @@ int RandomStream::overrun(int itemSize, int nItems, bool wait) { offset += ptr - start; ptr = start; - int length = start + DEFAULT_BUF_LEN - end; + size_t length = start + DEFAULT_BUF_LEN - end; #ifdef RFB_HAVE_WINCRYPT if (provider) { @@ -112,7 +112,7 @@ int RandomStream::overrun(int itemSize, int nItems, bool wait) { #else #ifndef WIN32 if (fp) { - int n = fread((U8*)end, length, 1, fp); + size_t n = fread((U8*)end, length, 1, fp); if (n != 1) throw rdr::SystemException("reading /dev/urandom or /dev/random failed", errno); @@ -122,12 +122,14 @@ int RandomStream::overrun(int itemSize, int nItems, bool wait) { { #endif #endif - for (int i=0; i<length; i++) + for (size_t i=0; i<length; i++) *(U8*)end++ = (int) (256.0*rand()/(RAND_MAX+1.0)); } - if (itemSize * nItems > end - ptr) - nItems = (end - ptr) / itemSize; + size_t nAvail; + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; return nItems; } diff --git a/common/rdr/RandomStream.h b/common/rdr/RandomStream.h index c33360d7..80b389b2 100644 --- a/common/rdr/RandomStream.h +++ b/common/rdr/RandomStream.h @@ -39,14 +39,14 @@ namespace rdr { RandomStream(); virtual ~RandomStream(); - int pos(); + size_t pos(); protected: - int overrun(int itemSize, int nItems, bool wait); + size_t overrun(size_t itemSize, size_t nItems, bool wait); private: U8* start; - int offset; + size_t offset; static unsigned int seed; #ifdef RFB_HAVE_WINCRYPT diff --git a/common/rdr/TLSInStream.cxx b/common/rdr/TLSInStream.cxx index 8cd07b6c..cd81f224 100644 --- a/common/rdr/TLSInStream.cxx +++ b/common/rdr/TLSInStream.cxx @@ -43,7 +43,7 @@ ssize_t TLSInStream::pull(gnutls_transport_ptr_t str, void* data, size_t size) return -1; } - if (in->getend() - in->getptr() < (ptrdiff_t)size) + if ((size_t)(in->getend() - in->getptr()) < size) size = in->getend() - in->getptr(); in->readBytes(data, size); @@ -75,12 +75,12 @@ TLSInStream::~TLSInStream() delete[] start; } -int TLSInStream::pos() +size_t TLSInStream::pos() { return offset + ptr - start; } -int TLSInStream::overrun(int itemSize, int nItems, bool wait) +size_t TLSInStream::overrun(size_t itemSize, size_t nItems, bool wait) { if (itemSize > bufSize) throw Exception("TLSInStream overrun: max itemSize exceeded"); @@ -92,20 +92,22 @@ int TLSInStream::overrun(int itemSize, int nItems, bool wait) end -= ptr - start; ptr = start; - while (end < start + itemSize) { - int n = readTLS((U8*) end, start + bufSize - end, wait); + while ((size_t)(end - start) < itemSize) { + size_t n = readTLS((U8*) end, start + bufSize - end, wait); if (!wait && n == 0) return 0; end += n; } - if (itemSize * nItems > end - ptr) - nItems = (end - ptr) / itemSize; + size_t nAvail; + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; return nItems; } -int TLSInStream::readTLS(U8* buf, int len, bool wait) +size_t TLSInStream::readTLS(U8* buf, size_t len, bool wait) { int n; diff --git a/common/rdr/TLSInStream.h b/common/rdr/TLSInStream.h index b16d9f5a..5f9dee7f 100644 --- a/common/rdr/TLSInStream.h +++ b/common/rdr/TLSInStream.h @@ -36,17 +36,17 @@ namespace rdr { TLSInStream(InStream* in, gnutls_session_t session); virtual ~TLSInStream(); - int pos(); + size_t pos(); private: - int overrun(int itemSize, int nItems, bool wait); - int readTLS(U8* buf, int len, bool wait); + size_t overrun(size_t itemSize, size_t nItems, bool wait); + size_t readTLS(U8* buf, size_t len, bool wait); static ssize_t pull(gnutls_transport_ptr_t str, void* data, size_t size); gnutls_session_t session; InStream* in; - int bufSize; - int offset; + size_t bufSize; + size_t offset; U8* start; }; }; diff --git a/common/rdr/TLSOutStream.cxx b/common/rdr/TLSOutStream.cxx index 44d2d9f3..7d7c3b56 100644 --- a/common/rdr/TLSOutStream.cxx +++ b/common/rdr/TLSOutStream.cxx @@ -75,7 +75,7 @@ TLSOutStream::~TLSOutStream() delete [] start; } -int TLSOutStream::length() +size_t TLSOutStream::length() { return offset + ptr - start; } @@ -84,7 +84,7 @@ void TLSOutStream::flush() { U8* sentUpTo = start; while (sentUpTo < ptr) { - int n = writeTLS(sentUpTo, ptr - sentUpTo); + size_t n = writeTLS(sentUpTo, ptr - sentUpTo); sentUpTo += n; offset += n; } @@ -93,20 +93,22 @@ void TLSOutStream::flush() out->flush(); } -int TLSOutStream::overrun(int itemSize, int nItems) +size_t TLSOutStream::overrun(size_t itemSize, size_t nItems) { if (itemSize > bufSize) throw Exception("TLSOutStream overrun: max itemSize exceeded"); flush(); - if (itemSize * nItems > end - ptr) - nItems = (end - ptr) / itemSize; + size_t nAvail; + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; return nItems; } -int TLSOutStream::writeTLS(const U8* data, int length) +size_t TLSOutStream::writeTLS(const U8* data, size_t length) { int n; diff --git a/common/rdr/TLSOutStream.h b/common/rdr/TLSOutStream.h index 81dd237a..71a7f3bf 100644 --- a/common/rdr/TLSOutStream.h +++ b/common/rdr/TLSOutStream.h @@ -36,20 +36,20 @@ namespace rdr { virtual ~TLSOutStream(); void flush(); - int length(); + size_t length(); protected: - int overrun(int itemSize, int nItems); + size_t overrun(size_t itemSize, size_t nItems); private: - int writeTLS(const U8* data, int length); + size_t writeTLS(const U8* data, size_t length); static ssize_t push(gnutls_transport_ptr_t str, const void* data, size_t size); gnutls_session_t session; OutStream* out; - int bufSize; + size_t bufSize; U8* start; - int offset; + size_t offset; }; }; diff --git a/common/rdr/ZlibInStream.cxx b/common/rdr/ZlibInStream.cxx index 4053bd19..9fcfaf6b 100644 --- a/common/rdr/ZlibInStream.cxx +++ b/common/rdr/ZlibInStream.cxx @@ -26,7 +26,7 @@ using namespace rdr; enum { DEFAULT_BUF_SIZE = 16384 }; -ZlibInStream::ZlibInStream(int bufSize_) +ZlibInStream::ZlibInStream(size_t bufSize_) : underlying(0), bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_SIZE), offset(0), zs(NULL), bytesIn(0) { @@ -40,28 +40,28 @@ ZlibInStream::~ZlibInStream() delete [] start; } -void ZlibInStream::setUnderlying(InStream* is, int bytesIn_) +void ZlibInStream::setUnderlying(InStream* is, size_t bytesIn_) { underlying = is; bytesIn = bytesIn_; ptr = end = start; } -int ZlibInStream::pos() +size_t ZlibInStream::pos() { return offset + ptr - start; } -void ZlibInStream::removeUnderlying() +void ZlibInStream::flushUnderlying() { ptr = end = start; - if (!underlying) return; while (bytesIn > 0) { decompress(true); end = start; // throw away any data } - underlying = 0; + + setUnderlying(NULL, 0); } void ZlibInStream::reset() @@ -90,18 +90,16 @@ void ZlibInStream::init() void ZlibInStream::deinit() { assert(zs != NULL); - removeUnderlying(); + setUnderlying(NULL, 0); inflateEnd(zs); delete zs; zs = NULL; } -int ZlibInStream::overrun(int itemSize, int nItems, bool wait) +size_t ZlibInStream::overrun(size_t itemSize, size_t nItems, bool wait) { if (itemSize > bufSize) throw Exception("ZlibInStream overrun: max itemSize exceeded"); - if (!underlying) - throw Exception("ZlibInStream overrun: no underlying stream"); if (end - ptr != 0) memmove(start, ptr, end - ptr); @@ -110,13 +108,15 @@ int ZlibInStream::overrun(int itemSize, int nItems, bool wait) end -= ptr - start; ptr = start; - while (end - ptr < itemSize) { + while ((size_t)(end - ptr) < itemSize) { if (!decompress(wait)) return 0; } - if (itemSize * nItems > end - ptr) - nItems = (end - ptr) / itemSize; + size_t nAvail; + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; return nItems; } @@ -127,14 +127,17 @@ int ZlibInStream::overrun(int itemSize, int nItems, bool wait) bool ZlibInStream::decompress(bool wait) { + if (!underlying) + throw Exception("ZlibInStream overrun: no underlying stream"); + zs->next_out = (U8*)end; zs->avail_out = start + bufSize - end; - int n = underlying->check(1, 1, wait); + size_t n = underlying->check(1, 1, wait); if (n == 0) return false; zs->next_in = (U8*)underlying->getptr(); zs->avail_in = underlying->getend() - underlying->getptr(); - if ((int)zs->avail_in > bytesIn) + if (zs->avail_in > bytesIn) zs->avail_in = bytesIn; int rc = inflate(zs, Z_SYNC_FLUSH); diff --git a/common/rdr/ZlibInStream.h b/common/rdr/ZlibInStream.h index 6bd4da4c..08784b0f 100644 --- a/common/rdr/ZlibInStream.h +++ b/common/rdr/ZlibInStream.h @@ -34,12 +34,12 @@ namespace rdr { public: - ZlibInStream(int bufSize=0); + ZlibInStream(size_t bufSize=0); virtual ~ZlibInStream(); - void setUnderlying(InStream* is, int bytesIn); - void removeUnderlying(); - int pos(); + void setUnderlying(InStream* is, size_t bytesIn); + void flushUnderlying(); + size_t pos(); void reset(); private: @@ -47,14 +47,14 @@ namespace rdr { void init(); void deinit(); - int overrun(int itemSize, int nItems, bool wait); + size_t overrun(size_t itemSize, size_t nItems, bool wait); bool decompress(bool wait); InStream* underlying; - int bufSize; - int offset; + size_t bufSize; + size_t offset; z_stream_s* zs; - int bytesIn; + size_t bytesIn; U8* start; }; diff --git a/common/rdr/ZlibOutStream.cxx b/common/rdr/ZlibOutStream.cxx index dd3c2367..7a0d692c 100644 --- a/common/rdr/ZlibOutStream.cxx +++ b/common/rdr/ZlibOutStream.cxx @@ -33,7 +33,7 @@ using namespace rdr; enum { DEFAULT_BUF_SIZE = 16384 }; -ZlibOutStream::ZlibOutStream(OutStream* os, int bufSize_, int compressLevel) +ZlibOutStream::ZlibOutStream(OutStream* os, size_t bufSize_, int compressLevel) : underlying(os), compressionLevel(compressLevel), newLevel(compressLevel), bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_SIZE), offset(0) { @@ -75,7 +75,7 @@ void ZlibOutStream::setCompressionLevel(int level) newLevel = level; } -int ZlibOutStream::length() +size_t ZlibOutStream::length() { return offset + ptr - start; } @@ -98,7 +98,7 @@ void ZlibOutStream::flush() ptr = start; } -int ZlibOutStream::overrun(int itemSize, int nItems) +size_t ZlibOutStream::overrun(size_t itemSize, size_t nItems) { #ifdef ZLIBOUT_DEBUG vlog.debug("overrun"); @@ -109,7 +109,7 @@ int ZlibOutStream::overrun(int itemSize, int nItems) checkCompressionLevel(); - while (end - ptr < itemSize) { + while ((size_t)(end - ptr) < itemSize) { zs->next_in = start; zs->avail_in = ptr - start; @@ -130,8 +130,10 @@ int ZlibOutStream::overrun(int itemSize, int nItems) } } - if (itemSize * nItems > end - ptr) - nItems = (end - ptr) / itemSize; + size_t nAvail; + nAvail = (end - ptr) / itemSize; + if (nAvail < nItems) + return nAvail; return nItems; } diff --git a/common/rdr/ZlibOutStream.h b/common/rdr/ZlibOutStream.h index 2d82a13b..11bb0468 100644 --- a/common/rdr/ZlibOutStream.h +++ b/common/rdr/ZlibOutStream.h @@ -35,25 +35,25 @@ namespace rdr { public: - ZlibOutStream(OutStream* os=0, int bufSize=0, int compressionLevel=-1); + ZlibOutStream(OutStream* os=0, size_t bufSize=0, int compressionLevel=-1); virtual ~ZlibOutStream(); void setUnderlying(OutStream* os); void setCompressionLevel(int level=-1); void flush(); - int length(); + size_t length(); private: - int overrun(int itemSize, int nItems); + size_t overrun(size_t itemSize, size_t nItems); void deflate(int flush); void checkCompressionLevel(); OutStream* underlying; int compressionLevel; int newLevel; - int bufSize; - int offset; + size_t bufSize; + size_t offset; z_stream_s* zs; U8* start; }; diff --git a/common/rfb/CMsgReader.cxx b/common/rfb/CMsgReader.cxx index a9e12d70..52d40ce7 100644 --- a/common/rfb/CMsgReader.cxx +++ b/common/rfb/CMsgReader.cxx @@ -242,7 +242,8 @@ void CMsgReader::readExtendedClipboard(rdr::S32 len) num++; } - zis.removeUnderlying(); + zis.flushUnderlying(); + zis.setUnderlying(NULL, 0); handler->handleClipboardProvide(flags, lengths, buffers); diff --git a/common/rfb/Configuration.cxx b/common/rfb/Configuration.cxx index ea000600..ceaefccb 100644 --- a/common/rfb/Configuration.cxx +++ b/common/rfb/Configuration.cxx @@ -421,7 +421,7 @@ StringParameter::operator const char *() const { // -=- BinaryParameter BinaryParameter::BinaryParameter(const char* name_, const char* desc_, - const void* v, int l, ConfigurationObject co) + const void* v, size_t l, ConfigurationObject co) : VoidParameter(name_, desc_, co), value(0), length(0), def_value((char*)v), def_length(l) { if (l) { value = new char[l]; @@ -441,7 +441,7 @@ bool BinaryParameter::setParam(const char* v) { return rdr::HexInStream::hexStrToBin(v, &value, &length); } -void BinaryParameter::setParam(const void* v, int len) { +void BinaryParameter::setParam(const void* v, size_t len) { LOCK_CONFIG; if (immutable) return; vlog.debug("set %s(Binary)", getName()); @@ -462,7 +462,7 @@ char* BinaryParameter::getValueStr() const { return rdr::HexOutStream::binToHexStr(value, length); } -void BinaryParameter::getData(void** data_, int* length_) const { +void BinaryParameter::getData(void** data_, size_t* length_) const { LOCK_CONFIG; if (length_) *length_ = length; if (data_) { diff --git a/common/rfb/Configuration.h b/common/rfb/Configuration.h index 6197317b..e23e8a51 100644 --- a/common/rfb/Configuration.h +++ b/common/rfb/Configuration.h @@ -256,24 +256,25 @@ namespace rfb { class BinaryParameter : public VoidParameter { public: - BinaryParameter(const char* name_, const char* desc_, const void* v, int l, - ConfigurationObject co=ConfGlobal); + BinaryParameter(const char* name_, const char* desc_, + const void* v, size_t l, + ConfigurationObject co=ConfGlobal); using VoidParameter::setParam; virtual ~BinaryParameter(); virtual bool setParam(const char* value); - virtual void setParam(const void* v, int l); + virtual void setParam(const void* v, size_t l); virtual char* getDefaultStr() const; virtual char* getValueStr() const; // getData() will return length zero if there is no data // NB: data may be set to zero, OR set to a zero-length buffer - void getData(void** data, int* length) const; + void getData(void** data, size_t* length) const; protected: char* value; - int length; + size_t length; char* def_value; - int def_length; + size_t def_length; }; // -=- ParameterIterator diff --git a/common/rfb/Cursor.cxx b/common/rfb/Cursor.cxx index d7b536de..3ca69f7c 100644 --- a/common/rfb/Cursor.cxx +++ b/common/rfb/Cursor.cxx @@ -272,8 +272,7 @@ void RenderedCursor::update(PixelBuffer* framebuffer, assert(cursor); format = framebuffer->getPF(); - width_ = framebuffer->width(); - height_ = framebuffer->height(); + setSize(framebuffer->width(), framebuffer->height()); rawOffset = pos.subtract(cursor->hotspot()); clippedRect = Rect(0, 0, cursor->width(), cursor->height()) diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx index 57020fee..a29502ae 100644 --- a/common/rfb/EncodeManager.cxx +++ b/common/rfb/EncodeManager.cxx @@ -28,6 +28,7 @@ #include <rfb/SMsgWriter.h> #include <rfb/UpdateTracker.h> #include <rfb/LogWriter.h> +#include <rfb/Exception.h> #include <rfb/RawEncoder.h> #include <rfb/RREEncoder.h> @@ -1051,11 +1052,13 @@ void EncodeManager::OffsetPixelBuffer::update(const PixelFormat& pf, int stride_) { format = pf; - width_ = width; - height_ = height; // Forced cast. We never write anything though, so it should be safe. - data = (rdr::U8*)data_; - stride = stride_; + setBuffer(width, height, (rdr::U8*)data_, stride_); +} + +rdr::U8* EncodeManager::OffsetPixelBuffer::getBufferRW(const Rect& r, int* stride) +{ + throw rfb::Exception("Invalid write attempt to OffsetPixelBuffer"); } // Preprocessor generated, optimised methods diff --git a/common/rfb/EncodeManager.h b/common/rfb/EncodeManager.h index bdae9063..f8201c34 100644 --- a/common/rfb/EncodeManager.h +++ b/common/rfb/EncodeManager.h @@ -148,6 +148,9 @@ namespace rfb { void update(const PixelFormat& pf, int width, int height, const rdr::U8* data_, int stride); + + private: + virtual rdr::U8* getBufferRW(const Rect& r, int* stride); }; OffsetPixelBuffer offsetPixelBuffer; diff --git a/common/rfb/Password.cxx b/common/rfb/Password.cxx index 240c9d4f..e4a508c8 100644 --- a/common/rfb/Password.cxx +++ b/common/rfb/Password.cxx @@ -38,7 +38,7 @@ PlainPasswd::PlainPasswd() {} PlainPasswd::PlainPasswd(char* pwd) : CharArray(pwd) { } -PlainPasswd::PlainPasswd(int len) : CharArray(len) { +PlainPasswd::PlainPasswd(size_t len) : CharArray(len) { } PlainPasswd::PlainPasswd(const ObfuscatedPasswd& obfPwd) : CharArray(9) { @@ -63,11 +63,11 @@ void PlainPasswd::replaceBuf(char* b) { ObfuscatedPasswd::ObfuscatedPasswd() : length(0) { } -ObfuscatedPasswd::ObfuscatedPasswd(int len) : CharArray(len), length(len) { +ObfuscatedPasswd::ObfuscatedPasswd(size_t len) : CharArray(len), length(len) { } ObfuscatedPasswd::ObfuscatedPasswd(const PlainPasswd& plainPwd) : CharArray(8), length(8) { - int l = strlen(plainPwd.buf), i; + size_t l = strlen(plainPwd.buf), i; for (i=0; i<8; i++) buf[i] = i<l ? plainPwd.buf[i] : 0; deskey(d3desObfuscationKey, EN0); diff --git a/common/rfb/Password.h b/common/rfb/Password.h index e5196ee0..712bc813 100644 --- a/common/rfb/Password.h +++ b/common/rfb/Password.h @@ -28,7 +28,7 @@ namespace rfb { public: PlainPasswd(); PlainPasswd(char* pwd); - PlainPasswd(int len); + PlainPasswd(size_t len); PlainPasswd(const ObfuscatedPasswd& obfPwd); ~PlainPasswd(); void replaceBuf(char* b); @@ -37,10 +37,10 @@ namespace rfb { class ObfuscatedPasswd : public CharArray { public: ObfuscatedPasswd(); - ObfuscatedPasswd(int l); + ObfuscatedPasswd(size_t l); ObfuscatedPasswd(const PlainPasswd& plainPwd); ~ObfuscatedPasswd(); - int length; + size_t length; }; } diff --git a/common/rfb/PixelBuffer.cxx b/common/rfb/PixelBuffer.cxx index 7f4c1ad3..fe406b96 100644 --- a/common/rfb/PixelBuffer.cxx +++ b/common/rfb/PixelBuffer.cxx @@ -31,12 +31,26 @@ using namespace rdr; static LogWriter vlog("PixelBuffer"); +// We do a lot of byte offset calculations that assume the result fits +// inside a signed 32 bit integer. Limit the maximum size of pixel +// buffers so that these calculations never overflow. + +const int maxPixelBufferWidth = 16384; +const int maxPixelBufferHeight = 16384; +const int maxPixelBufferStride = 16384; + // -=- Generic pixel buffer class PixelBuffer::PixelBuffer(const PixelFormat& pf, int w, int h) - : format(pf), width_(w), height_(h) {} -PixelBuffer::PixelBuffer() : width_(0), height_(0) {} + : format(pf), width_(0), height_(0) +{ + setSize(w, h); +} + +PixelBuffer::PixelBuffer() : width_(0), height_(0) +{ +} PixelBuffer::~PixelBuffer() {} @@ -53,7 +67,7 @@ PixelBuffer::getImage(void* imageBuf, const Rect& r, int outStride) const if (!r.enclosed_by(getRect())) throw rfb::Exception("Source rect %dx%d at %d,%d exceeds framebuffer %dx%d", r.width(), r.height(), - r.tl.x, r.tl.y, width_, height_); + r.tl.x, r.tl.y, width(), height()); data = getBuffer(r, &inStride); @@ -89,7 +103,7 @@ void PixelBuffer::getImage(const PixelFormat& pf, void* imageBuf, if (!r.enclosed_by(getRect())) throw rfb::Exception("Source rect %dx%d at %d,%d exceeds framebuffer %dx%d", r.width(), r.height(), - r.tl.x, r.tl.y, width_, height_); + r.tl.x, r.tl.y, width(), height()); if (stride == 0) stride = r.width(); @@ -100,6 +114,17 @@ void PixelBuffer::getImage(const PixelFormat& pf, void* imageBuf, stride, srcStride); } +void PixelBuffer::setSize(int width, int height) +{ + if ((width < 0) || (width > maxPixelBufferWidth)) + throw rfb::Exception("Invalid PixelBuffer width of %d pixels requested", width); + if ((height < 0) || (height > maxPixelBufferHeight)) + throw rfb::Exception("Invalid PixelBuffer height of %d pixels requested", height); + + width_ = width; + height_ = height; +} + // -=- Modifiable generic pixel buffer class ModifiablePixelBuffer::ModifiablePixelBuffer(const PixelFormat& pf, @@ -124,7 +149,7 @@ void ModifiablePixelBuffer::fillRect(const Rect& r, const void* pix) if (!r.enclosed_by(getRect())) throw rfb::Exception("Destination rect %dx%d at %d,%d exceeds framebuffer %dx%d", - r.width(), r.height(), r.tl.x, r.tl.y, width_, height_); + r.width(), r.height(), r.tl.x, r.tl.y, width(), height()); w = r.width(); h = r.height(); @@ -175,7 +200,7 @@ void ModifiablePixelBuffer::imageRect(const Rect& r, if (!r.enclosed_by(getRect())) throw rfb::Exception("Destination rect %dx%d at %d,%d exceeds framebuffer %dx%d", r.width(), r.height(), - r.tl.x, r.tl.y, width_, height_); + r.tl.x, r.tl.y, width(), height()); bytesPerPixel = getPF().bpp/8; @@ -214,13 +239,13 @@ void ModifiablePixelBuffer::copyRect(const Rect &rect, if (!drect.enclosed_by(getRect())) throw rfb::Exception("Destination rect %dx%d at %d,%d exceeds framebuffer %dx%d", drect.width(), drect.height(), - drect.tl.x, drect.tl.y, width_, height_); + drect.tl.x, drect.tl.y, width(), height()); srect = drect.translate(move_by_delta.negate()); if (!srect.enclosed_by(getRect())) throw rfb::Exception("Source rect %dx%d at %d,%d exceeds framebuffer %dx%d", srect.width(), srect.height(), - srect.tl.x, srect.tl.y, width_, height_); + srect.tl.x, srect.tl.y, width(), height()); bytesPerPixel = format.bpp/8; @@ -275,7 +300,7 @@ void ModifiablePixelBuffer::imageRect(const PixelFormat& pf, const Rect &dest, if (!dest.enclosed_by(getRect())) throw rfb::Exception("Destination rect %dx%d at %d,%d exceeds framebuffer %dx%d", dest.width(), dest.height(), - dest.tl.x, dest.tl.y, width_, height_); + dest.tl.x, dest.tl.y, width(), height()); if (stride == 0) stride = dest.width(); @@ -304,7 +329,7 @@ rdr::U8* FullFramePixelBuffer::getBufferRW(const Rect& r, int* stride_) if (!r.enclosed_by(getRect())) throw rfb::Exception("Pixel buffer request %dx%d at %d,%d exceeds framebuffer %dx%d", r.width(), r.height(), - r.tl.x, r.tl.y, width_, height_); + r.tl.x, r.tl.y, width(), height()); *stride_ = stride; return &data[(r.tl.x + (r.tl.y * stride)) * (format.bpp/8)]; @@ -319,55 +344,77 @@ const rdr::U8* FullFramePixelBuffer::getBuffer(const Rect& r, int* stride_) cons if (!r.enclosed_by(getRect())) throw rfb::Exception("Pixel buffer request %dx%d at %d,%d exceeds framebuffer %dx%d", r.width(), r.height(), - r.tl.x, r.tl.y, width_, height_); + r.tl.x, r.tl.y, width(), height()); *stride_ = stride; return &data[(r.tl.x + (r.tl.y * stride)) * (format.bpp/8)]; } +void FullFramePixelBuffer::setBuffer(int width, int height, + rdr::U8* data_, int stride_) +{ + if ((width < 0) || (width > maxPixelBufferWidth)) + throw rfb::Exception("Invalid PixelBuffer width of %d pixels requested", width); + if ((height < 0) || (height > maxPixelBufferHeight)) + throw rfb::Exception("Invalid PixelBuffer height of %d pixels requested", height); + if ((stride_ < 0) || (stride_ > maxPixelBufferStride) || (stride_ < width)) + throw rfb::Exception("Invalid PixelBuffer stride of %d pixels requested", stride_); + if ((width != 0) && (height != 0) && (data_ == NULL)) + throw rfb::Exception("PixelBuffer requested without a valid memory area"); + + ModifiablePixelBuffer::setSize(width, height); + stride = stride_; + data = data_; +} + +void FullFramePixelBuffer::setSize(int w, int h) +{ + // setBuffer() should be used + throw rfb::Exception("Invalid call to FullFramePixelBuffer::setSize()"); +} + // -=- Managed pixel buffer class // Automatically allocates enough space for the specified format & area ManagedPixelBuffer::ManagedPixelBuffer() - : datasize(0) + : data_(NULL), datasize(0) { - checkDataSize(); -}; +} ManagedPixelBuffer::ManagedPixelBuffer(const PixelFormat& pf, int w, int h) - : FullFramePixelBuffer(pf, w, h, NULL, w), datasize(0) + : FullFramePixelBuffer(pf, 0, 0, NULL, 0), data_(NULL), datasize(0) { - checkDataSize(); -}; - -ManagedPixelBuffer::~ManagedPixelBuffer() { - if (data) delete [] data; -}; + setSize(w, h); +} +ManagedPixelBuffer::~ManagedPixelBuffer() +{ + if (data_) + delete [] data_; +} -void -ManagedPixelBuffer::setPF(const PixelFormat &pf) { - format = pf; checkDataSize(); -}; -void -ManagedPixelBuffer::setSize(int w, int h) { - width_ = w; height_ = h; stride = w; checkDataSize(); -}; +void ManagedPixelBuffer::setPF(const PixelFormat &pf) +{ + format = pf; + setSize(width(), height()); +} +void ManagedPixelBuffer::setSize(int w, int h) +{ + unsigned long new_datasize = w * h * (format.bpp/8); -inline void -ManagedPixelBuffer::checkDataSize() { - unsigned long new_datasize = width_ * height_ * (format.bpp/8); + new_datasize = w * h * (format.bpp/8); if (datasize < new_datasize) { - if (data) { - delete [] data; - datasize = 0; data = 0; + if (data_) { + delete [] data_; + data_ = NULL; + datasize = 0; } if (new_datasize) { - data = new U8[new_datasize]; - if (!data) - throw Exception("rfb::ManagedPixelBuffer unable to allocate buffer"); + data_ = new U8[new_datasize]; datasize = new_datasize; } } -}; + + setBuffer(w, h, data_, w); +} diff --git a/common/rfb/PixelBuffer.h b/common/rfb/PixelBuffer.h index d89793f5..3e4018f9 100644 --- a/common/rfb/PixelBuffer.h +++ b/common/rfb/PixelBuffer.h @@ -90,7 +90,12 @@ namespace rfb { protected: PixelBuffer(); + virtual void setSize(int width, int height); + + protected: PixelFormat format; + + private: int width_, height_; }; @@ -154,7 +159,12 @@ namespace rfb { protected: FullFramePixelBuffer(); + virtual void setBuffer(int width, int height, rdr::U8* data, int stride); + private: + virtual void setSize(int w, int h); + + private: rdr::U8* data; int stride; }; @@ -172,12 +182,9 @@ namespace rfb { virtual void setPF(const PixelFormat &pf); virtual void setSize(int w, int h); - // Return the total number of bytes of pixel data in the buffer - int dataLen() const { return width_ * height_ * (format.bpp/8); } - - protected: + private: + rdr::U8* data_; // Mirrors FullFramePixelBuffer::data unsigned long datasize; - void checkDataSize(); }; }; diff --git a/common/rfb/PixelFormat.cxx b/common/rfb/PixelFormat.cxx index 883b0410..1b4ab1ba 100644 --- a/common/rfb/PixelFormat.cxx +++ b/common/rfb/PixelFormat.cxx @@ -81,7 +81,8 @@ PixelFormat::PixelFormat(int b, int d, bool e, bool t, redMax(rm), greenMax(gm), blueMax(bm), redShift(rs), greenShift(gs), blueShift(bs) { - assert(isSane()); + if (!isSane()) + throw Exception("invalid pixel format"); updateState(); } @@ -205,6 +206,12 @@ bool PixelFormat::is888(void) const return false; if (blueMax != 255) return false; + if ((redShift & 0x7) != 0) + return false; + if ((greenShift & 0x7) != 0) + return false; + if ((blueShift & 0x7) != 0) + return false; return true; } @@ -678,7 +685,14 @@ bool PixelFormat::isSane(void) return false; totalBits = bits(redMax) + bits(greenMax) + bits(blueMax); - if (totalBits > bpp) + if (totalBits > depth) + return false; + + if ((bits(redMax) + redShift) > bpp) + return false; + if ((bits(greenMax) + greenShift) > bpp) + return false; + if ((bits(blueMax) + blueShift) > bpp) return false; if (((redMax << redShift) & (greenMax << greenShift)) != 0) diff --git a/common/rfb/SMsgReader.cxx b/common/rfb/SMsgReader.cxx index ab42e59a..dc7ddea6 100644 --- a/common/rfb/SMsgReader.cxx +++ b/common/rfb/SMsgReader.cxx @@ -293,7 +293,8 @@ void SMsgReader::readExtendedClipboard(rdr::S32 len) num++; } - zis.removeUnderlying(); + zis.flushUnderlying(); + zis.setUnderlying(NULL, 0); handler->handleClipboardProvide(flags, lengths, buffers); diff --git a/common/rfb/TightDecoder.cxx b/common/rfb/TightDecoder.cxx index 5b7c553d..ebc98b06 100644 --- a/common/rfb/TightDecoder.cxx +++ b/common/rfb/TightDecoder.cxx @@ -341,7 +341,8 @@ void TightDecoder::decodeRect(const Rect& r, const void* buffer, zis[streamId].readBytes(netbuf, dataSize); - zis[streamId].removeUnderlying(); + zis[streamId].flushUnderlying(); + zis[streamId].setUnderlying(NULL, 0); delete ms; bufptr = netbuf; diff --git a/common/rfb/tightDecode.h b/common/rfb/tightDecode.h index b6e86ed5..8f77aebd 100644 --- a/common/rfb/tightDecode.h +++ b/common/rfb/tightDecode.h @@ -56,15 +56,17 @@ TightDecoder::FilterGradient24(const rdr::U8 *inbuf, int rectWidth = r.width(); for (y = 0; y < rectHeight; y++) { - /* First pixel in a row */ - for (c = 0; c < 3; c++) { - pix[c] = inbuf[y*rectWidth*3+c] + prevRow[c]; - thisRow[c] = pix[c]; - } - pf.bufferFromRGB((rdr::U8*)&outbuf[y*stride], pix, 1); + for (x = 0; x < rectWidth; x++) { + /* First pixel in a row */ + if (x == 0) { + for (c = 0; c < 3; c++) { + pix[c] = inbuf[y*rectWidth*3+c] + prevRow[c]; + thisRow[c] = pix[c]; + } + pf.bufferFromRGB((rdr::U8*)&outbuf[y*stride], pix, 1); + continue; + } - /* Remaining pixels of a row */ - for (x = 1; x < rectWidth; x++) { for (c = 0; c < 3; c++) { est[c] = prevRow[x*3+c] + pix[c] - prevRow[(x-1)*3+c]; if (est[c] > 0xff) { @@ -103,17 +105,20 @@ void TightDecoder::FilterGradient(const rdr::U8* inbuf, int rectWidth = r.width(); for (y = 0; y < rectHeight; y++) { - /* First pixel in a row */ - pf.rgbFromBuffer(pix, &inbuf[y*rectWidth], 1); - for (c = 0; c < 3; c++) - pix[c] += prevRow[c]; + for (x = 0; x < rectWidth; x++) { + /* First pixel in a row */ + if (x == 0) { + pf.rgbFromBuffer(pix, &inbuf[y*rectWidth], 1); + for (c = 0; c < 3; c++) + pix[c] += prevRow[c]; - memcpy(thisRow, pix, sizeof(pix)); + memcpy(thisRow, pix, sizeof(pix)); - pf.bufferFromRGB((rdr::U8*)&outbuf[y*stride], pix, 1); + pf.bufferFromRGB((rdr::U8*)&outbuf[y*stride], pix, 1); + + continue; + } - /* Remaining pixels of a row */ - for (x = 1; x < rectWidth; x++) { for (c = 0; c < 3; c++) { est[c] = prevRow[x*3+c] + pix[c] - prevRow[(x-1)*3+c]; if (est[c] > 255) { diff --git a/common/rfb/util.h b/common/rfb/util.h index 8503519d..3100f90f 100644 --- a/common/rfb/util.h +++ b/common/rfb/util.h @@ -50,7 +50,7 @@ namespace rfb { public: CharArray() : buf(0) {} CharArray(char* str) : buf(str) {} // note: assumes ownership - CharArray(int len) { + CharArray(size_t len) { buf = new char[len](); } ~CharArray() { diff --git a/common/rfb/zrleDecode.h b/common/rfb/zrleDecode.h index 32b5c92b..f4325385 100644 --- a/common/rfb/zrleDecode.h +++ b/common/rfb/zrleDecode.h @@ -174,7 +174,8 @@ void ZRLE_DECODE (const Rect& r, rdr::InStream* is, } } - zis->removeUnderlying(); + zis->flushUnderlying(); + zis->setUnderlying(NULL, 0); } #undef ZRLE_DECODE diff --git a/tests/perf/encperf.cxx b/tests/perf/encperf.cxx index e461197e..171564d1 100644 --- a/tests/perf/encperf.cxx +++ b/tests/perf/encperf.cxx @@ -71,11 +71,11 @@ class DummyOutStream : public rdr::OutStream { public: DummyOutStream(); - virtual int length(); + virtual size_t length(); virtual void flush(); private: - virtual int overrun(int itemSize, int nItems); + virtual size_t overrun(size_t itemSize, size_t nItems); int offset; rdr::U8 buf[131072]; @@ -141,7 +141,7 @@ DummyOutStream::DummyOutStream() end = buf + sizeof(buf); } -int DummyOutStream::length() +size_t DummyOutStream::length() { flush(); return offset; @@ -153,10 +153,10 @@ void DummyOutStream::flush() ptr = buf; } -int DummyOutStream::overrun(int itemSize, int nItems) +size_t DummyOutStream::overrun(size_t itemSize, size_t nItems) { flush(); - if (itemSize * nItems > end - ptr) + if (itemSize * nItems > (size_t)(end - ptr)) nItems = (end - ptr) / itemSize; return nItems; } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index c847238d..acc3adcd 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -8,3 +8,6 @@ target_link_libraries(convertlf rfb) add_executable(hostport hostport.cxx) target_link_libraries(hostport rfb) + +add_executable(pixelformat pixelformat.cxx) +target_link_libraries(pixelformat rfb) diff --git a/tests/unit/pixelformat.cxx b/tests/unit/pixelformat.cxx new file mode 100644 index 00000000..2e0c0bbb --- /dev/null +++ b/tests/unit/pixelformat.cxx @@ -0,0 +1,188 @@ +/* Copyright 2019 Pierre Ossman <ossman@cendio.se> for Cendio AB + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this software; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, + * USA. + */ + +#include <stdio.h> + +#include <rfb/PixelFormat.h> +#include <rfb/Exception.h> + +static void doTest(bool should_fail, int b, int d, bool e, bool t, + int rm, int gm, int bm, int rs, int gs, int bs) +{ + rfb::PixelFormat* pf; + + printf("PixelFormat(%d, %d, %s, %s, %d, %d, %d, %d, %d, %d): ", + b, d, e ? "true" : "false", t ? "true": "false", + rm, gm, bm, rs, gs, bs); + + try { + pf = new rfb::PixelFormat(b, d, e, t, rm, gm, bm, rs, gs, bs); + } catch(rfb::Exception &e) { + if (should_fail) + printf("OK"); + else + printf("FAILED"); + printf("\n"); + fflush(stdout); + return; + } + + delete pf; + + if (should_fail) + printf("FAILED"); + else + printf("OK"); + printf("\n"); + fflush(stdout); +} + +static void do888Test(bool expected, int b, int d, bool e, bool t, + int rm, int gm, int bm, int rs, int gs, int bs) +{ + rfb::PixelFormat* pf; + + printf("PixelFormat(%d, %d, %s, %s, %d, %d, %d, %d, %d, %d): ", + b, d, e ? "true" : "false", t ? "true": "false", + rm, gm, bm, rs, gs, bs); + + pf = new rfb::PixelFormat(b, d, e, t, rm, gm, bm, rs, gs, bs); + + if (pf->is888() == expected) + printf("OK"); + else + printf("FAILED"); + printf("\n"); + fflush(stdout); + + delete pf; +} + +static void sanityTests() +{ + printf("Sanity checks:\n\n"); + + /* Normal true color formats */ + + doTest(false, 32, 24, false, true, 255, 255, 255, 0, 8, 16); + doTest(false, 32, 24, false, true, 255, 255, 255, 24, 16, 8); + + doTest(false, 16, 16, false, true, 15, 31, 15, 0, 5, 11); + + doTest(false, 8, 8, false, true, 3, 7, 3, 0, 2, 5); + + /* Excessive bpp */ + + doTest(false, 32, 16, false, true, 15, 31, 15, 0, 5, 11); + + doTest(false, 16, 16, false, true, 15, 31, 15, 0, 5, 11); + + doTest(false, 32, 8, false, true, 3, 7, 3, 0, 2, 5); + + doTest(false, 16, 8, false, true, 3, 7, 3, 0, 2, 5); + + /* Colour map */ + + doTest(false, 8, 8, false, false, 0, 0, 0, 0, 0, 0); + + /* Invalid bpp */ + + doTest(true, 64, 24, false, true, 255, 255, 255, 0, 8, 16); + + doTest(true, 18, 16, false, true, 15, 31, 15, 0, 5, 11); + + doTest(true, 3, 3, false, true, 1, 1, 1, 0, 1, 2); + + /* Invalid depth */ + + doTest(true, 16, 24, false, true, 15, 31, 15, 0, 5, 11); + + doTest(true, 8, 24, false, true, 3, 7, 3, 0, 2, 5); + doTest(true, 8, 16, false, true, 3, 7, 3, 0, 2, 5); + + doTest(true, 32, 24, false, false, 0, 0, 0, 0, 0, 0); + + /* Invalid max values */ + + doTest(true, 32, 24, false, true, 254, 255, 255, 0, 8, 16); + doTest(true, 32, 24, false, true, 255, 253, 255, 0, 8, 16); + doTest(true, 32, 24, false, true, 255, 255, 252, 0, 8, 16); + + doTest(true, 32, 24, false, true, 511, 127, 127, 0, 16, 20); + doTest(true, 32, 24, false, true, 127, 511, 127, 0, 4, 20); + doTest(true, 32, 24, false, true, 127, 127, 511, 0, 4, 8); + + /* Insufficient depth */ + + doTest(true, 32, 16, false, true, 255, 255, 255, 0, 8, 16); + + /* Invalid shift values */ + + doTest(true, 32, 24, false, true, 255, 255, 255, 25, 8, 16); + doTest(true, 32, 24, false, true, 255, 255, 255, 0, 25, 16); + doTest(true, 32, 24, false, true, 255, 255, 255, 0, 8, 25); + + /* Overlapping channels */ + + doTest(true, 32, 24, false, true, 255, 255, 255, 0, 7, 16); + doTest(true, 32, 24, false, true, 255, 255, 255, 0, 8, 15); + doTest(true, 32, 24, false, true, 255, 255, 255, 0, 16, 7); + + printf("\n"); +} + +void is888Tests() +{ + printf("Simple format detection:\n\n"); + + /* Positive cases */ + + do888Test(true, 32, 24, false, true, 255, 255, 255, 0, 8, 16); + do888Test(true, 32, 24, false, true, 255, 255, 255, 24, 16, 8); + do888Test(true, 32, 24, false, true, 255, 255, 255, 24, 8, 0); + + /* Low depth */ + + do888Test(false, 32, 16, false, true, 15, 31, 15, 0, 8, 16); + do888Test(false, 32, 8, false, true, 3, 7, 3, 0, 8, 16); + + /* Low bpp and depth */ + + do888Test(false, 16, 16, false, true, 15, 31, 15, 0, 5, 11); + do888Test(false, 8, 8, false, true, 3, 7, 3, 0, 2, 5); + + /* Colour map */ + + do888Test(false, 8, 8, false, false, 0, 0, 0, 0, 0, 0); + + /* Odd shifts */ + + do888Test(false, 32, 24, false, true, 255, 255, 255, 0, 8, 18); + do888Test(false, 32, 24, false, true, 255, 255, 255, 0, 11, 24); + do888Test(false, 32, 24, false, true, 255, 255, 255, 4, 16, 24); + + printf("\n"); +} + +int main(int argc, char** argv) +{ + sanityTests(); + is888Tests(); + + return 0; +} diff --git a/unix/x0vncserver/XPixelBuffer.cxx b/unix/x0vncserver/XPixelBuffer.cxx index 4769b651..f0b06967 100644 --- a/unix/x0vncserver/XPixelBuffer.cxx +++ b/unix/x0vncserver/XPixelBuffer.cxx @@ -50,13 +50,8 @@ XPixelBuffer::XPixelBuffer(Display *dpy, ImageFactory &factory, ffs(m_image->xim->blue_mask) - 1); // Set up the remaining data of the parent class. - width_ = rect.width(); - height_ = rect.height(); - data = (rdr::U8 *)m_image->xim->data; - - // Calculate the distance in pixels between two subsequent scan - // lines of the framebuffer. This may differ from image width. - stride = m_image->xim->bytes_per_line * 8 / m_image->xim->bits_per_pixel; + setBuffer(rect.width(), rect.height(), (rdr::U8 *)m_image->xim->data, + m_image->xim->bytes_per_line * 8 / m_image->xim->bits_per_pixel); // Get initial screen image from the X display. m_image->get(DefaultRootWindow(m_dpy), m_offsetLeft, m_offsetTop); diff --git a/unix/xserver/hw/vnc/XserverDesktop.cc b/unix/xserver/hw/vnc/XserverDesktop.cc index 4edffec7..5c8b4cef 100644 --- a/unix/xserver/hw/vnc/XserverDesktop.cc +++ b/unix/xserver/hw/vnc/XserverDesktop.cc @@ -75,7 +75,7 @@ XserverDesktop::XserverDesktop(int screenIndex_, void* fbptr, int stride) : screenIndex(screenIndex_), server(0), listeners(listeners_), - directFbptr(true), + shadowFramebuffer(NULL), queryConnectId(0), queryConnectTimer(this) { format = pf; @@ -97,8 +97,8 @@ XserverDesktop::~XserverDesktop() delete listeners.back(); listeners.pop_back(); } - if (!directFbptr) - delete [] data; + if (shadowFramebuffer) + delete [] shadowFramebuffer; delete server; } @@ -116,22 +116,18 @@ void XserverDesktop::setFramebuffer(int w, int h, void* fbptr, int stride_) { ScreenSet layout; - width_ = w; - height_ = h; - - if (!directFbptr) { - delete [] data; - directFbptr = true; + if (shadowFramebuffer) { + delete [] shadowFramebuffer; + shadowFramebuffer = NULL; } if (!fbptr) { - fbptr = new rdr::U8[w * h * (format.bpp/8)]; + shadowFramebuffer = new rdr::U8[w * h * (format.bpp/8)]; + fbptr = shadowFramebuffer; stride_ = w; - directFbptr = false; } - data = (rdr::U8*)fbptr; - stride = stride_; + setBuffer(w, h, (rdr::U8*)fbptr, stride_); vncSetGlueContext(screenIndex); layout = ::computeScreenLayout(&outputIdMap); @@ -492,7 +488,7 @@ void XserverDesktop::handleClipboardData(const char* data_) void XserverDesktop::grabRegion(const rfb::Region& region) { - if (directFbptr) + if (shadowFramebuffer == NULL) return; std::vector<rfb::Rect> rects; diff --git a/unix/xserver/hw/vnc/XserverDesktop.h b/unix/xserver/hw/vnc/XserverDesktop.h index 6c670689..cc50f9e9 100644 --- a/unix/xserver/hw/vnc/XserverDesktop.h +++ b/unix/xserver/hw/vnc/XserverDesktop.h @@ -118,7 +118,7 @@ private: int screenIndex; rfb::VNCServer* server; std::list<network::SocketListener*> listeners; - bool directFbptr; + rdr::U8* shadowFramebuffer; uint32_t queryConnectId; network::Socket* queryConnectSocket; diff --git a/vncviewer/PlatformPixelBuffer.cxx b/vncviewer/PlatformPixelBuffer.cxx index ff1935e7..61f7b743 100644 --- a/vncviewer/PlatformPixelBuffer.cxx +++ b/vncviewer/PlatformPixelBuffer.cxx @@ -36,7 +36,7 @@ static rfb::LogWriter vlog("PlatformPixelBuffer"); PlatformPixelBuffer::PlatformPixelBuffer(int width, int height) : FullFramePixelBuffer(rfb::PixelFormat(32, 24, false, true, 255, 255, 255, 16, 8, 0), - width, height, NULL, 0), + 0, 0, NULL, 0), Surface(width, height) #if !defined(WIN32) && !defined(__APPLE__) , shminfo(NULL), xim(NULL) @@ -56,14 +56,13 @@ PlatformPixelBuffer::PlatformPixelBuffer(int width, int height) : vlog.debug("Using standard XImage"); } - data = (rdr::U8*)xim->data; - stride = xim->bytes_per_line / (getPF().bpp/8); + setBuffer(width, height, (rdr::U8*)xim->data, + xim->bytes_per_line / (getPF().bpp/8)); // On X11, the Pixmap backing this Surface is uninitialized. clear(0, 0, 0); #else - FullFramePixelBuffer::data = (rdr::U8*)Surface::data; - stride = width; + setBuffer(width, height, (rdr::U8*)Surface::data, width); #endif } diff --git a/win/rfb_win32/DIBSectionBuffer.cxx b/win/rfb_win32/DIBSectionBuffer.cxx index e2b0d641..e00cf233 100644 --- a/win/rfb_win32/DIBSectionBuffer.cxx +++ b/win/rfb_win32/DIBSectionBuffer.cxx @@ -52,39 +52,28 @@ void DIBSectionBuffer::setPF(const PixelFormat& pf) { if (!pf.trueColour) throw rfb::Exception("palette format not supported"); format = pf; - recreateBuffer(); + setSize(width(), height()); } -void DIBSectionBuffer::setSize(int w, int h) { - if (width_ == w && height_ == h) { - vlog.debug("size unchanged by setSize()"); - return; - } - width_ = w; - height_ = h; - recreateBuffer(); -} - - inline void initMaxAndShift(DWORD mask, int* max, int* shift) { for ((*shift) = 0; (mask & 1) == 0; (*shift)++) mask >>= 1; (*max) = (rdr::U16)mask; } -void DIBSectionBuffer::recreateBuffer() { +void DIBSectionBuffer::setSize(int w, int h) { HBITMAP new_bitmap = 0; rdr::U8* new_data = 0; - if (width_ && height_ && (format.depth != 0)) { + if (w && h && (format.depth != 0)) { BitmapInfo bi; memset(&bi, 0, sizeof(bi)); UINT iUsage = DIB_RGB_COLORS; bi.bmiHeader.biSize = sizeof(BITMAPINFOHEADER); bi.bmiHeader.biBitCount = format.bpp; - bi.bmiHeader.biSizeImage = (format.bpp / 8) * width_ * height_; + bi.bmiHeader.biSizeImage = (format.bpp / 8) * w * h; bi.bmiHeader.biPlanes = 1; - bi.bmiHeader.biWidth = width_; - bi.bmiHeader.biHeight = -height_; + bi.bmiHeader.biWidth = w; + bi.bmiHeader.biHeight = -h; bi.bmiHeader.biCompression = (format.bpp > 8) ? BI_BITFIELDS : BI_RGB; bi.mask.red = format.pixelFromRGB((rdr::U16)~0, 0, 0); bi.mask.green = format.pixelFromRGB(0, (rdr::U16)~0, 0); @@ -115,12 +104,12 @@ void DIBSectionBuffer::recreateBuffer() { if (device) { BitmapDC src_dev(device, bitmap); BitmapDC dest_dev(device, new_bitmap); - BitBlt(dest_dev, 0, 0, width_, height_, src_dev, 0, 0, SRCCOPY); + BitBlt(dest_dev, 0, 0, w, h, src_dev, 0, 0, SRCCOPY); } else { WindowDC wndDC(window); BitmapDC src_dev(wndDC, bitmap); BitmapDC dest_dev(wndDC, new_bitmap); - BitBlt(dest_dev, 0, 0, width_, height_, src_dev, 0, 0, SRCCOPY); + BitBlt(dest_dev, 0, 0, w, h, src_dev, 0, 0, SRCCOPY); } } @@ -128,17 +117,17 @@ void DIBSectionBuffer::recreateBuffer() { // Delete the old bitmap DeleteObject(bitmap); bitmap = 0; - data = 0; + setBuffer(0, 0, NULL, 0); } if (new_bitmap) { int bpp, depth; int redMax, greenMax, blueMax; int redShift, greenShift, blueShift; + int new_stride; // Set up the new bitmap bitmap = new_bitmap; - data = new_data; // Determine the *actual* DIBSection format DIBSECTION ds; @@ -147,14 +136,16 @@ void DIBSectionBuffer::recreateBuffer() { // Correct the "stride" of the DIB // *** This code DWORD aligns each row - is that right??? - stride = width_; - int bytesPerRow = stride * format.bpp/8; + new_stride = w; + int bytesPerRow = new_stride * format.bpp/8; if (bytesPerRow % 4) { bytesPerRow += 4 - (bytesPerRow % 4); - stride = (bytesPerRow * 8) / format.bpp; - vlog.info("adjusting DIB stride: %d to %d", width_, stride); + new_stride = (bytesPerRow * 8) / format.bpp; + vlog.info("adjusting DIB stride: %d to %d", w, new_stride); } + setBuffer(w, h, new_data, new_stride); + // Calculate the PixelFormat for the DIB bpp = depth = ds.dsBm.bmBitsPixel; diff --git a/win/rfb_win32/Registry.cxx b/win/rfb_win32/Registry.cxx index 9cd50184..5d78c4c4 100644 --- a/win/rfb_win32/Registry.cxx +++ b/win/rfb_win32/Registry.cxx @@ -146,7 +146,7 @@ void RegKey::setString(const TCHAR* valname, const TCHAR* value) const { if (result != ERROR_SUCCESS) throw rdr::SystemException("setString", result); } -void RegKey::setBinary(const TCHAR* valname, const void* value, int length) const { +void RegKey::setBinary(const TCHAR* valname, const void* value, size_t length) const { LONG result = RegSetValueEx(key, valname, 0, REG_BINARY, (const BYTE*)value, length); if (result != ERROR_SUCCESS) throw rdr::SystemException("setBinary", result); } @@ -169,12 +169,12 @@ TCHAR* RegKey::getString(const TCHAR* valname, const TCHAR* def) const { } } -void RegKey::getBinary(const TCHAR* valname, void** data, int* length) const { +void RegKey::getBinary(const TCHAR* valname, void** data, size_t* length) const { TCharArray hex(getRepresentation(valname)); if (!rdr::HexInStream::hexStrToBin(CStr(hex.buf), (char**)data, length)) throw rdr::Exception("getBinary failed"); } -void RegKey::getBinary(const TCHAR* valname, void** data, int* length, void* def, int deflen) const { +void RegKey::getBinary(const TCHAR* valname, void** data, size_t* length, void* def, size_t deflen) const { try { getBinary(valname, data, length); } catch(rdr::Exception&) { diff --git a/win/rfb_win32/Registry.h b/win/rfb_win32/Registry.h index 68d535cd..2bb16911 100644 --- a/win/rfb_win32/Registry.h +++ b/win/rfb_win32/Registry.h @@ -71,15 +71,15 @@ namespace rfb { void setExpandString(const TCHAR* valname, const TCHAR* s) const; void setString(const TCHAR* valname, const TCHAR* s) const; - void setBinary(const TCHAR* valname, const void* data, int length) const; + void setBinary(const TCHAR* valname, const void* data, size_t length) const; void setInt(const TCHAR* valname, int i) const; void setBool(const TCHAR* valname, bool b) const; TCHAR* getString(const TCHAR* valname) const; TCHAR* getString(const TCHAR* valname, const TCHAR* def) const; - void getBinary(const TCHAR* valname, void** data, int* length) const; - void getBinary(const TCHAR* valname, void** data, int* length, void* def, int deflength) const; + void getBinary(const TCHAR* valname, void** data, size_t* length) const; + void getBinary(const TCHAR* valname, void** data, size_t* length, void* def, size_t deflength) const; int getInt(const TCHAR* valname) const; int getInt(const TCHAR* valname, int def) const; |