Eliminate GCC signed/unsigned warnings related to encodings: The

encoding in the RFB protocol has always been signed, and signed values
are also used in the specification (ie DesktopName = -307 etc). In the
code, however, unsigned types were used in a number of places, but not
all, which causes warnings. This patch fixes the problem by switching
to signed values everywhere.



git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@3968 3789f03b-4d11-0410-bbf8-ca57d06f2519

common/rdr/types.h

@@ -61,6 +61,16 @@ namespace rdr {
     U32* buf;
   };
 
+  class S32Array {
+  public:
+    S32Array() : buf(0) {}
+    S32Array(S32* a) : buf(a) {} // note: assumes ownership
+    S32Array(int len) : buf(new S32[len]) {}
+    ~S32Array() { delete [] buf; }
+    S32* takeBuf() { S32* tmp = buf; buf = 0; return tmp; }
+    S32* buf;
+  };
+
 } // end of namespace rdr
 
 #endif

common/rfb/CMsgHandler.h

@@ -56,8 +56,8 @@ namespace rfb {
 
     virtual void framebufferUpdateStart() = 0;
     virtual void framebufferUpdateEnd() = 0;
-    virtual void beginRect(const Rect& r, unsigned int encoding) = 0;
-    virtual void endRect(const Rect& r, unsigned int encoding) = 0;
+    virtual void beginRect(const Rect& r, int encoding) = 0;
+    virtual void endRect(const Rect& r, int encoding) = 0;
 
     virtual void setColourMapEntries(int firstColour, int nColours,
 				     rdr::U16* rgbs) = 0;

common/rfb/CMsgReader.cxx

@@ -28,14 +28,14 @@ CMsgReader::CMsgReader(CMsgHandler* handler_, rdr::InStream* is_)
   : imageBufIdealSize(0), handler(handler_), is(is_),
     imageBuf(0), imageBufSize(0)
 {
-  for (unsigned int i = 0; i <= encodingMax; i++) {
+  for (int i = 0; i <= encodingMax; i++) {
     decoders[i] = 0;
   }
 }
 
 CMsgReader::~CMsgReader()
 {
-  for (unsigned int i = 0; i <= encodingMax; i++) {
+  for (int i = 0; i <= encodingMax; i++) {
     delete decoders[i];
   }
   delete [] imageBuf;
@@ -82,7 +82,7 @@ void CMsgReader::readFramebufferUpdateEnd()
   handler->framebufferUpdateEnd();
 }
 
-void CMsgReader::readRect(const Rect& r, unsigned int encoding)
+void CMsgReader::readRect(const Rect& r, int encoding)
 {
   if ((r.br.x > handler->cp.width) || (r.br.y > handler->cp.height)) {
     fprintf(stderr, "Rect too big: %dx%d at %d,%d exceeds %dx%d\n",

common/rfb/CMsgReader.h

@@ -55,7 +55,7 @@ namespace rfb {
 
     virtual void readFramebufferUpdateStart();
     virtual void readFramebufferUpdateEnd();
-    virtual void readRect(const Rect& r, unsigned int encoding);
+    virtual void readRect(const Rect& r, int encoding);
 
     virtual void readCopyRect(const Rect& r);
 

common/rfb/CMsgReaderV3.cxx

@@ -72,7 +72,7 @@ void CMsgReaderV3::readMsg()
     int y = is->readU16();
     int w = is->readU16();
     int h = is->readU16();
-    unsigned int encoding = is->readU32();
+    int encoding = is->readS32();
 
     switch (encoding) {
     case pseudoEncodingDesktopSize:

common/rfb/ConnParams.cxx

@@ -85,11 +85,11 @@ void ConnParams::setName(const char* name)
   name_ = strDup(name);
 }
 
-void ConnParams::setEncodings(int nEncodings, const rdr::U32* encodings)
+void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
 {
   if (nEncodings > nEncodings_) {
     delete [] encodings_;
-    encodings_ = new rdr::U32[nEncodings];
+    encodings_ = new rdr::S32[nEncodings];
   }
   nEncodings_ = nEncodings;
   useCopyRect = false;

common/rfb/ConnParams.h

@@ -66,10 +66,10 @@ namespace rfb {
     const char* name() { return name_; }
     void setName(const char* name);
 
-    rdr::U32 currentEncoding() { return currentEncoding_; }
+    rdr::S32 currentEncoding() { return currentEncoding_; }
     int nEncodings() { return nEncodings_; }
-    const rdr::U32* encodings() { return encodings_; }
-    void setEncodings(int nEncodings, const rdr::U32* encodings);
+    const rdr::S32* encodings() { return encodings_; }
+    void setEncodings(int nEncodings, const rdr::S32* encodings);
     bool useCopyRect;
 
     bool supportsLocalCursor;
@@ -91,7 +91,7 @@ namespace rfb {
     PixelFormat pf_;
     char* name_;
     int nEncodings_;
-    rdr::U32* encodings_;
+    rdr::S32* encodings_;
     int currentEncoding_;
     char verStr[13];
     int verStrPos;

common/rfb/Decoder.cxx

@@ -32,19 +32,19 @@ Decoder::~Decoder()
 
 DecoderCreateFnType Decoder::createFns[encodingMax+1] = { 0 };
 
-bool Decoder::supported(unsigned int encoding)
+bool Decoder::supported(int encoding)
 {
   return encoding <= encodingMax && createFns[encoding];
 }
 
-Decoder* Decoder::createDecoder(unsigned int encoding, CMsgReader* reader)
+Decoder* Decoder::createDecoder(int encoding, CMsgReader* reader)
 {
   if (encoding <= encodingMax && createFns[encoding])
     return (*createFns[encoding])(reader);
   return 0;
 }
 
-void Decoder::registerDecoder(unsigned int encoding,
+void Decoder::registerDecoder(int encoding,
                               DecoderCreateFnType createFn)
 {
   if (encoding > encodingMax)

common/rfb/Decoder.h

@@ -32,9 +32,9 @@ namespace rfb {
     virtual ~Decoder();
     virtual void readRect(const Rect& r, CMsgHandler* handler)=0;
 
-    static bool supported(unsigned int encoding);
-    static Decoder* createDecoder(unsigned int encoding, CMsgReader* reader);
-    static void registerDecoder(unsigned int encoding,
+    static bool supported(int encoding);
+    static Decoder* createDecoder(int encoding, CMsgReader* reader);
+    static void registerDecoder(int encoding,
                                 DecoderCreateFnType createFn);
   private:
     static DecoderCreateFnType createFns[encodingMax+1];

common/rfb/Encoder.cxx

@@ -32,19 +32,19 @@ Encoder::~Encoder()
 
 EncoderCreateFnType Encoder::createFns[encodingMax+1] = { 0 };
 
-bool Encoder::supported(unsigned int encoding)
+bool Encoder::supported(int encoding)
 {
   return encoding <= encodingMax && createFns[encoding];
 }
 
-Encoder* Encoder::createEncoder(unsigned int encoding, SMsgWriter* writer)
+Encoder* Encoder::createEncoder(int encoding, SMsgWriter* writer)
 {
   if (encoding <= encodingMax && createFns[encoding])
     return (*createFns[encoding])(writer);
   return 0;
 }
 
-void Encoder::registerEncoder(unsigned int encoding,
+void Encoder::registerEncoder(int encoding,
                               EncoderCreateFnType createFn)
 {
   if (encoding > encodingMax)
@@ -56,7 +56,7 @@ void Encoder::registerEncoder(unsigned int encoding,
   createFns[encoding] = createFn;
 }
 
-void Encoder::unregisterEncoder(unsigned int encoding)
+void Encoder::unregisterEncoder(int encoding)
 {
   if (encoding > encodingMax)
     throw Exception("Encoder::unregisterEncoder: encoding out of range");

common/rfb/Encoder.h

@@ -40,11 +40,11 @@ namespace rfb {
     // rectangle which was updated.
     virtual bool writeRect(const Rect& r, ImageGetter* ig, Rect* actual)=0;
 
-    static bool supported(unsigned int encoding);
-    static Encoder* createEncoder(unsigned int encoding, SMsgWriter* writer);
-    static void registerEncoder(unsigned int encoding,
+    static bool supported(int encoding);
+    static Encoder* createEncoder(int encoding, SMsgWriter* writer);
+    static void registerEncoder(int encoding,
                                 EncoderCreateFnType createFn);
-    static void unregisterEncoder(unsigned int encoding);
+    static void unregisterEncoder(int encoding);
   private:
     static EncoderCreateFnType createFns[encodingMax+1];
   };

common/rfb/SConnection.cxx

@@ -442,7 +442,7 @@ void SConnection::sendInteractionCaps()
   CapsList ecaps;
 
   // First, add true encodings.
-  for (unsigned int i = 1; i <= encodingMax; i++) {
+  for (int i = 1; i <= encodingMax; i++) {
     if (Encoder::supported(i)) {
       // FIXME: Capability info should be provided by Encoder objects.
       switch (i) {

common/rfb/SMsgHandler.cxx

@@ -39,7 +39,7 @@ void SMsgHandler::setPixelFormat(const PixelFormat& pf)
   cp.setPF(pf);
 }
 
-void SMsgHandler::setEncodings(int nEncodings, rdr::U32* encodings)
+void SMsgHandler::setEncodings(int nEncodings, rdr::S32* encodings)
 {
   cp.setEncodings(nEncodings, encodings);
   supportsLocalCursor();

common/rfb/SMsgHandler.h

@@ -46,7 +46,7 @@ namespace rfb {
     virtual void clientInit(bool shared);
 
     virtual void setPixelFormat(const PixelFormat& pf);
-    virtual void setEncodings(int nEncodings, rdr::U32* encodings);
+    virtual void setEncodings(int nEncodings, rdr::S32* encodings);
     virtual void framebufferUpdateRequest(const Rect& r, bool incremental) = 0;
     virtual void setDesktopSize(int fb_width, int fb_height,
                                 const ScreenSet& layout) = 0;

common/rfb/SMsgReader.cxx

@@ -51,7 +51,7 @@ void SMsgReader::readSetEncodings()
 {
   is->skip(1);
   int nEncodings = is->readU16();
-  rdr::U32Array encodings(nEncodings);
+  rdr::S32Array encodings(nEncodings);
   for (int i = 0; i < nEncodings; i++)
     encodings.buf[i] = is->readU32();
   handler->setEncodings(nEncodings, encodings.buf);

common/rfb/SMsgWriter.cxx

@@ -34,7 +34,7 @@ SMsgWriter::SMsgWriter(ConnParams* cp_, rdr::OutStream* os_)
     currentEncoding(0), updatesSent(0), rawBytesEquivalent(0),
     imageBuf(0), imageBufSize(0)
 {
-  for (unsigned int i = 0; i <= encodingMax; i++) {
+  for (int i = 0; i <= encodingMax; i++) {
     encoders[i] = 0;
     bytesSent[i] = 0;
     rectsSent[i] = 0;
@@ -45,7 +45,7 @@ SMsgWriter::~SMsgWriter()
 {
   vlog.info("framebuffer updates %d",updatesSent);
   int bytes = 0;
-  for (unsigned int i = 0; i <= encodingMax; i++) {
+  for (int i = 0; i <= encodingMax; i++) {
     delete encoders[i];
     if (i != encodingCopyRect)
       bytes += bytesSent[i];
@@ -92,7 +92,7 @@ void SMsgWriter::writeServerCutText(const char* str, int len)
 
 void SMsgWriter::setupCurrentEncoder()
 {
-  unsigned int encoding = cp->currentEncoding();
+  int encoding = cp->currentEncoding();
 
   // FIXME: Code duplication, see writeRect().
   if (!encoders[encoding]) {
@@ -106,7 +106,7 @@ void SMsgWriter::setupCurrentEncoder()
 
 int SMsgWriter::getNumRects(const Rect &r)
 {
-  unsigned int encoding = cp->currentEncoding();
+  int encoding = cp->currentEncoding();
 
   if (!encoders[encoding])
     setupCurrentEncoder();
@@ -169,7 +169,7 @@ bool SMsgWriter::writeRect(const Rect& r, ImageGetter* ig, Rect* actual)
   return writeRect(r, cp->currentEncoding(), ig, actual);
 }
 
-bool SMsgWriter::writeRect(const Rect& r, unsigned int encoding,
+bool SMsgWriter::writeRect(const Rect& r, int encoding,
                            ImageGetter* ig, Rect* actual)
 {
   if (!encoders[encoding]) {

common/rfb/SMsgWriter.h

@@ -145,12 +145,12 @@ namespace rfb {
     // write the whole rectangle it returns false and sets actual to the actual
     // rectangle which was updated.
     virtual bool writeRect(const Rect& r, ImageGetter* ig, Rect* actual);
-    virtual bool writeRect(const Rect& r, unsigned int encoding,
+    virtual bool writeRect(const Rect& r, int encoding,
                            ImageGetter* ig, Rect* actual);
 
     virtual void writeCopyRect(const Rect& r, int srcX, int srcY);
 
-    virtual void startRect(const Rect& r, unsigned int enc)=0;
+    virtual void startRect(const Rect& r, int enc)=0;
     virtual void endRect()=0;
 
     ConnParams* getConnParams() { return cp; }
@@ -176,7 +176,7 @@ namespace rfb {
 
     Encoder* encoders[encodingMax+1];
     int lenBeforeRect;
-    unsigned int currentEncoding;
+    int currentEncoding;
     int updatesSent;
     int bytesSent[encodingMax+1];
     int rectsSent[encodingMax+1];

common/rfb/SMsgWriterV3.cxx

@@ -221,7 +221,7 @@ void SMsgWriterV3::writeFramebufferUpdateEnd()
   endMsg();
 }
 
-void SMsgWriterV3::startRect(const Rect& r, unsigned int encoding)
+void SMsgWriterV3::startRect(const Rect& r, int encoding)
 {
   if (++nRectsInUpdate > nRectsInHeader && nRectsInHeader)
     throw Exception("SMsgWriterV3::startRect: nRects out of sync");

common/rfb/SMsgWriterV3.h

@@ -52,7 +52,7 @@ namespace rfb {
     virtual void writeFramebufferUpdateStart(int nRects);
     virtual void writeFramebufferUpdateStart();
     virtual void writeFramebufferUpdateEnd();
-    virtual void startRect(const Rect& r, unsigned int encoding);
+    virtual void startRect(const Rect& r, int encoding);
     virtual void endRect();
 
   protected:

common/rfb/encodings.cxx

@@ -34,7 +34,7 @@ int rfb::encodingNum(const char* name)
   return -1;
 }
 
-const char* rfb::encodingName(unsigned int num)
+const char* rfb::encodingName(int num)
 {
   switch (num) {
   case encodingRaw:      return "raw";

common/rfb/encodings.h

@@ -20,30 +20,30 @@
 
 namespace rfb {
 
-  const unsigned int encodingRaw = 0;
-  const unsigned int encodingCopyRect = 1;
-  const unsigned int encodingRRE = 2;
-  const unsigned int encodingCoRRE = 4;
-  const unsigned int encodingHextile = 5;
-  const unsigned int encodingTight = 7;
-  const unsigned int encodingZRLE = 16;
+  const int encodingRaw = 0;
+  const int encodingCopyRect = 1;
+  const int encodingRRE = 2;
+  const int encodingCoRRE = 4;
+  const int encodingHextile = 5;
+  const int encodingTight = 7;
+  const int encodingZRLE = 16;
 
-  const unsigned int encodingMax = 255;
+  const int encodingMax = 255;
 
-  const unsigned int pseudoEncodingXCursor = 0xffffff10;
-  const unsigned int pseudoEncodingCursor = 0xffffff11;
-  const unsigned int pseudoEncodingDesktopSize = 0xffffff21;
-  const unsigned int pseudoEncodingExtendedDesktopSize = 0xfffffecc;
-  const unsigned int pseudoEncodingDesktopName = 0xfffffecdl;
+  const int pseudoEncodingXCursor = -240;
+  const int pseudoEncodingCursor = -239;
+  const int pseudoEncodingDesktopSize = -223;
+  const int pseudoEncodingExtendedDesktopSize = -308;
+  const int pseudoEncodingDesktopName = -307;
 
   // TightVNC-specific
-  const unsigned int pseudoEncodingLastRect = 0xFFFFFF20;
-  const unsigned int pseudoEncodingQualityLevel0 = 0xFFFFFFE0;
-  const unsigned int pseudoEncodingQualityLevel9 = 0xFFFFFFE9;
-  const unsigned int pseudoEncodingCompressLevel0 = 0xFFFFFF00;
-  const unsigned int pseudoEncodingCompressLevel9 = 0xFFFFFF09;
+  const int pseudoEncodingLastRect = -224;
+  const int pseudoEncodingQualityLevel0 = -32;
+  const int pseudoEncodingQualityLevel9 = -23;
+  const int pseudoEncodingCompressLevel0 = -256;
+  const int pseudoEncodingCompressLevel9 = -247;
 
   int encodingNum(const char* name);
-  const char* encodingName(unsigned int num);
+  const char* encodingName(int num);
 }
 #endif

unix/vncviewer/CConn.cxx

@@ -396,7 +396,7 @@ void CConn::serverCutText(const char* str, rdr::U32 len) {
 // We start timing on beginRect and stop timing on endRect, to
 // avoid skewing the bandwidth estimation as a result of the server
 // being slow or the network having high latency
-void CConn::beginRect(const Rect& r, unsigned int encoding)
+void CConn::beginRect(const Rect& r, int encoding)
 {
   sock->inStream().startTiming();
   if (encoding != encodingCopyRect) {
@@ -404,7 +404,7 @@ void CConn::beginRect(const Rect& r, unsigned int encoding)
   }
 }
 
-void CConn::endRect(const Rect& r, unsigned int encoding)
+void CConn::endRect(const Rect& r, int encoding)
 {
   sock->inStream().stopTiming();
   if (debugDelay != 0) {
@@ -636,7 +636,7 @@ void CConn::getOptions() {
       formatChange = true;
     }
   }
-  unsigned int newEncoding = (options.tight.checked() ? encodingTight :
+  int newEncoding = (options.tight.checked() ? encodingTight :
 			      options.zrle.checked() ? encodingZRLE :
                               options.hextile.checked() ? encodingHextile :
                               encodingRaw);

unix/vncviewer/CConn.h

@@ -83,8 +83,8 @@ public:
   void serverCutText(const char* str, rdr::U32 len);
   void framebufferUpdateStart();
   void framebufferUpdateEnd();
-  void beginRect(const rfb::Rect& r, unsigned int encoding);
-  void endRect(const rfb::Rect& r, unsigned int encoding);
+  void beginRect(const rfb::Rect& r, int encoding);
+  void endRect(const rfb::Rect& r, int encoding);
   void fillRect(const rfb::Rect& r, rfb::Pixel p);
   void imageRect(const rfb::Rect& r, void* p);
   void copyRect(const rfb::Rect& r, int sx, int sy);
@@ -114,7 +114,7 @@ private:
   TXEventHandler* desktopEventHandler;
   rfb::PixelFormat fullColourPF;
   std::list<rfb::Rect> debugRects;
-  unsigned int currentEncoding, lastServerEncoding;
+  int currentEncoding, lastServerEncoding;
   bool fullColour;
   bool autoSelect;
   bool shared;

win/vncviewer/CConn.cxx

@@ -769,11 +769,11 @@ CConn::serverCutText(const char* str, rdr::U32 len) {
 }
 
 
-void CConn::beginRect(const Rect& r, unsigned int encoding) {
+void CConn::beginRect(const Rect& r, int encoding) {
   sock->inStream().startTiming();
 }
 
-void CConn::endRect(const Rect& r, unsigned int encoding) {
+void CConn::endRect(const Rect& r, int encoding) {
   sock->inStream().stopTiming();
   lastUsedEncoding_ = encoding;
   if (debugDelay != 0) {

win/vncviewer/CConn.h

@@ -106,8 +106,8 @@ namespace rfb {
       void setName(const char* name);
       void serverInit();
       void serverCutText(const char* str, rdr::U32 len);
-      void beginRect(const Rect& r, unsigned int encoding);
-      void endRect(const Rect& r, unsigned int encoding);
+      void beginRect(const Rect& r, int encoding);
+      void endRect(const Rect& r, int encoding);
       void fillRect(const Rect& r, Pixel pix);
       void imageRect(const Rect& r, void* pixels);
       void copyRect(const Rect& r, int srcX, int srcY);