Change fillRect() to take a buffer instead of a pixel

There has been some confusion if fillRect() should accept a buffer
or a pixel. This can cause misrendering if your data is not in the
native endian order. A buffer makes more sense here though, and
is what most of the callers are already assuming, so change the
API to follow that.

common/rfb/Cursor.cxx

@@ -38,6 +38,7 @@ void Cursor::setSize(int w, int h) {
 void Cursor::drawOutline(const Pixel& c)
 {
   Cursor outlined;
+  rdr::U8 cbuf[4];
 
   // Create a mirror of the existing cursor
   outlined.setPF(getPF());
@@ -45,7 +46,8 @@ void Cursor::drawOutline(const Pixel& c)
   outlined.hotspot = hotspot;
 
   // Clear the mirror's background to the outline colour
-  outlined.fillRect(getRect(), c);
+  outlined.getPF().bufferFromPixel(cbuf, c);
+  outlined.fillRect(getRect(), cbuf);
 
   // Blit the existing cursor, using its mask
   outlined.maskRect(getRect(), data, mask.buf);

common/rfb/Encoder.cxx

@@ -38,13 +38,11 @@ void Encoder::writeSolidRect(int width, int height,
                              const PixelFormat& pf, const rdr::U8* colour)
 {
   ManagedPixelBuffer buffer(pf, width, height);
-  Pixel pixel;
 
   Palette palette;
   rdr::U32 palcol;
 
-  pixel = pf.pixelFromBuffer(colour);
-  buffer.fillRect(buffer.getRect(), pixel);
+  buffer.fillRect(buffer.getRect(), colour);
 
   palcol = 0;
   memcpy(&palcol, colour, pf.bpp/8);

common/rfb/PixelBuffer.cxx

@@ -96,7 +96,7 @@ ModifiablePixelBuffer::~ModifiablePixelBuffer()
 {
 }
 
-void ModifiablePixelBuffer::fillRect(const Rect& r, Pixel pix)
+void ModifiablePixelBuffer::fillRect(const Rect& r, const void* pix)
 {
   int stride;
   U8 *buf;
@@ -113,20 +113,18 @@ void ModifiablePixelBuffer::fillRect(const Rect& r, Pixel pix)
 
   if (b == 1) {
     while (h--) {
-      memset(buf, pix, w);
+      memset(buf, *(const U8*)pix, w);
       buf += stride * b;
     }
   } else {
-    U8 pixbuf[4], *start;
+    U8 *start;
     int w1;
 
     start = buf;
 
-    format.bufferFromPixel(pixbuf, pix);
-
     w1 = w;
     while (w1--) {
-      memcpy(buf, pixbuf, b);
+      memcpy(buf, pix, b);
       buf += b;
     }
     buf += (stride - w) * b;
@@ -309,9 +307,11 @@ void ModifiablePixelBuffer::copyRect(const Rect &rect,
 }
 
 void ModifiablePixelBuffer::fillRect(const PixelFormat& pf, const Rect &dest,
-                                     Pixel pix)
+                                     const void* pix)
 {
-  fillRect(dest, format.pixelFromPixel(pf, pix));
+  rdr::U8 buf[4];
+  format.bufferFromBuffer(buf, pf, (const rdr::U8*)pix, 1);
+  fillRect(dest, buf);
 }
 
 void ModifiablePixelBuffer::imageRect(const PixelFormat& pf, const Rect &dest,

common/rfb/PixelBuffer.h

@@ -119,7 +119,7 @@ namespace rfb {
     // These operations DO NOT clip to the pixelbuffer area, or trap overruns.
 
     // Fill a rectangle
-    void fillRect(const Rect &dest, Pixel pix);
+    void fillRect(const Rect &dest, const void* pix);
 
     // Copy pixel data to the buffer
     void imageRect(const Rect &dest, const void* pixels, int stride=0);
@@ -140,7 +140,7 @@ namespace rfb {
     // Render in a specific format
     //   Does the exact same thing as the above methods, but the given
     //   pixel values are defined by the given PixelFormat.
-    void fillRect(const PixelFormat& pf, const Rect &dest, Pixel pix);
+    void fillRect(const PixelFormat& pf, const Rect &dest, const void* pix);
     void imageRect(const PixelFormat& pf, const Rect &dest,
                    const void* pixels, int stride=0);
 

common/rfb/SDesktop.h

@@ -92,12 +92,16 @@ namespace rfb {
   public:
     SStaticDesktop(const Point& size) : server(0), buffer(0) {
       PixelFormat pf;
+      const rdr::U8 black[4] = { 0, 0, 0, 0 };
       buffer = new ManagedPixelBuffer(pf, size.x, size.y);
-      if (buffer) buffer->fillRect(buffer->getRect(), 0);
+      if (buffer)
+        buffer->fillRect(buffer->getRect(), black);
     }
     SStaticDesktop(const Point& size, const PixelFormat& pf) : buffer(0) {
+      const rdr::U8 black[4] = { 0, 0, 0, 0 };
       buffer = new ManagedPixelBuffer(pf, size.x, size.y);
-      if (buffer) buffer->fillRect(buffer->getRect(), 0);
+      if (buffer)
+        buffer->fillRect(buffer->getRect(), black);
     }
     virtual ~SStaticDesktop() {
       if (buffer) delete buffer;

common/rfb/rreDecode.h

@@ -41,7 +41,7 @@ void RRE_DECODE (const Rect& r, rdr::InStream* is,
 {
   int nSubrects = is->readU32();
   PIXEL_T bg = is->READ_PIXEL();
-  pb->fillRect(pf, r, bg);
+  pb->fillRect(pf, r, &bg);
 
   for (int i = 0; i < nSubrects; i++) {
     PIXEL_T pix = is->READ_PIXEL();
@@ -49,7 +49,7 @@ void RRE_DECODE (const Rect& r, rdr::InStream* is,
     int y = is->readU16();
     int w = is->readU16();
     int h = is->readU16();
-    pb->fillRect(pf, Rect(r.tl.x+x, r.tl.y+y, r.tl.x+x+w, r.tl.y+y+h), pix);
+    pb->fillRect(pf, Rect(r.tl.x+x, r.tl.y+y, r.tl.x+x+w, r.tl.y+y+h), &pix);
   }
 }
 

common/rfb/tightDecode.h

@@ -77,7 +77,7 @@ void TIGHT_DECODE (const Rect& r)
     } else {
       pix = is->READ_PIXEL();
     }
-    pb->fillRect(serverpf, r, pix);
+    pb->fillRect(serverpf, r, &pix);
     return;
   }
 

common/rfb/zrleDecode.h

@@ -73,7 +73,7 @@ void ZRLE_DECODE (const Rect& r, rdr::InStream* is,
 
       if (palSize == 1) {
         PIXEL_T pix = palette[0];
-        pb->fillRect(pf, t, pix);
+        pb->fillRect(pf, t, &pix);
         continue;
       }
 

vncviewer/Viewport.cxx

@@ -281,6 +281,8 @@ void Viewport::resize(int x, int y, int w, int h)
   const rdr::U8* data;
   int stride;
 
+  const rdr::U8 black[4] = { 0, 0, 0, 0 };
+
   // FIXME: Resize should probably be a feature of the pixel buffer itself
 
   if ((w == frameBuffer->width()) && (h == frameBuffer->height()))
@@ -304,14 +306,14 @@ void Viewport::resize(int x, int y, int w, int h)
     rect.setXYWH(frameBuffer->width(), 0,
                  newBuffer->width() - frameBuffer->width(),
                  newBuffer->height());
-    newBuffer->fillRect(rect, 0);
+    newBuffer->fillRect(rect, black);
   }
 
   if (newBuffer->height() > frameBuffer->height()) {
     rect.setXYWH(0, frameBuffer->height(),
                  newBuffer->width(),
                  newBuffer->height() - frameBuffer->height());
-    newBuffer->fillRect(rect, 0);
+    newBuffer->fillRect(rect, black);
   }
 
   delete frameBuffer;