Consolidate pixel conversion into the PixelFormat class and optimise the

common cases.


git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@3636 3789f03b-4d11-0410-bbf8-ca57d06f2519

common/rfb/Makefile.am

@@ -11,7 +11,7 @@ HDRS = Blacklist.h CapsContainer.h CapsList.h CConnection.h CFTMsgReader.h \
 	HextileEncoder.h Hostname.h HTTPServer.h ImageGetter.h InputHandler.h \
 	KeyRemapper.h keysymdef.h \
 	ListConnInfo.h Logger_file.h Logger.h Logger_stdio.h LogWriter.h \
-	msgTypes.h Password.h PixelBuffer.h PixelFormat.h Pixel.h \
+	msgTypes.h Password.h PixelBuffer.h PixelFormat.h PixelFormat.inl Pixel.h \
 	RawDecoder.h RawEncoder.h Rect.h Region.h rreDecode.h RREDecoder.h \
 	rreEncode.h RREEncoder.h ScaledPixelBuffer.h ScaleFilters.h \
 	SConnection.h SDesktop.h secTypes.h ServerCore.h SFileTransfer.h \

common/rfb/PixelFormat.cxx

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2009 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -15,6 +16,7 @@
  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
  * USA.
  */
+#include <assert.h>
 #include <stdio.h>
 #include <string.h>
 #include <rdr/InStream.h>
@@ -30,17 +32,25 @@ using namespace rfb;
 
 PixelFormat::PixelFormat(int b, int d, bool e, bool t,
                          int rm, int gm, int bm, int rs, int gs, int bs)
-  : bpp(b), depth(d), bigEndian(e), trueColour(t),
+  : bpp(b), depth(d), trueColour(t), bigEndian(e),
     redMax(rm), greenMax(gm), blueMax(bm),
     redShift(rs), greenShift(gs), blueShift(bs)
 {
+  assert((bpp == 8) || (bpp == 16) || (bpp == 32));
+  assert(depth <= bpp);
+  assert((redMax & (redMax + 1)) == 0);
+  assert((greenMax & (greenMax + 1)) == 0);
+  assert((blueMax & (blueMax + 1)) == 0);
+
+  updateShifts();
 }
 
 PixelFormat::PixelFormat()
-  : bpp(8), depth(8), bigEndian(false), trueColour(true),
+  : bpp(8), depth(8), trueColour(true), bigEndian(false),
     redMax(7), greenMax(7), blueMax(3),
     redShift(0), greenShift(3), blueShift(6)
 {
+  updateShifts();
 }
 
 bool PixelFormat::equal(const PixelFormat& other) const
@@ -70,6 +80,8 @@ void PixelFormat::read(rdr::InStream* is)
   greenShift = is->readU8();
   blueShift = is->readU8();
   is->skip(3);
+
+  updateShifts();
 }
 
 void PixelFormat::write(rdr::OutStream* os) const
@@ -87,6 +99,36 @@ void PixelFormat::write(rdr::OutStream* os) const
   os->pad(3);
 }
 
+
+bool PixelFormat::is888(void) const
+{
+  if (bpp != 32)
+    return false;
+  if (depth != 24)
+    return false;
+  if (redMax != 255)
+    return false;
+  if (greenMax != 255)
+    return false;
+  if (blueMax != 255)
+    return false;
+
+  return true;
+}
+
+
+bool PixelFormat::isBigEndian(void) const
+{
+  return bigEndian;
+}
+
+
+bool PixelFormat::isLittleEndian(void) const
+{
+  return ! bigEndian;
+}
+
+
 Pixel PixelFormat::pixelFromRGB(rdr::U16 red, rdr::U16 green, rdr::U16 blue,
                                 ColourMap* cm) const
 {
@@ -120,14 +162,64 @@ Pixel PixelFormat::pixelFromRGB(rdr::U16 red, rdr::U16 green, rdr::U16 blue,
 }
 
 
-void PixelFormat::rgbFromPixel(Pixel p, ColourMap* cm, Colour* rgb) const
+Pixel PixelFormat::pixelFromRGB(rdr::U8 red, rdr::U8 green, rdr::U8 blue,
+                                ColourMap* cm) const
 {
   if (trueColour) {
-    rgb->r = (((p >> redShift  ) & redMax  ) * 65535 + redMax  /2) / redMax;
-    rgb->g = (((p >> greenShift) & greenMax) * 65535 + greenMax/2) / greenMax;
-    rgb->b = (((p >> blueShift ) & blueMax ) * 65535 + blueMax /2) / blueMax;
-  } else {
-    cm->lookup(p, &rgb->r, &rgb->g, &rgb->b);
+    rdr::U32 r = ((rdr::U32)red   * redMax   + 127) / 255;
+    rdr::U32 g = ((rdr::U32)green * greenMax + 127) / 255;
+    rdr::U32 b = ((rdr::U32)blue  * blueMax  + 127) / 255;
+
+    return (r << redShift) | (g << greenShift) | (b << blueShift);
+  }
+
+  return pixelFromRGB((rdr::U16)(red << 8), (rdr::U16)(green << 8),
+      (rdr::U16)(blue << 8), cm);
+}
+
+
+void PixelFormat::rgbFromPixel(Pixel p, ColourMap* cm, Colour* rgb) const
+{
+  rdr::U16 r, g, b;
+
+  rgbFromPixel(p, cm, &r, &g, &b);
+
+  rgb->r = r;
+  rgb->g = g;
+  rgb->b = b;
+}
+
+
+void PixelFormat::rgbFromBuffer(rdr::U16* dst, const rdr::U8* src, int pixels, ColourMap* cm) const
+{
+  Pixel p;
+  rdr::U16 r, g, b;
+
+  while (pixels--) {
+    p = pixelFromBuffer(src);
+    src += bpp/8;
+
+    rgbFromPixel(p, cm, &r, &g, &b);
+    *(dst++) = r;
+    *(dst++) = g;
+    *(dst++) = b;
+  }
+}
+
+
+void PixelFormat::rgbFromBuffer(rdr::U8* dst, const rdr::U8* src, int pixels, ColourMap* cm) const
+{
+  Pixel p;
+  rdr::U8 r, g, b;
+
+  while (pixels--) {
+    p = pixelFromBuffer(src);
+    src += bpp/8;
+
+    rgbFromPixel(p, cm, &r, &g, &b);
+    *(dst++) = r;
+    *(dst++) = g;
+    *(dst++) = b;
   }
 }
 
@@ -237,3 +329,43 @@ bool PixelFormat::parse(const char* str)
   }
   return true;
 }
+
+
+static int bits(rdr::U16 value)
+{
+  int bits;
+
+  bits = 16;
+
+  if (!(value & 0xff00)) {
+    bits -= 8;
+    value <<= 8;
+  }
+  if (!(value & 0xf000)) {
+    bits -= 4;
+    value <<= 4;
+  }
+  if (!(value & 0xc000)) {
+    bits -= 2;
+    value <<= 2;
+  }
+  if (!(value & 0x8000)) {
+    bits -= 1;
+    value <<= 1;
+  }
+
+  return bits;
+}
+
+void PixelFormat::updateShifts(void)
+{
+  int redBits, greenBits, blueBits;
+
+  redBits = bits(redMax);
+  greenBits = bits(greenMax);
+  blueBits = bits(blueMax);
+
+  redConvShift = 16 - redBits;
+  greenConvShift = 16 - greenBits;
+  blueConvShift = 16 - blueBits;
+}

common/rfb/PixelFormat.h

@@ -35,24 +35,56 @@ namespace rfb {
     PixelFormat(int b, int d, bool e, bool t,
                 int rm=0, int gm=0, int bm=0, int rs=0, int gs=0, int bs=0);
     PixelFormat();
+
     bool equal(const PixelFormat& other) const;
+
     void read(rdr::InStream* is);
     void write(rdr::OutStream* os) const;
+
+    bool is888(void) const;
+    bool isBigEndian(void) const;
+    bool isLittleEndian(void) const;
+
+    inline Pixel pixelFromBuffer(const rdr::U8* buffer) const;
+
     Pixel pixelFromRGB(rdr::U16 red, rdr::U16 green, rdr::U16 blue, ColourMap* cm=0) const;
+    Pixel pixelFromRGB(rdr::U8 red, rdr::U8 green, rdr::U8 blue, ColourMap* cm=0) const;
+
     void rgbFromPixel(Pixel pix, ColourMap* cm, Colour* rgb) const;
+    inline void rgbFromPixel(Pixel pix, ColourMap* cm, rdr::U16 *r, rdr::U16 *g, rdr::U16 *b) const;
+    inline void rgbFromPixel(Pixel pix, ColourMap* cm, rdr::U8 *r, rdr::U8 *g, rdr::U8 *b) const;
+
+    void rgbFromBuffer(rdr::U16* dst, const rdr::U8* src, int pixels, ColourMap* cm=0) const;
+    void rgbFromBuffer(rdr::U8* dst, const rdr::U8* src, int pixels, ColourMap* cm=0) const;
+
     void print(char* str, int len) const;
     bool parse(const char* str);
 
+  protected:
+    void updateShifts(void);
+
+  public:
     int bpp;
     int depth;
-    bool bigEndian;
     bool trueColour;
+
+  // FIXME: These should be protected, but we need to fix TransImageGetter first.
+  public:
+    bool bigEndian;
     int redMax;
     int greenMax;
     int blueMax;
     int redShift;
     int greenShift;
     int blueShift;
+
+  protected:
+    int redConvShift;
+    int greenConvShift;
+    int blueConvShift;
   };
 }
+
+#include <rfb/PixelFormat.inl>
+
 #endif

common/rfb/PixelFormat.inl

@@ -0,0 +1,92 @@
+/* Copyright 2009 Pierre Ossman for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+namespace rfb {
+
+
+inline Pixel PixelFormat::pixelFromBuffer(const rdr::U8* buffer) const
+{
+  Pixel p;
+
+  p = 0;
+
+  if (bigEndian) {
+    switch (bpp) {
+    case 32:
+      p |= ((Pixel)*(buffer++)) << 24;
+      p |= ((Pixel)*(buffer++)) << 16;
+    case 16:
+      p |= ((Pixel)*(buffer++)) << 8;
+    case 8:
+      p |= *buffer;
+    }
+  } else {
+    p |= buffer[0];
+    if (bpp >= 16) {
+      p |= ((Pixel)buffer[1]) << 8;
+      if (bpp == 32)
+        p |= ((Pixel)buffer[2]) << 16;
+    }
+  }
+
+  return p;
+}
+
+
+inline void PixelFormat::rgbFromPixel(Pixel p, ColourMap* cm, rdr::U16 *r, rdr::U16 *g, rdr::U16 *b) const
+{
+  if (trueColour) {
+    /* We don't need to mask since we shift out unwanted bits */
+    *r = (p >> redShift) << redConvShift;
+    *g = (p >> greenShift) << greenConvShift;
+    *b = (p >> blueShift) << blueConvShift;
+  } else if (cm) {
+    int ir, ig, ib;
+    cm->lookup(p, &ir, &ig, &ib);
+    *r = ir;
+    *g = ig;
+    *b = ib;
+  } else {
+    // XXX just return 0 for colour map?
+    *r = 0;
+    *g = 0;
+    *b = 0;
+  }
+}
+
+
+inline void PixelFormat::rgbFromPixel(Pixel p, ColourMap* cm, rdr::U8 *r, rdr::U8 *g, rdr::U8 *b) const
+{
+  if (trueColour) {
+    *r = (p >> redShift) << (redConvShift - 8);
+    *g = (p >> greenShift) << (greenConvShift - 8);
+    *b = (p >> blueShift) << (blueConvShift - 8);
+  } else {
+    rdr::U16 r2, g2, b2;
+
+    rgbFromPixel(p, cm, &r2, &g2, &b2);
+
+    *r = r2 >> 8;
+    *g = g2 >> 8;
+    *b = b2 >> 8;
+  }
+}
+
+
+}
+

common/rfb/ScaledPixelBuffer.cxx

@@ -115,12 +115,6 @@ void ScaledPixelBuffer::setScaleFilter(unsigned int scaleFilterID_) {
   }
 }
 
-inline void ScaledPixelBuffer::rgbFromPixel(U32 p, int &r, int &g, int &b) {
-  r = (((p >> pf.redShift  ) & pf.redMax  ) * 255 + pf.redMax  /2) / pf.redMax;
-  g = (((p >> pf.greenShift) & pf.greenMax) * 255 + pf.greenMax/2) / pf.greenMax;
-  b = (((p >> pf.blueShift ) & pf.blueMax ) * 255 + pf.blueMax /2) / pf.blueMax;
-}
-
 inline U32 ScaledPixelBuffer::getSourcePixel(int x, int y) {
   int bytes_per_pixel = pf.bpp / 8;
   U8 *ptr = &(*src_data)[(x + y*src_width)*bytes_per_pixel];
@@ -141,7 +135,8 @@ inline U32 ScaledPixelBuffer::getSourcePixel(int x, int y) {
 void ScaledPixelBuffer::scaleRect(const Rect& rect) {
   Rect changed_rect;
   U8 *ptr, *ptrs, *px, *pxs;
-  int r, g, b, red, green, blue;
+  U16 r, g, b;
+  int red, green, blue;
   short *xweight, *yweight, weight;
 
   // Calculate the changed pixel rect in the scaled image
@@ -169,7 +164,7 @@ void ScaledPixelBuffer::scaleRect(const Rect& rect) {
     for (int ys = yWeightTabs[y].i0; ys < yWeightTabs[y].i1; ys++) {
       px = pxs;
       for (int xs = xWeightTabs[changed_rect.tl.x].i0; xs < xWeightTabs[changed_rect.br.x-1].i1; xs++) {
-        rgbFromPixel(*((U32*)px), r, g, b);
+        pf.rgbFromPixel(*((U32*)px), NULL, &r, &g, &b);
         weight = *yweight;
         raccum[xs] += (int)(weight) * r;
         gaccum[xs] += (int)(weight) * g;

common/rfb/ScaledPixelBuffer.h

@@ -58,7 +58,6 @@ namespace rfb {
 
     // Pixel manipulation routines
     inline U32 getSourcePixel(int x, int y);
-    inline void rgbFromPixel(U32 p, int &r, int &g, int &b);
 
     // Get rectangle encompassing this buffer
     //   Top-left of rectangle is either at (0,0), or the specified point.

common/rfb/TightDecoder.cxx

@@ -26,24 +26,6 @@ extern "C" {
 
 using namespace rfb;
 
-#define RGB_TO_PIXEL(r,g,b)						\
-  (((PIXEL_T)(r) & myFormat.redMax) << myFormat.redShift |		\
-   ((PIXEL_T)(g) & myFormat.greenMax) << myFormat.greenShift |	        \
-   ((PIXEL_T)(b) & myFormat.blueMax) << myFormat.blueShift)
-
-#define RGB24_TO_PIXEL(r,g,b)                                       \
-   ((((PIXEL_T)(r) & 0xFF) * myFormat.redMax + 127) / 255             \
-    << myFormat.redShift |                                              \
-    (((PIXEL_T)(g) & 0xFF) * myFormat.greenMax + 127) / 255           \
-    << myFormat.greenShift |                                            \
-    (((PIXEL_T)(b) & 0xFF) * myFormat.blueMax + 127) / 255            \
-    << myFormat.blueShift)
-
-#define RGB24_TO_PIXEL32(r,g,b)						\
-  (((rdr::U32)(r) & 0xFF) << myFormat.redShift |				\
-   ((rdr::U32)(g) & 0xFF) << myFormat.greenShift |			\
-   ((rdr::U32)(b) & 0xFF) << myFormat.blueShift)
-
 #define TIGHT_MAX_WIDTH 2048
 
 static void JpegSetSrcManager(j_decompress_ptr cinfo, char *compressedData,

common/rfb/TrueColourMap.h

@@ -28,12 +28,11 @@ namespace rfb {
 
     virtual void lookup(int i, int* r, int* g, int* b)
     {
-      *r = (((i >> pf.redShift  ) & pf.redMax)
-            * 65535 + pf.redMax/2) / pf.redMax;
-      *g = (((i >> pf.greenShift) & pf.greenMax)
-            * 65535 + pf.greenMax/2) / pf.greenMax;
-      *b = (((i >> pf.blueShift) & pf.blueMax)
-            * 65535 + pf.blueMax/2) / pf.blueMax;
+      rdr::U16 _r, _g, _b;
+      pf.rgbFromPixel(i, NULL, &_r, &_g, &_b);
+      *r = _r;
+      *g = _g;
+      *b = _b;
     }
   private:
     PixelFormat pf;

common/rfb/ZRLEDecoder.cxx

@@ -63,21 +63,18 @@ void ZRLEDecoder::readRect(const Rect& r, CMsgHandler* handler)
   case 32:
     {
       const rfb::PixelFormat& pf = handler->cp.pf();
-      bool fitsInLS3Bytes = ((pf.redMax   << pf.redShift)   < (1<<24) &&
-                             (pf.greenMax << pf.greenShift) < (1<<24) &&
-                             (pf.blueMax  << pf.blueShift)  < (1<<24));
 
-      bool fitsInMS3Bytes = (pf.redShift   > 7  &&
-                             pf.greenShift > 7  &&
-                             pf.blueShift  > 7);
+      Pixel maxPixel = pf.pixelFromRGB((rdr::U16)-1, (rdr::U16)-1, (rdr::U16)-1);
+      bool fitsInLS3Bytes = maxPixel < (1<<24);
+      bool fitsInMS3Bytes = (maxPixel & 0xff) == 0;
 
-      if ((fitsInLS3Bytes && !pf.bigEndian) ||
-          (fitsInMS3Bytes && pf.bigEndian))
+      if ((fitsInLS3Bytes && pf.isLittleEndian()) ||
+          (fitsInMS3Bytes && pf.isBigEndian()))
       {
         zrleDecode24A(r, is, &zis, (rdr::U32*)buf, handler);
       }
-      else if ((fitsInLS3Bytes && pf.bigEndian) ||
-               (fitsInMS3Bytes && !pf.bigEndian))
+      else if ((fitsInLS3Bytes && pf.isBigEndian()) ||
+               (fitsInMS3Bytes && pf.isLittleEndian()))
       {
         zrleDecode24B(r, is, &zis, (rdr::U32*)buf, handler);
       }

common/rfb/ZRLEEncoder.cxx

@@ -87,21 +87,17 @@ bool ZRLEEncoder::writeRect(const Rect& r, ImageGetter* ig, Rect* actual)
     {
       const PixelFormat& pf = writer->getConnParams()->pf();
 
-      bool fitsInLS3Bytes = ((pf.redMax   << pf.redShift)   < (1<<24) &&
-                             (pf.greenMax << pf.greenShift) < (1<<24) &&
-                             (pf.blueMax  << pf.blueShift)  < (1<<24));
+      Pixel maxPixel = pf.pixelFromRGB((rdr::U16)-1, (rdr::U16)-1, (rdr::U16)-1);
+      bool fitsInLS3Bytes = maxPixel < (1<<24);
+      bool fitsInMS3Bytes = (maxPixel & 0xff) == 0;
 
-      bool fitsInMS3Bytes = (pf.redShift   > 7  &&
-                             pf.greenShift > 7  &&
-                             pf.blueShift  > 7);
-
-      if ((fitsInLS3Bytes && !pf.bigEndian) ||
-          (fitsInMS3Bytes && pf.bigEndian))
+      if ((fitsInLS3Bytes && pf.isLittleEndian()) ||
+          (fitsInMS3Bytes && pf.isBigEndian()))
       {
         wroteAll = zrleEncode24A(r, mos, &zos, imageBuf, maxLen, actual, ig);
       }
-      else if ((fitsInLS3Bytes && pf.bigEndian) ||
-               (fitsInMS3Bytes && !pf.bigEndian))
+      else if ((fitsInLS3Bytes && pf.isBigEndian()) ||
+               (fitsInMS3Bytes && pf.isLittleEndian()))
       {
         wroteAll = zrleEncode24B(r, mos, &zos, imageBuf, maxLen, actual, ig);
       }

common/rfb/rfb.dsp

@@ -597,6 +597,10 @@ SOURCE=.\PixelFormat.h
 # End Source File
 # Begin Source File
 
+SOURCE=.\PixelFormat.inl
+# End Source File
+# Begin Source File
+
 SOURCE=.\RawDecoder.h
 # End Source File
 # Begin Source File

common/rfb/tightDecode.h

@@ -67,8 +67,7 @@ void TIGHT_DECODE (const Rect& r, rdr::InStream* is,
   bool cutZeros = false;
   const rfb::PixelFormat& myFormat = handler->cp.pf();
 #if BPP == 32
-  if (myFormat.depth == 24 && myFormat.redMax == 0xFF &&
-      myFormat.greenMax == 0xFF && myFormat.blueMax == 0xFF) {
+  if (myFormat.is888()) {
     cutZeros = true;
   } 
 #endif
@@ -88,7 +87,7 @@ void TIGHT_DECODE (const Rect& r, rdr::InStream* is,
     PIXEL_T pix;
     if (cutZeros) {
       is->readBytes(bytebuf, 3);
-      pix = RGB24_TO_PIXEL32(bytebuf[0], bytebuf[1], bytebuf[2]);
+      pix = myFormat.pixelFromRGB(bytebuf[0], bytebuf[1], bytebuf[2]);
     } else {
       pix = is->READ_PIXEL();
     }
@@ -123,7 +122,7 @@ void TIGHT_DECODE (const Rect& r, rdr::InStream* is,
 	rdr::U8 *tightPalette = (rdr::U8*) palette;
 	is->readBytes(tightPalette, palSize*3);
 	for (int i = palSize - 1; i >= 0; i--) {
-	  palette[i] = RGB24_TO_PIXEL32(tightPalette[i*3],
+	  palette[i] = myFormat.pixelFromRGB(tightPalette[i*3],
 					tightPalette[i*3+1],
 					tightPalette[i*3+2]);
         }
@@ -179,7 +178,7 @@ void TIGHT_DECODE (const Rect& r, rdr::InStream* is,
       input->readBytes(buf, dataSize); 
       if (cutZeros) {
 	for (int p = r.height() * r.width() - 1; p >= 0; p--) {
-	  buf[p] = RGB24_TO_PIXEL32(bytebuf[p*3],
+	  buf[p] = myFormat.pixelFromRGB(bytebuf[p*3],
 				    bytebuf[p*3+1],
 				    bytebuf[p*3+2]);
 	}
@@ -270,7 +269,7 @@ DecompressJpegRect(const Rect& r, rdr::InStream* is,
 
     for (int dx = 0; dx < r.width(); dx++) {
       *pixelPtr++ = 
-	RGB24_TO_PIXEL(scanline[dx*3], scanline[dx*3+1], scanline[dx*3+2]);
+	myFormat.pixelFromRGB(scanline[dx*3], scanline[dx*3+1], scanline[dx*3+2]);
     }
   }
 
@@ -319,7 +318,7 @@ FilterGradient24(const Rect& r, rdr::InStream* is, int dataSize,
       pix[c] = netbuf[y*rectWidth*3+c] + prevRow[c];
       thisRow[c] = pix[c];
     }
-    buf[y*rectWidth] = RGB24_TO_PIXEL32(pix[0], pix[1], pix[2]);
+    buf[y*rectWidth] = myFormat.pixelFromRGB(pix[0], pix[1], pix[2]);
 
     /* Remaining pixels of a row */
     for (x = 1; x < rectWidth; x++) {
@@ -333,7 +332,7 @@ FilterGradient24(const Rect& r, rdr::InStream* is, int dataSize,
 	pix[c] = netbuf[(y*rectWidth+x)*3+c] + est[c];
 	thisRow[x*3+c] = pix[c];
       }
-      buf[y*rectWidth+x] = RGB24_TO_PIXEL32(pix[0], pix[1], pix[2]);
+      buf[y*rectWidth+x] = myFormat.pixelFromRGB(pix[0], pix[1], pix[2]);
     }
 
     memcpy(prevRow, thisRow, sizeof(prevRow));
@@ -351,9 +350,7 @@ FilterGradient(const Rect& r, rdr::InStream* is, int dataSize,
   int x, y, c;
   static rdr::U8 prevRow[TIGHT_MAX_WIDTH*sizeof(PIXEL_T)];
   static rdr::U8 thisRow[TIGHT_MAX_WIDTH*sizeof(PIXEL_T)];
-  int pix[3]; 
-  int max[3]; 
-  int shift[3]; 
+  rdr::U8 pix[3]; 
   int est[3]; 
 
   memset(prevRow, 0, sizeof(prevRow));
@@ -367,36 +364,39 @@ FilterGradient(const Rect& r, rdr::InStream* is, int dataSize,
 
   // Set up shortcut variables
   const rfb::PixelFormat& myFormat = handler->cp.pf();
-  max[0] = myFormat.redMax; 
-  max[1] = myFormat.greenMax; 
-  max[2] = myFormat.blueMax; 
-  shift[0] = myFormat.redShift; 
-  shift[1] = myFormat.greenShift; 
-  shift[2] = myFormat.blueShift; 
   int rectHeight = r.height();
   int rectWidth = r.width();
 
   for (y = 0; y < rectHeight; y++) {
     /* First pixel in a row */
-    for (c = 0; c < 3; c++) {
-      pix[c] = (netbuf[y*rectWidth] >> shift[c]) + prevRow[c] & max[c];
-      thisRow[c] = pix[c];
-    }
-    buf[y*rectWidth] = RGB_TO_PIXEL(pix[0], pix[1], pix[2]);
+    myFormat.rgbFromPixel(netbuf[y*rectWidth], NULL,
+                          &pix[0], &pix[1], &pix[2]);
+    for (c = 0; c < 3; c++)
+      pix[c] += prevRow[c];
+
+    memcpy(thisRow, pix, sizeof(pix));
+
+    buf[y*rectWidth] = myFormat.pixelFromRGB(pix[0], pix[1], pix[2]);
 
     /* Remaining pixels of a row */
     for (x = 1; x < rectWidth; x++) {
       for (c = 0; c < 3; c++) {
-	est[c] = prevRow[x*3+c] + pix[c] - prevRow[(x-1)*3+c];
-	if (est[c] > max[c]) {
-	  est[c] = max[c];
-	} else if (est[c] < 0) {
-	  est[c] = 0;
-	}
-	pix[c] = (netbuf[y*rectWidth+x] >> shift[c]) + est[c] & max[c];
-	thisRow[x*3+c] = pix[c];
+        est[c] = prevRow[x*3+c] + pix[c] - prevRow[(x-1)*3+c];
+        if (est[c] > 255) {
+          est[c] = 255;
+        } else if (est[c] < 0) {
+          est[c] = 0;
+        }
       }
-      buf[y*rectWidth+x] = RGB_TO_PIXEL(pix[0], pix[1], pix[2]);
+
+      myFormat.rgbFromPixel(netbuf[y*rectWidth+x], NULL,
+                            &pix[0], &pix[1], &pix[2]);
+      for (c = 0; c < 3; c++)
+        pix[c] += est[c];
+
+      memcpy(&thisRow[x*3], pix, sizeof(pix));
+
+      buf[y*rectWidth+x] = myFormat.pixelFromRGB(pix[0], pix[1], pix[2]);
     }
 
     memcpy(prevRow, thisRow, sizeof(prevRow));

common/rfb/tightEncode.h

@@ -79,9 +79,7 @@ struct TIGHT_PALETTE {
 };
 
 // FIXME: Is it really a good idea to use static variables for this?
-static int s_endianMismatch;      // local/remote formats differ in byte order
 static bool s_pack24;             // use 24-bit packing for 32-bit pixels
-static int s_rs, s_gs, s_bs;      // shifts for 24-bit pixel conversion
 
 // FIXME: Make a separate class for palette operations.
 static int s_palMaxColors, s_palNumColors;
@@ -251,14 +249,14 @@ JpegSetDstManager(j_compress_ptr cinfo, JOCTET *buf, size_t buflen)
 #endif  // #ifndef TIGHT_ONCE
 
 static void ENCODE_SOLID_RECT     (rdr::OutStream *os,
-                                   PIXEL_T *buf);
+                                   PIXEL_T *buf, const PixelFormat& pf);
 static void ENCODE_FULLCOLOR_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
-                                   PIXEL_T *buf, const Rect& r);
+                                   PIXEL_T *buf, const PixelFormat& pf, const Rect& r);
 static void ENCODE_MONO_RECT      (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
-                                   PIXEL_T *buf, const Rect& r);
+                                   PIXEL_T *buf, const PixelFormat& pf, const Rect& r);
 #if (BPP != 8)
 static void ENCODE_INDEXED_RECT   (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
-                                   PIXEL_T *buf, const Rect& r);
+                                   PIXEL_T *buf, const PixelFormat& pf, const Rect& r);
 static void ENCODE_JPEG_RECT      (rdr::OutStream *os,
                                    PIXEL_T *buf, const PixelFormat& pf, const Rect& r);
 #endif
@@ -271,7 +269,8 @@ static void FILL_PALETTE (PIXEL_T *data, int count);
 // Color components are assumed to be byte-aligned.
 //
 
-static inline unsigned int PACK_PIXELS (PIXEL_T *buf, unsigned int count)
+static inline unsigned int PACK_PIXELS (PIXEL_T *buf, unsigned int count,
+                                        const PixelFormat& pf)
 {
 #if (BPP != 32)
   return count * sizeof(PIXEL_T);
@@ -283,9 +282,8 @@ static inline unsigned int PACK_PIXELS (PIXEL_T *buf, unsigned int count)
   rdr::U8 *dst = (rdr::U8 *)buf;
   for (unsigned int i = 0; i < count; i++) {
     pix = *buf++;
-    *dst++ = (rdr::U8)(pix >> s_rs);
-    *dst++ = (rdr::U8)(pix >> s_gs);
-    *dst++ = (rdr::U8)(pix >> s_bs);
+    pf.rgbFromPixel(pix, NULL, &dst[0], &dst[1], &dst[2]);
+    dst += 3;
   }
   return count * 3;
 #endif
@@ -325,37 +323,14 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os,
 #endif
                   )
 {
-#if (BPP != 8) || (BPP == 32)
   const PixelFormat& pf = cp->pf();
-#endif
   GET_IMAGE_INTO_BUF(r, buf);
   PIXEL_T* pixels = (PIXEL_T*)buf;
 
-#if (BPP != 8)
-  union {
-    rdr::U32 value32;
-    rdr::U8 test;
-  } littleEndian;
-  littleEndian.value32 = 1;
-  s_endianMismatch = (littleEndian.test != !pf.bigEndian);
-#endif
-
 #if (BPP == 32)
   // Check if it's necessary to pack 24-bit pixels, and
   // compute appropriate shift values if necessary.
-  s_pack24 = (pf.depth == 24 && pf.redMax == 0xFF &&
-              pf.greenMax == 0xFF && pf.blueMax == 0xFF);
-  if (s_pack24) {
-    if (!s_endianMismatch) {
-      s_rs = pf.redShift;
-      s_gs = pf.greenShift;
-      s_bs = pf.blueShift;
-    } else {
-      s_rs = 24 - pf.redShift;
-      s_gs = 24 - pf.greenShift;
-      s_bs = 24 - pf.blueShift;
-    }
-  }
+  s_pack24 = pf.is888();
 #endif
 
   s_palMaxColors = r.area() / s_pconf->idxMaxColorsDivisor;
@@ -378,20 +353,20 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os,
       break;
     }
 #endif
-    ENCODE_FULLCOLOR_RECT(os, zos, pixels, r);
+    ENCODE_FULLCOLOR_RECT(os, zos, pixels, pf, r);
     break;
   case 1:
     // Solid rectangle
-    ENCODE_SOLID_RECT(os, pixels);
+    ENCODE_SOLID_RECT(os, pixels, pf);
     break;
   case 2:
     // Two-color rectangle
-    ENCODE_MONO_RECT(os, zos, pixels, r);
+    ENCODE_MONO_RECT(os, zos, pixels, pf, r);
     break;
 #if (BPP != 8)
   default:
     // Up to 256 different colors
-    ENCODE_INDEXED_RECT(os, zos, pixels, r);
+    ENCODE_INDEXED_RECT(os, zos, pixels, pf, r);
 #endif
   }
 }
@@ -400,26 +375,26 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os,
 // Subencoding implementations.
 //
 
-static void ENCODE_SOLID_RECT (rdr::OutStream *os, PIXEL_T *buf)
+static void ENCODE_SOLID_RECT (rdr::OutStream *os, PIXEL_T *buf, const PixelFormat& pf)
 {
   os->writeU8(0x08 << 4);
 
-  int length = PACK_PIXELS(buf, 1);
+  int length = PACK_PIXELS(buf, 1, pf);
   os->writeBytes(buf, length);
 }
 
 static void ENCODE_FULLCOLOR_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
-                                   PIXEL_T *buf, const Rect& r)
+                                   PIXEL_T *buf, const PixelFormat& pf, const Rect& r)
 {
   const int streamId = 0;
   os->writeU8(streamId << 4);
 
-  int length = PACK_PIXELS(buf, r.area());
+  int length = PACK_PIXELS(buf, r.area(), pf);
   compressData(os, &zos[streamId], buf, length, s_pconf->rawZlibLevel);
 }
 
 static void ENCODE_MONO_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
-                              PIXEL_T *buf, const Rect& r)
+                              PIXEL_T *buf, const PixelFormat& pf, const Rect& r)
 {
   const int streamId = 1;
   os->writeU8((streamId | 0x04) << 4);
@@ -428,7 +403,7 @@ static void ENCODE_MONO_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
   // Write the palette
   PIXEL_T pal[2] = { (PIXEL_T)s_monoBackground, (PIXEL_T)s_monoForeground };
   os->writeU8(1);
-  os->writeBytes(pal, PACK_PIXELS(pal, 2));
+  os->writeBytes(pal, PACK_PIXELS(pal, 2, pf));
 
   // Encode the data in-place
   PIXEL_T *src = buf;
@@ -486,7 +461,7 @@ static void ENCODE_MONO_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
 
 #if (BPP != 8)
 static void ENCODE_INDEXED_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
-                                 PIXEL_T *buf, const Rect& r)
+                                 PIXEL_T *buf, const PixelFormat& pf, const Rect& r)
 {
   const int streamId = 2;
   os->writeU8((streamId | 0x04) << 4);
@@ -498,7 +473,7 @@ static void ENCODE_INDEXED_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
     for (int i = 0; i < s_palNumColors; i++)
       pal[i] = (PIXEL_T)s_palette.entry[i].listNode->rgb;
     os->writeU8((rdr::U8)(s_palNumColors - 1));
-    os->writeBytes(pal, PACK_PIXELS(pal, s_palNumColors));
+    os->writeBytes(pal, PACK_PIXELS(pal, s_palNumColors, pf));
   }
 
   // Encode data in-place
@@ -537,23 +512,6 @@ static void ENCODE_INDEXED_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
 // JPEG compression.
 //
 
-#if (BPP != 8)
-static void PREPARE_JPEG_ROW (PIXEL_T *src, const PixelFormat& pf,
-                              rdr::U8 *dst, int count)
-{
-  // FIXME: Add a version of this function optimized for 24-bit colors?
-  PIXEL_T pix;
-  while (count--) {
-    pix = *src++;
-    if (s_endianMismatch)
-      pix = SWAP_PIXEL(pix);
-    *dst++ = (rdr::U8)((pix >> pf.redShift   & pf.redMax)   * 255 / pf.redMax);
-    *dst++ = (rdr::U8)((pix >> pf.greenShift & pf.greenMax) * 255 / pf.greenMax);
-    *dst++ = (rdr::U8)((pix >> pf.blueShift  & pf.blueMax)  * 255 / pf.blueMax);
-  }
-}
-#endif  // #if (BPP != 8)
-
 #if (BPP != 8)
 static void ENCODE_JPEG_RECT (rdr::OutStream *os, PIXEL_T *buf,
                               const PixelFormat& pf, const Rect& r)
@@ -585,7 +543,7 @@ static void ENCODE_JPEG_RECT (rdr::OutStream *os, PIXEL_T *buf,
 
   jpeg_start_compress(&cinfo, TRUE);
   for (int dy = 0; dy < h; dy++) {
-    PREPARE_JPEG_ROW(&buf[dy * w], pf, srcBuf, w);
+    pf.rgbFromBuffer(srcBuf, (const rdr::U8 *)(&buf[dy * w]), w);
     jpeg_write_scanlines(&cinfo, rowPointer, 1);
   }
   jpeg_finish_compress(&cinfo);