Fix some offenders that poke around in the PixelFormat internals

common/rfb/JpegCompressor.cxx

@@ -1,5 +1,6 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -32,6 +33,15 @@ extern "C" {
 
 using namespace rfb;
 
+//
+// Special formats that libjpeg can have optimised code paths for
+//
+
+static const PixelFormat pfRGBX(32, 24, false, true, 255, 255, 255, 0, 8, 16);
+static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0);
+static const PixelFormat pfXRGB(32, 24, false, true, 255, 255, 255, 8, 16, 24);
+static const PixelFormat pfXBGR(32, 24, false, true, 255, 255, 255, 24, 16, 8);
+
 //
 // Error manager implmentation for the JPEG library
 //
@@ -166,33 +176,20 @@ void JpegCompressor::compress(const rdr::U8 *buf, int stride, const Rect& r,
   pixelsize = 3;
 
 #ifdef JCS_EXTENSIONS
-  // Try to have libjpeg read directly from our native format
-  if(pf.is888()) {
-    int redShift, greenShift, blueShift;
-
-    if(pf.bigEndian) {
-      redShift = 24 - pf.redShift;
-      greenShift = 24 - pf.greenShift;
-      blueShift = 24 - pf.blueShift;
-    } else {
-      redShift = pf.redShift;
-      greenShift = pf.greenShift;
-      blueShift = pf.blueShift;
-    }
-
-    if(redShift == 0 && greenShift == 8 && blueShift == 16)
-      cinfo->in_color_space = JCS_EXT_RGBX;
-    if(redShift == 16 && greenShift == 8 && blueShift == 0)
-      cinfo->in_color_space = JCS_EXT_BGRX;
-    if(redShift == 24 && greenShift == 16 && blueShift == 8)
-      cinfo->in_color_space = JCS_EXT_XBGR;
-    if(redShift == 8 && greenShift == 16 && blueShift == 24)
-      cinfo->in_color_space = JCS_EXT_XRGB;
-
-    if (cinfo->in_color_space != JCS_RGB) {
-      srcBuf = (rdr::U8 *)buf;
-      pixelsize = 4;
-    }
+  // Try to have libjpeg output directly to our native format
+  // libjpeg can only handle some "standard" formats
+  if (pfRGBX.equal(pf))
+    cinfo->in_color_space = JCS_EXT_RGBX;
+  else if (pfBGRX.equal(pf))
+    cinfo->in_color_space = JCS_EXT_BGRX;
+  else if (pfXRGB.equal(pf))
+    cinfo->in_color_space = JCS_EXT_XRGB;
+  else if (pfXBGR.equal(pf))
+    cinfo->in_color_space = JCS_EXT_XBGR;
+
+  if (cinfo->in_color_space != JCS_RGB) {
+    srcBuf = (rdr::U8 *)buf;
+    pixelsize = 4;
   }
 #endif
 

common/rfb/JpegDecompressor.cxx

@@ -1,6 +1,7 @@
 /* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
  * Copyright (C) 2004-2005 Cendio AB. All rights reserved.
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -33,6 +34,14 @@ extern "C" {
 
 using namespace rfb;
 
+//
+// Special formats that libjpeg can have optimised code paths for
+//
+
+static const PixelFormat pfRGBX(32, 24, false, true, 255, 255, 255, 0, 8, 16);
+static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0);
+static const PixelFormat pfXRGB(32, 24, false, true, 255, 255, 255, 8, 16, 24);
+static const PixelFormat pfXBGR(32, 24, false, true, 255, 255, 255, 24, 16, 8);
 
 //
 // Error manager implmentation for the JPEG library
@@ -169,33 +178,19 @@ void JpegDecompressor::decompress(const rdr::U8 *jpegBuf, int jpegBufLen,
 
 #ifdef JCS_EXTENSIONS
   // Try to have libjpeg output directly to our native format
-  if (pf.is888()) {
-    int redShift, greenShift, blueShift;
-
-    if(pf.bigEndian) {
-      redShift = 24 - pf.redShift;
-      greenShift = 24 - pf.greenShift;
-      blueShift = 24 - pf.blueShift;
-    } else {
-      redShift = pf.redShift;
-      greenShift = pf.greenShift;
-      blueShift = pf.blueShift;
-    }
-
-    // libjpeg can only handle some "standard" formats
-    if(redShift == 0 && greenShift == 8 && blueShift == 16)
-      dinfo->out_color_space = JCS_EXT_RGBX;
-    if(redShift == 16 && greenShift == 8 && blueShift == 0)
-      dinfo->out_color_space = JCS_EXT_BGRX;
-    if(redShift == 24 && greenShift == 16 && blueShift == 8)
-      dinfo->out_color_space = JCS_EXT_XBGR;
-    if(redShift == 8 && greenShift == 16 && blueShift == 24)
-      dinfo->out_color_space = JCS_EXT_XRGB;
-
-    if (dinfo->out_color_space != JCS_RGB) {
-      dstBuf = (rdr::U8 *)buf;
-      pixelsize = 4;
-    }
+  // libjpeg can only handle some "standard" formats
+  if (pfRGBX.equal(pf))
+    dinfo->out_color_space = JCS_EXT_RGBX;
+  else if (pfBGRX.equal(pf))
+    dinfo->out_color_space = JCS_EXT_BGRX;
+  else if (pfXRGB.equal(pf))
+    dinfo->out_color_space = JCS_EXT_XRGB;
+  else if (pfXBGR.equal(pf))
+    dinfo->out_color_space = JCS_EXT_XBGR;
+
+  if (dinfo->out_color_space != JCS_RGB) {
+    dstBuf = (rdr::U8 *)buf;
+    pixelsize = 4;
   }
 #endif
 

common/rfb/tightEncode.h

@@ -540,12 +540,7 @@ void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r)
     *dataend = &data[stride * h];
   bool willTransform = ig->willTransform();
 
-  if (willTransform) {
-    mask = serverpf.redMax << serverpf.redShift;
-    mask |= serverpf.greenMax << serverpf.greenShift;
-    mask |= serverpf.blueMax << serverpf.blueShift;
-  }
-  else mask = ~0;
+  serverpf.bufferFromPixel((rdr::U8*)&mask, ~0);
 
   c0 = data[0] & mask;
   n0 = 0;

unix/x0vncserver/XPixelBuffer.cxx

@@ -1,4 +1,5 @@
 /* Copyright (C) 2007-2008 Constantin Kaplinsky.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  *    
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -38,16 +39,16 @@ XPixelBuffer::XPixelBuffer(Display *dpy, ImageFactory &factory,
     m_stride(0)
 {
   // Fill in the PixelFormat structure of the parent class.
-  format.bpp        = m_image->xim->bits_per_pixel;
-  format.depth      = m_image->xim->depth;
-  format.bigEndian  = (m_image->xim->byte_order == MSBFirst);
-  format.trueColour = m_image->isTrueColor();
-  format.redShift   = ffs(m_image->xim->red_mask) - 1;
-  format.greenShift = ffs(m_image->xim->green_mask) - 1;
-  format.blueShift  = ffs(m_image->xim->blue_mask) - 1;
-  format.redMax     = m_image->xim->red_mask   >> format.redShift;
-  format.greenMax   = m_image->xim->green_mask >> format.greenShift;
-  format.blueMax    = m_image->xim->blue_mask  >> format.blueShift;
+  format = PixelFormat(m_image->xim->bits_per_pixel,
+                       m_image->xim->depth,
+                       (m_image->xim->byte_order == MSBFirst),
+                       m_image->isTrueColor(),
+                       m_image->xim->red_mask   >> (ffs(m_image->xim->red_mask) - 1),
+                       m_image->xim->green_mask >> (ffs(m_image->xim->green_mask) - 1),
+                       m_image->xim->blue_mask  >> (ffs(m_image->xim->blue_mask) - 1),
+                       ffs(m_image->xim->red_mask) - 1,
+                       ffs(m_image->xim->green_mask) - 1,
+                       ffs(m_image->xim->blue_mask) - 1);
 
   // Set up the remaining data of the parent class.
   width_ = rect.width();

win/rfb_win32/DIBSectionBuffer.cxx

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -55,9 +56,11 @@ void DIBSectionBuffer::setPF(const PixelFormat& pf) {
   if ((pf.bpp <= 8) && pf.trueColour) {
     vlog.info("creating %d-bit TrueColour palette", pf.depth);
     for (int i=0; i < (1<<(pf.depth)); i++) {
-      palette[i].b = ((((i >> pf.blueShift) & pf.blueMax) * 65535) + pf.blueMax/2) / pf.blueMax;
-      palette[i].g = ((((i >> pf.greenShift) & pf.greenMax) * 65535) + pf.greenMax/2) / pf.greenMax;
-      palette[i].r = ((((i >> pf.redShift) & pf.redMax) * 65535) + pf.redMax/2) / pf.redMax;
+      rdr::U16 r, g, b;
+      pf.rgbFromPixel(i, NULL, &r, &g, &b);
+      palette[i].r = r;
+      palette[i].g = g;
+      palette[i].b = b;
     }
     refreshPalette();
   }
@@ -110,9 +113,9 @@ void DIBSectionBuffer::recreateBuffer() {
     bi.bmiHeader.biWidth = width_;
     bi.bmiHeader.biHeight = -height_;
     bi.bmiHeader.biCompression = (format.bpp > 8) ? BI_BITFIELDS : BI_RGB;
-    bi.mask.red = format.redMax << format.redShift;
-    bi.mask.green = format.greenMax << format.greenShift;
-    bi.mask.blue = format.blueMax << format.blueShift;
+    bi.mask.red = format.pixelFromRGB((rdr::U16)~0, 0, 0);
+    bi.mask.green = format.pixelFromRGB(0, (rdr::U16)~0, 0);
+    bi.mask.blue = format.pixelFromRGB(0, 0, (rdr::U16)~0);
 
     // Create a DIBSection to draw into
     if (device)
@@ -160,6 +163,11 @@ void DIBSectionBuffer::recreateBuffer() {
   }
 
   if (new_bitmap) {
+    int bpp, depth;
+    bool trueColour;
+    int redMax, greenMax, blueMax;
+    int redShift, greenShift, blueShift;
+
     // Set up the new bitmap
     bitmap = new_bitmap;
     data = new_data;
@@ -180,30 +188,34 @@ void DIBSectionBuffer::recreateBuffer() {
     }
 
     // Calculate the PixelFormat for the DIB
-    format.bigEndian = 0;
-    format.bpp = format.depth = ds.dsBm.bmBitsPixel;
-    format.trueColour = format.trueColour || format.bpp > 8;
-    if (format.bpp > 8) {
+    bpp = depth = ds.dsBm.bmBitsPixel;
+    trueColour = format.trueColour || format.bpp > 8;
 
+    if (trueColour) {
       // Get the truecolour format used by the DIBSection
-      initMaxAndShift(ds.dsBitfields[0], &format.redMax, &format.redShift);
-      initMaxAndShift(ds.dsBitfields[1], &format.greenMax, &format.greenShift);
-      initMaxAndShift(ds.dsBitfields[2], &format.blueMax, &format.blueShift);
+      initMaxAndShift(ds.dsBitfields[0], &redMax, &redShift);
+      initMaxAndShift(ds.dsBitfields[1], &greenMax, &greenShift);
+      initMaxAndShift(ds.dsBitfields[2], &blueMax, &blueShift);
 
       // Calculate the effective depth
-      format.depth = 0;
+      depth = 0;
       Pixel bits = ds.dsBitfields[0] | ds.dsBitfields[1] | ds.dsBitfields[2];
       while (bits) {
-        format.depth++;
+        depth++;
         bits = bits >> 1;
       }
-      if (format.depth > format.bpp)
+      if (depth > bpp)
         throw Exception("Bad DIBSection format (depth exceeds bpp)");
-    } else {
+    }
+
+    format = PixelFormat(bpp, depth, false, trueColour,
+                         redMax, greenMax, blueMax,
+                         redShift, greenShift, blueShift);
+
+    if (!trueColour) {
       // Set the DIBSection's palette
       refreshPalette();
     }
-
   }
 }
 

win/rfb_win32/DeviceContext.cxx

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -33,7 +34,11 @@ PixelFormat DeviceContext::getPF() const {
 }
 
 PixelFormat DeviceContext::getPF(HDC dc) {
-  PixelFormat format;
+  bool trueColour, bigEndian;
+  int bpp, depth;
+  int redMax, greenMax, blueMax;
+  int redShift, greenShift, blueShift;
+
   CompatibleBitmap bitmap(dc, 1, 1);
 
   // -=- Get the bitmap format information
@@ -49,11 +54,11 @@ PixelFormat DeviceContext::getPF(HDC dc) {
   }
 
   // Set the initial format information
-  format.trueColour = bi.bmiHeader.biBitCount > 8;
-  format.bigEndian = 0;
-  format.bpp = bi.bmiHeader.biBitCount;
+  trueColour = bi.bmiHeader.biBitCount > 8;
+  bigEndian = 0;
+  bpp = bi.bmiHeader.biBitCount;
 
-  if (format.trueColour) {
+  if (trueColour) {
     DWORD rMask=0, gMask=0, bMask=0;
 
     // Which true colour format is the DIB section using?
@@ -92,44 +97,42 @@ PixelFormat DeviceContext::getPF(HDC dc) {
     };
 
     // Convert the data we just retrieved
-    initMaxAndShift(rMask, &format.redMax, &format.redShift);
-    initMaxAndShift(gMask, &format.greenMax, &format.greenShift);
-    initMaxAndShift(bMask, &format.blueMax, &format.blueShift);
+    initMaxAndShift(rMask, &redMax, &redShift);
+    initMaxAndShift(gMask, &greenMax, &greenShift);
+    initMaxAndShift(bMask, &blueMax, &blueShift);
 
     // Calculate the depth from the colour shifts
-    format.depth = 0;
+    depth = 0;
     Pixel bits = rMask | gMask | bMask;
     while (bits) {
-      format.depth++;
+      depth++;
       bits = bits >> 1;
     }
 
     // Check that the depth & bpp are valid
-    if (format.depth > format.bpp) {
+    if (depth > bpp) {
       vlog.error("depth exceeds bits per pixel!");
-      format.bpp = format.depth;
+      bpp = depth;
     }
 
     // Correct the bits-per-pixel to something we're happy with
-    if (format.bpp <= 16)
-      format.bpp = 16;
-    else if (format.bpp <= 32)
-      format.bpp = 32;
+    if (bpp <= 16)
+      bpp = 16;
+    else if (bpp <= 32)
+      bpp = 32;
   } else {
     // Palettised format - depth reflects number of colours,
     // but bits-per-pixel is ALWAYS 8
-    format.depth = format.bpp;
-    if (format.bpp < 8)
-      format.bpp = 8;
-    vlog.info("%d-colour palettised", 1<<format.depth);
+    depth = bpp;
+    if (bpp < 8)
+      bpp = 8;
+    vlog.info("%d-colour palettised", 1<<depth);
   }
 
 
-  // Use 10 arguments constructor to trigger PixelFormat::updateState()
-  return PixelFormat(format.bpp, format.depth,
-		     format.bigEndian, format.trueColour,
-		     format.redMax, format.greenMax, format.blueMax,
-		     format.redShift, format.greenShift, format.blueShift);
+  return PixelFormat(bpp, depth, bigEndian, trueColour,
+		                 redMax, greenMax, blueMax,
+		                 redShift, greenShift, blueShift);
 }
 
 Rect DeviceContext::getClipBox() const {