Fix some offenders that poke around in the PixelFormat internals

3 files changed, 47 insertions, 60 deletions

diff --git a/common/rfb/JpegCompressor.cxx b/common/rfb/JpegCompressor.cxx
index c1ef3c7d..960bd378 100644
--- a/common/rfb/JpegCompressor.cxx
+++ b/common/rfb/JpegCompressor.cxx
@@ -1,5 +1,6 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -33,6 +34,15 @@ extern "C" {
 using namespace rfb;
 
 //
+// Special formats that libjpeg can have optimised code paths for
+//
+
+static const PixelFormat pfRGBX(32, 24, false, true, 255, 255, 255, 0, 8, 16);
+static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0);
+static const PixelFormat pfXRGB(32, 24, false, true, 255, 255, 255, 8, 16, 24);
+static const PixelFormat pfXBGR(32, 24, false, true, 255, 255, 255, 24, 16, 8);
+
+//
 // Error manager implmentation for the JPEG library
 //
 
@@ -166,33 +176,20 @@ void JpegCompressor::compress(const rdr::U8 *buf, int stride, const Rect& r,
   pixelsize = 3;
 
 #ifdef JCS_EXTENSIONS
-  // Try to have libjpeg read directly from our native format
-  if(pf.is888()) {
-    int redShift, greenShift, blueShift;
-
-    if(pf.bigEndian) {
-      redShift = 24 - pf.redShift;
-      greenShift = 24 - pf.greenShift;
-      blueShift = 24 - pf.blueShift;
-    } else {
-      redShift = pf.redShift;
-      greenShift = pf.greenShift;
-      blueShift = pf.blueShift;
-    }
-
-    if(redShift == 0 && greenShift == 8 && blueShift == 16)
-      cinfo->in_color_space = JCS_EXT_RGBX;
-    if(redShift == 16 && greenShift == 8 && blueShift == 0)
-      cinfo->in_color_space = JCS_EXT_BGRX;
-    if(redShift == 24 && greenShift == 16 && blueShift == 8)
-      cinfo->in_color_space = JCS_EXT_XBGR;
-    if(redShift == 8 && greenShift == 16 && blueShift == 24)
-      cinfo->in_color_space = JCS_EXT_XRGB;
-
-    if (cinfo->in_color_space != JCS_RGB) {
-      srcBuf = (rdr::U8 *)buf;
-      pixelsize = 4;
-    }
+  // Try to have libjpeg output directly to our native format
+  // libjpeg can only handle some "standard" formats
+  if (pfRGBX.equal(pf))
+    cinfo->in_color_space = JCS_EXT_RGBX;
+  else if (pfBGRX.equal(pf))
+    cinfo->in_color_space = JCS_EXT_BGRX;
+  else if (pfXRGB.equal(pf))
+    cinfo->in_color_space = JCS_EXT_XRGB;
+  else if (pfXBGR.equal(pf))
+    cinfo->in_color_space = JCS_EXT_XBGR;
+
+  if (cinfo->in_color_space != JCS_RGB) {
+    srcBuf = (rdr::U8 *)buf;
+    pixelsize = 4;
   }
 #endif
 
diff --git a/common/rfb/JpegDecompressor.cxx b/common/rfb/JpegDecompressor.cxx
index 4d230eb8..3f4d2d00 100644
--- a/common/rfb/JpegDecompressor.cxx
+++ b/common/rfb/JpegDecompressor.cxx
@@ -1,6 +1,7 @@
 /* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
  * Copyright (C) 2004-2005 Cendio AB. All rights reserved.
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -33,6 +34,14 @@ extern "C" {
 
 using namespace rfb;
 
+//
+// Special formats that libjpeg can have optimised code paths for
+//
+
+static const PixelFormat pfRGBX(32, 24, false, true, 255, 255, 255, 0, 8, 16);
+static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0);
+static const PixelFormat pfXRGB(32, 24, false, true, 255, 255, 255, 8, 16, 24);
+static const PixelFormat pfXBGR(32, 24, false, true, 255, 255, 255, 24, 16, 8);
 
 //
 // Error manager implmentation for the JPEG library
@@ -169,33 +178,19 @@ void JpegDecompressor::decompress(const rdr::U8 *jpegBuf, int jpegBufLen,
 
 #ifdef JCS_EXTENSIONS
   // Try to have libjpeg output directly to our native format
-  if (pf.is888()) {
-    int redShift, greenShift, blueShift;
-
-    if(pf.bigEndian) {
-      redShift = 24 - pf.redShift;
-      greenShift = 24 - pf.greenShift;
-      blueShift = 24 - pf.blueShift;
-    } else {
-      redShift = pf.redShift;
-      greenShift = pf.greenShift;
-      blueShift = pf.blueShift;
-    }
-
-    // libjpeg can only handle some "standard" formats
-    if(redShift == 0 && greenShift == 8 && blueShift == 16)
-      dinfo->out_color_space = JCS_EXT_RGBX;
-    if(redShift == 16 && greenShift == 8 && blueShift == 0)
-      dinfo->out_color_space = JCS_EXT_BGRX;
-    if(redShift == 24 && greenShift == 16 && blueShift == 8)
-      dinfo->out_color_space = JCS_EXT_XBGR;
-    if(redShift == 8 && greenShift == 16 && blueShift == 24)
-      dinfo->out_color_space = JCS_EXT_XRGB;
-
-    if (dinfo->out_color_space != JCS_RGB) {
-      dstBuf = (rdr::U8 *)buf;
-      pixelsize = 4;
-    }
+  // libjpeg can only handle some "standard" formats
+  if (pfRGBX.equal(pf))
+    dinfo->out_color_space = JCS_EXT_RGBX;
+  else if (pfBGRX.equal(pf))
+    dinfo->out_color_space = JCS_EXT_BGRX;
+  else if (pfXRGB.equal(pf))
+    dinfo->out_color_space = JCS_EXT_XRGB;
+  else if (pfXBGR.equal(pf))
+    dinfo->out_color_space = JCS_EXT_XBGR;
+
+  if (dinfo->out_color_space != JCS_RGB) {
+    dstBuf = (rdr::U8 *)buf;
+    pixelsize = 4;
   }
 #endif
 
diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h
index d5b2c660..c121b7aa 100644
--- a/common/rfb/tightEncode.h
+++ b/common/rfb/tightEncode.h
@@ -540,12 +540,7 @@ void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r)
     *dataend = &data[stride * h];
   bool willTransform = ig->willTransform();
 
-  if (willTransform) {
-    mask = serverpf.redMax << serverpf.redShift;
-    mask |= serverpf.greenMax << serverpf.greenShift;
-    mask |= serverpf.blueMax << serverpf.blueShift;
-  }
-  else mask = ~0;
+  serverpf.bufferFromPixel((rdr::U8*)&mask, ~0);
 
   c0 = data[0] & mask;
   n0 = 0;