From 19dbca236cfc807ffd7c90322dcda158781c289b Mon Sep 17 00:00:00 2001
From: Pierre Ossman <ossman@cendio.se>
Date: Tue, 21 Apr 2009 17:30:45 +0000
Subject: [PATCH] Make sure we handle endian problems in the conversion code.

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@3784 3789f03b-4d11-0410-bbf8-ca57d06f2519
---
 common/rfb/PixelFormat.cxx | 16 +++++++++++-----
 common/rfb/PixelFormat.h   |  3 ++-
 common/rfb/PixelFormat.inl |  9 +++++++++
 3 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/common/rfb/PixelFormat.cxx b/common/rfb/PixelFormat.cxx
index a106cf36..afcf4295 100644
--- a/common/rfb/PixelFormat.cxx
+++ b/common/rfb/PixelFormat.cxx
@@ -42,7 +42,7 @@ PixelFormat::PixelFormat(int b, int d, bool e, bool t,
   assert((greenMax & (greenMax + 1)) == 0);
   assert((blueMax & (blueMax + 1)) == 0);
 
-  updateShifts();
+  updateState();
 }
 
 PixelFormat::PixelFormat()
@@ -50,7 +50,7 @@ PixelFormat::PixelFormat()
     redMax(7), greenMax(7), blueMax(3),
     redShift(0), greenShift(3), blueShift(6)
 {
-  updateShifts();
+  updateState();
 }
 
 bool PixelFormat::equal(const PixelFormat& other) const
@@ -81,7 +81,7 @@ void PixelFormat::read(rdr::InStream* is)
   blueShift = is->readU8();
   is->skip(3);
 
-  updateShifts();
+  updateState();
 }
 
 void PixelFormat::write(rdr::OutStream* os) const
@@ -387,7 +387,7 @@ bool PixelFormat::parse(const char* str)
     return false;
   }
 
-  updateShifts();
+  updateState();
 
   return true;
 }
@@ -419,9 +419,10 @@ static int bits(rdr::U16 value)
   return bits;
 }
 
-void PixelFormat::updateShifts(void)
+void PixelFormat::updateState(void)
 {
   int redBits, greenBits, blueBits;
+  int endianTest = 1;
 
   redBits = bits(redMax);
   greenBits = bits(greenMax);
@@ -430,4 +431,9 @@ void PixelFormat::updateShifts(void)
   redConvShift = 16 - redBits;
   greenConvShift = 16 - greenBits;
   blueConvShift = 16 - blueBits;
+
+  if (((*(char*)&endianTest) == 0) != bigEndian)
+    endianMismatch = true;
+  else
+    endianMismatch = false;
 }
diff --git a/common/rfb/PixelFormat.h b/common/rfb/PixelFormat.h
index 8adbf911..c1de09a9 100644
--- a/common/rfb/PixelFormat.h
+++ b/common/rfb/PixelFormat.h
@@ -64,7 +64,7 @@ namespace rfb {
     bool parse(const char* str);
 
   protected:
-    void updateShifts(void);
+    void updateState(void);
 
   public:
     int bpp;
@@ -85,6 +85,7 @@ namespace rfb {
     int redConvShift;
     int greenConvShift;
     int blueConvShift;
+    bool endianMismatch;
   };
 }
 
diff --git a/common/rfb/PixelFormat.inl b/common/rfb/PixelFormat.inl
index 90d8e6da..ca83a305 100644
--- a/common/rfb/PixelFormat.inl
+++ b/common/rfb/PixelFormat.inl
@@ -75,9 +75,16 @@ inline void PixelFormat::bufferFromPixel(rdr::U8* buffer, Pixel p) const
 }
 
 
+#define _rfb_bswap_32(x) \
+        ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
+         (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
+
+
 inline void PixelFormat::rgbFromPixel(Pixel p, ColourMap* cm, rdr::U16 *r, rdr::U16 *g, rdr::U16 *b) const
 {
   if (trueColour) {
+    if (endianMismatch)
+      p = _rfb_bswap_32(p);
     /* We don't need to mask since we shift out unwanted bits */
     *r = (p >> redShift) << redConvShift;
     *g = (p >> greenShift) << greenConvShift;
@@ -100,6 +107,8 @@ inline void PixelFormat::rgbFromPixel(Pixel p, ColourMap* cm, rdr::U16 *r, rdr::
 inline void PixelFormat::rgbFromPixel(Pixel p, ColourMap* cm, rdr::U8 *r, rdr::U8 *g, rdr::U8 *b) const
 {
   if (trueColour) {
+    if (endianMismatch)
+      p = _rfb_bswap_32(p);
     *r = (p >> redShift) << (redConvShift - 8);
     *g = (p >> greenShift) << (greenConvShift - 8);
     *b = (p >> blueShift) << (blueConvShift - 8);
-- 
2.39.5