Fix rendering on big endian system

Our XRender code assumes a certain pixel layout which was not
guaranteed on big endian systems. The previous workaround only worked
for some cases, so fix this properly now.

2 files changed, 34 insertions, 8 deletions

diff --git a/vncviewer/PlatformPixelBuffer.cxx b/vncviewer/PlatformPixelBuffer.cxx
index c79b5c1a..e6a054ab 100644
--- a/vncviewer/PlatformPixelBuffer.cxx
+++ b/vncviewer/PlatformPixelBuffer.cxx
@@ -34,13 +34,8 @@
 static rfb::LogWriter vlog("PlatformPixelBuffer");
 
 PlatformPixelBuffer::PlatformPixelBuffer(int width, int height) :
-  FullFramePixelBuffer(rfb::PixelFormat(32, 24,
-#if !defined(WIN32) && !defined(__APPLE__)
-                                        ImageByteOrder(fl_display) == MSBFirst,
-#else
-                                        false,
-#endif
-                                        true, 255, 255, 255, 16, 8, 0),
+  FullFramePixelBuffer(rfb::PixelFormat(32, 24, false, true,
+                                        255, 255, 255, 16, 8, 0),
                        width, height, 0, stride),
   Surface(width, height)
 #if !defined(WIN32) && !defined(__APPLE__)
diff --git a/vncviewer/Surface_X11.cxx b/vncviewer/Surface_X11.cxx
index 3523da3d..6562634d 100644
--- a/vncviewer/Surface_X11.cxx
+++ b/vncviewer/Surface_X11.cxx
@@ -109,6 +109,7 @@ void Surface::blend(Surface* dst, int src_x, int src_y, int x, int y, int w, int
 
 void Surface::alloc()
 {
+  XRenderPictFormat templ;
   XRenderPictFormat* format;
 
   // Might not be open at this point
@@ -117,7 +118,37 @@ void Surface::alloc()
   pixmap = XCreatePixmap(fl_display, XDefaultRootWindow(fl_display),
                          width(), height(), 32);
 
-  format = XRenderFindStandardFormat(fl_display, PictStandardARGB32);
+  // Our code assumes a BGRA byte order, regardless of what the endian
+  // of the machine is or the native byte order of XImage, so make sure
+  // we find such a format
+  templ.type = PictTypeDirect;
+  templ.depth = 32;
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+  templ.direct.alpha = 0;
+  templ.direct.red   = 8;
+  templ.direct.green = 16;
+  templ.direct.blue  = 24;
+#else
+  templ.direct.alpha = 24;
+  templ.direct.red   = 16;
+  templ.direct.green = 8;
+  templ.direct.blue  = 0;
+#endif
+  templ.direct.alphaMask = 0xff;
+  templ.direct.redMask = 0xff;
+  templ.direct.greenMask = 0xff;
+  templ.direct.blueMask = 0xff;
+
+  format = XRenderFindFormat(fl_display, PictFormatType | PictFormatDepth |
+                             PictFormatRed | PictFormatRedMask |
+                             PictFormatGreen | PictFormatGreenMask |
+                             PictFormatBlue | PictFormatBlueMask |
+                             PictFormatAlpha | PictFormatAlphaMask,
+                             &templ, 0);
+
+  if (!format)
+    throw rdr::Exception("XRenderFindFormat");
+
   picture = XRenderCreatePicture(fl_display, pixmap, format, 0, NULL);
 
   visFormat = XRenderFindVisualFormat(fl_display, fl_visual->visual);