Avoid touching PixelFormat internals.

We forgot to update the internal variables with the previous version which
caused problems with some clients (first noticed with gtk-vnc).


git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@3746 3789f03b-4d11-0410-bbf8-ca57d06f2519

3 files changed, 65 insertions, 38 deletions

diff --git a/unix/xserver/hw/vnc/XserverDesktop.cc b/unix/xserver/hw/vnc/XserverDesktop.cc
index d4965d85..9ad283b8 100644
--- a/unix/xserver/hw/vnc/XserverDesktop.cc
+++ b/unix/xserver/hw/vnc/XserverDesktop.cc
@@ -172,7 +172,8 @@ public:
 XserverDesktop::XserverDesktop(ScreenPtr pScreen_,
                                network::TcpListener* listener_,
                                network::TcpListener* httpListener_,
-                               const char* name, void* fbptr, int stride)
+                               const char* name, const rfb::PixelFormat &pf,
+                               void* fbptr, int stride)
   : pScreen(pScreen_), deferredUpdateTimer(0), dummyTimer(0),
     server(0), httpServer(0),
     listener(listener_), httpListener(httpListener_),
@@ -181,41 +182,7 @@ XserverDesktop::XserverDesktop(ScreenPtr pScreen_,
     oldButtonMask(0),
     queryConnectId(0)
 {
-  int i;
-  format.depth = pScreen->rootDepth;
-  for (i = 0; i < screenInfo.numPixmapFormats; i++) {
-    if (screenInfo.formats[i].depth == format.depth) {
-      format.bpp = screenInfo.formats[i].bitsPerPixel;
-      break;
-    }
-  }
-  if (i == screenInfo.numPixmapFormats) {
-    fprintf(stderr,"no pixmap format for root depth???\n");
-    abort();
-  }
-  format.bigEndian = (screenInfo.imageByteOrder == MSBFirst);
-
-  VisualPtr vis = NULL;
-  for (i = 0; i < pScreen->numVisuals; i++) {
-    if (pScreen->visuals[i].vid == pScreen->rootVisual) {
-      vis = &pScreen->visuals[i];
-      break;
-    }
-  }
-  if (i == pScreen->numVisuals) {
-    fprintf(stderr,"no visual rec for root visual???\n");
-    abort();
-  }
-  format.trueColour = (vis->c_class == TrueColor);
-  if (!format.trueColour && format.bpp != 8)
-    throw rfb::Exception("X server uses unsupported visual");
-  format.redShift   = ffs(vis->redMask) - 1;
-  format.greenShift = ffs(vis->greenMask) - 1;
-  format.blueShift  = ffs(vis->blueMask) - 1;
-  format.redMax     = vis->redMask   >> format.redShift;
-  format.greenMax   = vis->greenMask >> format.greenShift;
-  format.blueMax    = vis->blueMask  >> format.blueShift;
-
+  format = pf;
   colourmap = this;
 
   serverReset(pScreen);
diff --git a/unix/xserver/hw/vnc/XserverDesktop.h b/unix/xserver/hw/vnc/XserverDesktop.h
index 8eecb6b3..6777f092 100644
--- a/unix/xserver/hw/vnc/XserverDesktop.h
+++ b/unix/xserver/hw/vnc/XserverDesktop.h
@@ -62,7 +62,8 @@ public:
 
   XserverDesktop(ScreenPtr pScreen, network::TcpListener* listener,
                  network::TcpListener* httpListener_,
-                 const char* name, void* fbptr, int stride);
+                 const char* name, const rfb::PixelFormat &pf,
+                 void* fbptr, int stride);
   virtual ~XserverDesktop();
 
   // methods called from X server code
diff --git a/unix/xserver/hw/vnc/vncExtInit.cc b/unix/xserver/hw/vnc/vncExtInit.cc
index 54e92f9a..a15a77b4 100644
--- a/unix/xserver/hw/vnc/vncExtInit.cc
+++ b/unix/xserver/hw/vnc/vncExtInit.cc
@@ -121,6 +121,61 @@ rfb::BoolParameter localhostOnly("localhost",
                                  "Only allow connections from localhost",
                                  false);
 
+static PixelFormat vncGetPixelFormat(ScreenPtr pScreen)
+{
+  int depth, bpp;
+  int trueColour, bigEndian;
+  int redShift, greenShift, blueShift;
+  int redMax, greenMax, blueMax;
+
+  int i;
+  VisualPtr vis = NULL;
+
+  depth = pScreen->rootDepth;
+
+  for (i = 0; i < screenInfo.numPixmapFormats; i++) {
+    if (screenInfo.formats[i].depth == depth) {
+      bpp = screenInfo.formats[i].bitsPerPixel;
+      break;
+    }
+  }
+
+  if (i == screenInfo.numPixmapFormats) {
+    fprintf(stderr,"no pixmap format for root depth???\n");
+    abort();
+  }
+
+  bigEndian = (screenInfo.imageByteOrder == MSBFirst);
+
+  for (i = 0; i < pScreen->numVisuals; i++) {
+    if (pScreen->visuals[i].vid == pScreen->rootVisual) {
+      vis = &pScreen->visuals[i];
+      break;
+    }
+  }
+
+  if (i == pScreen->numVisuals) {
+    fprintf(stderr,"no visual rec for root visual???\n");
+    abort();
+  }
+
+  trueColour = (vis->c_class == TrueColor);
+
+  if (!trueColour && bpp != 8)
+    throw rfb::Exception("X server uses unsupported visual");
+
+  redShift   = ffs(vis->redMask) - 1;
+  greenShift = ffs(vis->greenMask) - 1;
+  blueShift  = ffs(vis->blueMask) - 1;
+  redMax     = vis->redMask   >> redShift;
+  greenMax   = vis->greenMask >> greenShift;
+  blueMax    = vis->blueMask  >> blueShift;
+
+  return PixelFormat(bpp, depth, bigEndian, trueColour,
+                     redMax, greenMax, blueMax,
+                     redShift, greenShift, blueShift);
+}
+
 void vncExtensionInit()
 {
   if (vncExtGeneration == serverGeneration) {
@@ -186,9 +241,13 @@ void vncExtensionInit()
         }
 
         CharArray desktopNameStr(desktopName.getData());
-        desktop[scr] = new XserverDesktop(screenInfo.screens[scr], listener,
+        PixelFormat pf = vncGetPixelFormat(screenInfo.screens[scr]);
+
+        desktop[scr] = new XserverDesktop(screenInfo.screens[scr],
+                                          listener,
                                           httpListener,
                                           desktopNameStr.buf,
+                                          pf,
                                           vncFbptr[scr],
                                           vncFbstride[scr]);
         vlog.info("created VNC server for screen %d", scr);