Turns out that TXImage wasn't just being overly cautious with the custom error

handler; the shared memory extension is actually this broken so it will return
success all the way through even when you're on a remote connection where
things cannot possibly work.

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4505 3789f03b-4d11-0410-bbf8-ca57d06f2519

diff --git a/vncviewer/X11PixelBuffer.cxx b/vncviewer/X11PixelBuffer.cxx
index 79fa76a2eee25d5c712895e17df667267cb22c3f..a3f8ebfd39411d5c58101bf2ccdeee486e2d2932 100644 (file)
--- a/vncviewer/X11PixelBuffer.cxx
+++ b/vncviewer/X11PixelBuffer.cxx
@@ -140,11 +140,20 @@ int PlatformPixelBuffer::getStride() const
   return xim->bytes_per_line / (getPF().bpp/8);
 }
 
+static bool caughtError;
+
+static int XShmAttachErrorHandler(Display *dpy, XErrorEvent *error)
+{
+  caughtError = true;
+  return 0;
+}
 
 int PlatformPixelBuffer::setupShm()
 {
   int major, minor;
   Bool pixmaps;
+  XErrorHandler old_handler;
+  Status status;
 
   if (!XShmQueryVersion(fl_display, &major, &minor, &pixmaps))
     return 0;
@@ -167,12 +176,27 @@ int PlatformPixelBuffer::setupShm()
     goto free_shm;
 
   shminfo->readOnly = True;
+
+  // This is the only way we can detect that shared memory won't work
+  // (e.g. because we're accessing a remote X11 server)
+  caughtError = false;
+  old_handler = XSetErrorHandler(XShmAttachErrorHandler);
+
   XShmAttach(fl_display, shminfo);
+  XSync(fl_display, False);
+
+  XSetErrorHandler(old_handler);
+
+  if (caughtError)
+    goto free_shmaddr;
 
   vlog.debug("Using shared memory XImage");
 
   return 1;
 
+free_shmaddr:
+  shmdt(shminfo->shmaddr);
+
 free_shm:
   shmctl(shminfo->shmid, IPC_RMID, 0);