Add support for notifying clients about pointer movements

This change adds support for the VMware Mouse Position
pseudo-encoding[1], which is used to notify VNC clients when X11 clients
call `XWarpPointer()`[2]. This function is called by SDL (and other
similar libraries)  when they detect that the server does not support
native relative motion, like some RFB clients.

With this, RFB clients can choose to adjust the local cursor position
under certain circumstances to match what the server has set. For
instance, if pointer lock has been enabled on the client's machine and
the cursor is not being drawn locally, the local position of the cursor
is irrelevant, so the RFB client can use what the server sends as the
canonical absolute position of the cursor. This ultimately enables the
possibility of games (especially FPS games) to behave how users expect
(if the clients implement the corresponding change).

Part of: #619

1: https://github.com/rfbproto/rfbproto/blob/master/rfbproto.rst#vmware-cursor-position-pseudo-encoding
2: https://tronche.com/gui/x/xlib/input/XWarpPointer.html
3: https://hg.libsdl.org/SDL/file/28e3b60e2131/src/events/SDL_mouse.c#l804

diff --git a/common/rfb/ClientParams.cxx b/common/rfb/ClientParams.cxx
index 6f075a2461673392a2e364160fc7913fc20d3be3..c0cc36416bd507472e10d92c2b5099edc9e60397 100644 (file)
--- a/common/rfb/ClientParams.cxx
+++ b/common/rfb/ClientParams.cxx
@@ -30,7 +30,7 @@ ClientParams::ClientParams()
     compressLevel(2), qualityLevel(-1), fineQualityLevel(-1),
     subsampling(subsampleUndefined),
     width_(0), height_(0), name_(0),
-    ledState_(ledUnknown)
+    cursorPos_(0, 0), ledState_(ledUnknown)
 {
   setName("");
 
@@ -85,6 +85,11 @@ void ClientParams::setCursor(const Cursor& other)
   cursor_ = new Cursor(other);
 }
 
+void ClientParams::setCursorPos(const Point& pos)
+{
+  cursorPos_ = pos;
+}
+
 bool ClientParams::supportsEncoding(rdr::S32 encoding) const
 {
   return encodings_.count(encoding) != 0;
@@ -182,6 +187,13 @@ bool ClientParams::supportsLocalCursor() const
   return false;
 }
 
+bool ClientParams::supportsCursorPosition() const
+{
+  if (supportsEncoding(pseudoEncodingVMwareCursorPosition))
+    return true;
+  return false;
+}
+
 bool ClientParams::supportsDesktopSize() const
 {
   if (supportsEncoding(pseudoEncodingExtendedDesktopSize))

diff --git a/common/rfb/ClientParams.h b/common/rfb/ClientParams.h
index 3894ef2dd58e70a36f88f4806d5595e0634efe15..202cef4e2197042f1cfe239ad6f783686d65616e 100644 (file)
--- a/common/rfb/ClientParams.h
+++ b/common/rfb/ClientParams.h
@@ -77,6 +77,9 @@ namespace rfb {
     const Cursor& cursor() const { return *cursor_; }
     void setCursor(const Cursor& cursor);
 
+    const Point& cursorPos() const { return cursorPos_; }
+    void setCursorPos(const Point& pos);
+
     bool supportsEncoding(rdr::S32 encoding) const;
 
     void setEncodings(int nEncodings, const rdr::S32* encodings);
@@ -91,6 +94,7 @@ namespace rfb {
     // Wrappers to check for functionality rather than specific
     // encodings
     bool supportsLocalCursor() const;
+    bool supportsCursorPosition() const;
     bool supportsDesktopSize() const;
     bool supportsLEDState() const;
     bool supportsFence() const;
@@ -110,6 +114,7 @@ namespace rfb {
     PixelFormat pf_;
     char* name_;
     Cursor* cursor_;
+    Point cursorPos_;
     std::set<rdr::S32> encodings_;
     unsigned int ledState_;
     rdr::U32 clipFlags;

diff --git a/common/rfb/SMsgWriter.cxx b/common/rfb/SMsgWriter.cxx
index a29ed9d8623163bdb68e0b51628fdd6b45d7e701..2f40c0a67b672d068ee915d4fe13c2e863d0fc20 100644 (file)
--- a/common/rfb/SMsgWriter.cxx
+++ b/common/rfb/SMsgWriter.cxx
@@ -42,7 +42,8 @@ SMsgWriter::SMsgWriter(ClientParams* client_, rdr::OutStream* os_)
   : client(client_), os(os_),
     nRectsInUpdate(0), nRectsInHeader(0),
     needSetDesktopName(false), needCursor(false),
-    needLEDState(false), needQEMUKeyEvent(false)
+    needCursorPos(false), needLEDState(false),
+    needQEMUKeyEvent(false)
 {
 }
 
@@ -269,6 +270,14 @@ void SMsgWriter::writeCursor()
   needCursor = true;
 }
 
+void SMsgWriter::writeCursorPos()
+{
+  if (!client->supportsEncoding(pseudoEncodingVMwareCursorPosition))
+    throw Exception("Client does not support cursor position");
+
+  needCursorPos = true;
+}
+
 void SMsgWriter::writeLEDState()
 {
   if (!client->supportsEncoding(pseudoEncodingLEDState) &&
@@ -294,6 +303,8 @@ bool SMsgWriter::needFakeUpdate()
     return true;
   if (needCursor)
     return true;
+  if (needCursorPos)
+    return true;
   if (needLEDState)
     return true;
   if (needQEMUKeyEvent)
@@ -340,6 +351,8 @@ void SMsgWriter::writeFramebufferUpdateStart(int nRects)
       nRects++;
     if (needCursor)
       nRects++;
+    if (needCursorPos)
+      nRects++;
     if (needLEDState)
       nRects++;
     if (needQEMUKeyEvent)
@@ -455,6 +468,18 @@ void SMsgWriter::writePseudoRects()
     needCursor = false;
   }
 
+  if (needCursorPos) {
+    const Point& cursorPos = client->cursorPos();
+
+    if (client->supportsEncoding(pseudoEncodingVMwareCursorPosition)) {
+      writeSetVMwareCursorPositionRect(cursorPos.x, cursorPos.y);
+    } else {
+      throw Exception("Client does not support cursor position");
+    }
+
+    needCursorPos = false;
+  }
+
   if (needSetDesktopName) {
     writeSetDesktopNameRect(client->name());
     needSetDesktopName = false;
@@ -650,6 +675,20 @@ void SMsgWriter::writeSetVMwareCursorRect(int width, int height,
   os->writeBytes(data, width*height*4);
 }
 
+void SMsgWriter::writeSetVMwareCursorPositionRect(int hotspotX, int hotspotY)
+{
+  if (!client->supportsEncoding(pseudoEncodingVMwareCursorPosition))
+    throw Exception("Client does not support cursor position");
+  if (++nRectsInUpdate > nRectsInHeader && nRectsInHeader)
+    throw Exception("SMsgWriter::writeSetVMwareCursorRect: nRects out of sync");
+
+  os->writeS16(hotspotX);
+  os->writeS16(hotspotY);
+  os->writeU16(0);
+  os->writeU16(0);
+  os->writeU32(pseudoEncodingVMwareCursorPosition);
+}
+
 void SMsgWriter::writeLEDStateRect(rdr::U8 state)
 {
   if (!client->supportsEncoding(pseudoEncodingLEDState) &&

diff --git a/common/rfb/SMsgWriter.h b/common/rfb/SMsgWriter.h
index 2cea44d1e28d8e234438df29525739ccf181a4da..49381bad0456d90b153e0f9fb2df3714a64530fc 100644 (file)
--- a/common/rfb/SMsgWriter.h
+++ b/common/rfb/SMsgWriter.h
@@ -83,6 +83,9 @@ namespace rfb {
     // immediately. 
     void writeCursor();
 
+    // Notifies the client that the cursor pointer was moved by the server.
+    void writeCursorPos();
+
     // Same for LED state message
     void writeLEDState();
 
@@ -141,6 +144,7 @@ namespace rfb {
     void writeSetVMwareCursorRect(int width, int height,
                                   int hotspotX, int hotspotY,
                                   const rdr::U8* data);
+    void writeSetVMwareCursorPositionRect(int hotspotX, int hotspotY);
     void writeLEDStateRect(rdr::U8 state);
     void writeQEMUKeyEventRect();
 
@@ -152,6 +156,7 @@ namespace rfb {
 
     bool needSetDesktopName;
     bool needCursor;
+    bool needCursorPos;
     bool needLEDState;
     bool needQEMUKeyEvent;
 

diff --git a/common/rfb/VNCSConnectionST.cxx b/common/rfb/VNCSConnectionST.cxx
index 668bae0e50d4fd9d6ac0fd2de175f5750647c584..a966e66cb1b6d6b544335fe5ce77e609deda75b4 100644 (file)
--- a/common/rfb/VNCSConnectionST.cxx
+++ b/common/rfb/VNCSConnectionST.cxx
@@ -370,6 +370,15 @@ void VNCSConnectionST::renderedCursorChange()
   }
 }
 
+// cursorPositionChange() is called whenever the cursor has changed position by
+// the server.  If the client supports being informed about these changes then
+// it will arrange for the new cursor position to be sent to the client.
+
+void VNCSConnectionST::cursorPositionChange()
+{
+  setCursorPos();
+}
+
 // needRenderedCursor() returns true if this client needs the server-side
 // rendered cursor.  This may be because it does not support local cursor or
 // because the current cursor position has not been set by this client.
@@ -1123,6 +1132,21 @@ void VNCSConnectionST::setCursor()
     writer()->writeCursor();
 }
 
+// setCursorPos() is called whenever the cursor has changed position by the
+// server.  If the client supports being informed about these changes then it
+// will arrange for the new cursor position to be sent to the client.
+
+void VNCSConnectionST::setCursorPos()
+{
+  if (state() != RFBSTATE_NORMAL)
+    return;
+
+  if (client.supportsCursorPosition()) {
+    client.setCursorPos(server->getCursorPos());
+    writer()->writeCursorPos();
+  }
+}
+
 void VNCSConnectionST::setDesktopName(const char *name)
 {
   client.setName(name);

diff --git a/common/rfb/VNCSConnectionST.h b/common/rfb/VNCSConnectionST.h
index 6d95008d1847cbfb93e4957d323fd9c136d43be8..72b0c52908f312cd3d18664d94f7a57c84504383 100644 (file)
--- a/common/rfb/VNCSConnectionST.h
+++ b/common/rfb/VNCSConnectionST.h
@@ -93,6 +93,11 @@ namespace rfb {
     // cursor.
     void renderedCursorChange();
 
+    // cursorPositionChange() is called whenever the cursor has changed position by
+    // the server.  If the client supports being informed about these changes then
+    // it will arrange for the new cursor position to be sent to the client.
+    void cursorPositionChange();
+
     // needRenderedCursor() returns true if this client needs the server-side
     // rendered cursor.  This may be because it does not support local cursor
     // or because the current cursor position has not been set by this client.
@@ -155,6 +160,7 @@ namespace rfb {
 
     void screenLayoutChange(rdr::U16 reason);
     void setCursor();
+    void setCursorPos();
     void setDesktopName(const char *name);
     void setLEDState(unsigned int state);
 

diff --git a/common/rfb/VNCServer.h b/common/rfb/VNCServer.h
index 5d04da53eb2a3fe34099ea5faa6861bdc4d4d0e8..4535b56242dc3b43ea659adb6e1efc2e494ba1cd 100644 (file)
--- a/common/rfb/VNCServer.h
+++ b/common/rfb/VNCServer.h
@@ -97,8 +97,10 @@ namespace rfb {
     virtual void setCursor(int width, int height, const Point& hotspot,
                            const rdr::U8* cursorData) = 0;
 
-    // setCursorPos() tells the server the current position of the cursor.
-    virtual void setCursorPos(const Point& p) = 0;
+    // setCursorPos() tells the server the current position of the cursor, and
+    // whether the server initiated that change (e.g. through another X11
+    // client calling XWarpPointer()).
+    virtual void setCursorPos(const Point& p, bool warped) = 0;
 
     // setName() tells the server what desktop title to supply to clients
     virtual void setName(const char* name) = 0;

diff --git a/common/rfb/VNCServerST.cxx b/common/rfb/VNCServerST.cxx
index b32cac594d0e39592448292b3727110dfa155826..39cdde1fa7fe54e06348b1fe21fa203b71074675 100644 (file)
--- a/common/rfb/VNCServerST.cxx
+++ b/common/rfb/VNCServerST.cxx
@@ -429,14 +429,17 @@ void VNCServerST::setCursor(int width, int height, const Point& newHotspot,
   }
 }
 
-void VNCServerST::setCursorPos(const Point& pos)
+void VNCServerST::setCursorPos(const Point& pos, bool warped)
 {
   if (!cursorPos.equals(pos)) {
     cursorPos = pos;
     renderedCursorInvalid = true;
     std::list<VNCSConnectionST*>::iterator ci;
-    for (ci = clients.begin(); ci != clients.end(); ci++)
+    for (ci = clients.begin(); ci != clients.end(); ci++) {
       (*ci)->renderedCursorChange();
+      if (warped)
+        (*ci)->cursorPositionChange();
+    }
   }
 }
 

diff --git a/common/rfb/VNCServerST.h b/common/rfb/VNCServerST.h
index fd20cc37b18f2df375f9ee1c5542cfc4c16f7e95..159e3a4bee702942a372cb95e7aa928c28c09be6 100644 (file)
--- a/common/rfb/VNCServerST.h
+++ b/common/rfb/VNCServerST.h
@@ -99,7 +99,7 @@ namespace rfb {
     virtual void add_copied(const Region &dest, const Point &delta);
     virtual void setCursor(int width, int height, const Point& hotspot,
                            const rdr::U8* data);
-    virtual void setCursorPos(const Point& p);
+    virtual void setCursorPos(const Point& p, bool warped);
     virtual void setName(const char* name_);
     virtual void setLEDState(unsigned state);
 

diff --git a/common/rfb/encodings.h b/common/rfb/encodings.h
index cf0c8572fe7ae1e2e417e85cc07588f5a9830d64..f7ad7890e7db8c8b757335aefab4f24c4d6d4de5 100644 (file)
--- a/common/rfb/encodings.h
+++ b/common/rfb/encodings.h
@@ -61,6 +61,7 @@ namespace rfb {
 
   // VMware-specific
   const int pseudoEncodingVMwareCursor = 0x574d5664;
+  const int pseudoEncodingVMwareCursorPosition = 0x574d5666;
   const int pseudoEncodingVMwareLEDState = 0x574d5668;
 
   // UltraVNC-specific

diff --git a/unix/x0vncserver/XDesktop.cxx b/unix/x0vncserver/XDesktop.cxx
index eb36467e2d5404b6fbce9cae51b8202d07f1ee6c..e8e74fa7070cf686142188fe86b15324d3ce040a 100644 (file)
--- a/unix/x0vncserver/XDesktop.cxx
+++ b/unix/x0vncserver/XDesktop.cxx
@@ -217,7 +217,7 @@ void XDesktop::poll() {
                   &x, &y, &wx, &wy, &mask);
     x -= geometry->offsetLeft();
     y -= geometry->offsetTop();
-    server->setCursorPos(rfb::Point(x, y));
+    server->setCursorPos(rfb::Point(x, y), false);
   }
 }
 

diff --git a/unix/xserver/hw/vnc/XserverDesktop.cc b/unix/xserver/hw/vnc/XserverDesktop.cc
index 6f707299db9840a11db3be676f1a6a3fdb336f38..7ebad353ff0d9c223cbaf4044dc848772a43765a 100644 (file)
--- a/unix/xserver/hw/vnc/XserverDesktop.cc
+++ b/unix/xserver/hw/vnc/XserverDesktop.cc
@@ -261,6 +261,15 @@ void XserverDesktop::setCursor(int width, int height, int hotX, int hotY,
   delete [] cursorData;
 }
 
+void XserverDesktop::setCursorPos(int x, int y, bool warped)
+{
+  try {
+    server->setCursorPos(Point(x, y), warped);
+  } catch (rdr::Exception& e) {
+    vlog.error("XserverDesktop::setCursorPos: %s",e.str());
+  }
+}
+
 void XserverDesktop::add_changed(const rfb::Region &region)
 {
   try {
@@ -377,7 +386,7 @@ void XserverDesktop::blockHandler(int* timeout)
     if (oldCursorPos.x != cursorX || oldCursorPos.y != cursorY) {
       oldCursorPos.x = cursorX;
       oldCursorPos.y = cursorY;
-      server->setCursorPos(oldCursorPos);
+      server->setCursorPos(oldCursorPos, false);
     }
 
     // Trigger timers and check when the next will expire

diff --git a/unix/xserver/hw/vnc/XserverDesktop.h b/unix/xserver/hw/vnc/XserverDesktop.h
index cc50f9e90ca9df27e8a3d478db407f793cb4afea..383e0bbff446f51f7d467bfa106636f9f6676745 100644 (file)
--- a/unix/xserver/hw/vnc/XserverDesktop.h
+++ b/unix/xserver/hw/vnc/XserverDesktop.h
@@ -67,6 +67,7 @@ public:
   void setDesktopName(const char* name);
   void setCursor(int width, int height, int hotX, int hotY,
                  const unsigned char *rgbaData);
+  void setCursorPos(int x, int y, bool warped);
   void add_changed(const rfb::Region &region);
   void add_copied(const rfb::Region &dest, const rfb::Point &delta);
   void handleSocketEvent(int fd, bool read, bool write);

diff --git a/unix/xserver/hw/vnc/vncExtInit.cc b/unix/xserver/hw/vnc/vncExtInit.cc
index a45c5bdef1ed0763393ed0fe78ed95c80e1a4711..6c4612d1c708c675fe253f86efb08fc774064f9a 100644 (file)
--- a/unix/xserver/hw/vnc/vncExtInit.cc
+++ b/unix/xserver/hw/vnc/vncExtInit.cc
@@ -400,6 +400,11 @@ void vncSetCursor(int width, int height, int hotX, int hotY,
     desktop[scr]->setCursor(width, height, hotX, hotY, rgbaData);
 }
 
+void vncSetCursorPos(int scrIdx, int x, int y)
+{
+  desktop[scrIdx]->setCursorPos(x, y, true);
+}
+
 void vncPreScreenResize(int scrIdx)
 {
   // We need to prevent the RFB core from accessing the framebuffer

diff --git a/unix/xserver/hw/vnc/vncExtInit.h b/unix/xserver/hw/vnc/vncExtInit.h
index 23c0c6699fb98165f9564b5473ae03bae0bd7d4a..36e52032e3baf7d67cd551cc4bc1705143e1f36d 100644 (file)
--- a/unix/xserver/hw/vnc/vncExtInit.h
+++ b/unix/xserver/hw/vnc/vncExtInit.h
@@ -81,6 +81,7 @@ void vncAddCopied(int scrIdx, int nRects,
 
 void vncSetCursor(int width, int height, int hotX, int hotY,
                   const unsigned char *rgbaData);
+void vncSetCursorPos(int scrIdx, int x, int y);
 
 void vncPreScreenResize(int scrIdx);
 void vncPostScreenResize(int scrIdx, int success, int width, int height);

diff --git a/unix/xserver/hw/vnc/vncHooks.c b/unix/xserver/hw/vnc/vncHooks.c
index a8ab917b86605f45d50f7f038b1ba5572b19253d..d206e342098eab63e12a03a7e6e004803f07d6a9 100644 (file)
--- a/unix/xserver/hw/vnc/vncHooks.c
+++ b/unix/xserver/hw/vnc/vncHooks.c
@@ -62,6 +62,9 @@ typedef struct _vncHooksScreenRec {
   CopyWindowProcPtr            CopyWindow;
   ClearToBackgroundProcPtr     ClearToBackground;
   DisplayCursorProcPtr         DisplayCursor;
+#if XORG >= 119
+  CursorWarpedToProcPtr        CursorWarpedTo;
+#endif
   ScreenBlockHandlerProcPtr    BlockHandler;
 #ifdef RENDER
   CompositeProcPtr             Composite;
@@ -113,6 +116,12 @@ static void vncHooksClearToBackground(WindowPtr pWin, int x, int y, int w,
                                       int h, Bool generateExposures);
 static Bool vncHooksDisplayCursor(DeviceIntPtr pDev,
                                   ScreenPtr pScreen, CursorPtr cursor);
+#if XORG >= 119
+static void vncHooksCursorWarpedTo(DeviceIntPtr pDev,
+                                   ScreenPtr pScreen_, ClientPtr pClient,
+                                   WindowPtr pWindow, SpritePtr pSprite,
+                                   int x, int y);
+#endif
 #if XORG <= 118
 static void vncHooksBlockHandler(ScreenPtr pScreen, void * pTimeout,
                                  void * pReadmask);
@@ -271,6 +280,9 @@ int vncHooksInit(int scrIdx)
   wrap(vncHooksScreen, pScreen, CopyWindow, vncHooksCopyWindow);
   wrap(vncHooksScreen, pScreen, ClearToBackground, vncHooksClearToBackground);
   wrap(vncHooksScreen, pScreen, DisplayCursor, vncHooksDisplayCursor);
+#if XORG >= 119
+  wrap(vncHooksScreen, pScreen, CursorWarpedTo, vncHooksCursorWarpedTo);
+#endif
   wrap(vncHooksScreen, pScreen, BlockHandler, vncHooksBlockHandler);
 #ifdef RENDER
   ps = GetPictureScreenIfSet(pScreen);
@@ -631,6 +643,20 @@ out:
   return ret;
 }
 
+// CursorWarpedTo - notify that the cursor was warped
+
+#if XORG >= 119
+static void vncHooksCursorWarpedTo(DeviceIntPtr pDev,
+                                   ScreenPtr pScreen_, ClientPtr pClient,
+                                   WindowPtr pWindow, SpritePtr pSprite,
+                                   int x, int y)
+{
+  SCREEN_PROLOGUE(pScreen_, CursorWarpedTo);
+  vncSetCursorPos(pScreen->myNum, x, y);
+  SCREEN_EPILOGUE(CursorWarpedTo);
+}
+#endif
+
 // BlockHandler - ignore any changes during the block handler - it's likely
 // these are just drawing the cursor.
 

diff --git a/win/rfb_win32/SDisplay.cxx b/win/rfb_win32/SDisplay.cxx
index 06eccd9a21fb750a1fefc20ede0c8c1bb65e4406..a9ee361579bdbb449fcb676baa82f82b92b6633d 100644 (file)
--- a/win/rfb_win32/SDisplay.cxx
+++ b/win/rfb_win32/SDisplay.cxx
@@ -417,7 +417,7 @@ SDisplay::processEvent(HANDLE event) {
         // Update the cursor position
         // NB: First translate from Screen coordinates to Desktop
         Point desktopPos = info.position.translate(screenRect.tl.negate());
-        server->setCursorPos(desktopPos);
+        server->setCursorPos(desktopPos, false);
 
         old_cursor = info;
       }