Add server support for forward/back mouse buttons

This commit adds support for the pseudo-encoding ExtendedMouseButtons in
Xvnc and x0vncserver, which makes it possible to use to use the
back/forward mouse buttons.

This commit contains work originally done by
PixelSmith <manny33@frontbuffer.com>.

24 files changed, 128 insertions, 26 deletions

diff --git a/common/rfb/ClientParams.cxx b/common/rfb/ClientParams.cxx
index bc20c3d7..5ea104cf 100644
--- a/common/rfb/ClientParams.cxx
+++ b/common/rfb/ClientParams.cxx
@@ -228,3 +228,10 @@ bool ClientParams::supportsContinuousUpdates() const
     return true;
   return false;
 }
+
+bool ClientParams::supportsExtendedMouseButtons() const
+{
+  if (supportsEncoding(pseudoEncodingExtendedMouseButtons))
+    return true;
+  return false;
+}
\ No newline at end of file
diff --git a/common/rfb/ClientParams.h b/common/rfb/ClientParams.h
index ea86ea78..f715c47f 100644
--- a/common/rfb/ClientParams.h
+++ b/common/rfb/ClientParams.h
@@ -101,6 +101,7 @@ namespace rfb {
     bool supportsLEDState() const;
     bool supportsFence() const;
     bool supportsContinuousUpdates() const;
+    bool supportsExtendedMouseButtons() const;
 
     int compressLevel;
     int qualityLevel;
diff --git a/common/rfb/SConnection.cxx b/common/rfb/SConnection.cxx
index 905f88a4..9d481017 100644
--- a/common/rfb/SConnection.cxx
+++ b/common/rfb/SConnection.cxx
@@ -433,6 +433,11 @@ void SConnection::supportsQEMUKeyEvent()
   writer()->writeQEMUKeyEvent();
 }
 
+void SConnection::supportsExtendedMouseButtons()
+{
+  writer()->writeExtendedMouseButtonsSupport();
+}
+
 void SConnection::versionReceived()
 {
 }
diff --git a/common/rfb/SConnection.h b/common/rfb/SConnection.h
index 0a11f67b..a839f663 100644
--- a/common/rfb/SConnection.h
+++ b/common/rfb/SConnection.h
@@ -98,6 +98,8 @@ namespace rfb {
 
     void supportsQEMUKeyEvent() override;
 
+    virtual void supportsExtendedMouseButtons() override;
+
 
     // Methods to be overridden in a derived class
 
diff --git a/common/rfb/SDesktop.h b/common/rfb/SDesktop.h
index 1d3c325f..c97e788a 100644
--- a/common/rfb/SDesktop.h
+++ b/common/rfb/SDesktop.h
@@ -98,7 +98,7 @@ namespace rfb {
     // pointerEvent() is called whenever a client sends an event that
     // the pointer moved, or a button was pressed or released.
     virtual void pointerEvent(const Point& /*pos*/,
-                              uint8_t /*buttonMask*/) {};
+                              uint16_t /*buttonMask*/) {};
 
     // handleClipboardRequest() is called whenever a client requests
     // the server to send over its clipboard data. It will only be
diff --git a/common/rfb/SMsgHandler.cxx b/common/rfb/SMsgHandler.cxx
index 03917926..1dce634d 100644
--- a/common/rfb/SMsgHandler.cxx
+++ b/common/rfb/SMsgHandler.cxx
@@ -53,12 +53,13 @@ void SMsgHandler::setPixelFormat(const PixelFormat& pf)
 void SMsgHandler::setEncodings(int nEncodings, const int32_t* encodings)
 {
   bool firstFence, firstContinuousUpdates, firstLEDState,
-       firstQEMUKeyEvent;
+       firstQEMUKeyEvent, firstExtMouseButtonsEvent;
 
   firstFence = !client.supportsFence();
   firstContinuousUpdates = !client.supportsContinuousUpdates();
   firstLEDState = !client.supportsLEDState();
   firstQEMUKeyEvent = !client.supportsEncoding(pseudoEncodingQEMUKeyEvent);
+  firstExtMouseButtonsEvent = !client.supportsEncoding(pseudoEncodingExtendedMouseButtons);
 
   client.setEncodings(nEncodings, encodings);
 
@@ -72,6 +73,8 @@ void SMsgHandler::setEncodings(int nEncodings, const int32_t* encodings)
     supportsLEDState();
   if (client.supportsEncoding(pseudoEncodingQEMUKeyEvent) && firstQEMUKeyEvent)
     supportsQEMUKeyEvent();
+  if (client.supportsEncoding(pseudoEncodingExtendedMouseButtons) && firstExtMouseButtonsEvent)
+    supportsExtendedMouseButtons();
 }
 
 void SMsgHandler::keyEvent(uint32_t /*keysym*/, uint32_t /*keycode*/,
@@ -80,7 +83,7 @@ void SMsgHandler::keyEvent(uint32_t /*keysym*/, uint32_t /*keycode*/,
 }
 
 void SMsgHandler::pointerEvent(const Point& /*pos*/,
-                               uint8_t /*buttonMask*/)
+                               uint16_t /*buttonMask*/)
 {
 }
 
@@ -167,3 +170,7 @@ void SMsgHandler::supportsLEDState()
 void SMsgHandler::supportsQEMUKeyEvent()
 {
 }
+
+void SMsgHandler::supportsExtendedMouseButtons()
+{
+}
\ No newline at end of file
diff --git a/common/rfb/SMsgHandler.h b/common/rfb/SMsgHandler.h
index cff8b1bd..c5d13d78 100644
--- a/common/rfb/SMsgHandler.h
+++ b/common/rfb/SMsgHandler.h
@@ -57,7 +57,7 @@ namespace rfb {
     virtual void keyEvent(uint32_t keysym, uint32_t keycode,
                           bool down);
     virtual void pointerEvent(const Point& pos,
-                              uint8_t buttonMask);
+                              uint16_t buttonMask);
 
     virtual void clientCutText(const char* str);
 
@@ -98,6 +98,11 @@ namespace rfb {
     // handler will send a pseudo-rect back, signalling server support.
     virtual void supportsQEMUKeyEvent();
 
+    // supportsExtendedMouseButtons() is called the first time we detect that the
+    // client supports sending 16 bit mouse button state. This lets us pass more button
+    // states between server and client.
+    virtual void supportsExtendedMouseButtons();
+
     ClientParams client;
   };
 }
diff --git a/common/rfb/SMsgReader.cxx b/common/rfb/SMsgReader.cxx
index 9ddea53d..0aa83e3a 100644
--- a/common/rfb/SMsgReader.cxx
+++ b/common/rfb/SMsgReader.cxx
@@ -272,11 +272,32 @@ bool SMsgReader::readKeyEvent()
 
 bool SMsgReader::readPointerEvent()
 {
+  int mask;
+  int x;
+  int y;
+
   if (!is->hasData(1 + 2 + 2))
     return false;
-  int mask = is->readU8();
-  int x = is->readU16();
-  int y = is->readU16();
+
+  is->setRestorePoint();
+
+  mask = is->readU8();
+  x = is->readU16();
+  y = is->readU16();
+
+  if (handler->client.supportsExtendedMouseButtons() && mask & 0x80 ) {
+    int highBits;
+    int lowBits;
+
+    if (!is->hasDataOrRestore(1))
+      return false;
+
+    highBits = is->readU8();
+    lowBits = mask & 0x7f; /* Clear marker bit */
+    mask = (highBits << 7) | lowBits;
+  }
+
+  is->clearRestorePoint();
   handler->pointerEvent(Point(x, y), mask);
   return true;
 }
diff --git a/common/rfb/SMsgWriter.cxx b/common/rfb/SMsgWriter.cxx
index 0c03b51d..d1218c11 100644
--- a/common/rfb/SMsgWriter.cxx
+++ b/common/rfb/SMsgWriter.cxx
@@ -49,7 +49,7 @@ SMsgWriter::SMsgWriter(ClientParams* client_, rdr::OutStream* os_)
     nRectsInUpdate(0), nRectsInHeader(0),
     needSetDesktopName(false), needCursor(false),
     needCursorPos(false), needLEDState(false),
-    needQEMUKeyEvent(false)
+    needQEMUKeyEvent(false), needExtMouseButtonsEvent(false)
 {
 }
 
@@ -303,6 +303,14 @@ void SMsgWriter::writeQEMUKeyEvent()
   needQEMUKeyEvent = true;
 }
 
+void SMsgWriter::writeExtendedMouseButtonsSupport()
+{
+  if (!client->supportsEncoding(pseudoEncodingExtendedMouseButtons))
+    throw Exception("Client does not support Extended Mouse Buttons");
+
+  needExtMouseButtonsEvent = true;
+}
+
 bool SMsgWriter::needFakeUpdate()
 {
   if (needSetDesktopName)
@@ -315,6 +323,8 @@ bool SMsgWriter::needFakeUpdate()
     return true;
   if (needQEMUKeyEvent)
     return true;
+  if (needExtMouseButtonsEvent)
+    return true;
   if (needNoDataUpdate())
     return true;
 
@@ -363,6 +373,8 @@ void SMsgWriter::writeFramebufferUpdateStart(int nRects)
       nRects++;
     if (needQEMUKeyEvent)
       nRects++;
+    if (needExtMouseButtonsEvent)
+      nRects++;
   }
 
   os->writeU16(nRects);
@@ -502,6 +514,11 @@ void SMsgWriter::writePseudoRects()
     writeQEMUKeyEventRect();
     needQEMUKeyEvent = false;
   }
+
+  if (needExtMouseButtonsEvent) {
+    writeExtendedMouseButtonsRect();
+    needExtMouseButtonsEvent = false;
+  }
 }
 
 void SMsgWriter::writeNoDataRects()
@@ -734,3 +751,17 @@ void SMsgWriter::writeQEMUKeyEventRect()
   os->writeU16(0);
   os->writeU32(pseudoEncodingQEMUKeyEvent);
 }
+
+void SMsgWriter::writeExtendedMouseButtonsRect()
+{
+  if (!client->supportsEncoding(pseudoEncodingExtendedMouseButtons))
+    throw Exception("Client does not support extended mouse button events");
+  if (++nRectsInUpdate > nRectsInHeader && nRectsInHeader)
+    throw Exception("SMsgWriter::writeExtendedMouseButtonsRect: nRects out of sync");
+
+  os->writeS16(0);
+  os->writeS16(0);
+  os->writeU16(0);
+  os->writeU16(0);
+  os->writeU32(pseudoEncodingExtendedMouseButtons);
+}
diff --git a/common/rfb/SMsgWriter.h b/common/rfb/SMsgWriter.h
index c46551e9..7bc0ed6a 100644
--- a/common/rfb/SMsgWriter.h
+++ b/common/rfb/SMsgWriter.h
@@ -93,6 +93,9 @@ namespace rfb {
     // And QEMU keyboard event handshake
     void writeQEMUKeyEvent();
 
+    // let the client know we support extended mouse button support
+    void writeExtendedMouseButtonsSupport();
+
     // needFakeUpdate() returns true when an immediate update is needed in
     // order to flush out pseudo-rectangles to the client.
     bool needFakeUpdate();
@@ -148,6 +151,7 @@ namespace rfb {
     void writeSetVMwareCursorPositionRect(int hotspotX, int hotspotY);
     void writeLEDStateRect(uint8_t state);
     void writeQEMUKeyEventRect();
+    void writeExtendedMouseButtonsRect();
 
     ClientParams* client;
     rdr::OutStream* os;
@@ -160,6 +164,7 @@ namespace rfb {
     bool needCursorPos;
     bool needLEDState;
     bool needQEMUKeyEvent;
+    bool needExtMouseButtonsEvent;
 
     typedef struct {
       uint16_t reason, result;
diff --git a/common/rfb/VNCSConnectionST.cxx b/common/rfb/VNCSConnectionST.cxx
index 88deff8c..7a796dc2 100644
--- a/common/rfb/VNCSConnectionST.cxx
+++ b/common/rfb/VNCSConnectionST.cxx
@@ -477,7 +477,7 @@ void VNCSConnectionST::setPixelFormat(const PixelFormat& pf)
   setCursor();
 }
 
-void VNCSConnectionST::pointerEvent(const Point& pos, uint8_t buttonMask)
+void VNCSConnectionST::pointerEvent(const Point& pos, uint16_t buttonMask)
 {
   if (rfb::Server::idleTimeout)
     idleTimer.start(secsToMillis(rfb::Server::idleTimeout));
diff --git a/common/rfb/VNCSConnectionST.h b/common/rfb/VNCSConnectionST.h
index d857ef32..17de9d01 100644
--- a/common/rfb/VNCSConnectionST.h
+++ b/common/rfb/VNCSConnectionST.h
@@ -123,7 +123,7 @@ namespace rfb {
     void queryConnection(const char* userName) override;
     void clientInit(bool shared) override;
     void setPixelFormat(const PixelFormat& pf) override;
-    void pointerEvent(const Point& pos, uint8_t buttonMask) override;
+    void pointerEvent(const Point& pos, uint16_t buttonMask) override;
     void keyEvent(uint32_t keysym, uint32_t keycode,
                   bool down) override;
     void framebufferUpdateRequest(const Rect& r,
diff --git a/common/rfb/VNCServerST.cxx b/common/rfb/VNCServerST.cxx
index 114ff347..977fa937 100644
--- a/common/rfb/VNCServerST.cxx
+++ b/common/rfb/VNCServerST.cxx
@@ -482,7 +482,7 @@ void VNCServerST::keyEvent(uint32_t keysym, uint32_t keycode, bool down)
 }
 
 void VNCServerST::pointerEvent(VNCSConnectionST* client,
-                               const Point& pos, uint8_t buttonMask)
+                               const Point& pos, uint16_t buttonMask)
 {
   time_t now = time(nullptr);
   if (rfb::Server::maxIdleTime)
diff --git a/common/rfb/VNCServerST.h b/common/rfb/VNCServerST.h
index 6cc75a68..dc4f9aad 100644
--- a/common/rfb/VNCServerST.h
+++ b/common/rfb/VNCServerST.h
@@ -117,7 +117,7 @@ namespace rfb {
 
     // Event handlers
     void keyEvent(uint32_t keysym, uint32_t keycode, bool down);
-    void pointerEvent(VNCSConnectionST* client, const Point& pos, uint8_t buttonMask);
+    void pointerEvent(VNCSConnectionST* client, const Point& pos, uint16_t buttonMask);
 
     void handleClipboardRequest(VNCSConnectionST* client);
     void handleClipboardAnnounce(VNCSConnectionST* client, bool available);
diff --git a/common/rfb/encodings.h b/common/rfb/encodings.h
index e427572f..16868460 100644
--- a/common/rfb/encodings.h
+++ b/common/rfb/encodings.h
@@ -36,6 +36,7 @@ namespace rfb {
 
   const int pseudoEncodingXCursor = -240;
   const int pseudoEncodingCursor = -239;
+  const int pseudoEncodingExtendedMouseButtons = -316;
   const int pseudoEncodingDesktopSize = -223;
   const int pseudoEncodingLEDState = -261;
   const int pseudoEncodingExtendedDesktopSize = -308;
diff --git a/unix/x0vncserver/XDesktop.cxx b/unix/x0vncserver/XDesktop.cxx
index 29af059f..bf796452 100644
--- a/unix/x0vncserver/XDesktop.cxx
+++ b/unix/x0vncserver/XDesktop.cxx
@@ -242,9 +242,9 @@ void XDesktop::init(VNCServer* vs)
 void XDesktop::start()
 {
   // Determine actual number of buttons of the X pointer device.
-  unsigned char btnMap[8];
-  int numButtons = XGetPointerMapping(dpy, btnMap, 8);
-  maxButtons = (numButtons > 8) ? 8 : numButtons;
+  unsigned char btnMap[9];
+  int numButtons = XGetPointerMapping(dpy, btnMap, 9);
+  maxButtons = (numButtons > 9) ? 9 : numButtons;
   vlog.info("Enabling %d button%s of X pointer device",
             maxButtons, (maxButtons != 1) ? "s" : "");
 
@@ -342,7 +342,7 @@ void XDesktop::queryConnection(network::Socket* sock,
   queryConnectDialog->map();
 }
 
-void XDesktop::pointerEvent(const Point& pos, uint8_t buttonMask) {
+void XDesktop::pointerEvent(const Point& pos, uint16_t buttonMask) {
 #ifdef HAVE_XTEST
   if (!haveXtest) return;
   XTestFakeMotionEvent(dpy, DefaultScreen(dpy),
diff --git a/unix/x0vncserver/XDesktop.h b/unix/x0vncserver/XDesktop.h
index cf374fb9..711d6893 100644
--- a/unix/x0vncserver/XDesktop.h
+++ b/unix/x0vncserver/XDesktop.h
@@ -60,7 +60,7 @@ public:
   bool isRunning();
   void queryConnection(network::Socket* sock,
                        const char* userName) override;
-  void pointerEvent(const rfb::Point& pos, uint8_t buttonMask) override;
+  void pointerEvent(const rfb::Point& pos, uint16_t buttonMask) override;
   void keyEvent(uint32_t keysym, uint32_t xtcode, bool down) override;
   unsigned int setScreenLayout(int fb_width, int fb_height,
                                const rfb::ScreenSet& layout) override;
@@ -79,7 +79,7 @@ protected:
   rfb::VNCServer* server;
   QueryConnectDialog* queryConnectDialog;
   network::Socket* queryConnectSock;
-  uint8_t oldButtonMask;
+  uint16_t oldButtonMask;
   bool haveXtest;
   bool haveDamage;
   int maxButtons;
diff --git a/unix/xserver/hw/vnc/XserverDesktop.cc b/unix/xserver/hw/vnc/XserverDesktop.cc
index e3bc57d8..328b0886 100644
--- a/unix/xserver/hw/vnc/XserverDesktop.cc
+++ b/unix/xserver/hw/vnc/XserverDesktop.cc
@@ -463,7 +463,7 @@ void XserverDesktop::terminate()
   kill(getpid(), SIGTERM);
 }
 
-void XserverDesktop::pointerEvent(const Point& pos, uint8_t buttonMask)
+void XserverDesktop::pointerEvent(const Point& pos, uint16_t buttonMask)
 {
   vncPointerMove(pos.x + vncGetScreenX(screenIndex),
                  pos.y + vncGetScreenY(screenIndex));
diff --git a/unix/xserver/hw/vnc/XserverDesktop.h b/unix/xserver/hw/vnc/XserverDesktop.h
index d287b72f..8c543db7 100644
--- a/unix/xserver/hw/vnc/XserverDesktop.h
+++ b/unix/xserver/hw/vnc/XserverDesktop.h
@@ -95,7 +95,7 @@ public:
   void terminate() override;
   void queryConnection(network::Socket* sock,
                        const char* userName) override;
-  void pointerEvent(const rfb::Point& pos, uint8_t buttonMask) override;
+  void pointerEvent(const rfb::Point& pos, uint16_t buttonMask) override;
   void keyEvent(uint32_t keysym, uint32_t keycode, bool down) override;
   unsigned int setScreenLayout(int fb_width, int fb_height,
                                const rfb::ScreenSet& layout) override;
diff --git a/unix/xserver/hw/vnc/vncInput.c b/unix/xserver/hw/vnc/vncInput.c
index 1de41430..a705a85a 100644
--- a/unix/xserver/hw/vnc/vncInput.c
+++ b/unix/xserver/hw/vnc/vncInput.c
@@ -50,7 +50,7 @@ extern const unsigned int code_map_qnum_to_xorgevdev_len;
 extern const unsigned short code_map_qnum_to_xorgkbd[];
 extern const unsigned int code_map_qnum_to_xorgkbd_len;
 
-#define BUTTONS 7
+#define BUTTONS 9
 
 DeviceIntPtr vncKeyboardDev;
 DeviceIntPtr vncPointerDev;
@@ -207,6 +207,23 @@ static int vncPointerProc(DeviceIntPtr pDevice, int onoff)
 		btn_labels[5] = XIGetKnownProperty(BTN_LABEL_PROP_BTN_HWHEEL_LEFT);
 		btn_labels[6] = XIGetKnownProperty(BTN_LABEL_PROP_BTN_HWHEEL_RIGHT);
 
+		/*
+		 * The labels BTN_LABEL_PROP_BTN_SIDE and BTN_LABEL_PROP_BTN_EXTRA
+		 * represent the side buttons on mice that are typically used to
+		 * navigate back/forward respectively in web browsers.
+		 *
+		 * In X11, these labels are mapped to the BTN_SIDE and BTN_EXTRA
+		 * input codes, which are mapped in the Linux HID driver. These
+		 * are not to be confused with the BTN_FORWARD and BTN_BACK input
+		 * codes, which some applications also use for back/forward
+		 * navigation.
+		 *
+		 * It seems like most mice have their side buttons mapped to
+		 * BTN_SIDE and BTN_EXTRA.
+		 */
+		btn_labels[7] = XIGetKnownProperty(BTN_LABEL_PROP_BTN_SIDE);
+		btn_labels[8] = XIGetKnownProperty(BTN_LABEL_PROP_BTN_EXTRA);
+
 		axes_labels[0] = XIGetKnownProperty(AXIS_LABEL_PROP_REL_X);
 		axes_labels[1] = XIGetKnownProperty(AXIS_LABEL_PROP_REL_Y);
 
diff --git a/win/rfb_win32/SDisplay.cxx b/win/rfb_win32/SDisplay.cxx
index 0ec5e231..dee521e5 100644
--- a/win/rfb_win32/SDisplay.cxx
+++ b/win/rfb_win32/SDisplay.cxx
@@ -312,7 +312,7 @@ void SDisplay::handleClipboardData(const char* data) {
 }
 
 
-void SDisplay::pointerEvent(const Point& pos, uint8_t buttonmask) {
+void SDisplay::pointerEvent(const Point& pos, uint16_t buttonmask) {
   if (pb->getRect().contains(pos)) {
     Point screenPos = pos.translate(screenRect.tl);
     // - Check that the SDesktop doesn't need restarting
diff --git a/win/rfb_win32/SDisplay.h b/win/rfb_win32/SDisplay.h
index d4cf23e4..aa1a69e5 100644
--- a/win/rfb_win32/SDisplay.h
+++ b/win/rfb_win32/SDisplay.h
@@ -80,7 +80,7 @@ namespace rfb {
       void handleClipboardRequest() override;
       void handleClipboardAnnounce(bool available) override;
       void handleClipboardData(const char* data) override;
-      void pointerEvent(const Point& pos, uint8_t buttonmask) override;
+      void pointerEvent(const Point& pos, uint16_t buttonmask) override;
       void keyEvent(uint32_t keysym, uint32_t keycode, bool down) override;
 
       // -=- Clipboard events
diff --git a/win/rfb_win32/SInput.cxx b/win/rfb_win32/SInput.cxx
index 65d4a703..6441129d 100644
--- a/win/rfb_win32/SInput.cxx
+++ b/win/rfb_win32/SInput.cxx
@@ -65,7 +65,7 @@ win32::SPointer::SPointer()
 }
 
 void
-win32::SPointer::pointerEvent(const Point& pos, uint8_t buttonmask)
+win32::SPointer::pointerEvent(const Point& pos, uint16_t buttonmask)
 {
   // - We are specifying absolute coordinates
   DWORD flags = MOUSEEVENTF_ABSOLUTE;
diff --git a/win/rfb_win32/SInput.h b/win/rfb_win32/SInput.h
index 29e1df41..018bec55 100644
--- a/win/rfb_win32/SInput.h
+++ b/win/rfb_win32/SInput.h
@@ -44,10 +44,10 @@ namespace rfb {
       // - Create a pointer event at a the given coordinates, with the
       //   specified button state.  The event must be specified using
       //   Screen coordinates.
-      void pointerEvent(const Point& pos, uint8_t buttonmask);
+      void pointerEvent(const Point& pos, uint16_t buttonmask);
     protected:
       Point last_position;
-      uint8_t last_buttonmask;
+      uint16_t last_buttonmask;
     };
 
     // -=- Keyboard event handling