Implement touch gesture handling on Unix

Allows the user to perform certain important mouse operations using
touch gestures instead.

cmake/StaticBuild.cmake

@@ -123,7 +123,7 @@ if(BUILD_STATIC_GCC)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -nodefaultlibs")
   set(STATIC_BASE_LIBRARIES "")
   if(ENABLE_ASAN AND NOT WIN32 AND NOT APPLE)
-    set(STATIC_BASE_LIBRARIES "${STATIC_BASE_LIBRARIES} -Wl,-Bstatic -lasan -Wl,-Bdynamic -ldl -lm -lpthread")
+    set(STATIC_BASE_LIBRARIES "${STATIC_BASE_LIBRARIES} -Wl,-Bstatic -lasan -Wl,-Bdynamic -ldl -lpthread")
   endif()
   if(ENABLE_TSAN AND NOT WIN32 AND NOT APPLE AND CMAKE_SIZEOF_VOID_P MATCHES 8)
     # libtsan redefines some C++ symbols which then conflict with a
@@ -139,7 +139,7 @@ if(BUILD_STATIC_GCC)
     # these things again
     set(STATIC_BASE_LIBRARIES "${STATIC_BASE_LIBRARIES} -lmingw32 -lgcc_eh -lgcc -lmoldname -lmingwex -lmsvcrt")
   else()
-    set(STATIC_BASE_LIBRARIES "${STATIC_BASE_LIBRARIES} -lgcc -lgcc_eh -lc")
+    set(STATIC_BASE_LIBRARIES "${STATIC_BASE_LIBRARIES} -lm -lgcc -lgcc_eh -lc")
   endif()
   set(CMAKE_C_LINK_EXECUTABLE "${CMAKE_C_LINK_EXECUTABLE} ${STATIC_BASE_LIBRARIES}")
   set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -Wl,-Bstatic -lstdc++ -Wl,-Bdynamic ${STATIC_BASE_LIBRARIES}")

tests/unit/CMakeLists.txt

@@ -7,6 +7,9 @@ target_link_libraries(conv rfb)
 add_executable(convertlf convertlf.cxx)
 target_link_libraries(convertlf rfb)
 
+add_executable(gesturehandler gesturehandler.cxx ../../vncviewer/GestureHandler.cxx)
+target_link_libraries(gesturehandler rfb)
+
 add_executable(hostport hostport.cxx)
 target_link_libraries(hostport rfb)
 

vncviewer/BaseTouchHandler.cxx

@@ -0,0 +1,198 @@
+/* Copyright 2019 Aaron Sowry for Cendio AB
+ * Copyright 2019-2020 Pierre Ossman for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <math.h>
+
+#define XK_MISCELLANY
+#include <rfb/keysymdef.h>
+#include <rfb/util.h>
+
+#include "GestureHandler.h"
+#include "BaseTouchHandler.h"
+
+// Sensitivity threshold for gestures
+static const int ZOOMSENS = 30;
+static const int SCRLSENS = 50;
+
+static const unsigned DOUBLE_TAP_TIMEOUT   = 1000;
+static const unsigned DOUBLE_TAP_THRESHOLD = 50;
+
+BaseTouchHandler::BaseTouchHandler()
+{
+  gettimeofday(&lastTapTime, NULL);
+}
+
+BaseTouchHandler::~BaseTouchHandler()
+{
+}
+
+void BaseTouchHandler::handleGestureEvent(const GestureEvent& ev)
+{
+  double magnitude;
+
+  switch (ev.type) {
+  case GestureBegin:
+    switch (ev.gesture) {
+    case GestureOneTap:
+      handleTapEvent(ev, 1);
+      break;
+    case GestureTwoTap:
+      handleTapEvent(ev, 3);
+      break;
+    case GestureThreeTap:
+      handleTapEvent(ev, 2);
+      break;
+    case GestureDrag:
+      fakeMotionEvent(ev);
+      fakeButtonEvent(true, 1, ev);
+      break;
+    case GestureLongPress:
+      fakeMotionEvent(ev);
+      fakeButtonEvent(true, 3, ev);
+      break;
+    case GestureTwoDrag:
+      lastMagnitudeX = ev.magnitudeX;
+      lastMagnitudeY = ev.magnitudeY;
+      fakeMotionEvent(ev);
+      break;
+    case GesturePinch:
+      lastMagnitudeX = sqrt(ev.magnitudeX * ev.magnitudeX +
+                            ev.magnitudeY * ev.magnitudeY);
+      fakeMotionEvent(ev);
+      break;
+    }
+    break;
+
+  case GestureUpdate:
+    switch (ev.gesture) {
+    case GestureOneTap:
+    case GestureTwoTap:
+    case GestureThreeTap:
+      break;
+    case GestureDrag:
+    case GestureLongPress:
+      fakeMotionEvent(ev);
+      break;
+    case GestureTwoDrag:
+      // Always scroll in the same position.
+      // We don't know if the mouse was moved so we need to move it
+      // every update.
+      fakeMotionEvent(ev);
+      while ((ev.magnitudeY - lastMagnitudeY) > SCRLSENS) {
+        fakeButtonEvent(true, 4, ev);
+        fakeButtonEvent(false, 4, ev);
+        lastMagnitudeY += SCRLSENS;
+      }
+      while ((ev.magnitudeY - lastMagnitudeY) < -SCRLSENS) {
+        fakeButtonEvent(true, 5, ev);
+        fakeButtonEvent(false, 5, ev);
+        lastMagnitudeY -= SCRLSENS;
+      }
+      while ((ev.magnitudeX - lastMagnitudeX) > SCRLSENS) {
+        fakeButtonEvent(true, 6, ev);
+        fakeButtonEvent(false, 6, ev);
+        lastMagnitudeX += SCRLSENS;
+      }
+      while ((ev.magnitudeX - lastMagnitudeX) < -SCRLSENS) {
+        fakeButtonEvent(true, 7, ev);
+        fakeButtonEvent(false, 7, ev);
+        lastMagnitudeX -= SCRLSENS;
+      }
+      break;
+    case GesturePinch:
+      // Always scroll in the same position.
+      // We don't know if the mouse was moved so we need to move it
+      // every update.
+      fakeMotionEvent(ev);
+      magnitude = sqrt(ev.magnitudeX * ev.magnitudeX +
+                       ev.magnitudeY * ev.magnitudeY);
+      if (abs(magnitude - lastMagnitudeX) > ZOOMSENS) {
+        fakeKeyEvent(true, XK_Control_L, ev);
+
+        while ((magnitude - lastMagnitudeX) > ZOOMSENS) {
+          fakeButtonEvent(true, 4, ev);
+          fakeButtonEvent(false, 4, ev);
+          lastMagnitudeX += ZOOMSENS;
+        }
+        while ((magnitude - lastMagnitudeX) < -ZOOMSENS) {
+          fakeButtonEvent(true, 5, ev);
+          fakeButtonEvent(false, 5, ev);
+          lastMagnitudeX -= ZOOMSENS;
+        }
+
+        fakeKeyEvent(false, XK_Control_L, ev);
+      }
+    }
+    break;
+
+  case GestureEnd:
+    switch (ev.gesture) {
+    case GestureOneTap:
+    case GestureTwoTap:
+    case GestureThreeTap:
+    case GesturePinch:
+    case GestureTwoDrag:
+      break;
+    case GestureDrag:
+      fakeMotionEvent(ev);
+      fakeButtonEvent(false, 1, ev);
+      break;
+    case GestureLongPress:
+      fakeMotionEvent(ev);
+      fakeButtonEvent(false, 3, ev);
+      break;
+    }
+    break;
+  }
+}
+
+void BaseTouchHandler::handleTapEvent(const GestureEvent& ev,
+                                      int buttonEvent)
+{
+  GestureEvent newEv = ev;
+
+  // If the user quickly taps multiple times we assume they meant to
+  // hit the same spot, so slightly adjust coordinates
+  if ((rfb::msSince(&lastTapTime) < DOUBLE_TAP_TIMEOUT) &&
+      (firstDoubleTapEvent.type == ev.type)) {
+
+    double dx = firstDoubleTapEvent.eventX - ev.eventX;
+    double dy = firstDoubleTapEvent.eventY - ev.eventY;
+    double distance = sqrt((dx * dx) + (dy * dy));
+
+    if (distance < DOUBLE_TAP_THRESHOLD) {
+     newEv.eventX = firstDoubleTapEvent.eventX;
+     newEv.eventY = firstDoubleTapEvent.eventY;
+    } else {
+      firstDoubleTapEvent = ev;
+    }
+  } else {
+    firstDoubleTapEvent = ev;
+  }
+  gettimeofday(&lastTapTime, NULL);
+
+  fakeMotionEvent(newEv);
+  fakeButtonEvent(true, buttonEvent, newEv);
+  fakeButtonEvent(false, buttonEvent, newEv);
+}

vncviewer/BaseTouchHandler.h

@@ -0,0 +1,51 @@
+/* Copyright 2019 Aaron Sowry for Cendio AB
+ * Copyright 2019-2020 Pierre Ossman for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#ifndef __BASETOUCHHANDLER_H__
+#define __BASETOUCHHANDLER_H__
+
+#include "GestureEvent.h"
+
+class BaseTouchHandler {
+  public:
+    virtual ~BaseTouchHandler();
+
+  protected:
+    BaseTouchHandler();
+
+  protected:
+    virtual void fakeMotionEvent(const GestureEvent origEvent) = 0;
+    virtual void fakeButtonEvent(bool press, int button,
+                                 const GestureEvent origEvent) = 0;
+    virtual void fakeKeyEvent(bool press, int keycode,
+                              const GestureEvent origEvent) = 0;
+
+    virtual void handleGestureEvent(const GestureEvent& event);
+
+  private:
+    void handleTapEvent(const GestureEvent& ev, int buttonEvent);
+
+    double lastMagnitudeX;
+    double lastMagnitudeY;
+
+    GestureEvent firstDoubleTapEvent;
+    struct timeval lastTapTime;
+};
+
+#endif

vncviewer/CMakeLists.txt

@@ -4,6 +4,7 @@ include_directories(${GETTEXT_INCLUDE_DIR})
 include_directories(${CMAKE_SOURCE_DIR}/common)
 set(VNCVIEWER_SOURCES
   menukey.cxx
+  BaseTouchHandler.cxx
   CConn.cxx
   DesktopWindow.cxx
   EmulateMB.cxx
@@ -33,7 +34,7 @@ if(WIN32)
 elseif(APPLE)
   set(VNCVIEWER_SOURCES ${VNCVIEWER_SOURCES} cocoa.mm osx_to_qnum.c)
 else()
-  set(VNCVIEWER_SOURCES ${VNCVIEWER_SOURCES} XInputTouchHandler.cxx xkb_to_qnum.c)
+  set(VNCVIEWER_SOURCES ${VNCVIEWER_SOURCES} GestureHandler.cxx XInputTouchHandler.cxx xkb_to_qnum.c)
 endif()
 
 if(WIN32)

vncviewer/GestureEvent.h

@@ -0,0 +1,51 @@
+/* Copyright 2019 Aaron Sowry for Cendio AB
+ * Copyright 2020 Samuel Mannehed for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#ifndef __GESTUREEVENT_H__
+#define __GESTUREEVENT_H__
+
+enum GestureEventGesture {
+  GestureOneTap,
+  GestureTwoTap,
+  GestureThreeTap,
+  GestureDrag,
+  GestureLongPress,
+  GestureTwoDrag,
+  GesturePinch,
+};
+
+enum GestureEventType {
+  GestureBegin,
+  GestureUpdate,
+  GestureEnd,
+};
+
+// magnitude is used by two gestures:
+//  GestureTwoDrag: distance moved since GestureBegin
+//  GesturePinch: distance between fingers
+struct GestureEvent {
+  double eventX;
+  double eventY;
+  double magnitudeX;
+  double magnitudeY;
+  GestureEventGesture gesture;
+  GestureEventType type;
+};
+
+#endif // __GESTUREEVENT_H__

vncviewer/GestureHandler.cxx

@@ -0,0 +1,512 @@
+/* Copyright 2019 Aaron Sowry for Cendio AB
+ * Copyright 2020 Pierre Ossman for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+ #ifdef HAVE_CONFIG_H
+ #include <config.h>
+ #endif
+
+#include <assert.h>
+#include <math.h>
+
+#include <rfb/util.h>
+#include <rfb/LogWriter.h>
+
+#include "GestureHandler.h"
+
+static rfb::LogWriter vlog("GestureHandler");
+
+static const unsigned char GH_NOGESTURE = 0;
+static const unsigned char GH_ONETAP    = 1;
+static const unsigned char GH_TWOTAP    = 2;
+static const unsigned char GH_THREETAP  = 4;
+static const unsigned char GH_DRAG      = 8;
+static const unsigned char GH_LONGPRESS = 16;
+static const unsigned char GH_TWODRAG   = 32;
+static const unsigned char GH_PINCH     = 64;
+
+static const unsigned char GH_INITSTATE = 127;
+
+const unsigned GH_MOVE_THRESHOLD = 50;
+const unsigned GH_ANGLE_THRESHOLD = 90; // Degrees
+
+// Timeout when waiting for gestures (ms)
+const unsigned GH_MULTITOUCH_TIMEOUT = 250;
+
+// Maximum time between press and release for a tap (ms)
+const unsigned GH_TAP_TIMEOUT = 1000;
+
+// Timeout when waiting for longpress (ms)
+const unsigned GH_LONGPRESS_TIMEOUT = 1000;
+
+GestureHandler::GestureHandler() :
+  state(GH_INITSTATE), waitingRelease(false),
+  longpressTimer(this), twoTouchTimer(this)
+{
+}
+
+GestureHandler::~GestureHandler()
+{
+}
+
+void GestureHandler::handleTouchBegin(int id, double x, double y)
+{
+  GHTouch ght;
+
+  // Ignore any new touches if there is already an active gesture,
+  // or we're in a cleanup state
+  if (hasDetectedGesture() || (state == GH_NOGESTURE)) {
+    ignored.insert(id);
+    return;
+  }
+
+  // Did it take too long between touches that we should no longer
+  // consider this a single gesture?
+  if ((tracked.size() > 0) &&
+      (rfb::msSince(&tracked.begin()->second.started) > GH_MULTITOUCH_TIMEOUT)) {
+    state = GH_NOGESTURE;
+    ignored.insert(id);
+    return;
+  }
+
+  // If we're waiting for fingers to release then we should no longer
+  // recognize new touches
+  if (waitingRelease) {
+    state = GH_NOGESTURE;
+    ignored.insert(id);
+    return;
+  }
+
+  gettimeofday(&ght.started, NULL);
+  ght.active = true;
+  ght.lastX = ght.firstX = x;
+  ght.lastY = ght.firstY = y;
+  ght.angle = 0;
+
+  tracked[id] = ght;
+
+  switch (tracked.size()) {
+    case 1:
+      longpressTimer.start(GH_LONGPRESS_TIMEOUT);
+      break;
+
+    case 2:
+      state &= ~(GH_ONETAP | GH_DRAG | GH_LONGPRESS);
+      longpressTimer.stop();
+      break;
+
+    case 3:
+      state &= ~(GH_TWOTAP | GH_TWODRAG | GH_PINCH);
+      break;
+
+    default:
+      state = GH_NOGESTURE;
+  }
+}
+
+void GestureHandler::handleTouchUpdate(int id, double x, double y)
+{
+  GHTouch *touch, *prevTouch;
+  double deltaX, deltaY, prevDeltaMove;
+  unsigned deltaAngle;
+
+  // If this is an update for a touch we're not tracking, ignore it
+  if (tracked.count(id) == 0)
+    return;
+
+  touch = &tracked[id];
+
+  // Update the touches last position with the event coordinates
+  touch->lastX = x;
+  touch->lastY = y;
+
+  deltaX = x - touch->firstX;
+  deltaY = y - touch->firstY;
+
+  // Update angle when the touch has moved
+  if ((touch->firstX != touch->lastX) ||
+      (touch->firstY != touch->lastY))
+    touch->angle = atan2(deltaY, deltaX) * 180 / M_PI;
+
+  if (!hasDetectedGesture()) {
+    // Ignore moves smaller than the minimum threshold
+    if (hypot(deltaX, deltaY) < GH_MOVE_THRESHOLD)
+      return;
+
+    // Can't be a tap or long press as we've seen movement
+    state &= ~(GH_ONETAP | GH_TWOTAP | GH_THREETAP | GH_LONGPRESS);
+    longpressTimer.stop();
+
+    if (tracked.size() != 1)
+      state &= ~(GH_DRAG);
+    if (tracked.size() != 2)
+      state &= ~(GH_TWODRAG | GH_PINCH);
+
+    // We need to figure out which of our different two touch gestures
+    // this might be
+    if (tracked.size() == 2) {
+
+      // The other touch can be first or last in tracked
+      // depending on which event came first
+      prevTouch = &tracked.rbegin()->second;
+      if (prevTouch == touch)
+        prevTouch = &tracked.begin()->second;
+
+      // How far the previous touch point has moved since start
+      prevDeltaMove = hypot(prevTouch->firstX - prevTouch->lastX,
+                            prevTouch->firstY - prevTouch->lastY);
+
+      // We know that the current touch moved far enough,
+      // but unless both touches moved further than their
+      // threshold we don't want to disqualify any gestures
+      if (prevDeltaMove > GH_MOVE_THRESHOLD) {
+
+        // The angle difference between the direction of the touch points
+        deltaAngle = fabs(touch->angle - prevTouch->angle);
+        deltaAngle = fabs(((deltaAngle + 180) % 360) - 180);
+
+        // PINCH or TWODRAG can be eliminated depending on the angle
+        if (deltaAngle > GH_ANGLE_THRESHOLD)
+          state &= ~GH_TWODRAG;
+        else
+          state &= ~GH_PINCH;
+
+        if (twoTouchTimer.isStarted())
+          twoTouchTimer.stop();
+
+      } else if(!twoTouchTimer.isStarted()) {
+        // We can't determine the gesture right now, let's
+        // wait and see if more events are on their way
+        twoTouchTimer.start(50);
+      }
+    }
+
+    if (!hasDetectedGesture())
+      return;
+
+    pushEvent(GestureBegin);
+  }
+
+  pushEvent(GestureUpdate);
+}
+
+void GestureHandler::handleTouchEnd(int id)
+{
+  std::map<int, GHTouch>::const_iterator iter;
+
+  // Check if this is an ignored touch
+  if(ignored.count(id)) {
+      ignored.erase(id);
+      if (ignored.empty() && tracked.empty()) {
+        state = GH_INITSTATE;
+        waitingRelease = false;
+      }
+      return;
+  }
+
+  // We got a TouchEnd before the timer triggered,
+  // this cannot result in a gesture anymore.
+  if (!hasDetectedGesture() && twoTouchTimer.isStarted()) {
+    twoTouchTimer.stop();
+    state = GH_NOGESTURE;
+  }
+
+  // Some gesture don't trigger until a touch is released
+  if (!hasDetectedGesture()) {
+    // Can't be a gesture that relies on movement
+    state &= ~(GH_DRAG | GH_TWODRAG | GH_PINCH);
+    // Or something that relies on more time
+    state &= ~GH_LONGPRESS;
+    longpressTimer.stop();
+
+    if (!waitingRelease) {
+      gettimeofday(&releaseStart, NULL);
+      waitingRelease = true;
+
+      // Can't be a tap that requires more touches than we current have
+      switch (tracked.size()) {
+        case 1:
+          state &= ~(GH_TWOTAP | GH_THREETAP);
+          break;
+
+        case 2:
+          state &= ~(GH_ONETAP | GH_THREETAP);
+          break;
+      }
+    }
+  }
+
+  // Waiting for all touches to release? (i.e. some tap)
+  if (waitingRelease) {
+    // Were all touches release roughly the same time?
+    if (rfb::msSince(&releaseStart) > GH_MULTITOUCH_TIMEOUT)
+      state = GH_NOGESTURE;
+
+    // Did too long time pass between press and release?
+    for (iter = tracked.begin(); iter != tracked.end(); ++iter) {
+      if (rfb::msSince(&iter->second.started) > GH_TAP_TIMEOUT) {
+        state = GH_NOGESTURE;
+        break;
+      }
+    }
+
+    tracked[id].active = false;
+
+    // Are we still waiting for more releases?
+    if (hasDetectedGesture()) {
+      pushEvent(GestureBegin);
+    } else {
+      // Have we reached a dead end?
+      if (state != GH_NOGESTURE)
+        return;
+    }
+  }
+
+  if (hasDetectedGesture())
+    pushEvent(GestureEnd);
+
+  // Ignore any remaining touches until they are ended
+  for (iter = tracked.begin(); iter != tracked.end(); ++iter) {
+    if (iter->second.active)
+      ignored.insert(iter->first);
+  }
+  tracked.clear();
+
+  state = GH_NOGESTURE;
+
+  ignored.erase(id);
+  if (ignored.empty()) {
+    state = GH_INITSTATE;
+    waitingRelease = false;
+  }
+}
+
+bool GestureHandler::hasDetectedGesture()
+{
+  if (state == GH_NOGESTURE)
+    return false;
+  // Check to see if the bitmask value is a power of 2
+  // (i.e. only one bit set). If it is, we have a state.
+  if (state & (state - 1))
+    return false;
+
+  // For taps we also need to have all touches released
+  // before we've fully detected the gesture
+  if (state & (GH_ONETAP | GH_TWOTAP | GH_THREETAP)) {
+    std::map<int, GHTouch>::const_iterator iter;
+
+    // Any touch still active/pressed?
+    for (iter = tracked.begin(); iter != tracked.end(); ++iter) {
+      if (iter->second.active)
+        return false;
+    }
+  }
+
+  return true;
+}
+
+bool GestureHandler::handleTimeout(rfb::Timer* t)
+{
+  if (t == &longpressTimer)
+    longpressTimeout();
+  else if (t == &twoTouchTimer)
+    twoTouchTimeout();
+
+  return false;
+}
+
+void GestureHandler::longpressTimeout()
+{
+  assert(!hasDetectedGesture());
+
+  state = GH_LONGPRESS;
+  pushEvent(GestureBegin);
+}
+
+void GestureHandler::twoTouchTimeout()
+{
+  double avgMoveH, avgMoveV, fdx, fdy, ldx, ldy, deltaTouchDistance;
+
+  assert(!tracked.empty());
+
+  // How far each touch point has moved since start
+  getAverageMovement(&avgMoveH, &avgMoveV);
+  avgMoveH = fabs(avgMoveH);
+  avgMoveV = fabs(avgMoveV);
+
+  // The difference in the distance between where
+  // the touch points started and where they are now
+  getAverageDistance(&fdx, &fdy, &ldx, &ldy);
+  deltaTouchDistance = fabs(hypot(fdx, fdy) - hypot(ldx, ldy));
+
+  if ((avgMoveV < deltaTouchDistance) &&
+      (avgMoveH < deltaTouchDistance))
+    state = GH_PINCH;
+  else
+    state = GH_TWODRAG;
+
+  pushEvent(GestureBegin);
+  pushEvent(GestureUpdate);
+}
+
+void GestureHandler::pushEvent(GestureEventType t)
+{
+  GestureEvent gev;
+  double avgX, avgY;
+
+  gev.type = t;
+  gev.gesture = stateToGesture(state);
+
+  // For most gesture events the current (average) position is the
+  // most useful
+  getPosition(NULL, NULL, &avgX, &avgY);
+
+  // However we have a slight distance to detect gestures, so for the
+  // first gesture event we want to use the first positions we saw
+  if (t == GestureBegin)
+    getPosition(&avgX, &avgY, NULL, NULL);
+
+  // For these gestures, we always want the event coordinates
+  // to be where the gesture began, not the current touch location.
+  switch (state) {
+    case GH_TWODRAG:
+    case GH_PINCH:
+      getPosition(&avgX, &avgY, NULL, NULL);
+      break;
+  }
+
+  gev.eventX = avgX;
+  gev.eventY = avgY;
+
+  // Some gestures also have a magnitude
+  if (state == GH_PINCH) {
+    if (t == GestureBegin)
+      getAverageDistance(&gev.magnitudeX, &gev.magnitudeY,
+                         NULL, NULL);
+    else
+      getAverageDistance(NULL, NULL,
+                         &gev.magnitudeX, &gev.magnitudeY);
+  } else if (state == GH_TWODRAG) {
+    if (t == GestureBegin)
+      gev.magnitudeX = gev.magnitudeY = 0;
+    else
+      getAverageMovement(&gev.magnitudeX, &gev.magnitudeY);
+  }
+
+  handleGestureEvent(gev);
+}
+
+GestureEventGesture GestureHandler::stateToGesture(unsigned char state)
+{
+  switch (state) {
+    case GH_ONETAP:
+      return GestureOneTap;
+    case GH_TWOTAP:
+      return GestureTwoTap;
+    case GH_THREETAP:
+      return GestureThreeTap;
+    case GH_DRAG:
+      return GestureDrag;
+    case GH_LONGPRESS:
+      return GestureLongPress;
+    case GH_TWODRAG:
+      return GestureTwoDrag;
+    case GH_PINCH:
+      return GesturePinch;
+  }
+
+  assert(false);
+
+  return (GestureEventGesture)0;
+}
+
+void GestureHandler::getPosition(double *firstX, double *firstY,
+                                 double *lastX, double *lastY)
+{
+  size_t size;
+  double fx = 0, fy = 0, lx = 0, ly = 0;
+
+  assert(!tracked.empty());
+
+  size = tracked.size();
+
+  std::map<int, GHTouch>::const_iterator iter;
+  for (iter = tracked.begin(); iter != tracked.end(); ++iter) {
+    fx += iter->second.firstX;
+    fy += iter->second.firstY;
+    lx += iter->second.lastX;
+    ly += iter->second.lastY;
+  }
+
+  if (firstX)
+    *firstX = fx / size;
+  if (firstY)
+    *firstY = fy / size;
+  if (lastX)
+    *lastX = lx / size;
+  if (lastY)
+    *lastY = ly / size;
+}
+
+void GestureHandler::getAverageMovement(double *h, double *v)
+{
+  double totalH, totalV;
+  size_t size;
+
+  assert(!tracked.empty());
+
+  totalH = totalV = 0;
+  size = tracked.size();
+
+  std::map<int, GHTouch>::const_iterator iter;
+  for (iter = tracked.begin(); iter != tracked.end(); ++iter) {
+    totalH += iter->second.lastX - iter->second.firstX;
+    totalV += iter->second.lastY - iter->second.firstY;
+  }
+
+  if (h)
+    *h = totalH / size;
+  if (v)
+    *v = totalV / size;
+}
+
+void GestureHandler::getAverageDistance(double *firstX, double *firstY,
+                                        double *lastX, double *lastY)
+{
+  double dx, dy;
+
+  assert(!tracked.empty());
+
+  // Distance between the first and last tracked touches
+
+  dx = fabs(tracked.rbegin()->second.firstX - tracked.begin()->second.firstX);
+  dy = fabs(tracked.rbegin()->second.firstY - tracked.begin()->second.firstY);
+
+  if (firstX)
+    *firstX = dx;
+  if (firstY)
+    *firstY = dy;
+
+  dx = fabs(tracked.rbegin()->second.lastX - tracked.begin()->second.lastX);
+  dy = fabs(tracked.rbegin()->second.lastY - tracked.begin()->second.lastY);
+
+  if (lastX)
+    *lastX = dx;
+  if (lastY)
+    *lastY = dy;
+}

vncviewer/GestureHandler.h

@@ -0,0 +1,81 @@
+/* Copyright 2019 Aaron Sowry for Cendio AB
+ * Copyright 2020 Pierre Ossman for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#ifndef __GESTUREHANDLER_H__
+#define __GESTUREHANDLER_H__
+
+#include <set>
+#include <map>
+
+#include <rfb/Timer.h>
+
+#include "GestureEvent.h"
+
+class GestureHandler : public rfb::Timer::Callback {
+  public:
+    GestureHandler();
+    virtual ~GestureHandler();
+
+    void handleTouchBegin(int id, double x, double y);
+    void handleTouchUpdate(int id, double x, double y);
+    void handleTouchEnd(int id);
+
+  protected:
+    virtual void handleGestureEvent(const GestureEvent& event) = 0;
+
+  private:
+    bool hasDetectedGesture();
+
+    virtual bool handleTimeout(rfb::Timer* t);
+    void longpressTimeout();
+    void twoTouchTimeout();
+
+    void pushEvent(GestureEventType t);
+    GestureEventGesture stateToGesture(unsigned char state);
+
+    void getPosition(double *firstX, double *firstY,
+                     double *lastX, double *lastY);
+    void getAverageMovement(double *h, double *v);
+    void getAverageDistance(double *firstX, double *firstY,
+                            double *lastX, double *lastY);
+
+  private:
+    struct GHTouch {
+      struct timeval started;
+      bool active;
+      double firstX;
+      double firstY;
+      double lastX;
+      double lastY;
+      int angle;
+    };
+
+    unsigned char state;
+
+    std::map<int, GHTouch> tracked;
+    std::set<int> ignored;
+
+    bool waitingRelease;
+    struct timeval releaseStart;
+
+    rfb::Timer longpressTimer;
+    rfb::Timer twoTouchTimer;
+};
+
+#endif // __GESTUREHANDLER_H__

vncviewer/XInputTouchHandler.cxx

@@ -26,9 +26,14 @@
 
 #include <X11/extensions/XInput2.h>
 #include <X11/extensions/XI2.h>
+#include <X11/XKBlib.h>
 
 #include <FL/x.H>
 
+#ifndef XK_MISCELLANY
+#define XK_MISCELLANY
+#include <rfb/keysymdef.h>
+#endif
 #include <rfb/LogWriter.h>
 
 #include "i18n.h"
@@ -39,7 +44,7 @@ static rfb::LogWriter vlog("XInputTouchHandler");
 static bool grabbed = false;
 
 XInputTouchHandler::XInputTouchHandler(Window wnd)
-  : wnd(wnd), fakeStateMask(0), trackingTouch(false)
+  : wnd(wnd), fakeStateMask(0)
 {
   XIEventMask eventmask;
   unsigned char flags[XIMaskLen(XI_LASTEVENT)] = { 0 };
@@ -232,9 +237,6 @@ void XInputTouchHandler::processEvent(const XIDeviceEvent* devev)
     fakeButtonEvent(false, devev->detail, devev);
     break;
   case XI_TouchBegin:
-    if (trackingTouch)
-      break;
-
     // XInput2 wants us to explicitly accept touch sequences
     // for grabbed devices before it will pass events
     if (grabbed) {
@@ -245,22 +247,13 @@ void XInputTouchHandler::processEvent(const XIDeviceEvent* devev)
                          XIAcceptTouch);
     }
 
-    trackingTouch = true;
-    trackedTouchPoint = devev->detail;
-
-    fakeMotionEvent(devev);
-    fakeButtonEvent(true, Button1, devev);
+    handleTouchBegin(devev->detail, devev->event_x, devev->event_y);
     break;
   case XI_TouchUpdate:
-    if (!trackingTouch || (devev->detail != trackedTouchPoint))
-      break;
-    fakeMotionEvent(devev);
+    handleTouchUpdate(devev->detail, devev->event_x, devev->event_y);
     break;
   case XI_TouchEnd:
-    if (!trackingTouch || (devev->detail != trackedTouchPoint))
-      break;
-    fakeButtonEvent(false, Button1, devev);
-    trackingTouch = false;
+    handleTouchEnd(devev->detail);
     break;
   case XI_TouchOwnership:
     // FIXME: Currently ignored, see constructor
@@ -297,13 +290,76 @@ void XInputTouchHandler::fakeMotionEvent(const XIDeviceEvent* origEvent)
   fakeEvent.xmotion.is_hint = False;
   preparePointerEvent(&fakeEvent, origEvent);
 
+  pushFakeEvent(&fakeEvent);
+}
+
+void XInputTouchHandler::fakeButtonEvent(bool press, int button,
+                                         const XIDeviceEvent* origEvent)
+{
+  XEvent fakeEvent;
+
+  memset(&fakeEvent, 0, sizeof(XEvent));
+
+  fakeEvent.type = press ? ButtonPress : ButtonRelease;
+  fakeEvent.xbutton.button = button;
+
+  // Apply the fake mask before pushing so it will be in sync
+  fakeEvent.xbutton.state |= fakeStateMask;
+  preparePointerEvent(&fakeEvent, origEvent);
+
+  pushFakeEvent(&fakeEvent);
+}
+
+void XInputTouchHandler::preparePointerEvent(XEvent* dst, const GestureEvent src)
+{
+  Window root, child;
+  int rootX, rootY;
+  XkbStateRec state;
+
+  // We don't have a real event to steal things from, so we'll have
+  // to fake these events based on the current state of things
+
+  root = XDefaultRootWindow(fl_display);
+  XTranslateCoordinates(fl_display, wnd, root,
+                        src.eventX,
+                        src.eventY,
+                        &rootX, &rootY, &child);
+  XkbGetState(fl_display, XkbUseCoreKbd, &state);
+
+  // XButtonEvent and XMotionEvent are almost identical, so we
+  // don't have to care which it is for these fields
+  dst->xbutton.serial = XLastKnownRequestProcessed(fl_display);
+  dst->xbutton.display = fl_display;
+  dst->xbutton.window = wnd;
+  dst->xbutton.root = root;
+  dst->xbutton.subwindow = None;
+  dst->xbutton.time = fl_event_time;
+  dst->xbutton.x = src.eventX;
+  dst->xbutton.y = src.eventY;
+  dst->xbutton.x_root = rootX;
+  dst->xbutton.y_root = rootY;
+  dst->xbutton.state = state.mods;
+  dst->xbutton.state |= ((state.ptr_buttons >> 1) & 0x1f) << 8;
+  dst->xbutton.same_screen = True;
+}
+
+void XInputTouchHandler::fakeMotionEvent(const GestureEvent origEvent)
+{
+  XEvent fakeEvent;
+
+  memset(&fakeEvent, 0, sizeof(XEvent));
+
+  fakeEvent.type = MotionNotify;
+  fakeEvent.xmotion.is_hint = False;
+  preparePointerEvent(&fakeEvent, origEvent);
+
   fakeEvent.xbutton.state |= fakeStateMask;
 
   pushFakeEvent(&fakeEvent);
 }
 
 void XInputTouchHandler::fakeButtonEvent(bool press, int button,
-                                         const XIDeviceEvent* origEvent)
+                                         const GestureEvent origEvent)
 {
   XEvent fakeEvent;
 
@@ -315,6 +371,8 @@ void XInputTouchHandler::fakeButtonEvent(bool press, int button,
 
   fakeEvent.xbutton.state |= fakeStateMask;
 
+  // The button mask should indicate the button state just prior to
+  // the event, we update the button mask after pushing
   pushFakeEvent(&fakeEvent);
 
   // Set/unset the bit for the correct button
@@ -325,6 +383,77 @@ void XInputTouchHandler::fakeButtonEvent(bool press, int button,
   }
 }
 
+void XInputTouchHandler::fakeKeyEvent(bool press, int keysym,
+                                      const GestureEvent origEvent)
+{
+  XEvent fakeEvent;
+
+  Window root, child;
+  int rootX, rootY;
+  XkbStateRec state;
+
+  int modmask;
+
+  root = XDefaultRootWindow(fl_display);
+  XTranslateCoordinates(fl_display, wnd, root,
+                        origEvent.eventX,
+                        origEvent.eventY,
+                        &rootX, &rootY, &child);
+  XkbGetState(fl_display, XkbUseCoreKbd, &state);
+
+  KeyCode kc = XKeysymToKeycode(fl_display, keysym);
+
+  memset(&fakeEvent, 0, sizeof(XEvent));
+
+  fakeEvent.type = press ? KeyPress : KeyRelease;
+  fakeEvent.xkey.type = press ? KeyPress : KeyRelease;
+  fakeEvent.xkey.keycode = kc;
+  fakeEvent.xkey.serial = XLastKnownRequestProcessed(fl_display);
+  fakeEvent.xkey.display = fl_display;
+  fakeEvent.xkey.window = wnd;
+  fakeEvent.xkey.root = root;
+  fakeEvent.xkey.subwindow = None;
+  fakeEvent.xkey.time = fl_event_time;
+  fakeEvent.xkey.x = origEvent.eventX;
+  fakeEvent.xkey.y = origEvent.eventY;
+  fakeEvent.xkey.x_root = rootX;
+  fakeEvent.xkey.y_root = rootY;
+  fakeEvent.xkey.state = state.mods;
+  fakeEvent.xkey.state |= ((state.ptr_buttons >> 1) & 0x1f) << 8;
+  fakeEvent.xkey.same_screen = True;
+
+  // Apply our fake mask
+  fakeEvent.xkey.state |= fakeStateMask;
+
+  pushFakeEvent(&fakeEvent);
+
+  switch(keysym) {
+    case XK_Shift_L:
+    case XK_Shift_R:
+      modmask = ShiftMask;
+      break;
+    case XK_Caps_Lock:
+      modmask = LockMask;
+      break;
+    case XK_Control_L:
+    case XK_Control_R:
+      modmask = ControlMask;
+      break;
+    default:
+      modmask = 0;
+  }
+
+  if (press)
+    fakeStateMask |= modmask;
+  else
+    fakeStateMask &= ~modmask;
+}
+
+void XInputTouchHandler::handleGestureEvent(const GestureEvent& event)
+{
+  BaseTouchHandler::handleGestureEvent(event);
+}
+
 void XInputTouchHandler::pushFakeEvent(XEvent* event)
 {
   // Perhaps use XPutBackEvent() to avoid round trip latency?

vncviewer/XInputTouchHandler.h

@@ -20,7 +20,10 @@
 #ifndef __XINPUTTOUCHHANDLER_H__
 #define __XINPUTTOUCHHANDLER_H__
 
-class XInputTouchHandler {
+#include "BaseTouchHandler.h"
+#include "GestureHandler.h"
+
+class XInputTouchHandler: public BaseTouchHandler, GestureHandler {
   public:
     XInputTouchHandler(Window wnd);
 
@@ -35,14 +38,21 @@ class XInputTouchHandler {
     void fakeButtonEvent(bool press, int button,
                          const XIDeviceEvent* origEvent);
 
+    void preparePointerEvent(XEvent* dst, const GestureEvent src);
+    virtual void fakeMotionEvent(const GestureEvent origEvent);
+    virtual void fakeButtonEvent(bool press, int button,
+                                 const GestureEvent origEvent);
+    virtual void fakeKeyEvent(bool press, int keycode,
+                              const GestureEvent origEvent);
+
+    virtual void handleGestureEvent(const GestureEvent& event);
+
   private:
     void pushFakeEvent(XEvent* event);
 
   private:
     Window wnd;
     int fakeStateMask;
-    bool trackingTouch;
-    int trackedTouchPoint;
 };
 
 #endif