Some applications (e.g. wine and rdesktop) tend to misbehave

when we use fake shift to get different symbols on the numpad.
So avoid using fake shifts on NumLock related keys and hope
that other applications won't get upset by getting the
non-numpad keysym counterparts.


git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@5119 3789f03b-4d11-0410-bbf8-ca57d06f2519

4 files changed, 148 insertions, 0 deletions

diff --git a/unix/xserver/hw/vnc/Input.cc b/unix/xserver/hw/vnc/Input.cc
index 83f2ef3e..a6a51d7f 100644
--- a/unix/xserver/hw/vnc/Input.cc
+++ b/unix/xserver/hw/vnc/Input.cc
@@ -64,6 +64,8 @@ using namespace rfb;
 
 static LogWriter vlog("Input");
 
+rfb::BoolParameter avoidShiftNumLock("AvoidShiftNumLock", "Avoid fake Shift presses for keys affected by NumLock.", true);
+
 #define BUTTONS 7
 
 /* Event queue is shared between all devices. */
@@ -520,6 +522,53 @@ void InputDevice::keyEvent(rdr::U32 keysym, bool down)
 	}
 
 	/*
+	 * X11 generally lets shift toggle the keys on the numeric pad
+	 * the same way NumLock does. This is however not the case on
+	 * other systems like Windows. As a result, some applications
+	 * get confused when we do a fake shift to get the same effect
+	 * that having NumLock active would produce.
+	 *
+	 * Until we have proper NumLock synchronisation (so we can
+	 * avoid faking shift), we try to avoid the fake shifts if we
+	 * can use an alternative keysym.
+	 */
+	if (((state & ShiftMask) != (new_state & ShiftMask)) &&
+	    avoidShiftNumLock && isAffectedByNumLock(keycode)) {
+	    	KeyCode keycode2;
+	    	unsigned new_state2;
+
+		vlog.debug("Finding alternative to keysym 0x%x to avoid fake shift for numpad", keysym);
+
+		for (i = 0;i < sizeof(altKeysym)/sizeof(altKeysym[0]);i++) {
+			KeySym altsym;
+
+			if (altKeysym[i].a == keysym)
+				altsym = altKeysym[i].b;
+			else if (altKeysym[i].b == keysym)
+				altsym = altKeysym[i].a;
+			else
+				continue;
+
+			keycode2 = keysymToKeycode(altsym, state, &new_state2);
+			if (keycode2 == 0)
+				continue;
+
+			if (((state & ShiftMask) != (new_state2 & ShiftMask)) &&
+			    isAffectedByNumLock(keycode2))
+				continue;
+
+			break;
+		}
+
+		if (i == sizeof(altKeysym)/sizeof(altKeysym[0]))
+			vlog.debug("No alternative keysym found");
+		else {
+			keycode = keycode2;
+			new_state = new_state2;
+		}
+	}
+
+	/*
 	 * "Shifted Tab" is a bit of a mess. Some systems have varying,
 	 * special keysyms for this symbol. VNC mandates that clients
 	 * should always send the plain XK_Tab keysym and the server
diff --git a/unix/xserver/hw/vnc/Input.h b/unix/xserver/hw/vnc/Input.h
index 3a0e9039..e893049e 100644
--- a/unix/xserver/hw/vnc/Input.h
+++ b/unix/xserver/hw/vnc/Input.h
@@ -91,6 +91,8 @@ private:
 
 	bool isLockModifier(KeyCode keycode, unsigned state);
 
+	bool isAffectedByNumLock(KeyCode keycode);
+
 	KeyCode addKeysym(KeySym keysym, unsigned state);
 
 private:
diff --git a/unix/xserver/hw/vnc/InputCore.cc b/unix/xserver/hw/vnc/InputCore.cc
index de7653ef..a880ca07 100644
--- a/unix/xserver/hw/vnc/InputCore.cc
+++ b/unix/xserver/hw/vnc/InputCore.cc
@@ -514,6 +514,26 @@ done:
 	return false;
 }
 
+bool InputDevice::isAffectedByNumLock(KeyCode keycode)
+{
+	KeySymsPtr keymap;
+	int i, per;
+	KeySym *syms;
+
+	keymap = &keyboardDev->key->curKeySyms;
+	per = keymap->mapWidth;
+	syms = &keymap->map[(keycode - keymap->minKeyCode) * per];
+
+	for (i = 0;i < per;i++) {
+		if (IsKeypadKey(syms[i]))
+			return true;
+		if (IsPrivateKeypadKey(syms[i]))
+			return true;
+	}
+
+	return false;
+}
+
 KeyCode InputDevice::addKeysym(KeySym keysym, unsigned state)
 {
 	KeyCode kc;
diff --git a/unix/xserver/hw/vnc/InputXKB.cc b/unix/xserver/hw/vnc/InputXKB.cc
index 4c04ef9f..ed93afcc 100644
--- a/unix/xserver/hw/vnc/InputXKB.cc
+++ b/unix/xserver/hw/vnc/InputXKB.cc
@@ -177,6 +177,36 @@ static XkbAction *XkbKeyActionPtr(XkbDescPtr xkb, KeyCode key, unsigned int mods
 	return &acts[col];
 }
 
+static unsigned XkbKeyEffectiveGroup(XkbDescPtr xkb, KeyCode key, unsigned int mods)
+{
+	int nKeyGroups;
+	unsigned effectiveGroup;
+
+	nKeyGroups= XkbKeyNumGroups(xkb,key);
+	if ((!XkbKeycodeInRange(xkb,key))||(nKeyGroups==0))
+		return 0;
+
+	effectiveGroup= XkbGroupForCoreState(mods);
+	if ( effectiveGroup>=nKeyGroups ) {
+		unsigned groupInfo= XkbKeyGroupInfo(xkb,key);
+		switch (XkbOutOfRangeGroupAction(groupInfo)) {
+		default:
+			effectiveGroup %= nKeyGroups;
+			break;
+		case XkbClampIntoRange:
+			effectiveGroup = nKeyGroups-1;
+			break;
+		case XkbRedirectIntoRange:
+			effectiveGroup = XkbOutOfRangeGroupNumber(groupInfo);
+			if (effectiveGroup>=nKeyGroups)
+				effectiveGroup= 0;
+			break;
+		}
+	}
+
+	return effectiveGroup;
+}
+
 void InputDevice::PrepareInputDevices(void)
 {
 #if XORG < 19
@@ -479,6 +509,53 @@ bool InputDevice::isLockModifier(KeyCode keycode, unsigned state)
 	return true;
 }
 
+bool InputDevice::isAffectedByNumLock(KeyCode keycode)
+{
+	unsigned state;
+
+	KeyCode numlock_keycode;
+	unsigned numlock_mask;
+
+	XkbDescPtr xkb;
+	XkbAction *act;
+
+	unsigned group;
+	XkbKeyTypeRec *type;
+
+	/* Group state is still important */
+	state = getKeyboardState();
+	state &= ~0xff;
+
+	/*
+	 * Not sure if hunting for a virtual modifier called "NumLock",
+	 * or following the keysym Num_Lock is the best approach. We
+	 * try the latter.
+	 */
+	numlock_keycode = keysymToKeycode(XK_Num_Lock, state, NULL);
+	if (numlock_keycode == 0)
+		return false;
+
+	xkb = GetMaster(keyboardDev, KEYBOARD_OR_FLOAT)->key->xkbInfo->desc;
+
+	act = XkbKeyActionPtr(xkb, numlock_keycode, state);
+	if (act == NULL)
+		return false;
+	if (act->type != XkbSA_LockMods)
+		return false;
+
+	if (act->mods.flags & XkbSA_UseModMapMods)
+		numlock_mask = xkb->map->modmap[keycode];
+	else
+		numlock_mask = act->mods.mask;
+
+	group = XkbKeyEffectiveGroup(xkb, keycode, state);
+	type = XkbKeyKeyType(xkb, keycode, group);
+	if ((type->mods.mask & numlock_mask) == 0)
+		return false;
+
+	return true;
+}
+
 KeyCode InputDevice::addKeysym(KeySym keysym, unsigned state)
 {
 	DeviceIntPtr master;