Merge branch 'vmware' of https://github.com/CendioOssman/tigervnc

7 files changed, 163 insertions, 6 deletions

diff --git a/common/rfb/CConnection.cxx b/common/rfb/CConnection.cxx
index 3ea217fa..fcad4053 100644
--- a/common/rfb/CConnection.cxx
+++ b/common/rfb/CConnection.cxx
@@ -596,6 +596,7 @@ void CConnection::updateEncodings()
 
   if (supportsLocalCursor) {
     encodings.push_back(pseudoEncodingCursorWithAlpha);
+    encodings.push_back(pseudoEncodingVMwareCursor);
     encodings.push_back(pseudoEncodingCursor);
     encodings.push_back(pseudoEncodingXCursor);
   }
@@ -603,8 +604,10 @@ void CConnection::updateEncodings()
     encodings.push_back(pseudoEncodingDesktopSize);
     encodings.push_back(pseudoEncodingExtendedDesktopSize);
   }
-  if (supportsLEDState)
+  if (supportsLEDState) {
     encodings.push_back(pseudoEncodingLEDState);
+    encodings.push_back(pseudoEncodingVMwareLEDState);
+  }
 
   encodings.push_back(pseudoEncodingDesktopName);
   encodings.push_back(pseudoEncodingLastRect);
diff --git a/common/rfb/CMsgReader.cxx b/common/rfb/CMsgReader.cxx
index 3422ebf5..e8e7afb2 100644
--- a/common/rfb/CMsgReader.cxx
+++ b/common/rfb/CMsgReader.cxx
@@ -97,6 +97,9 @@ void CMsgReader::readMsg()
     case pseudoEncodingCursorWithAlpha:
       readSetCursorWithAlpha(w, h, Point(x,y));
       break;
+    case pseudoEncodingVMwareCursor:
+      readSetVMwareCursor(w, h, Point(x,y));
+      break;
     case pseudoEncodingDesktopName:
       readSetDesktopName(x, y, w, h);
       break;
@@ -108,6 +111,10 @@ void CMsgReader::readMsg()
       break;
     case pseudoEncodingLEDState:
       readLEDState();
+      break;
+    case pseudoEncodingVMwareLEDState:
+      readVMwareLEDState();
+      break;
     case pseudoEncodingQEMUKeyEvent:
       handler->supportsQEMUKeyEvent();
       break;
@@ -349,6 +356,94 @@ void CMsgReader::readSetCursorWithAlpha(int width, int height, const Point& hots
                      pb.getBuffer(pb.getRect(), &stride));
 }
 
+void CMsgReader::readSetVMwareCursor(int width, int height, const Point& hotspot)
+{
+  if (width > maxCursorSize || height > maxCursorSize)
+    throw Exception("Too big cursor");
+
+  rdr::U8 type;
+
+  type = is->readU8();
+  is->skip(1);
+
+  if (type == 0) {
+    int len = width * height * (handler->server.pf().bpp/8);
+    rdr::U8Array andMask(len);
+    rdr::U8Array xorMask(len);
+
+    rdr::U8Array data(width*height*4);
+
+    rdr::U8* andIn;
+    rdr::U8* xorIn;
+    rdr::U8* out;
+    int Bpp;
+
+    is->readBytes(andMask.buf, len);
+    is->readBytes(xorMask.buf, len);
+
+    andIn = andMask.buf;
+    xorIn = xorMask.buf;
+    out = data.buf;
+    Bpp = handler->server.pf().bpp/8;
+    for (int y = 0;y < height;y++) {
+      for (int x = 0;x < width;x++) {
+        Pixel andPixel, xorPixel;
+
+        andPixel = handler->server.pf().pixelFromBuffer(andIn);
+        xorPixel = handler->server.pf().pixelFromBuffer(xorIn);
+        andIn += Bpp;
+        xorIn += Bpp;
+
+        if (andPixel == 0) {
+          rdr::U8 r, g, b;
+
+          // Opaque pixel
+
+          handler->server.pf().rgbFromPixel(xorPixel, &r, &g, &b);
+          *out++ = r;
+          *out++ = g;
+          *out++ = b;
+          *out++ = 0xff;
+        } else if (xorPixel == 0) {
+          // Fully transparent pixel
+          *out++ = 0;
+          *out++ = 0;
+          *out++ = 0;
+          *out++ = 0;
+        } else if (andPixel == xorPixel) {
+          // Inverted pixel
+
+          // We don't really support this, so just turn the pixel black
+          // FIXME: Do an outline like WinVNC does?
+          *out++ = 0;
+          *out++ = 0;
+          *out++ = 0;
+          *out++ = 0xff;
+        } else {
+          // Partially transparent/inverted pixel
+
+          // We _really_ can't handle this, just make it black
+          *out++ = 0;
+          *out++ = 0;
+          *out++ = 0;
+          *out++ = 0xff;
+        }
+      }
+    }
+
+    handler->setCursor(width, height, hotspot, data.buf);
+  } else if (type == 1) {
+    rdr::U8Array data(width*height*4);
+
+    // FIXME: Is alpha premultiplied?
+    is->readBytes(data.buf, width*height*4);
+
+    handler->setCursor(width, height, hotspot, data.buf);
+  } else {
+    throw Exception("Unknown cursor type");
+  }
+}
+
 void CMsgReader::readSetDesktopName(int x, int y, int w, int h)
 {
   char* name = is->readString();
@@ -394,3 +489,15 @@ void CMsgReader::readLEDState()
 
   handler->setLEDState(state);
 }
+
+void CMsgReader::readVMwareLEDState()
+{
+  rdr::U32 state;
+
+  state = is->readU32();
+
+  // As luck has it, this extension uses the same bit definitions,
+  // so no conversion required
+
+  handler->setLEDState(state);
+}
diff --git a/common/rfb/CMsgReader.h b/common/rfb/CMsgReader.h
index 99638276..03f3d8d2 100644
--- a/common/rfb/CMsgReader.h
+++ b/common/rfb/CMsgReader.h
@@ -63,9 +63,11 @@ namespace rfb {
     void readSetXCursor(int width, int height, const Point& hotspot);
     void readSetCursor(int width, int height, const Point& hotspot);
     void readSetCursorWithAlpha(int width, int height, const Point& hotspot);
+    void readSetVMwareCursor(int width, int height, const Point& hotspot);
     void readSetDesktopName(int x, int y, int w, int h);
     void readExtendedDesktopSize(int x, int y, int w, int h);
     void readLEDState();
+    void readVMwareLEDState();
 
     CMsgHandler* handler;
     rdr::InStream* is;
diff --git a/common/rfb/ClientParams.cxx b/common/rfb/ClientParams.cxx
index 2f8783bb..e42d494b 100644
--- a/common/rfb/ClientParams.cxx
+++ b/common/rfb/ClientParams.cxx
@@ -140,6 +140,8 @@ bool ClientParams::supportsLocalCursor() const
 {
   if (supportsEncoding(pseudoEncodingCursorWithAlpha))
     return true;
+  if (supportsEncoding(pseudoEncodingVMwareCursor))
+    return true;
   if (supportsEncoding(pseudoEncodingCursor))
     return true;
   if (supportsEncoding(pseudoEncodingXCursor))
@@ -160,6 +162,8 @@ bool ClientParams::supportsLEDState() const
 {
   if (supportsEncoding(pseudoEncodingLEDState))
     return true;
+  if (supportsEncoding(pseudoEncodingVMwareLEDState))
+    return true;
   return false;
 }
 
diff --git a/common/rfb/SMsgWriter.cxx b/common/rfb/SMsgWriter.cxx
index a12a6d40..6a2c2ba0 100644
--- a/common/rfb/SMsgWriter.cxx
+++ b/common/rfb/SMsgWriter.cxx
@@ -144,7 +144,8 @@ void SMsgWriter::writeCursor()
 {
   if (!client->supportsEncoding(pseudoEncodingCursor) &&
       !client->supportsEncoding(pseudoEncodingXCursor) &&
-      !client->supportsEncoding(pseudoEncodingCursorWithAlpha))
+      !client->supportsEncoding(pseudoEncodingCursorWithAlpha) &&
+      !client->supportsEncoding(pseudoEncodingVMwareCursor))
     throw Exception("Client does not support local cursor");
 
   needCursor = true;
@@ -152,7 +153,8 @@ void SMsgWriter::writeCursor()
 
 void SMsgWriter::writeLEDState()
 {
-  if (!client->supportsEncoding(pseudoEncodingLEDState))
+  if (!client->supportsEncoding(pseudoEncodingLEDState) &&
+      !client->supportsEncoding(pseudoEncodingVMwareLEDState))
     throw Exception("Client does not support LED state");
   if (client->ledState() == ledUnknown)
     throw Exception("Server has not specified LED state");
@@ -299,6 +301,10 @@ void SMsgWriter::writePseudoRects()
       writeSetCursorWithAlphaRect(cursor.width(), cursor.height(),
                                   cursor.hotspot().x, cursor.hotspot().y,
                                   cursor.getBuffer());
+    } else if (client->supportsEncoding(pseudoEncodingVMwareCursor)) {
+      writeSetVMwareCursorRect(cursor.width(), cursor.height(),
+                               cursor.hotspot().x, cursor.hotspot().y,
+                               cursor.getBuffer());
     } else if (client->supportsEncoding(pseudoEncodingCursor)) {
       rdr::U8Array data(cursor.width()*cursor.height() * (client->pf().bpp/8));
       rdr::U8Array mask(cursor.getMask());
@@ -503,9 +509,32 @@ void SMsgWriter::writeSetCursorWithAlphaRect(int width, int height,
   }
 }
 
+void SMsgWriter::writeSetVMwareCursorRect(int width, int height,
+                                          int hotspotX, int hotspotY,
+                                          const rdr::U8* data)
+{
+  if (!client->supportsEncoding(pseudoEncodingVMwareCursor))
+    throw Exception("Client does not support local cursors");
+  if (++nRectsInUpdate > nRectsInHeader && nRectsInHeader)
+    throw Exception("SMsgWriter::writeSetVMwareCursorRect: nRects out of sync");
+
+  os->writeS16(hotspotX);
+  os->writeS16(hotspotY);
+  os->writeU16(width);
+  os->writeU16(height);
+  os->writeU32(pseudoEncodingVMwareCursor);
+
+  os->writeU8(1); // Alpha cursor
+  os->pad(1);
+
+  // FIXME: Should alpha be premultiplied?
+  os->writeBytes(data, width*height*4);
+}
+
 void SMsgWriter::writeLEDStateRect(rdr::U8 state)
 {
-  if (!client->supportsEncoding(pseudoEncodingLEDState))
+  if (!client->supportsEncoding(pseudoEncodingLEDState) &&
+      !client->supportsEncoding(pseudoEncodingVMwareLEDState))
     throw Exception("Client does not support LED state updates");
   if (client->ledState() == ledUnknown)
     throw Exception("Server does not support LED state updates");
@@ -516,8 +545,13 @@ void SMsgWriter::writeLEDStateRect(rdr::U8 state)
   os->writeS16(0);
   os->writeU16(0);
   os->writeU16(0);
-  os->writeU32(pseudoEncodingLEDState);
-  os->writeU8(state);
+  if (client->supportsEncoding(pseudoEncodingLEDState)) {
+    os->writeU32(pseudoEncodingLEDState);
+    os->writeU8(state);
+  } else {
+    os->writeU32(pseudoEncodingVMwareLEDState);
+    os->writeU32(state);
+  }
 }
 
 void SMsgWriter::writeQEMUKeyEventRect()
diff --git a/common/rfb/SMsgWriter.h b/common/rfb/SMsgWriter.h
index 80f6de91..4f4c9cc0 100644
--- a/common/rfb/SMsgWriter.h
+++ b/common/rfb/SMsgWriter.h
@@ -130,6 +130,9 @@ namespace rfb {
     void writeSetCursorWithAlphaRect(int width, int height,
                                      int hotspotX, int hotspotY,
                                      const rdr::U8* data);
+    void writeSetVMwareCursorRect(int width, int height,
+                                  int hotspotX, int hotspotY,
+                                  const rdr::U8* data);
     void writeLEDStateRect(rdr::U8 state);
     void writeQEMUKeyEventRect();
 
diff --git a/common/rfb/encodings.h b/common/rfb/encodings.h
index 122afe7f..acb86ecc 100644
--- a/common/rfb/encodings.h
+++ b/common/rfb/encodings.h
@@ -59,6 +59,10 @@ namespace rfb {
   const int pseudoEncodingSubsamp8X = -764;
   const int pseudoEncodingSubsamp16X = -763;
 
+  // VMware-specific
+  const int pseudoEncodingVMwareCursor = 0x574d5664;
+  const int pseudoEncodingVMwareLEDState = 0x574d5668;
+
   int encodingNum(const char* name);
   const char* encodingName(int num);
 }