Server support for cursor with alpha

5 files changed, 69 insertions, 6 deletions

diff --git a/common/rfb/ConnParams.cxx b/common/rfb/ConnParams.cxx
index dc9ebf58..9ee1d9c0 100644
--- a/common/rfb/ConnParams.cxx
+++ b/common/rfb/ConnParams.cxx
@@ -31,6 +31,7 @@ ConnParams::ConnParams()
   : majorVersion(0), minorVersion(0),
     width(0), height(0), useCopyRect(false),
     supportsLocalCursor(false), supportsLocalXCursor(false),
+    supportsLocalCursorWithAlpha(false),
     supportsDesktopResize(false), supportsExtendedDesktopSize(false),
     supportsDesktopRename(false), supportsLastRect(false),
     supportsSetDesktopSize(false), supportsFence(false),
@@ -101,6 +102,7 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
 {
   useCopyRect = false;
   supportsLocalCursor = false;
+  supportsLocalCursorWithAlpha = false;
   supportsDesktopResize = false;
   supportsExtendedDesktopSize = false;
   supportsLocalXCursor = false;
@@ -124,6 +126,9 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
     case pseudoEncodingXCursor:
       supportsLocalXCursor = true;
       break;
+    case pseudoEncodingCursorWithAlpha:
+      supportsLocalCursorWithAlpha = true;
+      break;
     case pseudoEncodingDesktopSize:
       supportsDesktopResize = true;
       break;
diff --git a/common/rfb/ConnParams.h b/common/rfb/ConnParams.h
index 517e6490..5e538933 100644
--- a/common/rfb/ConnParams.h
+++ b/common/rfb/ConnParams.h
@@ -88,6 +88,7 @@ namespace rfb {
 
     bool supportsLocalCursor;
     bool supportsLocalXCursor;
+    bool supportsLocalCursorWithAlpha;
     bool supportsDesktopResize;
     bool supportsExtendedDesktopSize;
     bool supportsDesktopRename;
diff --git a/common/rfb/SMsgWriter.cxx b/common/rfb/SMsgWriter.cxx
index 51e1105a..cf3264e8 100644
--- a/common/rfb/SMsgWriter.cxx
+++ b/common/rfb/SMsgWriter.cxx
@@ -36,7 +36,8 @@ SMsgWriter::SMsgWriter(ConnParams* cp_, rdr::OutStream* os_)
   : cp(cp_), os(os_),
     nRectsInUpdate(0), nRectsInHeader(0),
     needSetDesktopSize(false), needExtendedDesktopSize(false),
-    needSetDesktopName(false), needSetCursor(false), needSetXCursor(false)
+    needSetDesktopName(false), needSetCursor(false),
+    needSetXCursor(false), needSetCursorWithAlpha(false)
 {
 }
 
@@ -180,11 +181,21 @@ bool SMsgWriter::writeSetXCursor()
   return true;
 }
 
+bool SMsgWriter::writeSetCursorWithAlpha()
+{
+  if (!cp->supportsLocalCursorWithAlpha)
+    return false;
+
+  needSetCursorWithAlpha = true;
+
+  return true;
+}
+
 bool SMsgWriter::needFakeUpdate()
 {
   if (needSetDesktopName)
     return true;
-  if (needSetCursor || needSetXCursor)
+  if (needSetCursor || needSetXCursor || needSetCursorWithAlpha)
     return true;
   if (needNoDataUpdate())
     return true;
@@ -232,6 +243,8 @@ void SMsgWriter::writeFramebufferUpdateStart(int nRects)
       nRects++;
     if (needSetXCursor)
       nRects++;
+    if (needSetCursorWithAlpha)
+      nRects++;
   }
 
   os->writeU16(nRects);
@@ -334,6 +347,15 @@ void SMsgWriter::writePseudoRects()
     needSetXCursor = false;
   }
 
+  if (needSetCursorWithAlpha) {
+    const Cursor& cursor = cp->cursor();
+
+    writeSetCursorWithAlphaRect(cursor.width(), cursor.height(),
+                                cursor.hotspot().x, cursor.hotspot().y,
+                                cursor.getBuffer());
+    needSetCursorWithAlpha = false;
+  }
+
   if (needSetDesktopName) {
     writeSetDesktopNameRect(cp->name());
     needSetDesktopName = false;
@@ -473,3 +495,31 @@ void SMsgWriter::writeSetXCursorRect(int width, int height,
     os->writeBytes(mask, (width+7)/8 * height);
   }
 }
+
+void SMsgWriter::writeSetCursorWithAlphaRect(int width, int height,
+                                             int hotspotX, int hotspotY,
+                                             const rdr::U8* data)
+{
+  if (!cp->supportsLocalCursorWithAlpha)
+    throw Exception("Client does not support local cursors");
+  if (++nRectsInUpdate > nRectsInHeader && nRectsInHeader)
+    throw Exception("SMsgWriter::writeSetCursorWithAlphaRect: nRects out of sync");
+
+  os->writeS16(hotspotX);
+  os->writeS16(hotspotY);
+  os->writeU16(width);
+  os->writeU16(height);
+  os->writeU32(pseudoEncodingCursorWithAlpha);
+
+  // FIXME: Use an encoder with compression?
+  os->writeU32(encodingRaw);
+
+  // Alpha needs to be pre-multiplied
+  for (int i = 0;i < width*height;i++) {
+    os->writeU8((unsigned)data[0] * data[3] / 255);
+    os->writeU8((unsigned)data[1] * data[3] / 255);
+    os->writeU8((unsigned)data[2] * data[3] / 255);
+    os->writeU8(data[3]);
+    data += 4;
+  }
+}
diff --git a/common/rfb/SMsgWriter.h b/common/rfb/SMsgWriter.h
index a516e103..548b8e8e 100644
--- a/common/rfb/SMsgWriter.h
+++ b/common/rfb/SMsgWriter.h
@@ -80,6 +80,7 @@ namespace rfb {
     // immediately. 
     bool writeSetCursor();
     bool writeSetXCursor();
+    bool writeSetCursorWithAlpha();
 
     // needFakeUpdate() returns true when an immediate update is needed in
     // order to flush out pseudo-rectangles to the client.
@@ -127,6 +128,9 @@ namespace rfb {
     void writeSetXCursorRect(int width, int height,
                              int hotspotX, int hotspotY,
                              const void* data, const void* mask);
+    void writeSetCursorWithAlphaRect(int width, int height,
+                                     int hotspotX, int hotspotY,
+                                     const rdr::U8* data);
 
     ConnParams* cp;
     rdr::OutStream* os;
@@ -140,6 +144,7 @@ namespace rfb {
     bool needLastRect;
     bool needSetCursor;
     bool needSetXCursor;
+    bool needSetCursorWithAlpha;
 
     typedef struct {
       rdr::U16 reason, result;
diff --git a/common/rfb/VNCSConnectionST.cxx b/common/rfb/VNCSConnectionST.cxx
index e7a5dcf1..5ab1b4b2 100644
--- a/common/rfb/VNCSConnectionST.cxx
+++ b/common/rfb/VNCSConnectionST.cxx
@@ -1121,10 +1121,12 @@ void VNCSConnectionST::setCursor()
 
   cp.setCursor(*server->cursor);
 
-  if (!writer()->writeSetCursor()) {
-    if (!writer()->writeSetXCursor()) {
-      // No client support
-      return;
+  if (!writer()->writeSetCursorWithAlpha()) {
+    if (!writer()->writeSetCursor()) {
+      if (!writer()->writeSetXCursor()) {
+        // No client support
+        return;
+      }
     }
   }