Client support for cursors with full alpha

common/rfb/CConnection.cxx

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2011-2017 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -320,6 +321,13 @@ void CConnection::setExtendedDesktopSize(unsigned reason,
   CMsgHandler::setExtendedDesktopSize(reason, result, w, h, layout);
 }
 
+void CConnection::readAndDecodeRect(const Rect& r, int encoding,
+                                    ModifiablePixelBuffer* pb)
+{
+  decoder.decodeRect(r, encoding, pb);
+  decoder.flush();
+}
+
 void CConnection::framebufferUpdateStart()
 {
   CMsgHandler::framebufferUpdateStart();

common/rfb/CConnection.h

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2011-2017 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -99,6 +100,9 @@ namespace rfb {
                                         int w, int h,
                                         const ScreenSet& layout);
 
+    virtual void readAndDecodeRect(const Rect& r, int encoding,
+                                   ModifiablePixelBuffer* pb);
+
     virtual void framebufferUpdateStart();
     virtual void framebufferUpdateEnd();
     virtual void dataRect(const Rect& r, int encoding);

common/rfb/CMsgHandler.h

@@ -57,6 +57,9 @@ namespace rfb {
     virtual void endOfContinuousUpdates();
     virtual void serverInit() = 0;
 
+    virtual void readAndDecodeRect(const Rect& r, int encoding,
+                                   ModifiablePixelBuffer* pb) = 0;
+
     virtual void framebufferUpdateStart();
     virtual void framebufferUpdateEnd();
     virtual void dataRect(const Rect& r, int encoding) = 0;

common/rfb/CMsgReader.cxx

@@ -16,7 +16,10 @@
  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
  * USA.
  */
+
+#include <assert.h>
 #include <stdio.h>
+
 #include <rfb/msgTypes.h>
 #include <rdr/InStream.h>
 #include <rfb/Exception.h>
@@ -94,6 +97,9 @@ void CMsgReader::readMsg()
     case pseudoEncodingCursor:
       readSetCursor(w, h, Point(x,y));
       break;
+    case pseudoEncodingCursorWithAlpha:
+      readSetCursorWithAlpha(w, h, Point(x,y));
+      break;
     case pseudoEncodingDesktopName:
       readSetDesktopName(x, y, w, h);
       break;
@@ -287,6 +293,50 @@ void CMsgReader::readSetCursor(int width, int height, const Point& hotspot)
   handler->setCursor(width, height, hotspot, buf);
 }
 
+void CMsgReader::readSetCursorWithAlpha(int width, int height, const Point& hotspot)
+{
+  int encoding;
+
+  const PixelFormat rgbaPF(32, 32, false, true, 255, 255, 255, 16, 8, 0);
+  ManagedPixelBuffer pb(rgbaPF, width, height);
+  PixelFormat origPF;
+
+  rdr::U8* buf;
+  int stride;
+
+  encoding = is->readS32();
+
+  origPF = handler->cp.pf();
+  handler->cp.setPF(rgbaPF);
+  handler->readAndDecodeRect(pb.getRect(), encoding, &pb);
+  handler->cp.setPF(origPF);
+
+  // On-wire data has pre-multiplied alpha, but we store it
+  // non-pre-multiplied
+  buf = pb.getBufferRW(pb.getRect(), &stride);
+  assert(stride == width);
+
+  for (int i = 0;i < pb.area();i++) {
+    rdr::U8 alpha;
+
+    alpha = buf[3];
+    if (alpha == 0)
+      alpha = 1; // Avoid division by zero
+
+    buf[0] = (unsigned)buf[0] * 255/alpha;
+    buf[1] = (unsigned)buf[1] * 255/alpha;
+    buf[2] = (unsigned)buf[2] * 255/alpha;
+    buf[3] = alpha;
+
+    buf += 4;
+  }
+
+  pb.commitBufferRW(pb.getRect());
+
+  handler->setCursor(width, height, hotspot,
+                     pb.getBuffer(pb.getRect(), &stride));
+}
+
 void CMsgReader::readSetDesktopName(int x, int y, int w, int h)
 {
   char* name = is->readString();

common/rfb/CMsgReader.h

@@ -62,6 +62,7 @@ namespace rfb {
 
     void readSetXCursor(int width, int height, const Point& hotspot);
     void readSetCursor(int width, int height, const Point& hotspot);
+    void readSetCursorWithAlpha(int width, int height, const Point& hotspot);
     void readSetDesktopName(int x, int y, int w, int h);
     void readExtendedDesktopSize(int x, int y, int w, int h);
 

common/rfb/CMsgWriter.cxx

@@ -74,6 +74,7 @@ void CMsgWriter::writeSetEncodings(int preferredEncoding, bool useCopyRect)
   if (cp->supportsLocalCursor) {
     encodings[nEncodings++] = pseudoEncodingXCursor;
     encodings[nEncodings++] = pseudoEncodingCursor;
+    encodings[nEncodings++] = pseudoEncodingCursorWithAlpha;
   }
   if (cp->supportsDesktopResize)
     encodings[nEncodings++] = pseudoEncodingDesktopSize;

common/rfb/encodings.h

@@ -38,6 +38,7 @@ namespace rfb {
   const int pseudoEncodingDesktopName = -307;
   const int pseudoEncodingFence = -312;
   const int pseudoEncodingContinuousUpdates = -313;
+  const int pseudoEncodingCursorWithAlpha = -314;
 
   // TightVNC-specific
   const int pseudoEncodingLastRect = -224;