Java client support for cursors with full alpha

9 files changed, 126 insertions, 79 deletions

diff --git a/java/com/tigervnc/rfb/CConnection.java b/java/com/tigervnc/rfb/CConnection.java
index 0b38aeaf..aefc2760 100644
--- a/java/com/tigervnc/rfb/CConnection.java
+++ b/java/com/tigervnc/rfb/CConnection.java
@@ -336,6 +336,13 @@ abstract public class CConnection extends CMsgHandler {
     super.setExtendedDesktopSize(reason, result, w, h, layout);
   }
 
+  public void readAndDecodeRect(Rect r, int encoding,
+                                ModifiablePixelBuffer pb)
+  {
+    decoder.decodeRect(r, encoding, pb);
+    decoder.flush();
+  }
+
   // getIdVerifier() returns the identity verifier associated with the connection.
   // Ownership of the IdentityVerifier is retained by the CConnection instance.
   //public IdentityVerifier getIdentityVerifier() { return 0; }
diff --git a/java/com/tigervnc/rfb/CMsgHandler.java b/java/com/tigervnc/rfb/CMsgHandler.java
index 99405983..2f3151b7 100644
--- a/java/com/tigervnc/rfb/CMsgHandler.java
+++ b/java/com/tigervnc/rfb/CMsgHandler.java
@@ -74,21 +74,24 @@ abstract public class CMsgHandler {
     cp.supportsContinuousUpdates = true;
   }
 
-  public void clientRedirect(int port, String host,
-                             String x509subject) {}
+  abstract public void clientRedirect(int port, String host,
+                                      String x509subject);
 
-  public void setCursor(int width, int height, Point hotspot,
-                        byte[] data, byte[] mask) {}
-  public void serverInit() {}
+  abstract public void setCursor(int width, int height, Point hotspot,
+                                 byte[] data);
+  abstract public void serverInit();
 
-  public void framebufferUpdateStart() {}
-  public void framebufferUpdateEnd() {}
-  public void dataRect(Rect r, int encoding) {}
+  abstract public void readAndDecodeRect(Rect r, int encoding,
+                                         ModifiablePixelBuffer pb);
 
-  public void setColourMapEntries(int firstColour, int nColours,
-    int[] rgbs) { }
-  public void bell() {}
-  public void serverCutText(String str, int len) {}
+  public void framebufferUpdateStart() {};
+  public void framebufferUpdateEnd() {};
+  abstract public void dataRect(Rect r, int encoding);
+
+  abstract public void setColourMapEntries(int firstColour, int nColours,
+    int[] rgbs);
+  abstract public void bell();
+  abstract public void serverCutText(String str, int len);
 
   public ConnParams cp;
 
diff --git a/java/com/tigervnc/rfb/CMsgReader.java b/java/com/tigervnc/rfb/CMsgReader.java
index e9cad89d..12862c14 100644
--- a/java/com/tigervnc/rfb/CMsgReader.java
+++ b/java/com/tigervnc/rfb/CMsgReader.java
@@ -1,4 +1,6 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright (C) 2011-2017 Brian P. Hinz
+ * Copyright (C) 2017 Pierre Ossman for Cendio AB
  *
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -23,9 +25,11 @@
 
 package com.tigervnc.rfb;
 
+import java.awt.image.*;
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
 import java.nio.charset.Charset;
+
 import com.tigervnc.rdr.*;
 
 public class CMsgReader {
@@ -78,7 +82,6 @@ public class CMsgReader {
         readEndOfContinuousUpdates();
         break;
       default:
-        //fprintf(stderr, "unknown message type %d\n", type);
         throw new Exception("unknown message type");
       }
     } else {
@@ -95,6 +98,9 @@ public class CMsgReader {
       case Encodings.pseudoEncodingCursor:
         readSetCursor(w, h, new Point(x,y));
         break;
+      case Encodings.pseudoEncodingCursorWithAlpha:
+        readSetCursorWithAlpha(w, h, new Point(x,y));
+        break;
       case Encodings.pseudoEncodingDesktopName:
         readSetDesktopName(x, y, w, h);
         break;
@@ -185,20 +191,6 @@ public class CMsgReader {
     handler.framebufferUpdateStart();
   }
 
-
-
-  /*
-  protected void readFramebufferUpdateStart()
-  {
-    handler.framebufferUpdateStart();
-  }
-
-  protected void readFramebufferUpdateEnd()
-  {
-    handler.framebufferUpdateEnd();
-  }
-  */
-
   protected void readRect(Rect r, int encoding)
   {
     if ((r.br.x > handler.cp.width) || (r.br.y > handler.cp.height)) {
@@ -218,13 +210,82 @@ public class CMsgReader {
   {
     int data_len = width * height * (handler.cp.pf().bpp/8);
     int mask_len = ((width+7)/8) * height;
-    byte[] data = new byte[data_len];
-    byte[] mask = new byte[mask_len];
+    ByteBuffer data = ByteBuffer.allocate(data_len);
+    ByteBuffer mask = ByteBuffer.allocate(mask_len);
+
+    int x, y;
+    byte[] buf = new byte[width*height*4];
+    ByteBuffer in;
+    ByteBuffer out;
+
+    is.readBytes(data, data_len);
+    is.readBytes(mask, mask_len);
+
+    int maskBytesPerRow = (width+7)/8;
+    in = (ByteBuffer)data.duplicate().mark();
+    out = (ByteBuffer)ByteBuffer.wrap(buf).mark();
+    for (y = 0;y < height;y++) {
+      for (x = 0;x < width;x++) {
+        int byte_ = y * maskBytesPerRow + x / 8;
+        int bit = 7 - x % 8;
+
+        // NOTE: BufferedImage needs ARGB, rather than RGBA
+        if ((mask.get(byte_) & (1 << bit)) != 0)
+          out.put((byte)255);
+        else
+          out.put((byte)0);
+
+        handler.cp.pf().rgbFromBuffer(out, in.duplicate(), 1);
+
+        in.position(in.position() + handler.cp.pf().bpp/8);
+        out.position(out.reset().position() + 4).mark();
+      }
+    }
 
-    is.readBytes(data, 0, data_len);
-    is.readBytes(mask, 0, mask_len);
+    handler.setCursor(width, height, hotspot, buf);
+  }
+
+  protected void readSetCursorWithAlpha(int width, int height, Point hotspot)
+  {
+    int encoding;
+
+    PixelFormat rgbaPF =
+      new PixelFormat(32, 32, false, true, 255, 255, 255, 16, 8, 0);
+    ManagedPixelBuffer pb =
+      new ManagedPixelBuffer(rgbaPF, width, height);
+    PixelFormat origPF;
+
+    DataBufferInt buf;
+
+    encoding = is.readS32();
+
+    origPF = handler.cp.pf();
+    handler.cp.setPF(rgbaPF);
+    handler.readAndDecodeRect(pb.getRect(), encoding, pb);
+    handler.cp.setPF(origPF);
+
+    if (pb.getRect().area() == 0)
+      return;
+
+    // ARGB with pre-multiplied alpha works best for BufferedImage
+    buf = (DataBufferInt)pb.getBufferRW(pb.getRect()).getDataBuffer();
+    ByteBuffer bbuf =
+      ByteBuffer.allocate(pb.area()*4).order(rgbaPF.getByteOrder());
+    bbuf.asIntBuffer().put(buf.getData()).flip().mark();
+
+    for (int i = 0;i < pb.area();i++) {
+      byte alpha = bbuf.get(bbuf.position()+3);
+
+      bbuf.put(i*4+3, (byte)(bbuf.get(i*4+2)));
+      bbuf.put(i*4+2, (byte)(bbuf.get(i*4+1)));
+      bbuf.put(i*4+1, (byte)(bbuf.get(i*4+0))); 
+      bbuf.put(i*4+0, (byte)alpha);
+
+      bbuf.position(bbuf.position() + 4);
+    }
 
-    handler.setCursor(width, height, hotspot, data, mask);
+    handler.setCursor(width, height, hotspot,
+                      bbuf.array());
   }
 
   protected void readSetDesktopName(int x, int y, int w, int h)
diff --git a/java/com/tigervnc/rfb/CMsgWriter.java b/java/com/tigervnc/rfb/CMsgWriter.java
index bdf50c28..701f2d3d 100644
--- a/java/com/tigervnc/rfb/CMsgWriter.java
+++ b/java/com/tigervnc/rfb/CMsgWriter.java
@@ -64,8 +64,10 @@ public class CMsgWriter {
     int nEncodings = 0;
     int[] encodings = new int[Encodings.encodingMax+3];
 
-    if (cp.supportsLocalCursor)
+    if (cp.supportsLocalCursor) {
       encodings[nEncodings++] = Encodings.pseudoEncodingCursor;
+      encodings[nEncodings++] = Encodings.pseudoEncodingCursorWithAlpha;
+    }
     if (cp.supportsDesktopResize)
       encodings[nEncodings++] = Encodings.pseudoEncodingDesktopSize;
     if (cp.supportsExtendedDesktopSize)
diff --git a/java/com/tigervnc/rfb/Encodings.java b/java/com/tigervnc/rfb/Encodings.java
index ec2331e1..8a499e03 100644
--- a/java/com/tigervnc/rfb/Encodings.java
+++ b/java/com/tigervnc/rfb/Encodings.java
@@ -40,6 +40,7 @@ public class Encodings {
   public static final int pseudoEncodingClientRedirect = -311;
   public static final int pseudoEncodingFence = -312;
   public static final int pseudoEncodingContinuousUpdates = -313;
+  public static final int pseudoEncodingCursorWithAlpha = -314;
 
   // TightVNC-specific
   public static final int pseudoEncodingLastRect = -224;
diff --git a/java/com/tigervnc/rfb/ModifiablePixelBuffer.java b/java/com/tigervnc/rfb/ModifiablePixelBuffer.java
index bcc559d5..1a1dcc65 100644
--- a/java/com/tigervnc/rfb/ModifiablePixelBuffer.java
+++ b/java/com/tigervnc/rfb/ModifiablePixelBuffer.java
@@ -90,7 +90,9 @@ public abstract class ModifiablePixelBuffer extends PixelBuffer
   {
     WritableRaster dest = getBufferRW(r);
 
-    ByteBuffer src = ByteBuffer.wrap(pixels).order(format.getByteOrder());
+    int length = r.area()*format.bpp/8;
+    ByteBuffer src =
+      ByteBuffer.wrap(pixels, 0, length).order(format.getByteOrder());
     Raster raster = format.rasterFromBuffer(r, src);
     dest.setDataElements(0, 0, raster);
 
@@ -239,7 +241,8 @@ public abstract class ModifiablePixelBuffer extends PixelBuffer
         cm.isCompatibleSampleModel(dstBuffer.getSampleModel())) {
       imageRect(dest, pixels);
     } else {
-      ByteBuffer src = ByteBuffer.wrap(pixels).order(pf.getByteOrder());
+      int length = dest.area()*pf.bpp/8;
+      ByteBuffer src = ByteBuffer.wrap(pixels, 0, length).order(pf.getByteOrder());
       Raster raster = pf.rasterFromBuffer(dest, src);
       ColorConvertOp converter = format.getColorConvertOp(cm.getColorSpace());
       converter.filter(raster, dstBuffer);
diff --git a/java/com/tigervnc/vncviewer/CConn.java b/java/com/tigervnc/vncviewer/CConn.java
index 128867b9..c53f8058 100644
--- a/java/com/tigervnc/vncviewer/CConn.java
+++ b/java/com/tigervnc/vncviewer/CConn.java
@@ -434,9 +434,9 @@ public class CConn extends CConnection implements
   }
 
   public void setCursor(int width, int height, Point hotspot,
-                        byte[] data, byte[] mask)
+                        byte[] data)
   {
-    desktop.setCursor(width, height, hotspot, data, mask);
+    desktop.setCursor(width, height, hotspot, data);
   }
 
   public void fence(int flags, int len, byte[] data)
diff --git a/java/com/tigervnc/vncviewer/DesktopWindow.java b/java/com/tigervnc/vncviewer/DesktopWindow.java
index 187fbad0..4169c769 100644
--- a/java/com/tigervnc/vncviewer/DesktopWindow.java
+++ b/java/com/tigervnc/vncviewer/DesktopWindow.java
@@ -282,9 +282,9 @@ public class DesktopWindow extends JFrame
   }
 
   public void setCursor(int width, int height, Point hotspot,
-                        byte[] data, byte[] mask)
+                        byte[] data)
   {
-    viewport.setCursor(width, height, hotspot, data, mask);
+    viewport.setCursor(width, height, hotspot, data);
   }
 
   public void fullscreen_on()
diff --git a/java/com/tigervnc/vncviewer/Viewport.java b/java/com/tigervnc/vncviewer/Viewport.java
index bf07d2d5..f7448bc8 100644
--- a/java/com/tigervnc/vncviewer/Viewport.java
+++ b/java/com/tigervnc/vncviewer/Viewport.java
@@ -2,7 +2,7 @@
  * Copyright (C) 2006 Constantin Kaplinsky.  All Rights Reserved.
  * Copyright (C) 2009 Paul Donohue.  All Rights Reserved.
  * Copyright (C) 2010, 2012-2013 D. R. Commander.  All Rights Reserved.
- * Copyright (C) 2011-2014 Brian P. Hinz
+ * Copyright (C) 2011-2017 Brian P. Hinz
  *
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -125,16 +125,14 @@ class Viewport extends JPanel implements MouseListener,
   };
 
   public void setCursor(int width, int height, Point hotspot,
-                        byte[] data, byte[] mask)
+                        byte[] data)
   {
-
-    int mask_len = ((width+7)/8) * height;
     int i;
 
-    for (i = 0; i < mask_len; i++)
-      if ((mask[i] & 0xff) != 0) break;
+    for (i = 0; i < width*height; i++)
+      if (data[i*4 + 3] != 0) break;
 
-    if ((i == mask_len) && dotWhenNoCursor.getValue()) {
+    if ((i == width*height) && dotWhenNoCursor.getValue()) {
       vlog.debug("cursor is empty - using dot");
       cursor = new BufferedImage(5, 5, BufferedImage.TYPE_INT_ARGB_PRE);
       cursor.setRGB(0, 0, 5, 5, dotcursor_xpm, 0, 5);
@@ -146,39 +144,11 @@ class Viewport extends JPanel implements MouseListener,
                                    BufferedImage.TYPE_INT_ARGB_PRE);
         cursorHotspot.x = cursorHotspot.y = 0;
       } else {
-        ByteBuffer buffer = ByteBuffer.allocate(width*height*4);
-        ByteBuffer in, o, m;
-        int m_width;
-
-        PixelFormat pf;
-
-        pf = cc.cp.pf();
-
-        in = (ByteBuffer)ByteBuffer.wrap(data).mark();
-        o = (ByteBuffer)buffer.duplicate().mark();
-        m = ByteBuffer.wrap(mask);
-        m_width = (width+7)/8;
-
-        for (int y = 0; y < height; y++) {
-          for (int x = 0; x < width; x++) {
-            // NOTE: BufferedImage needs ARGB, rather than RGBA
-            if ((m.get((m_width*y)+(x/8)) & 0x80>>(x%8)) != 0)
-              o.put((byte)255);
-            else
-              o.put((byte)0);
-
-            pf.rgbFromBuffer(o, in.duplicate(), 1);
-
-            o.position(o.reset().position() + 4).mark();
-            in.position(in.position() + pf.bpp/8);
-          }
-        }
-
-        IntBuffer rgb =
-          IntBuffer.allocate(width*height).put(buffer.asIntBuffer());
-        cursor = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB_PRE);
-        cursor.setRGB(0, 0, width, height, rgb.array(), 0, width);
-
+        IntBuffer buffer = IntBuffer.allocate(width*height);
+        buffer.put(ByteBuffer.wrap(data).asIntBuffer());
+        cursor =
+          new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB_PRE);
+        cursor.setRGB(0, 0, width, height, buffer.array(), 0, width);
         cursorHotspot = hotspot;
 
       }