Add alpha blending support to surfaces

7 files changed, 137 insertions, 21 deletions

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 5fbaa11a..8ea99255 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -33,6 +33,9 @@ else()
 endif()
 add_executable(fbperf ${FBPERF_SOURCES})
 target_link_libraries(fbperf test_util rfb ${FLTK_LIBRARIES} ${GETTEXT_LIBRARIES})
+if(WIN32)
+  target_link_libraries(fbperf msimg32)
+endif()
 if(APPLE)
   target_link_libraries(fbperf "-framework Cocoa" "-framework Carbon")
 endif()
diff --git a/tests/fbperf.cxx b/tests/fbperf.cxx
index 7f1e2c1d..a19ee479 100644
--- a/tests/fbperf.cxx
+++ b/tests/fbperf.cxx
@@ -214,8 +214,9 @@ void OverlayTestWindow::start(int width, int height)
   overlay = new Surface(400, 200);
   overlay->clear(0xff, 0x80, 0x00, 0xcc);
 
-  // X11 needs an off screen buffer for compositing to avoid flicker
-#if !defined(WIN32) && !defined(__APPLE__)
+  // X11 needs an off screen buffer for compositing to avoid flicker,
+  // and alpha blending doesn't work for windows on Win32
+#if !defined(__APPLE__)
   offscreen = new Surface(w(), h());
 #else
   offscreen = NULL;
@@ -270,9 +271,9 @@ void OverlayTestWindow::draw()
   fl_clip_box(ox, oy, ow, oh, X, Y, W, H);
   if ((W != 0) && (H != 0)) {
     if (offscreen)
-      overlay->draw(offscreen, X - ox, Y - oy, X, Y, W, H);
+      overlay->blend(offscreen, X - ox, Y - oy, X, Y, W, H);
     else
-      overlay->draw(X - ox, Y - oy, X, Y, W, H);
+      overlay->blend(X - ox, Y - oy, X, Y, W, H);
   }
 
   fl_pop_clip();
diff --git a/vncviewer/CMakeLists.txt b/vncviewer/CMakeLists.txt
index d6a1b585..1833e704 100644
--- a/vncviewer/CMakeLists.txt
+++ b/vncviewer/CMakeLists.txt
@@ -48,6 +48,10 @@ endif()
 
 target_link_libraries(vncviewer rfb network rdr os ${FLTK_LIBRARIES} ${GETTEXT_LIBRARIES})
 
+if(WIN32)
+  target_link_libraries(vncviewer msimg32)
+endif()
+
 if(APPLE)
   target_link_libraries(vncviewer "-framework Cocoa" "-framework Carbon")
 endif()
diff --git a/vncviewer/Surface.h b/vncviewer/Surface.h
index 9b1788a5..7d164681 100644
--- a/vncviewer/Surface.h
+++ b/vncviewer/Surface.h
@@ -44,6 +44,9 @@ public:
   void draw(int src_x, int src_y, int x, int y, int w, int h);
   void draw(Surface* dst, int src_x, int src_y, int x, int y, int w, int h);
 
+  void blend(int src_x, int src_y, int x, int y, int w, int h);
+  void blend(Surface* dst, int src_x, int src_y, int x, int y, int w, int h);
+
 protected:
   void alloc();
   void dealloc();
diff --git a/vncviewer/Surface_OSX.cxx b/vncviewer/Surface_OSX.cxx
index dbf0c418..73c10f46 100644
--- a/vncviewer/Surface_OSX.cxx
+++ b/vncviewer/Surface_OSX.cxx
@@ -28,7 +28,7 @@
 
 #include "Surface.h"
 
-static void render(CGContextRef gc, CGImageRef image,
+static void render(CGContextRef gc, CGImageRef image, CGBlendMode mode,
                    int src_x, int src_y, int src_w, int src_h,
                    int x, int y, int w, int h)
 {
@@ -36,6 +36,8 @@ static void render(CGContextRef gc, CGImageRef image,
 
   CGContextSaveGState(gc);
 
+  CGContextSetBlendMode(gc, mode);
+
   // We have to use clipping to partially display an image
   rect.origin.x = x;
   rect.origin.y = y;
@@ -54,6 +56,27 @@ static void render(CGContextRef gc, CGImageRef image,
   CGContextRestoreGState(gc);
 }
 
+static CGContextRef make_bitmap(int width, int height, unsigned char* data)
+{
+  CGColorSpaceRef lut;
+  CGContextRef bitmap;
+
+  lut = CGDisplayCopyColorSpace(kCGDirectMainDisplay);
+  if (!lut) {
+    lut = CGColorSpaceCreateDeviceRGB();
+    if (!lut)
+      throw rdr::Exception("CGColorSpaceCreateDeviceRGB");
+  }
+
+  bitmap = CGBitmapContextCreate(data, width, height, 8, width*4, lut,
+                                 kCGImageAlphaPremultipliedFirst | kCGBitmapByteOrder32Little);
+  CGColorSpaceRelease(lut);
+  if (!bitmap)
+    throw rdr::Exception("CGBitmapContextCreate");
+
+  return bitmap;
+}
+
 void Surface::clear(unsigned char r, unsigned char g, unsigned char b, unsigned char a)
 {
   unsigned char* out;
@@ -86,35 +109,58 @@ void Surface::draw(int src_x, int src_y, int x, int y, int w, int h)
   src_y = height() - (src_y + h);
   y = Fl_Window::current()->h() - (y + h);
 
-  render(fl_gc, image, src_x, src_y, width(), height(), x, y, w, h);
+  render(fl_gc, image, kCGBlendModeCopy,
+         src_x, src_y, width(), height(), x, y, w, h);
 
   CGContextRestoreGState(fl_gc);
 }
 
 void Surface::draw(Surface* dst, int src_x, int src_y, int x, int y, int w, int h)
 {
-  CGColorSpaceRef lut;
   CGContextRef bitmap;
 
-  lut = CGDisplayCopyColorSpace(kCGDirectMainDisplay);
-  if (!lut) {
-    lut = CGColorSpaceCreateDeviceRGB();
-    if (!lut)
-      throw rdr::Exception("CGColorSpaceCreateDeviceRGB");
-  }
+  bitmap = make_bitmap(dst->width(), dst->height(), dst->data);
 
-  bitmap = CGBitmapContextCreate(dst->data, dst->width(),
-                                 dst->height(), 8, dst->width()*4, lut,
-                                 kCGImageAlphaNoneSkipFirst | kCGBitmapByteOrder32Little);
-  CGColorSpaceRelease(lut);
-  if (!bitmap)
-    throw rdr::Exception("CGBitmapContextCreate");
+  // macOS Coordinates are from bottom left, not top left
+  src_y = height() - (src_y + h);
+  y = dst->height() - (y + h);
+
+  render(bitmap, image, kCGBlendModeCopy,
+         src_x, src_y, width(), height(), x, y, w, h);
+
+  CGContextRelease(bitmap);
+}
+
+void Surface::blend(int src_x, int src_y, int x, int y, int w, int h)
+{
+  CGContextSaveGState(fl_gc);
+
+  // Reset the transformation matrix back to the default identity
+  // matrix as otherwise we get a massive performance hit
+  CGContextConcatCTM(fl_gc, CGAffineTransformInvert(CGContextGetCTM(fl_gc)));
+
+  // macOS Coordinates are from bottom left, not top left
+  src_y = height() - (src_y + h);
+  y = Fl_Window::current()->h() - (y + h);
+
+  render(fl_gc, image, kCGBlendModeNormal,
+         src_x, src_y, width(), height(), x, y, w, h);
+
+  CGContextRestoreGState(fl_gc);
+}
+
+void Surface::blend(Surface* dst, int src_x, int src_y, int x, int y, int w, int h)
+{
+  CGContextRef bitmap;
+
+  bitmap = make_bitmap(dst->width(), dst->height(), dst->data);
 
   // macOS Coordinates are from bottom left, not top left
   src_y = height() - (src_y + h);
   y = dst->height() - (y + h);
 
-  render(bitmap, image, src_x, src_y, width(), height(), x, y, w, h);
+  render(bitmap, image, kCGBlendModeNormal,
+         src_x, src_y, width(), height(), x, y, w, h);
 
   CGContextRelease(bitmap);
 }
@@ -139,7 +185,7 @@ void Surface::alloc()
     throw rdr::Exception("CGDataProviderCreateWithData");
 
   image = CGImageCreate(width(), height(), 8, 32, width() * 4, lut,
-                        kCGImageAlphaNoneSkipFirst | kCGBitmapByteOrder32Little,
+                        kCGImageAlphaPremultipliedFirst | kCGBitmapByteOrder32Little,
                         provider, NULL, false, kCGRenderingIntentDefault);
   CGColorSpaceRelease(lut);
   CGDataProviderRelease(provider);
diff --git a/vncviewer/Surface_Win32.cxx b/vncviewer/Surface_Win32.cxx
index 5eea2d1d..87b0cb2f 100644
--- a/vncviewer/Surface_Win32.cxx
+++ b/vncviewer/Surface_Win32.cxx
@@ -89,6 +89,48 @@ void Surface::draw(Surface* dst, int src_x, int src_y, int x, int y, int w, int
   DeleteDC(dstdc);
 }
 
+void Surface::blend(int src_x, int src_y, int x, int y, int w, int h)
+{
+  // Compositing doesn't work properly for window DC:s
+  assert(false);
+}
+
+void Surface::blend(Surface* dst, int src_x, int src_y, int x, int y, int w, int h)
+{
+  HDC dstdc, srcdc;
+  BLENDFUNCTION blend;
+
+  dstdc = CreateCompatibleDC(NULL);
+  if (!dstdc)
+    throw rdr::SystemException("CreateCompatibleDC", GetLastError());
+  srcdc = CreateCompatibleDC(NULL);
+  if (!srcdc)
+    throw rdr::SystemException("CreateCompatibleDC", GetLastError());
+
+  if (!SelectObject(dstdc, dst->bitmap))
+    throw rdr::SystemException("SelectObject", GetLastError());
+  if (!SelectObject(srcdc, bitmap))
+    throw rdr::SystemException("SelectObject", GetLastError());
+
+  blend.BlendOp = AC_SRC_OVER;
+  blend.BlendFlags = 0;
+  blend.SourceConstantAlpha = 255;
+  blend.AlphaFormat = AC_SRC_ALPHA;
+
+  if (!AlphaBlend(dstdc, x, y, w, h, srcdc, src_x, src_y, w, h, blend)) {
+    // If the desktop we're rendering to is inactive (like when the screen
+    // is locked or the UAC is active), then GDI calls will randomly fail.
+    // This is completely undocumented so we have no idea how best to deal
+    // with it. For now, we've only seen this error and for this function
+    // so only ignore this combination.
+    if (GetLastError() != ERROR_INVALID_HANDLE)
+      throw rdr::SystemException("BitBlt", GetLastError());
+  }
+
+  DeleteDC(srcdc);
+  DeleteDC(dstdc);
+}
+
 void Surface::alloc()
 {
   BITMAPINFOHEADER bih;
diff --git a/vncviewer/Surface_X11.cxx b/vncviewer/Surface_X11.cxx
index e7616143..f36976fa 100644
--- a/vncviewer/Surface_X11.cxx
+++ b/vncviewer/Surface_X11.cxx
@@ -54,6 +54,23 @@ void Surface::draw(Surface* dst, int src_x, int src_y, int x, int y, int w, int
                    src_x, src_y, 0, 0, x, y, w, h);
 }
 
+void Surface::blend(int src_x, int src_y, int x, int y, int w, int h)
+{
+  Picture winPict;
+
+  winPict = XRenderCreatePicture(fl_display, fl_window, visFormat, 0, NULL);
+  XRenderComposite(fl_display, PictOpOver, picture, None, winPict,
+                   src_x, src_y, 0, 0, x, y, w, h);
+  XRenderFreePicture(fl_display, winPict);
+}
+
+void Surface::blend(Surface* dst, int src_x, int src_y, int x, int y, int w, int h)
+{
+  XRenderComposite(fl_display, PictOpOver, picture, None, dst->picture,
+                   src_x, src_y, 0, 0, x, y, w, h);
+}
+
+
 void Surface::alloc()
 {
   XRenderPictFormat* format;