From 2289342de7222e4ff2483eae95bb9898830e6097 Mon Sep 17 00:00:00 2001
From: Hugo Lundin <hugo@lundin.dev>
Date: Tue, 13 Jul 2021 13:07:41 +0200
Subject: [PATCH] Capture all displays inside the viewport (macOS)

It was only possible to capture either the current, or all displays
previously. This could become an issue if you want to show a vnc session
over only a selected set of displays, because then we only want to
capture those displays.

The solution in the commit was chosen because it handles all cases - by
looking at what monitors are enclosed by the viewport the implementation
is independent on any configuration, but instead captures the monitors
actually being used.
---
 vncviewer/DesktopWindow.cxx |  4 ++--
 vncviewer/cocoa.h           |  6 ++++--
 vncviewer/cocoa.mm          | 37 +++++++++++++++++++++----------------
 3 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/vncviewer/DesktopWindow.cxx b/vncviewer/DesktopWindow.cxx
index dddd74d7..1278e16c 100644
--- a/vncviewer/DesktopWindow.cxx
+++ b/vncviewer/DesktopWindow.cxx
@@ -932,7 +932,7 @@ void DesktopWindow::grabKeyboard()
 #elif defined(__APPLE__)
   int ret;
   
-  ret = cocoa_capture_display(this, fullScreenAllMonitors);
+  ret = cocoa_capture_displays(this);
   if (ret != 0) {
     vlog.error(_("Failure grabbing keyboard"));
     return;
@@ -988,7 +988,7 @@ void DesktopWindow::ungrabKeyboard()
 #if defined(WIN32)
   win32_disable_lowlevel_keyboard(fl_xid(this));
 #elif defined(__APPLE__)
-  cocoa_release_display(this);
+  cocoa_release_displays(this);
 #else
   // FLTK has a grab so lets not mess with it
   if (Fl::grab())
diff --git a/vncviewer/cocoa.h b/vncviewer/cocoa.h
index 98e5fafd..08340038 100644
--- a/vncviewer/cocoa.h
+++ b/vncviewer/cocoa.h
@@ -19,8 +19,10 @@
 #ifndef __VNCVIEWER_COCOA_H__
 #define __VNCVIEWER_COCOA_H__
 
-int cocoa_capture_display(Fl_Window *win, bool all_displays);
-void cocoa_release_display(Fl_Window *win);
+class Fl_Window;
+
+int cocoa_capture_displays(Fl_Window *win);
+void cocoa_release_displays(Fl_Window *win);
 
 typedef struct CGColorSpace *CGColorSpaceRef;
 
diff --git a/vncviewer/cocoa.mm b/vncviewer/cocoa.mm
index c3fe98e5..d6596d34 100644
--- a/vncviewer/cocoa.mm
+++ b/vncviewer/cocoa.mm
@@ -35,6 +35,7 @@
 #define XK_XKB_KEYS
 #include <rfb/keysymdef.h>
 #include <rfb/XF86keysym.h>
+#include <rfb/Rect.h>
 
 #include "keysym2ucs.h"
 
@@ -49,31 +50,35 @@ const int kVK_Menu = 0x6E;
 
 static bool captured = false;
 
-int cocoa_capture_display(Fl_Window *win, bool all_displays)
+int cocoa_capture_displays(Fl_Window *win)
 {
   NSWindow *nsw;
 
   nsw = (NSWindow*)fl_xid(win);
 
   if (!captured) {
-    if (all_displays) {
-      if (CGCaptureAllDisplays() != kCGErrorSuccess)
-        return 1;
-    } else {
-      CGDirectDisplayID displays[16];
-      CGDisplayCount count;
-      int index;
+    CGDisplayCount count;
+    CGDirectDisplayID displays[16];
 
-      if (CGGetActiveDisplayList(16, displays, &count) != kCGErrorSuccess)
-        return 1;
+    int sx, sy, sw, sh;
+    rfb::Rect windows_rect, screen_rect;
 
-      if (count != (unsigned)Fl::screen_count())
-        return 1;
+    windows_rect.setXYWH(win->x(), win->y(), win->w(), win->h());
 
-      index = Fl::screen_num(win->x(), win->y(), win->w(), win->h());
+    if (CGGetActiveDisplayList(16, displays, &count) != kCGErrorSuccess)
+      return 1;
+
+    if (count != (unsigned)Fl::screen_count())
+      return 1;
+
+    for (int i = 0; i < Fl::screen_count(); i++) {
+      Fl::screen_xywh(sx, sy, sw, sh, i);
 
-      if (CGDisplayCapture(displays[index]) != kCGErrorSuccess)
-        return 1;
+      screen_rect.setXYWH(sx, sy, sw, sh);
+      if (screen_rect.enclosed_by(windows_rect)) {
+        if (CGDisplayCapture(displays[i]) != kCGErrorSuccess)
+          return 1;
+      }
     }
 
     captured = true;
@@ -87,7 +92,7 @@ int cocoa_capture_display(Fl_Window *win, bool all_displays)
   return 0;
 }
 
-void cocoa_release_display(Fl_Window *win)
+void cocoa_release_displays(Fl_Window *win)
 {
   NSWindow *nsw;
   int newlevel;
-- 
2.39.5