From 94ab2db366b2607391358d472dfb6f8f5eab9644 Mon Sep 17 00:00:00 2001
From: =?utf8?q?Peter=20=C3=85strand=20=28astrand=29?= <astrand@cendio.se>
Date: Wed, 7 Mar 2018 12:32:30 +0100
Subject: [PATCH] Corrected RandR wrapper return codes

In RandR land, there's a lot of return code confusion. Our wrappers
are using the same return codes as RRCrtcSet, RRScreenSizeSet: 1/TRUE
for success. Fixes:

* vncRandRCreateOutputs did not follow this convention

* A lot of code returned -1 upon failure

* vncRandRDisableOutput returned 0 for already disabled outputs
---
 unix/common/randr.cxx           |  2 +-
 unix/xserver/hw/vnc/RandrGlue.c | 12 ++++++------
 unix/xserver/hw/vnc/vncModule.c |  2 +-
 unix/xserver/hw/vnc/xvnc.c      |  4 ++--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/unix/common/randr.cxx b/unix/common/randr.cxx
index 44e46de7..08bd45e6 100644
--- a/unix/common/randr.cxx
+++ b/unix/common/randr.cxx
@@ -113,7 +113,7 @@ unsigned int setScreenLayout(int screenIndex,
                layout.num_screens() - availableOutputs);
     ret = vncRandRCreateOutputs(screenIndex,
                                 layout.num_screens() - availableOutputs);
-    if (ret < 0) {
+    if (!ret) {
       vlog.error("Unable to create more screens, as needed by the new client layout.");
       return rfb::resultInvalid;
     }
diff --git a/unix/xserver/hw/vnc/RandrGlue.c b/unix/xserver/hw/vnc/RandrGlue.c
index 9f770fa9..81065710 100644
--- a/unix/xserver/hw/vnc/RandrGlue.c
+++ b/unix/xserver/hw/vnc/RandrGlue.c
@@ -50,7 +50,7 @@ int vncRandRResizeScreen(int scrIdx, int width, int height)
                          pScreen->mmWidth * width / pScreen->width,
                          pScreen->mmHeight * height / pScreen->height);
 #else
-  return -1;
+  return 0;
 #endif
 }
 
@@ -199,11 +199,11 @@ int vncRandRDisableOutput(int scrIdx, int outputIdx)
 
   crtc = rp->outputs[outputIdx]->crtc;
   if (crtc == NULL)
-    return 0;
+    return 1;
 
   return RRCrtcSet(crtc, NULL, crtc->x, crtc->y, crtc->rotation, 0, NULL);
 #else
-  return -1;
+  return 0;
 #endif
 }
 
@@ -267,17 +267,17 @@ int vncRandRReconfigureOutput(int scrIdx, int outputIdx, int x, int y,
 
     /* Couldn't find one... */
     if (crtc == NULL)
-      return -1;
+      return 0;
   }
 
   /* Make sure we have the mode we want */
   mode = vncRandRCreatePreferredMode(output, width, height);
   if (mode == NULL)
-    return -1;
+    return 0;
 
   /* Reconfigure new mode and position */
   return RRCrtcSet(crtc, mode, x, y, crtc->rotation, 1, &output);
 #else
-  return -1;
+  return 0;
 #endif
 }
diff --git a/unix/xserver/hw/vnc/vncModule.c b/unix/xserver/hw/vnc/vncModule.c
index 3fb07769..704e56ee 100644
--- a/unix/xserver/hw/vnc/vncModule.c
+++ b/unix/xserver/hw/vnc/vncModule.c
@@ -115,7 +115,7 @@ void vncClientGone(int fd)
 #ifdef RANDR
 int vncRandRCreateOutputs(int scrIdx, int extraOutputs)
 {
-  return -1;
+  return 0;
 }
 
 void *vncRandRCreatePreferredMode(void *out, int width, int height)
diff --git a/unix/xserver/hw/vnc/xvnc.c b/unix/xserver/hw/vnc/xvnc.c
index 56a6ca58..4133689b 100644
--- a/unix/xserver/hw/vnc/xvnc.c
+++ b/unix/xserver/hw/vnc/xvnc.c
@@ -1387,11 +1387,11 @@ int vncRandRCreateOutputs(int scrIdx, int extraOutputs)
     while (extraOutputs > 0) {
         crtc = vncRandRCrtcCreate(screenInfo.screens[scrIdx]);
         if (crtc == NULL)
-            return -1;
+            return 0;
         extraOutputs--;
     }
 
-    return 0;
+    return 1;
 }
 
 /* Used to create a preferred mode from various places */
-- 
2.39.5