Add a tryScreenLayout function

This can be used to test if a layout if possible.

unix/common/RandrGlue.h

@@ -33,6 +33,7 @@ extern "C" {
 int vncGetScreenWidth(void);
 int vncGetScreenHeight(void);
 
+int vncRandRIsValidScreenSize(int width, int height);
 int vncRandRResizeScreen(int width, int height);
 void vncRandRUpdateSetTime(void);
 
@@ -47,6 +48,8 @@ int vncRandRIsOutputEnabled(int outputIdx);
 int vncRandRIsOutputUsable(int outputIdx);
 int vncRandRIsOutputConnected(int outputIdx);
 
+int vncRandRCheckOutputMode(int outputIdx, int width, int height);
+
 int vncRandRDisableOutput(int outputIdx);
 int vncRandRReconfigureOutput(int outputIdx, int x, int y,
                               int width, int height);
@@ -55,8 +58,8 @@ unsigned int vncRandRGetOutputId(int outputIdx);
 int vncRandRGetOutputDimensions(int outputIdx,
                                  int *x, int *y, int *width, int *height);
 
+int vncRandRCanCreateOutputs(int extraOutputs);
 int vncRandRCreateOutputs(int extraOutputs);
-void *vncRandRCreatePreferredMode(void *output, int width, int height);
 
 #ifdef __cplusplus
 }

unix/common/randr.cxx

@@ -31,7 +31,7 @@
 #include <RandrGlue.h>
 static rfb::LogWriter vlog("RandR");
 
-static int ResizeScreen(int fb_width, int fb_height,
+static int ResizeScreen(bool dryrun, int fb_width, int fb_height,
                         std::set<unsigned int>* disabledOutputs)
 {
   vlog.debug("Resizing screen framebuffer to %dx%d", fb_width, fb_height);
@@ -43,17 +43,25 @@ static int ResizeScreen(int fb_width, int fb_height,
     int x, y, width, height;
     if (vncRandRGetOutputDimensions(i, &x, &y, &width, &height) == 0) {
       if (x + width > fb_width || y + height > fb_height) {
-        /* Currently ignoring errors */
-        /* FIXME: Save output rotation and restore when configuring output */
         char *name = vncRandRGetOutputName(i);
         vlog.debug("Temporarily disabling output '%s'", name);
         free(name);
-        vncRandRDisableOutput(i);
-        disabledOutputs->insert(vncRandRGetOutputId(i));
+        if (!dryrun) {
+          /* Currently ignoring errors */
+          /* FIXME: Save output rotation and restore when configuring output */
+          vncRandRDisableOutput(i);
+          disabledOutputs->insert(vncRandRGetOutputId(i));
+        }
       }
     }
   }
 
+  if (!vncRandRIsValidScreenSize(fb_width, fb_height))
+    return 0;
+
+  if (dryrun)
+    return 1;
+
   return vncRandRResizeScreen(fb_width, fb_height);
 }
 
@@ -164,12 +172,15 @@ rfb::ScreenSet computeScreenLayout(OutputIdMap *outputIdMap)
   return layout;
 }
 
-unsigned int setScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet& layout,
-                             OutputIdMap *outputIdMap)
+static unsigned int _setScreenLayout(bool dryrun,
+                                     int fb_width, int fb_height, const rfb::ScreenSet& layout,
+                                     OutputIdMap *outputIdMap)
 {
   int ret;
   int availableOutputs;
   std::set<unsigned int> disabledOutputs;
+  /* Printing errors in the dryrun pass might be confusing */
+  const bool logErrors = !dryrun || vlog.getLevel() >= vlog.LEVEL_DEBUG;
 
   // RandR support?
   if (vncRandRGetOutputCount() == 0)
@@ -180,7 +191,9 @@ unsigned int setScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet&
    * too messy to deal with.
    */
   if (vncRandRHasOutputClones()) {
-    vlog.error("Clone mode active. Refusing to touch screen layout.");
+    if (logErrors) {
+      vlog.error("Clone mode active. Refusing to touch screen layout.");
+    }
     return rfb::resultInvalid;
   }
 
@@ -191,19 +204,31 @@ unsigned int setScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet&
   if (layout.num_screens() > availableOutputs) {
     vlog.debug("Insufficient screens. Need to create %d more.",
                layout.num_screens() - availableOutputs);
-    ret = vncRandRCreateOutputs(layout.num_screens() - availableOutputs);
-    if (!ret) {
-      vlog.error("Unable to create more screens, as needed by the new client layout.");
+
+    if (!vncRandRCanCreateOutputs(layout.num_screens() - availableOutputs)) {
+      if (logErrors)
+        vlog.error("Unable to create more screens, as needed by the new client layout.");
       return rfb::resultInvalid;
     }
+
+    if (!dryrun) {
+      ret = vncRandRCreateOutputs(layout.num_screens() - availableOutputs);
+      if (!ret) {
+        if (logErrors)
+          vlog.error("Unable to create more screens, as needed by the new client layout.");
+        return rfb::resultInvalid;
+      }
+    }
   }
 
   /* First we might need to resize the screen */
   if ((fb_width != vncGetScreenWidth()) ||
       (fb_height != vncGetScreenHeight())) {
-    ret = ResizeScreen(fb_width, fb_height, &disabledOutputs);
+    ret = ResizeScreen(dryrun, fb_width, fb_height, &disabledOutputs);
     if (!ret) {
-      vlog.error("Failed to resize screen to %dx%d", fb_width, fb_height);
+      if (logErrors) {
+        vlog.error("Failed to resize screen to %dx%d", fb_width, fb_height);
+      }
       return rfb::resultInvalid;
     }
   }
@@ -232,25 +257,53 @@ unsigned int setScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet&
       continue;
     }
 
+    /* Probably not needed, but let's be safe */
+    if (!vncRandRIsOutputUsable(i)) {
+      if (logErrors) {
+        char *name = vncRandRGetOutputName(i);
+        vlog.error("Required output '%s' cannot be used", name);
+        free(name);
+      }
+      return rfb::resultInvalid;
+    }
+
+    /* Possible mode? */
+    if (!vncRandRCheckOutputMode(i, iter->dimensions.width(),
+                                 iter->dimensions.height())) {
+      if (logErrors) {
+        char *name = vncRandRGetOutputName(i);
+        vlog.error("Output '%s' does not support required mode %dx%d", name,
+                   iter->dimensions.width(), iter->dimensions.height());
+        free(name);
+      }
+      return rfb::resultInvalid;
+    }
+
+    char *name = vncRandRGetOutputName(i);
+    vlog.debug("Reconfiguring output '%s' to %dx%d+%d+%d", name,
+               iter->dimensions.width(), iter->dimensions.height(),
+               iter->dimensions.tl.x, iter->dimensions.tl.y);
+    free(name);
+
+    if (dryrun)
+      continue;
+
     /* Reconfigure new mode and position */
     ret = vncRandRReconfigureOutput(i,
                                     iter->dimensions.tl.x,
                                     iter->dimensions.tl.y,
                                     iter->dimensions.width(),
                                     iter->dimensions.height());
-    char *name = vncRandRGetOutputName(i);
-    if (ret) {
-      vlog.debug("Reconfigured output '%s' to %dx%d+%d+%d", name,
-                 iter->dimensions.width(), iter->dimensions.height(),
-                 iter->dimensions.tl.x, iter->dimensions.tl.y);
-    } else {
-      vlog.error("Failed to reconfigure output '%s' to %dx%d+%d+%d", name,
-                 iter->dimensions.width(), iter->dimensions.height(),
-                 iter->dimensions.tl.x, iter->dimensions.tl.y);
-      free(name);
+    if (!ret) {
+      if (logErrors) {
+        char *name = vncRandRGetOutputName(i);
+        vlog.error("Failed to reconfigure output '%s' to %dx%d+%d+%d", name,
+                   iter->dimensions.width(), iter->dimensions.height(),
+                   iter->dimensions.tl.x, iter->dimensions.tl.y);
+        free(name);
+      }
       return rfb::resultInvalid;
     }
-    free(name);
   }
 
   /* Allocate new outputs for new screens */
@@ -284,25 +337,53 @@ unsigned int setScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet&
      */
     (*outputIdMap)[output] = iter->id;
 
+    /* Probably not needed, but let's be safe */
+    if (!vncRandRIsOutputUsable(i)) {
+      if (logErrors) {
+        char *name = vncRandRGetOutputName(i);
+        vlog.error("Required new output '%s' cannot be used", name);
+        free(name);
+      }
+      return rfb::resultInvalid;
+    }
+
+    /* Possible mode? */
+    if (!vncRandRCheckOutputMode(i, iter->dimensions.width(),
+                                 iter->dimensions.height())) {
+      if (logErrors) {
+        char *name = vncRandRGetOutputName(i);
+        vlog.error("New output '%s' does not support required mode %dx%d", name,
+                   iter->dimensions.width(), iter->dimensions.height());
+        free(name);
+      }
+      return rfb::resultInvalid;
+    }
+
+    char *name = vncRandRGetOutputName(i);
+    vlog.debug("Reconfiguring new output '%s' to %dx%d+%d+%d", name,
+               iter->dimensions.width(), iter->dimensions.height(),
+               iter->dimensions.tl.x, iter->dimensions.tl.y);
+    free(name);
+
+    if (dryrun)
+      continue;
+
     /* Reconfigure new mode and position */
     ret = vncRandRReconfigureOutput(i,
                                     iter->dimensions.tl.x,
                                     iter->dimensions.tl.y,
                                     iter->dimensions.width(),
                                     iter->dimensions.height());
-    char *name = vncRandRGetOutputName(i);
-    if (ret) {
-      vlog.debug("Reconfigured new output '%s' to %dx%d+%d+%d", name,
-                 iter->dimensions.width(), iter->dimensions.height(),
-                 iter->dimensions.tl.x, iter->dimensions.tl.y);
-    } else {
-      vlog.error("Failed to reconfigure new output '%s' to %dx%d+%d+%d", name,
-                 iter->dimensions.width(), iter->dimensions.height(),
-                 iter->dimensions.tl.x, iter->dimensions.tl.y);
-      free(name);
+    if (!ret) {
+      if (logErrors) {
+        char *name = vncRandRGetOutputName(i);
+        vlog.error("Failed to reconfigure new output '%s' to %dx%d+%d+%d", name,
+                   iter->dimensions.width(), iter->dimensions.height(),
+                   iter->dimensions.tl.x, iter->dimensions.tl.y);
+        free(name);
+      }
       return rfb::resultInvalid;
     }
-    free(name);
   }
 
   /* Turn off unused outputs */
@@ -323,7 +404,9 @@ unsigned int setScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet&
     if (ret) {
       vlog.debug("Disabled unused output '%s'", name);
     } else {
-      vlog.error("Failed to disable unused output '%s'", name);
+      if (logErrors) {
+        vlog.error("Failed to disable unused output '%s'", name);
+      }
       free(name);
       return rfb::resultInvalid;
     }
@@ -339,3 +422,18 @@ unsigned int setScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet&
 
   return rfb::resultSuccess;
 }
+
+
+unsigned int setScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet& layout,
+                             OutputIdMap *outputIdMap)
+{
+    return _setScreenLayout(false, fb_width, fb_height, layout, outputIdMap);
+}
+
+
+unsigned int tryScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet& layout,
+                             OutputIdMap *outputIdMap)
+{
+    OutputIdMap dryrunIdMap = *outputIdMap;
+    return _setScreenLayout(true, fb_width, fb_height, layout, &dryrunIdMap);
+}

unix/common/unixcommon.h

@@ -35,6 +35,9 @@ rfb::ScreenSet computeScreenLayout(OutputIdMap *outputIdMap);
 unsigned int setScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet& layout,
                              OutputIdMap *outputIdMap);
 
+unsigned int tryScreenLayout(int fb_width, int fb_height, const rfb::ScreenSet& layout,
+                             OutputIdMap *outputIdMap);
+
 /*
  * FIXME: This is only exposed because we still have logic in XDesktop
  *        that we haven't integrated in setScreenLayout()

unix/xserver/hw/vnc/RandrGlue.c

@@ -49,9 +49,22 @@ int vncGetScreenHeight(void)
   return screenInfo.screens[scrIdx]->height;
 }
 
+int vncRandRIsValidScreenSize(int width, int height)
+{
+  rrScrPrivPtr rp = rrGetScrPriv(screenInfo.screens[scrIdx]);
+
+  if (width < rp->minWidth || rp->maxWidth < width)
+    return 0;
+  if (height < rp->minHeight || rp->maxHeight < height)
+    return 0;
+
+  return 1;
+}
+
 int vncRandRResizeScreen(int width, int height)
 {
   ScreenPtr pScreen = screenInfo.screens[scrIdx];
+
   /* Try to retain DPI when we resize */
   return RRScreenSizeSet(pScreen, width, height,
                          pScreen->mmWidth * width / pScreen->width,
@@ -183,6 +196,44 @@ int vncRandRIsOutputConnected(int outputIdx)
   return (output->connection == RR_Connected);
 }
 
+static RRModePtr vncRandRGetMatchingMode(int outputIdx, int width, int height)
+{
+  rrScrPrivPtr rp = rrGetScrPriv(screenInfo.screens[scrIdx]);
+
+  RROutputPtr output;
+
+  output = rp->outputs[outputIdx];
+
+  if (output->crtc != NULL) {
+    unsigned int swap;
+    switch (output->crtc->rotation) {
+    case RR_Rotate_90:
+    case RR_Rotate_270:
+      swap = width;
+      width = height;
+      height = swap;
+      break;
+    }
+  }
+
+  for (int i = 0; i < output->numModes; i++) {
+    if ((output->modes[i]->mode.width == width) &&
+        (output->modes[i]->mode.height == height))
+      return output->modes[i];
+  }
+
+  return NULL;
+}
+
+int vncRandRCheckOutputMode(int outputIdx, int width, int height)
+{
+  if (vncRandRGetMatchingMode(outputIdx, width, height) != NULL)
+    return 1;
+  if (vncRandRCanCreateModes())
+    return 1;
+  return 0;
+}
+
 int vncRandRDisableOutput(int outputIdx)
 {
   rrScrPrivPtr rp = rrGetScrPriv(screenInfo.screens[scrIdx]);
@@ -284,7 +335,13 @@ int vncRandRReconfigureOutput(int outputIdx, int x, int y,
   }
 
   /* Make sure we have the mode we want */
-  mode = vncRandRCreatePreferredMode(output, width, height);
+  mode = vncRandRGetMatchingMode(outputIdx, width, height);
+  if (mode == NULL) {
+    mode = vncRandRCreateMode(output, width, height);
+    if (mode == NULL)
+      return 0;
+  }
+  mode = vncRandRSetPreferredMode(output, mode);
   if (mode == NULL)
     return 0;
 
@@ -292,6 +349,11 @@ int vncRandRReconfigureOutput(int outputIdx, int x, int y,
   return RRCrtcSet(crtc, mode, x, y, crtc->rotation, 1, &output);
 }
 
+int vncRandRCanCreateOutputs(int extraOutputs)
+{
+  return vncRandRCanCreateScreenOutputs(scrIdx, extraOutputs);
+}
+
 int vncRandRCreateOutputs(int extraOutputs)
 {
   return vncRandRCreateScreenOutputs(scrIdx, extraOutputs);

unix/xserver/hw/vnc/XorgGlue.h

@@ -48,7 +48,11 @@ int vncGetScreenY(int scrIdx);
 
 // These hide in xvnc.c or vncModule.c
 void vncClientGone(int fd);
+int vncRandRCanCreateScreenOutputs(int scrIdx, int extraOutputs);
 int vncRandRCreateScreenOutputs(int scrIdx, int extraOutputs);
+int vncRandRCanCreateModes(void);
+void* vncRandRCreateMode(void* output, int width, int height);
+void* vncRandRSetPreferredMode(void* output, void* mode);
 
 #ifdef __cplusplus
 }

unix/xserver/hw/vnc/vncModule.c

@@ -110,28 +110,31 @@ void vncClientGone(int fd)
 {
 }
 
+int vncRandRCanCreateScreenOutputs(int scrIdx, int extraOutputs)
+{
+  return 0;
+}
+
 int vncRandRCreateScreenOutputs(int scrIdx, int extraOutputs)
 {
   return 0;
 }
 
-void *vncRandRCreatePreferredMode(void *out, int width, int height)
+int vncRandRCanCreateModes(void)
+{
+  return 0;
+}
+
+void* vncRandRCreateMode(void* output, int width, int height)
 {
-  RROutputPtr output;
+  return 0;
+}
 
+void* vncRandRSetPreferredMode(void* output, void* mode)
+{
   /*
-   * We're not going to change which modes are preferred, but let's
-   * see if we can at least find a mode with matching dimensions.
+   * We're not going to change which modes are preferred,
+   * so just return the incoming mode.
    */
-
-  output = out;
-
-  for (int i = 0;i < output->numModes;i++) {
-    if ((output->modes[i]->mode.width == width) &&
-        (output->modes[i]->mode.height == height))
-      return output->modes[i];
-  }
-
-  return NULL;
+  return mode;
 }
-

unix/xserver/hw/vnc/xvnc.c

@@ -1377,6 +1377,12 @@ static RRCrtcPtr vncRandRCrtcCreate(ScreenPtr pScreen)
 }
 
 /* Used from XserverDesktop when it needs more outputs... */
+
+int vncRandRCanCreateScreenOutputs(int scrIdx, int extraOutputs)
+{
+    return 1;
+}
+
 int vncRandRCreateScreenOutputs(int scrIdx, int extraOutputs)
 {
     RRCrtcPtr crtc;
@@ -1391,27 +1397,67 @@ int vncRandRCreateScreenOutputs(int scrIdx, int extraOutputs)
     return 1;
 }
 
-/* Used to create a preferred mode from various places */
-void *vncRandRCreatePreferredMode(void *out, int width, int height)
+/* Creating and modifying modes, used by XserverDesktop and init here */
+
+int vncRandRCanCreateModes()
+{
+    return 1;
+}
+
+void* vncRandRCreateMode(void* out, int width, int height)
 {
     RROutputPtr output;
 
     output = out;
 
+    /* Do we already have the mode? */
+    for (int i = 0; i < output->numModes; i++) {
+        if ((output->modes[i]->mode.width == width) &&
+            (output->modes[i]->mode.height == height))
+            return output->modes[i];
+    }
+
+    /* Just recreate the entire list */
+    vncRandRSetModes(output, width, height);
+
+    /* Find the new mode */
+    for (int i = 0; i < output->numModes; i++) {
+        if ((output->modes[i]->mode.width == width) &&
+            (output->modes[i]->mode.height == height))
+            return output->modes[i];
+    }
+
+    /* Something went horribly wrong */
+    return NULL;
+}
+
+void* vncRandRSetPreferredMode(void* out, void* m)
+{
+    RRModePtr mode;
+    RROutputPtr output;
+    int width, height;
+
+    mode = m;
+    output = out;
+
+    width = mode->mode.width;
+    height = mode->mode.height;
+
     /* Already the preferred mode? */
     if ((output->numModes >= 1) && (output->numPreferred == 1) &&
-        (output->modes[0]->mode.width == width) &&
-        (output->modes[0]->mode.height == height))
-        return output->modes[0];
+        (output->modes[0] == mode))
+        return mode;
 
     /* Recreate the list, with the mode we want as preferred */
     vncRandRSetModes(output, width, height);
 
+    /* Sanity check */
     if ((output->numModes >= 1) && (output->numPreferred == 1) &&
         (output->modes[0]->mode.width == width) &&
         (output->modes[0]->mode.height == height))
         return output->modes[0];
 
+    /* Something went horribly wrong */
     return NULL;
 }
 
@@ -1434,8 +1480,11 @@ static Bool vncRandRInit(ScreenPtr pScreen)
     crtc = vncRandRCrtcCreate(pScreen);
 
     /* Make sure the current screen size is the active mode */
-    mode = vncRandRCreatePreferredMode(crtc->outputs[0],
-                                       pScreen->width, pScreen->height);
+    mode = vncRandRCreateMode(crtc->outputs[0],
+                              pScreen->width, pScreen->height);
+    if (mode == NULL)
+        return FALSE;
+    mode = vncRandRSetPreferredMode(crtc->outputs[0], mode);
     if (mode == NULL)
         return FALSE;