Add unit test for PixelFormat sanity checks

(cherry picked from commit 014c501237)

common/rfb/PixelFormat.cxx

@@ -81,7 +81,8 @@ PixelFormat::PixelFormat(int b, int d, bool e, bool t,
     redMax(rm), greenMax(gm), blueMax(bm),
     redShift(rs), greenShift(gs), blueShift(bs)
 {
-  assert(isSane());
+  if (!isSane())
+    throw Exception("invalid pixel format");
 
   updateState();
 }

tests/unit/CMakeLists.txt

@@ -8,3 +8,6 @@ target_link_libraries(convertlf rfb)
 
 add_executable(hostport hostport.cxx)
 target_link_libraries(hostport rfb)
+
+add_executable(pixelformat pixelformat.cxx)
+target_link_libraries(pixelformat rfb)

tests/unit/pixelformat.cxx

@@ -0,0 +1,114 @@
+/* Copyright 2019 Pierre Ossman <ossman@cendio.se> for Cendio AB
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#include <stdio.h>
+
+#include <rfb/PixelFormat.h>
+#include <rfb/Exception.h>
+
+static void doTest(bool should_fail, int b, int d, bool e, bool t,
+                   int rm, int gm, int bm, int rs, int gs, int bs)
+{
+    rfb::PixelFormat* pf;
+
+    printf("PixelFormat(%d, %d, %s, %s, %d, %d, %d, %d, %d, %d): ",
+           b, d, e ? "true" : "false", t ? "true": "false",
+           rm, gm, bm, rs, gs, bs);
+
+    try {
+        pf = new rfb::PixelFormat(b, d, e, t, rm, gm, bm, rs, gs, bs);
+    } catch(rfb::Exception &e) {
+        if (should_fail)
+            printf("OK");
+        else
+            printf("FAILED");
+        printf("\n");
+        fflush(stdout);
+        return;
+    }
+
+    delete pf;
+
+    if (should_fail)
+        printf("FAILED");
+    else
+        printf("OK");
+    printf("\n");
+    fflush(stdout);
+}
+
+int main(int argc, char** argv)
+{
+    /* Normal true color formats */
+
+    doTest(false, 32, 24, false, true, 255, 255, 255, 0, 8, 16);
+    doTest(false, 32, 24, false, true, 255, 255, 255, 24, 16, 8);
+
+    doTest(false, 16, 16, false, true, 15, 31, 15, 0, 5, 11);
+
+    doTest(false, 8, 8, false, true, 3, 7, 3, 0, 2, 5);
+
+    /* Excessive bpp */
+
+    doTest(false, 32, 16, false, true, 15, 31, 15, 0, 5, 11);
+
+    doTest(false, 16, 16, false, true, 15, 31, 15, 0, 5, 11);
+
+    doTest(false, 32, 8, false, true, 3, 7, 3, 0, 2, 5);
+
+    doTest(false, 16, 8, false, true, 3, 7, 3, 0, 2, 5);
+
+    /* Colour map */
+
+    doTest(false, 8, 8, false, false, 0, 0, 0, 0, 0, 0);
+
+    /* Invalid bpp */
+
+    doTest(true, 64, 24, false, true, 255, 255, 255, 0, 8, 16);
+
+    doTest(true, 18, 16, false, true, 15, 31, 15, 0, 5, 11);
+
+    doTest(true, 3, 3, false, true, 1, 1, 1, 0, 1, 2);
+
+    /* Invalid depth */
+
+    doTest(true, 16, 24, false, true, 15, 31, 15, 0, 5, 11);
+
+    doTest(true, 8, 24, false, true, 3, 7, 3, 0, 2, 5);
+    doTest(true, 8, 16, false, true, 3, 7, 3, 0, 2, 5);
+
+    doTest(true, 32, 24, false, false, 0, 0, 0, 0, 0, 0);
+
+    /* Invalid max values */
+
+    doTest(true, 32, 24, false, true, 254, 255, 255, 0, 8, 16);
+    doTest(true, 32, 24, false, true, 255, 253, 255, 0, 8, 16);
+    doTest(true, 32, 24, false, true, 255, 255, 252, 0, 8, 16);
+
+    doTest(true, 32, 24, false, true, 511, 127, 127, 0, 16, 20);
+    doTest(true, 32, 24, false, true, 127, 511, 127, 0, 4, 20);
+    doTest(true, 32, 24, false, true, 127, 127, 511, 0, 4, 8);
+
+    /* Overlapping channels */
+
+    doTest(true, 32, 24, false, true, 255, 255, 255, 0, 7, 16);
+    doTest(true, 32, 24, false, true, 255, 255, 255, 0, 8, 15);
+    doTest(true, 32, 24, false, true, 255, 255, 255, 0, 16, 7);
+
+    return 0;
+}