Handle pixel formats with odd shift values

Our fast paths assume that each channel fits in to a separate byte.
That means the shift needs to be a multiple of 8. Start actually
checking this so that a client cannot trip us up and possibly cause
incorrect code exection.

Issue found by Pavel Cheremushkin from Kaspersky Lab.

diff --git a/common/rfb/PixelFormat.cxx b/common/rfb/PixelFormat.cxx
index 789c43edaa4e5496f59c28941e9e1cdc02aa2f5c..1b4ab1ba16701d0078c0b0e089c28575629e63b1 100644 (file)
--- a/common/rfb/PixelFormat.cxx
+++ b/common/rfb/PixelFormat.cxx
@@ -206,6 +206,12 @@ bool PixelFormat::is888(void) const
     return false;
   if (blueMax != 255)
     return false;
+  if ((redShift & 0x7) != 0)
+    return false;
+  if ((greenShift & 0x7) != 0)
+    return false;
+  if ((blueShift & 0x7) != 0)
+    return false;
 
   return true;
 }

diff --git a/tests/unit/pixelformat.cxx b/tests/unit/pixelformat.cxx
index cfae2f9d0a83d535ba58752be777d82957521e90..2e0c0bbb6d170de96cd2521ab07b0fcb4d83b6d3 100644 (file)
--- a/tests/unit/pixelformat.cxx
+++ b/tests/unit/pixelformat.cxx
@@ -170,6 +170,12 @@ void is888Tests()
 
     do888Test(false, 8, 8, false, false, 0, 0, 0, 0, 0, 0);
 
+    /* Odd shifts */
+
+    do888Test(false, 32, 24, false, true, 255, 255, 255, 0, 8, 18);
+    do888Test(false, 32, 24, false, true, 255, 255, 255, 0, 11, 24);
+    do888Test(false, 32, 24, false, true, 255, 255, 255, 4, 16, 24);
+
     printf("\n");
 }