From: Pierre Ossman <ossman@cendio.se>
Date: Wed, 2 Oct 2019 14:06:08 +0000 (+0200)
Subject: Handle pixel formats with odd shift values
X-Git-Tag: v1.10.90~76^2
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=refs%2Fpull%2F921%2Fhead;p=tigervnc.git

Handle pixel formats with odd shift values

Our fast paths assume that each channel fits in to a separate byte.
That means the shift needs to be a multiple of 8. Start actually
checking this so that a client cannot trip us up and possibly cause
incorrect code exection.

Issue found by Pavel Cheremushkin from Kaspersky Lab.
---

diff --git a/common/rfb/PixelFormat.cxx b/common/rfb/PixelFormat.cxx
index 789c43ed..1b4ab1ba 100644
--- a/common/rfb/PixelFormat.cxx
+++ b/common/rfb/PixelFormat.cxx
@@ -206,6 +206,12 @@ bool PixelFormat::is888(void) const
     return false;
   if (blueMax != 255)
     return false;
+  if ((redShift & 0x7) != 0)
+    return false;
+  if ((greenShift & 0x7) != 0)
+    return false;
+  if ((blueShift & 0x7) != 0)
+    return false;
 
   return true;
 }
diff --git a/tests/unit/pixelformat.cxx b/tests/unit/pixelformat.cxx
index cfae2f9d..2e0c0bbb 100644
--- a/tests/unit/pixelformat.cxx
+++ b/tests/unit/pixelformat.cxx
@@ -170,6 +170,12 @@ void is888Tests()
 
     do888Test(false, 8, 8, false, false, 0, 0, 0, 0, 0, 0);
 
+    /* Odd shifts */
+
+    do888Test(false, 32, 24, false, true, 255, 255, 255, 0, 8, 18);
+    do888Test(false, 32, 24, false, true, 255, 255, 255, 0, 11, 24);
+    do888Test(false, 32, 24, false, true, 255, 255, 255, 4, 16, 24);
+
     printf("\n");
 }