From 6c47340e095258a959c95db9aa2a6c715d62bf7c Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Wed, 2 Oct 2019 16:06:08 +0200 Subject: [PATCH] Handle pixel formats with odd shift values Our fast paths assume that each channel fits in to a separate byte. That means the shift needs to be a multiple of 8. Start actually checking this so that a client cannot trip us up and possibly cause incorrect code exection. Issue found by Pavel Cheremushkin from Kaspersky Lab. (cherry picked from commit 05e28490873a861379c943bf616614b78b558b89) --- common/rfb/PixelFormat.cxx | 6 ++++++ tests/unit/pixelformat.cxx | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/common/rfb/PixelFormat.cxx b/common/rfb/PixelFormat.cxx index 789c43ed..1b4ab1ba 100644 --- a/common/rfb/PixelFormat.cxx +++ b/common/rfb/PixelFormat.cxx @@ -206,6 +206,12 @@ bool PixelFormat::is888(void) const return false; if (blueMax != 255) return false; + if ((redShift & 0x7) != 0) + return false; + if ((greenShift & 0x7) != 0) + return false; + if ((blueShift & 0x7) != 0) + return false; return true; } diff --git a/tests/unit/pixelformat.cxx b/tests/unit/pixelformat.cxx index cfae2f9d..2e0c0bbb 100644 --- a/tests/unit/pixelformat.cxx +++ b/tests/unit/pixelformat.cxx @@ -170,6 +170,12 @@ void is888Tests() do888Test(false, 8, 8, false, false, 0, 0, 0, 0, 0, 0); + /* Odd shifts */ + + do888Test(false, 32, 24, false, true, 255, 255, 255, 0, 8, 18); + do888Test(false, 32, 24, false, true, 255, 255, 255, 0, 11, 24); + do888Test(false, 32, 24, false, true, 255, 255, 255, 4, 16, 24); + printf("\n"); } -- 2.39.5