From e6810fa3e598c06f64f82af0141c7da3b8f9b9a8 Mon Sep 17 00:00:00 2001
From: Pierre Ossman <ossman@cendio.se>
Date: Thu, 1 Mar 2018 14:00:06 +0100
Subject: Fix calculation of conversion test target values

Bit shifting gives an approximate, but not properly accurate value
when upconverting. Do a proper calculation.
---
 tests/conv.cxx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'tests/conv.cxx')

diff --git a/tests/conv.cxx b/tests/conv.cxx
index 840f18dc..3901b9d1 100644
--- a/tests/conv.cxx
+++ b/tests/conv.cxx
@@ -72,9 +72,9 @@ bool verifyPixel(const rfb::PixelFormat &dstpf,
   g = (p >> dstpf.greenShift) & dstpf.greenMax;
   b = (p >> dstpf.blueShift) & dstpf.blueMax;
 
-  r <<= 8 - dstpf.redBits;
-  g <<= 8 - dstpf.greenBits;
-  b <<= 8 - dstpf.blueBits;
+  r = (r * 255 + dstpf.redMax/2) / dstpf.redMax;
+  g = (g * 255 + dstpf.greenMax/2) / dstpf.greenMax;
+  b = (b * 255 + dstpf.blueMax/2) / dstpf.blueMax;
 
   // The allowed error depends on:
   //
-- 
cgit v1.2.3