From 9da4f05748c1ec084ee28982c3d1299a5f775664 Mon Sep 17 00:00:00 2001
From: Pierre Ossman <ossman@cendio.se>
Date: Sat, 7 Dec 2024 15:38:14 +0100
Subject: Fix encoder solid area detection

This got completely broken in baca73d0 where the templated method got
incorrectly called instead of the wrapper/multiplexer. It seems to have
been harmless on most systems, but caused crashes on CPUs with strict
memory alignment requirements.
---
 common/rfb/EncodeManager.cxx | 39 ++++++++++++++++++++++-----------------
 common/rfb/EncodeManager.h   |  5 +++--
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx
index 4526c0b3..67a32f5b 100644
--- a/common/rfb/EncodeManager.cxx
+++ b/common/rfb/EncodeManager.cxx
@@ -881,13 +881,24 @@ void EncodeManager::writeSubRect(const Rect& rect, const PixelBuffer *pb)
 bool EncodeManager::checkSolidTile(const Rect& r, const uint8_t* colourValue,
                                    const PixelBuffer *pb)
 {
+  const uint8_t* buffer;
+  int stride;
+
+  buffer = pb->getBuffer(r, &stride);
+
   switch (pb->getPF().bpp) {
   case 32:
-    return checkSolidTile(r, *(const uint32_t*)colourValue, pb);
+    return checkSolidTile(r.width(), r.height(),
+                          (const uint32_t*)buffer, stride,
+                          *(const uint32_t*)colourValue);
   case 16:
-    return checkSolidTile(r, *(const uint16_t*)colourValue, pb);
+    return checkSolidTile(r.width(), r.height(),
+                          (const uint16_t*)buffer, stride,
+                          *(const uint16_t*)colourValue);
   default:
-    return checkSolidTile(r, *(const uint8_t*)colourValue, pb);
+    return checkSolidTile(r.width(), r.height(),
+                          (const uint8_t*)buffer, stride,
+                          *(const uint8_t*)colourValue);
   }
 }
 
@@ -1058,23 +1069,17 @@ uint8_t* EncodeManager::OffsetPixelBuffer::getBufferRW(const Rect& /*r*/, int* /
 }
 
 template<class T>
-inline bool EncodeManager::checkSolidTile(const Rect& r,
-                                          const T colourValue,
-                                          const PixelBuffer *pb)
+inline bool EncodeManager::checkSolidTile(int width, int height,
+                                          const T* buffer, int stride,
+                                          const T colourValue)
 {
-  int w, h;
-  const T* buffer;
-  int stride, pad;
-
-  w = r.width();
-  h = r.height();
+  int pad;
 
-  buffer = (const T*)pb->getBuffer(r, &stride);
-  pad = stride - w;
+  pad = stride - width;
 
-  while (h--) {
-    int w_ = w;
-    while (w_--) {
+  while (height--) {
+    int width_ = width;
+    while (width_--) {
       if (*buffer != colourValue)
         return false;
       buffer++;
diff --git a/common/rfb/EncodeManager.h b/common/rfb/EncodeManager.h
index a01a1614..7ae9b5b8 100644
--- a/common/rfb/EncodeManager.h
+++ b/common/rfb/EncodeManager.h
@@ -100,8 +100,9 @@ namespace rfb {
   protected:
     // Templated, optimised methods
     template<class T>
-    inline bool checkSolidTile(const Rect& r, const T,
-                               const PixelBuffer *pb);
+    inline bool checkSolidTile(int width, int height,
+                               const T* buffer, int stride,
+                               const T colourValue);
     template<class T>
     inline bool analyseRect(int width, int height,
                             const T* buffer, int stride,
-- 
cgit v1.2.3