From 1349e42e395a0a88b67447580d526daf31dba591 Mon Sep 17 00:00:00 2001
From: Pierre Ossman <ossman@cendio.se>
Date: Wed, 5 Oct 2016 11:00:37 +0200
Subject: [PATCH] Fix temporary decoder buffer sizes

Some of these were excessively large because of redundant factors
in the size calculation.
---
 common/rfb/ZRLEDecoder.cxx | 11 +++++------
 common/rfb/hextileDecode.h |  2 +-
 common/rfb/zrleDecode.h    |  3 ++-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/common/rfb/ZRLEDecoder.cxx b/common/rfb/ZRLEDecoder.cxx
index c13f2861..b891ba52 100644
--- a/common/rfb/ZRLEDecoder.cxx
+++ b/common/rfb/ZRLEDecoder.cxx
@@ -86,10 +86,9 @@ void ZRLEDecoder::decodeRect(const Rect& r, const void* buffer,
 {
   rdr::MemInStream is(buffer, buflen);
   const rfb::PixelFormat& pf = cp.pf();
-  rdr::U8* buf[64 * 64 * 4 * pf.bpp/8];
   switch (pf.bpp) {
-  case 8:  zrleDecode8 (r, &is, &zis, (rdr::U8*) buf, pf, pb); break;
-  case 16: zrleDecode16(r, &is, &zis, (rdr::U16*)buf, pf, pb); break;
+  case 8:  zrleDecode8 (r, &is, &zis, pf, pb); break;
+  case 16: zrleDecode16(r, &is, &zis, pf, pb); break;
   case 32:
     {
       Pixel maxPixel = pf.pixelFromRGB((rdr::U16)-1, (rdr::U16)-1, (rdr::U16)-1);
@@ -99,16 +98,16 @@ void ZRLEDecoder::decodeRect(const Rect& r, const void* buffer,
       if ((fitsInLS3Bytes && pf.isLittleEndian()) ||
           (fitsInMS3Bytes && pf.isBigEndian()))
       {
-        zrleDecode24A(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+        zrleDecode24A(r, &is, &zis, pf, pb);
       }
       else if ((fitsInLS3Bytes && pf.isBigEndian()) ||
                (fitsInMS3Bytes && pf.isLittleEndian()))
       {
-        zrleDecode24B(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+        zrleDecode24B(r, &is, &zis, pf, pb);
       }
       else
       {
-        zrleDecode32(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+        zrleDecode32(r, &is, &zis, pf, pb);
       }
       break;
     }
diff --git a/common/rfb/hextileDecode.h b/common/rfb/hextileDecode.h
index 7affa157..47006a04 100644
--- a/common/rfb/hextileDecode.h
+++ b/common/rfb/hextileDecode.h
@@ -44,7 +44,7 @@ static void HEXTILE_DECODE (const Rect& r, rdr::InStream* is,
   Rect t;
   PIXEL_T bg = 0;
   PIXEL_T fg = 0;
-  PIXEL_T buf[16 * 16 * 4];
+  PIXEL_T buf[16 * 16];
 
   for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 16) {
 
diff --git a/common/rfb/zrleDecode.h b/common/rfb/zrleDecode.h
index 07d6795a..0bfbbe15 100644
--- a/common/rfb/zrleDecode.h
+++ b/common/rfb/zrleDecode.h
@@ -47,12 +47,13 @@ namespace rfb {
 #endif
 
 void ZRLE_DECODE (const Rect& r, rdr::InStream* is,
-                  rdr::ZlibInStream* zis, PIXEL_T* buf,
+                  rdr::ZlibInStream* zis,
                   const PixelFormat& pf, ModifiablePixelBuffer* pb)
 {
   int length = is->readU32();
   zis->setUnderlying(is, length);
   Rect t;
+  PIXEL_T buf[64 * 64];
 
   for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 64) {
 
-- 
2.39.5