Fix temporary decoder buffer sizes

Some of these were excessively large because of redundant factors
in the size calculation.

diff --git a/common/rfb/ZRLEDecoder.cxx b/common/rfb/ZRLEDecoder.cxx
index c13f28613cd765f0e1c77d68a7aa6a87d297d67b..b891ba522942d659c56f252dacf3743ca088387f 100644 (file)
--- a/common/rfb/ZRLEDecoder.cxx
+++ b/common/rfb/ZRLEDecoder.cxx
@@ -86,10 +86,9 @@ void ZRLEDecoder::decodeRect(const Rect& r, const void* buffer,
 {
   rdr::MemInStream is(buffer, buflen);
   const rfb::PixelFormat& pf = cp.pf();
-  rdr::U8* buf[64 * 64 * 4 * pf.bpp/8];
   switch (pf.bpp) {
-  case 8:  zrleDecode8 (r, &is, &zis, (rdr::U8*) buf, pf, pb); break;
-  case 16: zrleDecode16(r, &is, &zis, (rdr::U16*)buf, pf, pb); break;
+  case 8:  zrleDecode8 (r, &is, &zis, pf, pb); break;
+  case 16: zrleDecode16(r, &is, &zis, pf, pb); break;
   case 32:
     {
       Pixel maxPixel = pf.pixelFromRGB((rdr::U16)-1, (rdr::U16)-1, (rdr::U16)-1);
@@ -99,16 +98,16 @@ void ZRLEDecoder::decodeRect(const Rect& r, const void* buffer,
       if ((fitsInLS3Bytes && pf.isLittleEndian()) ||
           (fitsInMS3Bytes && pf.isBigEndian()))
       {
-        zrleDecode24A(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+        zrleDecode24A(r, &is, &zis, pf, pb);
       }
       else if ((fitsInLS3Bytes && pf.isBigEndian()) ||
                (fitsInMS3Bytes && pf.isLittleEndian()))
       {
-        zrleDecode24B(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+        zrleDecode24B(r, &is, &zis, pf, pb);
       }
       else
       {
-        zrleDecode32(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+        zrleDecode32(r, &is, &zis, pf, pb);
       }
       break;
     }

diff --git a/common/rfb/hextileDecode.h b/common/rfb/hextileDecode.h
index 7affa157fe6c4d9400498c3bc5dfcbfd85ac71be..47006a042828469a3581fd7ed74b42439446c032 100644 (file)
--- a/common/rfb/hextileDecode.h
+++ b/common/rfb/hextileDecode.h
@@ -44,7 +44,7 @@ static void HEXTILE_DECODE (const Rect& r, rdr::InStream* is,
   Rect t;
   PIXEL_T bg = 0;
   PIXEL_T fg = 0;
-  PIXEL_T buf[16 * 16 * 4];
+  PIXEL_T buf[16 * 16];
 
   for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 16) {
 

diff --git a/common/rfb/zrleDecode.h b/common/rfb/zrleDecode.h
index 07d6795a01f261dbd125ed47d510f82fb827d372..0bfbbe152dd443ede674001572ddbbe18ae05dce 100644 (file)
--- a/common/rfb/zrleDecode.h
+++ b/common/rfb/zrleDecode.h
@@ -47,12 +47,13 @@ namespace rfb {
 #endif
 
 void ZRLE_DECODE (const Rect& r, rdr::InStream* is,
-                  rdr::ZlibInStream* zis, PIXEL_T* buf,
+                  rdr::ZlibInStream* zis,
                   const PixelFormat& pf, ModifiablePixelBuffer* pb)
 {
   int length = is->readU32();
   zis->setUnderlying(is, length);
   Rect t;
+  PIXEL_T buf[64 * 64];
 
   for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 64) {