Avoid integer overflows in pixel size calculations

diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx
index 0ce611e9b0509573327db55b193e94e0c8ca6a34..53e0365de52a00dae13653cd48aaf42ade6f84b0 100644 (file)
--- a/common/rfb/EncodeManager.cxx
+++ b/common/rfb/EncodeManager.cxx
@@ -519,7 +519,7 @@ Encoder *EncodeManager::startRect(const Rect& rect, int type)
 
   stats[klass][activeType].rects++;
   stats[klass][activeType].pixels += rect.area();
-  equiv = 12 + rect.area() * conn->cp.pf().bpp/8;
+  equiv = 12 + rect.area() * (conn->cp.pf().bpp/8);
   stats[klass][activeType].equivalent += equiv;
 
   encoder = encoders[klass];
@@ -561,7 +561,7 @@ void EncodeManager::writeCopyRects(const Region& copied, const Point& delta)
 
     copyStats.rects++;
     copyStats.pixels += rect->area();
-    equiv = 12 + rect->area() * conn->cp.pf().bpp/8;
+    equiv = 12 + rect->area() * (conn->cp.pf().bpp/8);
     copyStats.equivalent += equiv;
 
     conn->writer()->writeCopyRect(*rect, rect->tl.x - delta.x,

diff --git a/common/rfb/PixelBuffer.cxx b/common/rfb/PixelBuffer.cxx
index 007b6c84ed6d6bf1bc3851d3b6031490a1c83cc7..7f4c1ad30c24a4e0e67918f26b2f02445a514d4c 100644 (file)
--- a/common/rfb/PixelBuffer.cxx
+++ b/common/rfb/PixelBuffer.cxx
@@ -204,6 +204,7 @@ void ModifiablePixelBuffer::copyRect(const Rect &rect,
                                      const Point &move_by_delta)
 {
   int srcStride, dstStride;
+  int bytesPerPixel;
   const U8* srcData;
   U8* dstData;
 
@@ -221,6 +222,8 @@ void ModifiablePixelBuffer::copyRect(const Rect &rect,
                          srect.width(), srect.height(),
                          srect.tl.x, srect.tl.y, width_, height_);
 
+  bytesPerPixel = format.bpp/8;
+
   srcData = getBuffer(srect, &srcStride);
   dstData = getBufferRW(drect, &dstStride);
 
@@ -228,27 +231,27 @@ void ModifiablePixelBuffer::copyRect(const Rect &rect,
     // Possible overlap. Be careful and use memmove().
     int h = drect.height();
     while (h--) {
-      memmove(dstData, srcData, drect.width() * format.bpp/8);
-      dstData += dstStride * format.bpp/8;
-      srcData += srcStride * format.bpp/8;
+      memmove(dstData, srcData, drect.width() * bytesPerPixel);
+      dstData += dstStride * bytesPerPixel;
+      srcData += srcStride * bytesPerPixel;
     }
   } else if (move_by_delta.y < 0) {
     // The data shifted upwards. Copy from top to bottom.
     int h = drect.height();
     while (h--) {
-      memcpy(dstData, srcData, drect.width() * format.bpp/8);
-      dstData += dstStride * format.bpp/8;
-      srcData += srcStride * format.bpp/8;
+      memcpy(dstData, srcData, drect.width() * bytesPerPixel);
+      dstData += dstStride * bytesPerPixel;
+      srcData += srcStride * bytesPerPixel;
     }
   } else {
     // The data shifted downwards. Copy from bottom to top.
     int h = drect.height();
-    dstData += (h-1) * dstStride * format.bpp/8;
-    srcData += (h-1) * srcStride * format.bpp/8;
+    dstData += (h-1) * dstStride * bytesPerPixel;
+    srcData += (h-1) * srcStride * bytesPerPixel;
     while (h--) {
-      memcpy(dstData, srcData, drect.width() * format.bpp/8);
-      dstData -= dstStride * format.bpp/8;
-      srcData -= srcStride * format.bpp/8;
+      memcpy(dstData, srcData, drect.width() * bytesPerPixel);
+      dstData -= dstStride * bytesPerPixel;
+      srcData -= srcStride * bytesPerPixel;
     }
   }
 
@@ -304,7 +307,7 @@ rdr::U8* FullFramePixelBuffer::getBufferRW(const Rect& r, int* stride_)
                          r.tl.x, r.tl.y, width_, height_);
 
   *stride_ = stride;
-  return &data[(r.tl.x + (r.tl.y * stride)) * format.bpp/8];
+  return &data[(r.tl.x + (r.tl.y * stride)) * (format.bpp/8)];
 }
 
 void FullFramePixelBuffer::commitBufferRW(const Rect& r)
@@ -319,7 +322,7 @@ const rdr::U8* FullFramePixelBuffer::getBuffer(const Rect& r, int* stride_) cons
                          r.tl.x, r.tl.y, width_, height_);
 
   *stride_ = stride;
-  return &data[(r.tl.x + (r.tl.y * stride)) * format.bpp/8];
+  return &data[(r.tl.x + (r.tl.y * stride)) * (format.bpp/8)];
 }
 
 // -=- Managed pixel buffer class

diff --git a/common/rfb/RawDecoder.cxx b/common/rfb/RawDecoder.cxx
index 786f15451eda969b536221db1f79f677499793fe..ec0c68ee4ae37788b0782ae72c003e99eb40d6e8 100644 (file)
--- a/common/rfb/RawDecoder.cxx
+++ b/common/rfb/RawDecoder.cxx
@@ -36,13 +36,13 @@ RawDecoder::~RawDecoder()
 void RawDecoder::readRect(const Rect& r, rdr::InStream* is,
                           const ConnParams& cp, rdr::OutStream* os)
 {
-  os->copyBytes(is, r.area() * cp.pf().bpp/8);
+  os->copyBytes(is, r.area() * (cp.pf().bpp/8));
 }
 
 void RawDecoder::decodeRect(const Rect& r, const void* buffer,
                             size_t buflen, const ConnParams& cp,
                             ModifiablePixelBuffer* pb)
 {
-  assert(buflen >= (size_t)r.area() * cp.pf().bpp/8);
+  assert(buflen >= (size_t)r.area() * (cp.pf().bpp/8));
   pb->imageRect(cp.pf(), r, buffer);
 }

diff --git a/common/rfb/SMsgWriter.cxx b/common/rfb/SMsgWriter.cxx
index 6ef7692eeb4c2805a8aa83772b7806621c2b53ab..3da9413f4da18a25bb5e94d4bff4a84ac92b587b 100644 (file)
--- a/common/rfb/SMsgWriter.cxx
+++ b/common/rfb/SMsgWriter.cxx
@@ -350,7 +350,7 @@ void SMsgWriter::writePseudoRects()
   if (needSetCursor) {
     const Cursor& cursor = cp->cursor();
 
-    rdr::U8Array data(cursor.width()*cursor.height() * cp->pf().bpp/8);
+    rdr::U8Array data(cursor.width()*cursor.height() * (cp->pf().bpp/8));
     rdr::U8Array mask(cursor.getMask());
 
     const rdr::U8* in;

diff --git a/common/rfb/TightDecoder.cxx b/common/rfb/TightDecoder.cxx
index 3a1254a2a66e5fe456db991e8978ac9996f87e46..cc786f5b8cb4f8bee556d775c823248712736bcd 100644 (file)
--- a/common/rfb/TightDecoder.cxx
+++ b/common/rfb/TightDecoder.cxx
@@ -364,7 +364,7 @@ void TightDecoder::decodeRect(const Rect& r, const void* buffer,
   if (directDecode)
     outbuf = pb->getBufferRW(r, &stride);
   else {
-    outbuf = new rdr::U8[r.area() * pf.bpp/8];
+    outbuf = new rdr::U8[r.area() * (pf.bpp/8)];
     stride = r.width();
   }
 

diff --git a/common/rfb/ZRLEEncoder.cxx b/common/rfb/ZRLEEncoder.cxx
index d3afe747e18fabdeb1da71635586014fb94e5597..8917d8ffe7df2b89c672c654202a992490ab1ff0 100644 (file)
--- a/common/rfb/ZRLEEncoder.cxx
+++ b/common/rfb/ZRLEEncoder.cxx
@@ -223,7 +223,7 @@ void ZRLEEncoder::writePixels(const rdr::U8* buffer, const PixelFormat& pf,
   pf.bufferFromPixel(pixBuf, maxPixel);
 
   if ((pf.bpp != 32) || ((pixBuf[0] != 0) && (pixBuf[3] != 0))) {
-    zos.writeBytes(buffer, count * pf.bpp/8);
+    zos.writeBytes(buffer, count * (pf.bpp/8));
     return;
   }