Be defensive about overflows in stream objects

We use a lot of lengths given to us over the network, so be more
paranoid about them causing an overflow as otherwise an attacker
might trick us in to overwriting other memory.

This primarily affects the client which often gets lengths from the
server, but there are also some scenarios where the server might
theoretically be vulnerable.

Issue found by Pavel Cheremushkin from Kaspersky Lab.

(cherry picked from commit 75e6e0653a)

common/rdr/FdInStream.cxx

   ptr = start;
 
   size_t bytes_to_read;
-  while (end < start + itemSize) {
+  while ((size_t)(end - start) < itemSize) {
     bytes_to_read = start + bufSize - end;
     if (!timing) {
       // When not timing, we must be careful not to read too much
     end += n;
   }
 
-  if (itemSize * nItems > (size_t)(end - ptr))
-    nItems = (end - ptr) / itemSize;
+  size_t nAvail;
+  nAvail = (end - ptr) / itemSize;
+  if (nAvail < nItems)
+    return nAvail;
 
   return nItems;
 }

common/rdr/FdOutStream.cxx

     }
   }
 
-  // Can we fit all the items asked for?
-  if (itemSize * nItems > (size_t)(end - ptr))
-    nItems = (end - ptr) / itemSize;
+  size_t nAvail;
+  nAvail = (end - ptr) / itemSize;
+  if (nAvail < nItems)
+    return nAvail;
 
   return nItems;
 }

common/rdr/FileInStream.cxx

   ptr = b;
 
 
-  while (end < b + itemSize) {
+  while ((size_t)(end - b) < itemSize) {
     size_t n = fread((U8 *)end, b + sizeof(b) - end, 1, file);
     if (n == 0) {
       if (ferror(file))
     end += b + sizeof(b) - end;
   }
 
-  if (itemSize * nItems > (size_t)(end - ptr))
-    nItems = (end - ptr) / itemSize;
+  size_t nAvail;
+  nAvail = (end - ptr) / itemSize;
+  if (nAvail < nItems)
+    return nAvail;
 
   return nItems;
 }

common/rdr/HexInStream.cxx

   offset += ptr - start;
   ptr = start;
 
-  while (end < ptr + itemSize) {
+  while ((size_t)(end - ptr) < itemSize) {
     size_t n = in_stream.check(2, 1, wait);
     if (n == 0) return 0;
     const U8* iptr = in_stream.getptr();
     end += length;
   }
 
-  if (itemSize * nItems > (size_t)(end - ptr))
-    nItems = (end - ptr) / itemSize;
+  size_t nAvail;
+  nAvail = (end - ptr) / itemSize;
+  if (nAvail < nItems)
+    return nAvail;
 
   return nItems;
 }

common/rdr/HexOutStream.cxx

 
   writeBuffer();
 
-  if (itemSize * nItems > (size_t)(end - ptr))
-    nItems = (end - ptr) / itemSize;
+  size_t nAvail;
+  nAvail = (end - ptr) / itemSize;
+  if (nAvail < nItems)
+    return nAvail;
 
   return nItems;
 }

common/rdr/InStream.h

 
     inline size_t check(size_t itemSize, size_t nItems=1, bool wait=true)
     {
-      if (ptr + itemSize * nItems > end) {
-        if (ptr + itemSize > end)
-          return overrun(itemSize, nItems, wait);
+      size_t nAvail;
+
+      if (itemSize > (size_t)(end - ptr))
+        return overrun(itemSize, nItems, wait);
+
+      nAvail = (end - ptr) / itemSize;
+      if (nAvail < nItems)
+        return nAvail;
 
-        nItems = (end - ptr) / itemSize;
-      }
       return nItems;
     }
 
     // readBytes() reads an exact number of bytes.
 
     void readBytes(void* data, size_t length) {
-      U8* dataPtr = (U8*)data;
-      U8* dataEnd = dataPtr + length;
-      while (dataPtr < dataEnd) {
-        size_t n = check(1, dataEnd - dataPtr);
-        memcpy(dataPtr, ptr, n);
+      while (length > 0) {
+        size_t n = check(1, length);
+        memcpy(data, ptr, n);
         ptr += n;
-        dataPtr += n;
+        data = (U8*)data + n;
+        length -= n;
       }
     }
 

common/rdr/MemOutStream.h

 #ifndef __RDR_MEMOUTSTREAM_H__
 #define __RDR_MEMOUTSTREAM_H__
 
+#include <rdr/Exception.h>
 #include <rdr/OutStream.h>
 
 namespace rdr {
       if (len < (size_t)(end - start) * 2)
         len = (end - start) * 2;
 
+      if (len < (size_t)(end - start))
+        throw Exception("Overflow in MemOutStream::overrun()");
+
       U8* newStart = new U8[len];
       memcpy(newStart, start, ptr - start);
       ptr = newStart + (ptr - start);

common/rdr/OutStream.h

 
     inline size_t check(size_t itemSize, size_t nItems=1)
     {
-      if (ptr + itemSize * nItems > end) {
-        if (ptr + itemSize > end)
-          return overrun(itemSize, nItems);
+      size_t nAvail;
+
+      if (itemSize > (size_t)(end - ptr))
+        return overrun(itemSize, nItems);
+
+      nAvail = (end - ptr) / itemSize;
+      if (nAvail < nItems)
+        return nAvail;
 
-        nItems = (end - ptr) / itemSize;
-      }
       return nItems;
     }
 
     // writeBytes() writes an exact number of bytes.
 
     void writeBytes(const void* data, size_t length) {
-      const U8* dataPtr = (const U8*)data;
-      const U8* dataEnd = dataPtr + length;
-      while (dataPtr < dataEnd) {
-        size_t n = check(1, dataEnd - dataPtr);
-        memcpy(ptr, dataPtr, n);
+      while (length > 0) {
+        size_t n = check(1, length);
+        memcpy(ptr, data, n);
         ptr += n;
-        dataPtr += n;
+        data = (U8*)data + n;
+        length -= n;
       }
     }
 

common/rdr/RandomStream.cxx

       *(U8*)end++ = (int) (256.0*rand()/(RAND_MAX+1.0));
   }
 
-  if (itemSize * nItems > (size_t)(end - ptr))
-    nItems = (end - ptr) / itemSize;
+  size_t nAvail;
+  nAvail = (end - ptr) / itemSize;
+  if (nAvail < nItems)
+    return nAvail;
 
   return nItems;
 }

common/rdr/TLSInStream.cxx

       return -1;
     }
 
-    if (in->getend() - in->getptr() < (ptrdiff_t)size)
+    if ((size_t)(in->getend() - in->getptr()) < size)
       size = in->getend() - in->getptr();
   
     in->readBytes(data, size);
   end -= ptr - start;
   ptr = start;
 
-  while (end < start + itemSize) {
+  while ((size_t)(end - start) < itemSize) {
     size_t n = readTLS((U8*) end, start + bufSize - end, wait);
     if (!wait && n == 0)
       return 0;
     end += n;
   }
 
-  if (itemSize * nItems > (size_t)(end - ptr))
-    nItems = (end - ptr) / itemSize;
+  size_t nAvail;
+  nAvail = (end - ptr) / itemSize;
+  if (nAvail < nItems)
+    return nAvail;
 
   return nItems;
 }

common/rdr/TLSOutStream.cxx

 
   flush();
 
-  if (itemSize * nItems > (size_t)(end - ptr))
-    nItems = (end - ptr) / itemSize;
+  size_t nAvail;
+  nAvail = (end - ptr) / itemSize;
+  if (nAvail < nItems)
+    return nAvail;
 
   return nItems;
 }

common/rdr/ZlibInStream.cxx

       return 0;
   }
 
-  if (itemSize * nItems > (size_t)(end - ptr))
-    nItems = (end - ptr) / itemSize;
+  size_t nAvail;
+  nAvail = (end - ptr) / itemSize;
+  if (nAvail < nItems)
+    return nAvail;
 
   return nItems;
 }

common/rdr/ZlibOutStream.cxx

     }
   }
 
-  if (itemSize * nItems > (size_t)(end - ptr))
-    nItems = (end - ptr) / itemSize;
+  size_t nAvail;
+  nAvail = (end - ptr) / itemSize;
+  if (nAvail < nItems)
+    return nAvail;
 
   return nItems;
 }