From: DRC <dcommander@users.sourceforge.net>
Date: Tue, 9 Aug 2011 11:12:55 +0000 (+0000)
Subject: Revert r4498 and fix #3305357 properly.	 The issue was two-fold:  (1) the compress... 
X-Git-Tag: v1.1.90~183
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=f4a341bc85c931e24e27a088f27f19f574b0c455;p=tigervnc.git

Revert r4498 and fix #3305357 properly.	 The issue was two-fold:  (1) the compress buffer allocated by	the Tight encoder was not large	enough,	and (2) Zlib 1.2.5 can	sometimes call deflate(..., Z_BLOCK) within the	body of	deflateParams(), so we	need to	check avail_in after calling checkCompressionLevel() to	ensure that there is still data left to compress before we call deflate() again.


git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4617 3789f03b-4d11-0410-bbf8-ca57d06f2519
---

diff --git a/common/rdr/ZlibOutStream.cxx b/common/rdr/ZlibOutStream.cxx
index c39aa55d..c86a5a55 100644
--- a/common/rdr/ZlibOutStream.cxx
+++ b/common/rdr/ZlibOutStream.cxx
@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -88,6 +89,10 @@ void ZlibOutStream::flush()
 //        fprintf(stderr,"zos flush: calling deflate, avail_in %d, avail_out %d\n",
 //                zs->avail_in,zs->avail_out);
       checkCompressionLevel();
+      if (zs->avail_in != 0) {
+        int rc = deflate(zs, Z_SYNC_FLUSH);
+        if (rc != Z_OK) throw Exception("ZlibOutStream: deflate failed");
+      }
 
 //        fprintf(stderr,"zos flush: after deflate: %d bytes\n",
 //                zs->next_out-underlying->getptr());
@@ -119,7 +124,11 @@ int ZlibOutStream::overrun(int itemSize, int nItems)
 //        fprintf(stderr,"zos overrun: calling deflate, avail_in %d, avail_out %d\n",
 //                zs->avail_in,zs->avail_out);
 
-      checkCompressionLevel();
+     checkCompressionLevel();
+     if (zs->avail_in != 0) {
+       int rc = deflate(zs, 0);
+       if (rc != Z_OK) throw Exception("ZlibOutStream: deflate failed");
+     }
 
 //        fprintf(stderr,"zos overrun: after deflate: %d bytes\n",
 //                zs->next_out-underlying->getptr());
@@ -151,15 +160,9 @@ int ZlibOutStream::overrun(int itemSize, int nItems)
 void ZlibOutStream::checkCompressionLevel()
 {
   if (newLevel != compressionLevel) {
-    int rc = deflate(zs, Z_SYNC_FLUSH);
-    if (rc != Z_OK) 
-      throw Exception("ZlibOutStream: deflate failed");
-    if (deflateParams (zs, newLevel, Z_DEFAULT_STRATEGY) != Z_OK)
+    if (deflateParams (zs, newLevel, Z_DEFAULT_STRATEGY) != Z_OK) {
       throw Exception("ZlibOutStream: deflateParams failed");
+    }
     compressionLevel = newLevel;
-  } else {
-    int rc = deflate(zs, Z_SYNC_FLUSH);
-    if (rc != Z_OK) 
-      throw Exception("ZlibOutStream: deflate failed");
   }
 }
diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h
index 2e6d0d88..9877759a 100644
--- a/common/rfb/tightEncode.h
+++ b/common/rfb/tightEncode.h
@@ -1,4 +1,5 @@
 /* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -188,7 +189,9 @@ static void compressData(rdr::OutStream *os, rdr::ZlibOutStream *zos,
   } else {
     // FIXME: Using a temporary MemOutStream may be not efficient.
     //        Maybe use the same static object used in the JPEG coder?
-    rdr::MemOutStream mem_os;
+    int maxBeforeSize = s_pconf->maxRectSize * (BPP / 8);
+    int maxAfterSize = maxBeforeSize + (maxBeforeSize + 99) / 100 + 12;
+    rdr::MemOutStream mem_os(maxAfterSize);
     zos->setUnderlying(&mem_os);
     zos->setCompressionLevel(zlibLevel);
     zos->writeBytes(buf, length);