Merge further fixes to Zlib encoder from 1.1 branch

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4623 3789f03b-4d11-0410-bbf8-ca57d06f2519

3 files changed, 35 insertions, 11 deletions

diff --git a/common/rdr/ZlibOutStream.cxx b/common/rdr/ZlibOutStream.cxx
index c86a5a55..f04536fa 100644
--- a/common/rdr/ZlibOutStream.cxx
+++ b/common/rdr/ZlibOutStream.cxx
@@ -29,7 +29,8 @@ enum { DEFAULT_BUF_SIZE = 16384 };
 
 ZlibOutStream::ZlibOutStream(OutStream* os, int bufSize_, int compressLevel)
   : underlying(os), compressionLevel(compressLevel), newLevel(compressLevel),
-    bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_SIZE), offset(0)
+    bufSize(bufSize_ ? bufSize_ : DEFAULT_BUF_SIZE), offset(0),
+    newBehavior(false)
 {
   zs = new z_stream;
   zs->zalloc    = Z_NULL;
@@ -41,6 +42,8 @@ ZlibOutStream::ZlibOutStream(OutStream* os, int bufSize_, int compressLevel)
   }
   ptr = start = new U8[bufSize];
   end = start + bufSize;
+  const char *version = zlibVersion();
+  if (strcmp(version, "1.2.3") > 0) newBehavior = true;
 }
 
 ZlibOutStream::~ZlibOutStream()
@@ -79,6 +82,9 @@ void ZlibOutStream::flush()
 
 //    fprintf(stderr,"zos flush: avail_in %d\n",zs->avail_in);
 
+  if (!underlying)
+    throw Exception("ZlibOutStream: underlying OutStream has not been set");
+
   while (zs->avail_in != 0) {
 
     do {
@@ -112,6 +118,9 @@ int ZlibOutStream::overrun(int itemSize, int nItems)
   if (itemSize > bufSize)
     throw Exception("ZlibOutStream overrun: max itemSize exceeded");
 
+  if (!underlying)
+    throw Exception("ZlibOutStream: underlying OutStream has not been set");
+
   while (end - ptr < itemSize) {
     zs->next_in = start;
     zs->avail_in = ptr - start;
@@ -124,11 +133,11 @@ int ZlibOutStream::overrun(int itemSize, int nItems)
 //        fprintf(stderr,"zos overrun: calling deflate, avail_in %d, avail_out %d\n",
 //                zs->avail_in,zs->avail_out);
 
-     checkCompressionLevel();
-     if (zs->avail_in != 0) {
-       int rc = deflate(zs, 0);
-       if (rc != Z_OK) throw Exception("ZlibOutStream: deflate failed");
-     }
+      checkCompressionLevel();
+      if (zs->avail_in != 0) {
+        int rc = deflate(zs, 0);
+        if (rc != Z_OK) throw Exception("ZlibOutStream: deflate failed");
+      }
 
 //        fprintf(stderr,"zos overrun: after deflate: %d bytes\n",
 //                zs->next_out-underlying->getptr());
@@ -160,6 +169,17 @@ int ZlibOutStream::overrun(int itemSize, int nItems)
 void ZlibOutStream::checkCompressionLevel()
 {
   if (newLevel != compressionLevel) {
+
+    // This is a horrible hack, but after many hours of trying, I couldn't find
+    // a better way to make this class work properly with both Zlib 1.2.3 and
+    // 1.2.5.  1.2.3 does a Z_PARTIAL_FLUSH in the body of deflateParams() if
+    // the compression level has changed, and 1.2.5 does a Z_BLOCK flush.
+
+    if (newBehavior) {
+      int rc = deflate(zs, Z_SYNC_FLUSH);
+      if (rc != Z_OK) throw Exception("ZlibOutStream: deflate failed");
+    }
+
     if (deflateParams (zs, newLevel, Z_DEFAULT_STRATEGY) != Z_OK) {
       throw Exception("ZlibOutStream: deflateParams failed");
     }
diff --git a/common/rdr/ZlibOutStream.h b/common/rdr/ZlibOutStream.h
index 8027d27a..250bb035 100644
--- a/common/rdr/ZlibOutStream.h
+++ b/common/rdr/ZlibOutStream.h
@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -54,6 +55,7 @@ namespace rdr {
     int offset;
     z_stream_s* zs;
     U8* start;
+    bool newBehavior;
   };
 
 } // end of namespace rdr
diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h
index 9877759a..28d6dc2b 100644
--- a/common/rfb/tightEncode.h
+++ b/common/rfb/tightEncode.h
@@ -182,20 +182,22 @@ static int paletteInsert(rdr::U32 rgb, int numPixels, int bpp)
 //
 
 static void compressData(rdr::OutStream *os, rdr::ZlibOutStream *zos,
-                         const void *buf, unsigned int length, int zlibLevel)
+                         const void *buf, const PixelFormat& pf,
+                         unsigned int length, int zlibLevel)
 {
   if (length < TIGHT_MIN_TO_COMPRESS) {
     os->writeBytes(buf, length);
   } else {
     // FIXME: Using a temporary MemOutStream may be not efficient.
     //        Maybe use the same static object used in the JPEG coder?
-    int maxBeforeSize = s_pconf->maxRectSize * (BPP / 8);
+    int maxBeforeSize = s_pconf->maxRectSize * (pf.bpp / 8);
     int maxAfterSize = maxBeforeSize + (maxBeforeSize + 99) / 100 + 12;
     rdr::MemOutStream mem_os(maxAfterSize);
     zos->setUnderlying(&mem_os);
     zos->setCompressionLevel(zlibLevel);
     zos->writeBytes(buf, length);
     zos->flush();
+    zos->setUnderlying(NULL);
     os->writeCompactLength(mem_os.length());
     os->writeBytes(mem_os.data(), mem_os.length());
   }
@@ -393,7 +395,7 @@ static void ENCODE_FULLCOLOR_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4]
   os->writeU8(streamId << 4);
 
   int length = PACK_PIXELS(buf, r.area(), pf);
-  compressData(os, &zos[streamId], buf, length, s_pconf->rawZlibLevel);
+  compressData(os, &zos[streamId], buf, pf, length, s_pconf->rawZlibLevel);
 }
 
 static void ENCODE_MONO_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
@@ -459,7 +461,7 @@ static void ENCODE_MONO_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
   // Write the data
   int length = (w + 7) / 8;
   length *= h;
-  compressData(os, &zos[streamId], buf, length, s_pconf->monoZlibLevel);
+  compressData(os, &zos[streamId], buf, pf, length, s_pconf->monoZlibLevel);
 }
 
 #if (BPP != 8)
@@ -507,7 +509,7 @@ static void ENCODE_INDEXED_RECT (rdr::OutStream *os, rdr::ZlibOutStream zos[4],
   }
 
   // Write the data
-  compressData(os, &zos[streamId], buf, r.area(), s_pconf->idxZlibLevel);
+  compressData(os, &zos[streamId], buf, pf, r.area(), s_pconf->idxZlibLevel);
 }
 #endif  // #if (BPP != 8)