Add support for TurboVNC pseudo-encodings and Grayscale JPEG compression so that, when a TurboVNC viewer is connected, the TigerVNC Server will behave exactly like the TurboVNC Server.

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4641 3789f03b-4d11-0410-bbf8-ca57d06f2519

10 files changed, 77 insertions, 12 deletions

diff --git a/common/rfb/ConnParams.cxx b/common/rfb/ConnParams.cxx
index a6368851..e44facc9 100644
--- a/common/rfb/ConnParams.cxx
+++ b/common/rfb/ConnParams.cxx
@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -34,7 +35,8 @@ ConnParams::ConnParams()
     supportsDesktopRename(false), supportsLastRect(false),
     supportsSetDesktopSize(false),
     customCompressLevel(false), compressLevel(6),
-    noJpeg(false), qualityLevel(-1), 
+    noJpeg(false), qualityLevel(-1), fineQualityLevel(-1),
+    subsampling(SUBSAMP_UNDEFINED),
     name_(0), nEncodings_(0), encodings_(0),
     currentEncoding_(encodingRaw), verStrPos(0)
 {
@@ -102,6 +104,8 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
   compressLevel = -1;
   noJpeg = true;
   qualityLevel = -1;
+  fineQualityLevel = -1;
+  subsampling = SUBSAMP_UNDEFINED;
   currentEncoding_ = encodingRaw;
 
   for (int i = nEncodings-1; i >= 0; i--) {
@@ -132,4 +136,18 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
     } else if (Encoder::supported(encodings[i]))
       currentEncoding_ = encodings[i];
   }
+
+  // If the TurboVNC fine quality/subsampling encodings exist, let them
+  // override the coarse TightVNC quality level
+  for (int i = nEncodings-1; i >= 0; i--) {
+    if (encodings[i] >= pseudoEncodingFineQualityLevel0 + 1 &&
+        encodings[i] <= pseudoEncodingFineQualityLevel100) {
+      noJpeg = false;
+      fineQualityLevel = encodings[i] - pseudoEncodingFineQualityLevel0;
+    } else if (encodings[i] >= pseudoEncodingSubsamp1X &&
+               encodings[i] <= pseudoEncodingSubsampGray) {
+      noJpeg = false;
+      subsampling = (JPEG_SUBSAMP)(encodings[i] - pseudoEncodingSubsamp1X);
+    }
+  }
 }
diff --git a/common/rfb/ConnParams.h b/common/rfb/ConnParams.h
index c25e5630..f81a3584 100644
--- a/common/rfb/ConnParams.h
+++ b/common/rfb/ConnParams.h
@@ -25,6 +25,7 @@
 #include <rdr/types.h>
 #include <rfb/PixelFormat.h>
 #include <rfb/ScreenSet.h>
+#include <rfb/JpegCompressor.h>
 
 namespace rdr { class InStream; }
 
@@ -84,6 +85,8 @@ namespace rfb {
     int compressLevel;
     bool noJpeg;
     int qualityLevel;
+    int fineQualityLevel;
+    JPEG_SUBSAMP subsampling;
 
   private:
 
diff --git a/common/rfb/Encoder.h b/common/rfb/Encoder.h
index 893c0138..da2c5c09 100644
--- a/common/rfb/Encoder.h
+++ b/common/rfb/Encoder.h
@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,6 +22,7 @@
 #include <rfb/Rect.h>
 #include <rfb/encodings.h>
 #include <rfb/TransImageGetter.h>
+#include <rfb/JpegCompressor.h>
 
 namespace rfb {
   class SMsgWriter;
@@ -34,6 +36,7 @@ namespace rfb {
 
     virtual void setCompressLevel(int level) {};
     virtual void setQualityLevel(int level) {};
+    virtual void setFineQualityLevel(int quality, JPEG_SUBSAMP subsampling) {};
     virtual int getNumRects(const Rect &r) { return 1; }
 
     // writeRect() tries to write the given rectangle.  If it is unable to
diff --git a/common/rfb/JpegCompressor.cxx b/common/rfb/JpegCompressor.cxx
index 7a870b0d..33f6c418 100644
--- a/common/rfb/JpegCompressor.cxx
+++ b/common/rfb/JpegCompressor.cxx
@@ -194,6 +194,8 @@ void JpegCompressor::compress(rdr::U8 *buf, int pitch, const Rect& r,
     cinfo.comp_info[0].h_samp_factor = 2;
     cinfo.comp_info[0].v_samp_factor = 1;
     break;
+  case SUBSAMP_GRAY:
+    jpeg_set_colorspace(&cinfo, JCS_GRAYSCALE);
   default:
     cinfo.comp_info[0].h_samp_factor = 1;
     cinfo.comp_info[0].v_samp_factor = 1;
diff --git a/common/rfb/JpegCompressor.h b/common/rfb/JpegCompressor.h
index 6860b415..668c9a00 100644
--- a/common/rfb/JpegCompressor.h
+++ b/common/rfb/JpegCompressor.h
@@ -51,9 +51,11 @@ namespace rfb {
   } JPEG_DEST_MGR;
 
   enum JPEG_SUBSAMP {
-    SUBSAMP_NONE,
+    SUBSAMP_UNDEFINED = -1,
+    SUBSAMP_NONE = 0,
+    SUBSAMP_420,
     SUBSAMP_422,
-    SUBSAMP_420
+    SUBSAMP_GRAY
   };
 
   class JpegCompressor : public rdr::MemOutStream {
diff --git a/common/rfb/SMsgWriter.cxx b/common/rfb/SMsgWriter.cxx
index 07ae37d5..f47f3eeb 100644
--- a/common/rfb/SMsgWriter.cxx
+++ b/common/rfb/SMsgWriter.cxx
@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -102,6 +103,8 @@ void SMsgWriter::setupCurrentEncoder()
 
   encoders[encoding]->setCompressLevel(cp->compressLevel);
   encoders[encoding]->setQualityLevel(cp->qualityLevel);
+  encoders[encoding]->setFineQualityLevel(cp->fineQualityLevel,
+                                          cp->subsampling);
 }
 
 int SMsgWriter::getNumRects(const Rect &r)
diff --git a/common/rfb/TightEncoder.cxx b/common/rfb/TightEncoder.cxx
index e35bad62..b7ef2236 100644
--- a/common/rfb/TightEncoder.cxx
+++ b/common/rfb/TightEncoder.cxx
@@ -117,9 +117,21 @@ void TightEncoder::setCompressLevel(int level)
 void TightEncoder::setQualityLevel(int level)
 {
   if (level >= 0 && level <= 9) {
-    pjconf = &conf[level];
+    jpegQuality = conf[level].jpegQuality;
+    jpegSubsampling = conf[level].jpegSubsampling;
   } else {
-    pjconf = NULL;
+    jpegQuality = -1;
+    jpegSubsampling = SUBSAMP_UNDEFINED;
+  }
+}
+
+void TightEncoder::setFineQualityLevel(int quality, JPEG_SUBSAMP subsampling)
+{
+  if (quality >= 1 && quality <= 100) {
+    jpegQuality = quality;
+  }
+  if (subsampling >= SUBSAMP_NONE && subsampling <= SUBSAMP_GRAY) {
+    jpegSubsampling = subsampling;
   }
 }
 
@@ -333,6 +345,14 @@ bool TightEncoder::writeRect(const Rect& _r, TransImageGetter* _ig,
       sr.setXYWH(dx, dy, dw, dh);
       if (checkSolidTile(sr, &colorValue, false)) {
 
+         if (jpegSubsampling == SUBSAMP_GRAY && jpegQuality != -1) {
+           Colour rgb;
+           serverpf.rgbFromPixel(colorValue, NULL, &rgb);
+           rdr::U32 lum = ((257 * rgb.r) + (504 * rgb.g) + (98 * rgb.b)
+                           + 16500) / 1000;
+           colorValue = lum + (lum << 8) + (lum << 16);
+         }
+
         // Get dimensions of solid-color area.
         sr.setXYWH(dx, dy, r.br.x - dx, r.br.y - dy);
         findBestSolidArea(sr, colorValue, bestr);
diff --git a/common/rfb/TightEncoder.h b/common/rfb/TightEncoder.h
index ae9672c0..755d8829 100644
--- a/common/rfb/TightEncoder.h
+++ b/common/rfb/TightEncoder.h
@@ -21,7 +21,6 @@
 
 #include <rdr/MemOutStream.h>
 #include <rdr/ZlibOutStream.h>
-#include <rfb/JpegCompressor.h>
 #include <rfb/TransImageGetter.h>
 #include <rfb/Encoder.h>
 
@@ -40,7 +39,7 @@ namespace rfb {
     int idxMaxColorsDivisor;
     int palMaxColorsWithJPEG;
     int jpegQuality;
-    JPEG_SUBSAMP jpegSubSample;
+    JPEG_SUBSAMP jpegSubsampling;
   };
 
   //
@@ -81,6 +80,7 @@ namespace rfb {
     static Encoder* create(SMsgWriter* writer);
     virtual void setCompressLevel(int level);
     virtual void setQualityLevel(int level);
+    virtual void setFineQualityLevel(int quality, JPEG_SUBSAMP subsampling);
     virtual int getNumRects(const Rect &r);
     virtual bool writeRect(const Rect& r, TransImageGetter* ig, Rect* actual);
     virtual ~TightEncoder();
@@ -153,7 +153,8 @@ namespace rfb {
     static const TIGHT_CONF conf[];
 
     const TIGHT_CONF* pconf;
-    const TIGHT_CONF* pjconf;
+    int jpegQuality;
+    JPEG_SUBSAMP jpegSubsampling;
   };
 
 }
diff --git a/common/rfb/encodings.h b/common/rfb/encodings.h
index 16cd73a4..40f5f101 100644
--- a/common/rfb/encodings.h
+++ b/common/rfb/encodings.h
@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyeight (C) 2011 D. R. Commander.  All Rights Reserved.
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -43,6 +44,16 @@ namespace rfb {
   const int pseudoEncodingCompressLevel0 = -256;
   const int pseudoEncodingCompressLevel9 = -247;
 
+  // TurboVNC-specific
+  const int pseudoEncodingFineQualityLevel0 = -512;
+  const int pseudoEncodingFineQualityLevel100 = -412;
+  const int pseudoEncodingSubsamp1X = -768;
+  const int pseudoEncodingSubsamp4X = -767;
+  const int pseudoEncodingSubsamp2X = -766;
+  const int pseudoEncodingSubsampGray = -765;
+  const int pseudoEncodingSubsamp8X = -764;
+  const int pseudoEncodingSubsamp16X = -763;
+
   int encodingNum(const char* name);
   const char* encodingName(int num);
 }
diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h
index 9b2b29d1..7cabb0a7 100644
--- a/common/rfb/tightEncode.h
+++ b/common/rfb/tightEncode.h
@@ -200,9 +200,11 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
 
   if (forceSolid)
     palNumColors = 1;
+  else if (jpegSubsampling == SUBSAMP_GRAY && jpegQuality != -1)
+    palNumColors = 0;
   else {
     palMaxColors = r.area() / pconf->idxMaxColorsDivisor;
-    if (pjconf != NULL) palMaxColors = pconf->palMaxColorsWithJPEG;
+    if (jpegQuality != -1) palMaxColors = pconf->palMaxColorsWithJPEG;
     if (palMaxColors < 2 && r.area() >= pconf->monoMinRectSize) {
       palMaxColors = 2;
     }
@@ -211,7 +213,7 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
       // This is so we can avoid translating the pixels when compressing
       // with JPEG, since it is unnecessary
       FAST_FILL_PALETTE(r, pixels, stride);
-      if(palNumColors != 0 || pjconf == NULL) {
+      if(palNumColors != 0 || jpegQuality == -1) {
         pixels = (PIXEL_T *)writer->getImageBuf(r.area());
         stride = r.width();
         ig->getImage(pixels, r);
@@ -229,7 +231,7 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
   case 0:
     // Truecolor image
 #if (BPP != 8)
-    if (pjconf != NULL) {
+    if (jpegQuality != -1) {
       ENCODE_JPEG_RECT(os, pixels, stride, r);
       break;
     }
@@ -396,7 +398,7 @@ void ENCODE_JPEG_RECT (rdr::OutStream *os, PIXEL_T *buf, int stride,
 {
   jc.clear();
   jc.compress((rdr::U8 *)buf, stride * serverpf.bpp / 8, r, serverpf,
-    pjconf->jpegQuality, pjconf->jpegSubSample);
+    jpegQuality, jpegSubsampling);
   os->writeU8(0x09 << 4);
   os->writeCompactLength(jc.length());
   os->writeBytes(jc.data(), jc.length());