Make the subsampling setting follow the common style

8 files changed, 64 insertions, 45 deletions

diff --git a/common/rfb/ConnParams.cxx b/common/rfb/ConnParams.cxx
index 7f39e2c6..4d14dac5 100644
--- a/common/rfb/ConnParams.cxx
+++ b/common/rfb/ConnParams.cxx
@@ -1,5 +1,6 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -37,7 +38,7 @@ ConnParams::ConnParams()
     supportsContinuousUpdates(false),
     customCompressLevel(false), compressLevel(2),
     noJpeg(false), qualityLevel(-1), fineQualityLevel(-1),
-    subsampling(SUBSAMP_UNDEFINED), name_(0),
+    subsampling(subsampleUndefined), name_(0),
     currentEncoding_(encodingRaw), verStrPos(0)
 {
   setName("");
@@ -99,7 +100,7 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
   noJpeg = true;
   qualityLevel = -1;
   fineQualityLevel = -1;
-  subsampling = SUBSAMP_UNDEFINED;
+  subsampling = subsampleUndefined;
   currentEncoding_ = encodingRaw;
 
   for (int i = nEncodings-1; i >= 0; i--) {
@@ -141,9 +142,28 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
       noJpeg = false;
       fineQualityLevel = encodings[i] - pseudoEncodingFineQualityLevel0;
     } else if (encodings[i] >= pseudoEncodingSubsamp1X &&
-               encodings[i] <= pseudoEncodingSubsampGray) {
+               encodings[i] <= pseudoEncodingSubsamp16X) {
       noJpeg = false;
-      subsampling = (JPEG_SUBSAMP)(encodings[i] - pseudoEncodingSubsamp1X);
+      switch (encodings[i]) {
+      case pseudoEncodingSubsamp1X:
+        subsampling = subsampleNone;
+        break;
+      case pseudoEncodingSubsampGray:
+        subsampling = subsampleGray;
+        break;
+      case pseudoEncodingSubsamp2X:
+        subsampling = subsample2X;
+        break;
+      case pseudoEncodingSubsamp4X:
+        subsampling = subsample4X;
+        break;
+      case pseudoEncodingSubsamp8X:
+        subsampling = subsample8X;
+        break;
+      case pseudoEncodingSubsamp16X:
+        subsampling = subsample16X;
+        break;
+      }
     }
   }
 }
diff --git a/common/rfb/ConnParams.h b/common/rfb/ConnParams.h
index 47ec14a3..68e20754 100644
--- a/common/rfb/ConnParams.h
+++ b/common/rfb/ConnParams.h
@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -25,12 +26,19 @@
 #include <rdr/types.h>
 #include <rfb/PixelFormat.h>
 #include <rfb/ScreenSet.h>
-#include <rfb/JpegCompressor.h>
 
 namespace rdr { class InStream; }
 
 namespace rfb {
 
+  const int subsampleUndefined = -1;
+  const int subsampleNone = 0;
+  const int subsampleGray = 1;
+  const int subsample2X = 2;
+  const int subsample4X = 3;
+  const int subsample8X = 4;
+  const int subsample16X = 5;
+
   class ConnParams {
   public:
     ConnParams();
@@ -88,7 +96,7 @@ namespace rfb {
     bool noJpeg;
     int qualityLevel;
     int fineQualityLevel;
-    JPEG_SUBSAMP subsampling;
+    int subsampling;
 
   private:
 
diff --git a/common/rfb/Encoder.h b/common/rfb/Encoder.h
index 882074d5..26d57963 100644
--- a/common/rfb/Encoder.h
+++ b/common/rfb/Encoder.h
@@ -22,7 +22,6 @@
 
 #include <rfb/Rect.h>
 #include <rfb/TransImageGetter.h>
-#include <rfb/JpegCompressor.h>
 
 namespace rfb {
   class SMsgWriter;
@@ -34,7 +33,7 @@ namespace rfb {
 
     virtual void setCompressLevel(int level) {};
     virtual void setQualityLevel(int level) {};
-    virtual void setFineQualityLevel(int quality, JPEG_SUBSAMP subsampling) {};
+    virtual void setFineQualityLevel(int quality, int subsampling) {};
     virtual int getNumRects(const Rect &r) { return 1; }
 
     // writeRect() tries to write the given rectangle.  If it is unable to
diff --git a/common/rfb/JpegCompressor.cxx b/common/rfb/JpegCompressor.cxx
index 47f09d81..7dbb7d94 100644
--- a/common/rfb/JpegCompressor.cxx
+++ b/common/rfb/JpegCompressor.cxx
@@ -21,6 +21,7 @@
 #include <rdr/Exception.h>
 #include <rfb/Rect.h>
 #include <rfb/PixelFormat.h>
+#include <rfb/ConnParams.h>
 #include <os/print.h>
 
 #include <stdio.h>
@@ -142,7 +143,7 @@ JpegCompressor::~JpegCompressor(void)
 }
 
 void JpegCompressor::compress(const rdr::U8 *buf, int pitch, const Rect& r,
-  const PixelFormat& pf, int quality, JPEG_SUBSAMP subsamp)
+  const PixelFormat& pf, int quality, int subsamp)
 {
   int w = r.width();
   int h = r.height();
@@ -217,15 +218,18 @@ void JpegCompressor::compress(const rdr::U8 *buf, int pitch, const Rect& r,
   }
 
   switch (subsamp) {
-  case SUBSAMP_420:
+  case subsample16X:
+  case subsample8X:
+    // FIXME (fall through)
+  case subsample4X:
     cinfo->comp_info[0].h_samp_factor = 2;
     cinfo->comp_info[0].v_samp_factor = 2;
     break;
-  case SUBSAMP_422:
+  case subsample2X:
     cinfo->comp_info[0].h_samp_factor = 2;
     cinfo->comp_info[0].v_samp_factor = 1;
     break;
-  case SUBSAMP_GRAY:
+  case subsampleGray:
     jpeg_set_colorspace(cinfo, JCS_GRAYSCALE);
   default:
     cinfo->comp_info[0].h_samp_factor = 1;
diff --git a/common/rfb/JpegCompressor.h b/common/rfb/JpegCompressor.h
index d50e5871..8fbb7a9e 100644
--- a/common/rfb/JpegCompressor.h
+++ b/common/rfb/JpegCompressor.h
@@ -36,14 +36,6 @@ struct JPEG_DEST_MGR;
 
 namespace rfb {
 
-  enum JPEG_SUBSAMP {
-    SUBSAMP_UNDEFINED = -1,
-    SUBSAMP_NONE = 0,
-    SUBSAMP_420,
-    SUBSAMP_422,
-    SUBSAMP_GRAY
-  };
-
   class JpegCompressor : public rdr::MemOutStream {
 
   public:
@@ -51,8 +43,7 @@ namespace rfb {
     JpegCompressor(int bufferLen = 128*1024);
     virtual ~JpegCompressor();
 
-    void compress(const rdr::U8 *, int, const Rect&, const PixelFormat&, int,
-      JPEG_SUBSAMP);
+    void compress(const rdr::U8 *, int, const Rect&, const PixelFormat&, int, int);
 
     void writeBytes(const void*, int);
 
diff --git a/common/rfb/TightEncoder.cxx b/common/rfb/TightEncoder.cxx
index 61a1c590..733365e0 100644
--- a/common/rfb/TightEncoder.cxx
+++ b/common/rfb/TightEncoder.cxx
@@ -64,16 +64,16 @@ using namespace rfb;
 // 0 = JPEG quality 15,  4:2:0 subsampling (ratio ~= 100:1)
 
 const TIGHT_CONF TightEncoder::conf[10] = {
-  { 65536, 2048,   6, 0, 0, 0,   4, 24, 15, SUBSAMP_420 }, // 0
-  { 65536, 2048,   6, 1, 1, 1,   8, 24, 29, SUBSAMP_420 }, // 1
-  { 65536, 2048,   8, 3, 3, 2,  24, 96, 41, SUBSAMP_420 }, // 2
-  { 65536, 2048,  12, 5, 5, 2,  32, 96, 42, SUBSAMP_422 }, // 3
-  { 65536, 2048,  12, 6, 7, 3,  32, 96, 62, SUBSAMP_422 }, // 4
-  { 65536, 2048,  12, 7, 8, 4,  32, 96, 77, SUBSAMP_422 }, // 5
-  { 65536, 2048,  16, 7, 8, 5,  32, 96, 79, SUBSAMP_NONE }, // 6
-  { 65536, 2048,  16, 8, 9, 6,  64, 96, 86, SUBSAMP_NONE }, // 7
-  { 65536, 2048,  24, 9, 9, 7,  64, 96, 92, SUBSAMP_NONE }, // 8
-  { 65536, 2048,  32, 9, 9, 9,  96, 96,100, SUBSAMP_NONE }  // 9
+  { 65536, 2048,   6, 0, 0, 0,   4, 24, 15, subsample4X }, // 0
+  { 65536, 2048,   6, 1, 1, 1,   8, 24, 29, subsample4X }, // 1
+  { 65536, 2048,   8, 3, 3, 2,  24, 96, 41, subsample4X }, // 2
+  { 65536, 2048,  12, 5, 5, 2,  32, 96, 42, subsample2X }, // 3
+  { 65536, 2048,  12, 6, 7, 3,  32, 96, 62, subsample2X }, // 4
+  { 65536, 2048,  12, 7, 8, 4,  32, 96, 77, subsample2X }, // 5
+  { 65536, 2048,  16, 7, 8, 5,  32, 96, 79, subsampleNone }, // 6
+  { 65536, 2048,  16, 8, 9, 6,  64, 96, 86, subsampleNone }, // 7
+  { 65536, 2048,  24, 9, 9, 7,  64, 96, 92, subsampleNone }, // 8
+  { 65536, 2048,  32, 9, 9, 9,  96, 96,100, subsampleNone }  // 9
 };
 
 const int TightEncoder::defaultCompressLevel = 2;
@@ -118,18 +118,14 @@ void TightEncoder::setQualityLevel(int level)
     jpegSubsampling = conf[level].jpegSubsampling;
   } else {
     jpegQuality = -1;
-    jpegSubsampling = SUBSAMP_UNDEFINED;
+    jpegSubsampling = subsampleUndefined;
   }
 }
 
-void TightEncoder::setFineQualityLevel(int quality, JPEG_SUBSAMP subsampling)
+void TightEncoder::setFineQualityLevel(int quality, int subsampling)
 {
-  if (quality >= 1 && quality <= 100) {
-    jpegQuality = quality;
-  }
-  if (subsampling >= SUBSAMP_NONE && subsampling <= SUBSAMP_GRAY) {
-    jpegSubsampling = subsampling;
-  }
+  jpegQuality = quality;
+  jpegSubsampling = subsampling;
 }
 
 bool TightEncoder::checkSolidTile(Rect& r, rdr::U32* colorPtr,
@@ -341,7 +337,7 @@ bool TightEncoder::writeRect(const Rect& _r, TransImageGetter* _ig,
       sr.setXYWH(dx, dy, dw, dh);
       if (checkSolidTile(sr, &colorValue, false)) {
 
-         if (jpegSubsampling == SUBSAMP_GRAY && jpegQuality != -1) {
+         if (jpegSubsampling == subsampleGray && jpegQuality != -1) {
            Colour rgb;
            serverpf.rgbFromPixel(colorValue, NULL, &rgb);
            rdr::U32 lum = ((257 * rgb.r) + (504 * rgb.g) + (98 * rgb.b)
diff --git a/common/rfb/TightEncoder.h b/common/rfb/TightEncoder.h
index 5faef444..3632539d 100644
--- a/common/rfb/TightEncoder.h
+++ b/common/rfb/TightEncoder.h
@@ -22,6 +22,7 @@
 #include <rdr/MemOutStream.h>
 #include <rdr/ZlibOutStream.h>
 #include <rfb/Encoder.h>
+#include <rfb/JpegCompressor.h>
 
 // FIXME: Check if specifying extern "C" is really necessary.
 #include <stdio.h>
@@ -40,7 +41,7 @@ namespace rfb {
     int idxMaxColorsDivisor;
     int palMaxColorsWithJPEG;
     int jpegQuality;
-    JPEG_SUBSAMP jpegSubsampling;
+    int jpegSubsampling;
   };
 
   //
@@ -83,7 +84,7 @@ namespace rfb {
 
     virtual void setCompressLevel(int level);
     virtual void setQualityLevel(int level);
-    virtual void setFineQualityLevel(int quality, JPEG_SUBSAMP subsampling);
+    virtual void setFineQualityLevel(int quality, int subsampling);
     virtual int getNumRects(const Rect &r);
     virtual bool writeRect(const Rect& r, TransImageGetter* ig, Rect* actual);
 
@@ -158,7 +159,7 @@ namespace rfb {
 
     const TIGHT_CONF* pconf;
     int jpegQuality;
-    JPEG_SUBSAMP jpegSubsampling;
+    int jpegSubsampling;
   };
 
 }
diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h
index 0c616f53..6a59ba81 100644
--- a/common/rfb/tightEncode.h
+++ b/common/rfb/tightEncode.h
@@ -193,7 +193,7 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
   rdr::U32 solidColor;
   const PIXEL_T *rawPixels = (const PIXEL_T *)ig->getRawBufferR(r, &stride);
   PIXEL_T *pixels = NULL;
-  bool grayScaleJPEG = (jpegSubsampling == SUBSAMP_GRAY && jpegQuality != -1);
+  bool grayScaleJPEG = (jpegSubsampling == subsampleGray && jpegQuality != -1);
 
 #if (BPP == 32)
   // Check if it's necessary to pack 24-bit pixels, and