Fix regression caused by r4841. That patch assumed that JPEG encoding always uses...

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4852 3789f03b-4d11-0410-bbf8-ca57d06f2519

diff --git a/common/rfb/TightEncoder.cxx b/common/rfb/TightEncoder.cxx
index fcd9d9b001694916c449c34b40e2f6a67329cf9f..9be4581e9bd0bf5fb877f2c4738656532b3141c2 100644 (file)
--- a/common/rfb/TightEncoder.cxx
+++ b/common/rfb/TightEncoder.cxx
@@ -418,19 +418,3 @@ void TightEncoder::writeSubrect(const Rect& r, bool forceSolid)
   os->writeBytes(mos.data(), mos.length());
   writer->endRect();
 }
-
-void TightEncoder::encodeJpegRect(const Rect& r, rdr::OutStream *os)
-{
-  const rdr::U8 *buf;
-  int stride;
-
-  buf = ig->getRawPixelsR(r, &stride);
-
-  jc.clear();
-  jc.compress(buf, stride * serverpf.bpp / 8, r, serverpf,
-              jpegQuality, jpegSubsampling);
-
-  os->writeU8(0x09 << 4);
-  os->writeCompactLength(jc.length());
-  os->writeBytes(jc.data(), jc.length());
-}

diff --git a/common/rfb/TightEncoder.h b/common/rfb/TightEncoder.h
index 10ca761651f52bfa4d363ccb6afec17f275fb2f2..4fff0832b8896eb459edf2c0b6346a33e293808a 100644 (file)
--- a/common/rfb/TightEncoder.h
+++ b/common/rfb/TightEncoder.h
@@ -100,9 +100,9 @@ namespace rfb {
     int paletteInsert(rdr::U32 rgb, int numPixels, int bpp);
     void paletteReset(void);
 
-    void fastFillPalette8(const rdr::U8 *buffer, int stride, const Rect &r);
-    void fastFillPalette16(const rdr::U8 *buffer, int stride, const Rect &r);
-    void fastFillPalette32(const rdr::U8 *buffer, int stride, const Rect &r);
+    void fastFillPalette8(const rdr::U8 *data, int stride, const Rect &r);
+    void fastFillPalette16(const rdr::U16 *data, int stride, const Rect &r);
+    void fastFillPalette32(const rdr::U32 *data, int stride, const Rect &r);
 
     void fillPalette8(rdr::U8 *data, int count);
     void fillPalette16(rdr::U16 *data, int count);
@@ -135,7 +135,10 @@ namespace rfb {
     void encodeIndexedRect16(rdr::U16 *buf, const Rect& r, rdr::OutStream *os);
     void encodeIndexedRect32(rdr::U32 *buf, const Rect& r, rdr::OutStream *os);
 
-    void encodeJpegRect(const Rect& r, rdr::OutStream *os);
+    void encodeJpegRect16(rdr::U16 *buf, int stride, const Rect& r,
+                          rdr::OutStream *os);
+    void encodeJpegRect32(rdr::U32 *buf, int stride, const Rect& r,
+                          rdr::OutStream *os);
 
     SMsgWriter* writer;
     rdr::MemOutStream mos;

diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h
index 446d45e7f5b52d6721e932fe5b77c0e4e3988ed2..8f900b580713d681d3fb0fba4693b2f722f18a8b 100644 (file)
--- a/common/rfb/tightEncode.h
+++ b/common/rfb/tightEncode.h
@@ -191,8 +191,9 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
 {
   int stride;
   rdr::U32 solidColor;
-  const rdr::U8 *rawPixels;
-  PIXEL_T *pixels;
+  const PIXEL_T *rawPixels = (const PIXEL_T *)ig->getRawPixelsR(r, &stride);
+  PIXEL_T *pixels = NULL;
+  bool grayScaleJPEG = (jpegSubsampling == SUBSAMP_GRAY && jpegQuality != -1);
 
 #if (BPP == 32)
   // Check if it's necessary to pack 24-bit pixels, and
@@ -200,45 +201,41 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
   pack24 = clientpf.is888();
 #endif
 
-  rawPixels = ig->getRawPixelsR(r, &stride);
-  pixels = NULL;
-
   if (forceSolid) {
-    // Forced solid block
+    // Subrectangle has already been determined to be solid.
     palNumColors = 1;
     ig->translatePixels(rawPixels, &solidColor, 1);
     pixels = (PIXEL_T *)&solidColor;
-  } else if (jpegSubsampling == SUBSAMP_GRAY && jpegQuality != -1) {
-    // Forced gray scale (JPEG)
-    palNumColors = 0;
   } else {
-    // Normal analysis
+    // Analyze subrectangle's colors to determine best encoding method.
     palMaxColors = r.area() / pconf->idxMaxColorsDivisor;
-
     if (jpegQuality != -1)
       palMaxColors = pconf->palMaxColorsWithJPEG;
-
     if (palMaxColors < 2 && r.area() >= pconf->monoMinRectSize)
       palMaxColors = 2;
 
     if (clientpf.equal(serverpf) && clientpf.bpp >= 16) {
-      // No conversion, so just analyse the raw buffer
-      FAST_FILL_PALETTE(rawPixels, stride, r);
+      // Count the colors in the raw buffer, so we can avoid unnecessary pixel
+      // translation when encoding with JPEG.
+      if (grayScaleJPEG) palNumColors = 0;
+      else FAST_FILL_PALETTE(rawPixels, stride, r);
 
-      // JPEG reads from the raw buffer and has its own output buffer,
-      // but the other encodings need some help.
+      // JPEG can read from the raw buffer, but for the other methods, we need
+      // to translate the raw pixels into an intermediate buffer.
       if(palNumColors != 0 || jpegQuality == -1) {
         pixels = (PIXEL_T *)writer->getImageBuf(r.area());
         stride = r.width();
         ig->getImage(pixels, r);
       }
     } else {
-      // Need to do the conversion first so we have something to analyse
+      // Pixel translation will be required, so create an intermediate buffer,
+      // translate the raw pixels into it, and count its colors.
       pixels = (PIXEL_T *)writer->getImageBuf(r.area());
       stride = r.width();
       ig->getImage(pixels, r);
 
-      FILL_PALETTE(pixels, r.area());
+      if (grayScaleJPEG) palNumColors = 0;
+      else FILL_PALETTE(pixels, r.area());
     }
   }
 
@@ -247,7 +244,10 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
     // Truecolor image
 #if (BPP != 8)
     if (jpegQuality != -1) {
-      encodeJpegRect(r, os);
+      if (pixels)
+        ENCODE_JPEG_RECT(pixels, stride, r, os);
+      else
+        ENCODE_JPEG_RECT((PIXEL_T *)rawPixels, stride, r, os);
       break;
     }
 #endif
@@ -403,6 +403,23 @@ void ENCODE_INDEXED_RECT (PIXEL_T *buf, const Rect& r, rdr::OutStream *os)
 }
 #endif  // #if (BPP != 8)
 
+//
+// JPEG compression.
+//
+
+#if (BPP != 8)
+void ENCODE_JPEG_RECT (PIXEL_T *buf, int stride, const Rect& r,
+                       rdr::OutStream *os)
+{
+  jc.clear();
+  jc.compress((rdr::U8 *)buf, stride * clientpf.bpp / 8, r, clientpf,
+    jpegQuality, jpegSubsampling);
+  os->writeU8(0x09 << 4);
+  os->writeCompactLength(jc.length());
+  os->writeBytes(jc.data(), jc.length());
+}
+#endif  // #if (BPP != 8)
+
 //
 // Determine the number of colors in the rectangle, and fill in the palette.
 //
@@ -449,7 +466,7 @@ void FILL_PALETTE (PIXEL_T *data, int count)
   }
 }
 
-void FAST_FILL_PALETTE (const rdr::U8 *buffer, int stride, const Rect& r)
+void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r)
 {
 }
 
@@ -514,17 +531,15 @@ void FILL_PALETTE (PIXEL_T *data, int count)
   paletteInsert (ci, (rdr::U32)ni, BPP);
 }
 
-void FAST_FILL_PALETTE (const rdr::U8 *buffer, int stride, const Rect& r)
+void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r)
 {
   PIXEL_T c0, c1, ci = 0, mask, c0t, c1t, cit;
   int n0, n1, ni;
   int w = r.width(), h = r.height();
-  const PIXEL_T *data, *rowptr, *colptr, *rowptr2, *colptr2, *dataend;
+  const PIXEL_T *rowptr, *colptr, *rowptr2, *colptr2,
+    *dataend = &data[stride * h];
   bool willTransform = ig->willTransform();
 
-  data = (const PIXEL_T*)buffer;
-  dataend = &data[stride * h];
-
   if (willTransform) {
     mask = serverpf.redMax << serverpf.redShift;
     mask |= serverpf.greenMax << serverpf.greenShift;