Prevent the Tight encoder (specifically packPixels*()) from accidentally using the...

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/branches/1_2@4851 3789f03b-4d11-0410-bbf8-ca57d06f2519

diff --git a/common/rfb/TightEncoder.h b/common/rfb/TightEncoder.h
index 95c2b9845d9322b150c9da15da20fa1596bccd2d..4fff0832b8896eb459edf2c0b6346a33e293808a 100644 (file)
--- a/common/rfb/TightEncoder.h
+++ b/common/rfb/TightEncoder.h
@@ -100,9 +100,9 @@ namespace rfb {
     int paletteInsert(rdr::U32 rgb, int numPixels, int bpp);
     void paletteReset(void);
 
-    void fastFillPalette8(rdr::U8 *data, int stride, const Rect &r);
-    void fastFillPalette16(rdr::U16 *data, int stride, const Rect &r);
-    void fastFillPalette32(rdr::U32 *data, int stride, const Rect &r);
+    void fastFillPalette8(const rdr::U8 *data, int stride, const Rect &r);
+    void fastFillPalette16(const rdr::U16 *data, int stride, const Rect &r);
+    void fastFillPalette32(const rdr::U32 *data, int stride, const Rect &r);
 
     void fillPalette8(rdr::U8 *data, int count);
     void fillPalette16(rdr::U16 *data, int count);

diff --git a/common/rfb/TransImageGetter.cxx b/common/rfb/TransImageGetter.cxx
index bf26b44d5d8e3f472ccb56942ac753e06bcdc71e..c1add1ced6a7628d0c26d450e4137e3510fd3988 100644 (file)
--- a/common/rfb/TransImageGetter.cxx
+++ b/common/rfb/TransImageGetter.cxx
@@ -56,12 +56,12 @@ void TransImageGetter::setColourMapEntries(int firstCol, int nCols)
   PixelTransformer::setColourMapEntries(firstCol, nCols);
 }
 
-rdr::U8 *TransImageGetter::getRawPixelsRW(const Rect &r, int *stride)
+const rdr::U8 *TransImageGetter::getRawPixelsR(const Rect &r, int *stride)
 {
   if (!offset.equals(Point(0, 0)))
-    return pb->getPixelsRW(r.translate(offset.negate()), stride);
+    return pb->getPixelsR(r.translate(offset.negate()), stride);
   else
-    return pb->getPixelsRW(r, stride);
+    return pb->getPixelsR(r, stride);
 }
 
 void TransImageGetter::getImage(void* outPtr, const Rect& r, int outStride)

diff --git a/common/rfb/TransImageGetter.h b/common/rfb/TransImageGetter.h
index 1ad49b7ce7314e964d06e9205d10201396c631f0..f2b35b45c148085ab0232b1ab7d9a3468b5ef2bf 100644 (file)
--- a/common/rfb/TransImageGetter.h
+++ b/common/rfb/TransImageGetter.h
@@ -72,11 +72,11 @@ namespace rfb {
     // padding will be outStride-r.width() pixels).
     void getImage(void* outPtr, const Rect& r, int outStride=0);
 
-    // getRawPixelsRW() gets the given rectangle of data directly from the
+    // getRawPixelsR() gets the given rectangle of data directly from the
     // underlying PixelBuffer, bypassing the translation logic. Only use
     // this when doing something that's independent of the client's pixel
     // format.
-    rdr::U8 *getRawPixelsRW(const Rect &r, int *stride);
+    const rdr::U8 *getRawPixelsR(const Rect &r, int *stride);
 
     // setPixelBuffer() changes the pixel buffer to be used.  The new pixel
     // buffer MUST have the same pixel format as the old one - if not you

diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h
index a1d5dbe9a74f0e30508f995d32fd8fc38457bfc0..8f900b580713d681d3fb0fba4693b2f722f18a8b 100644 (file)
--- a/common/rfb/tightEncode.h
+++ b/common/rfb/tightEncode.h
@@ -189,9 +189,10 @@ unsigned int PACK_PIXELS (PIXEL_T *buf, unsigned int count)
 
 void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
 {
-  int stride = r.width();
+  int stride;
   rdr::U32 solidColor;
-  PIXEL_T *pixels = (PIXEL_T *)ig->getRawPixelsRW(r, &stride);
+  const PIXEL_T *rawPixels = (const PIXEL_T *)ig->getRawPixelsR(r, &stride);
+  PIXEL_T *pixels = NULL;
   bool grayScaleJPEG = (jpegSubsampling == SUBSAMP_GRAY && jpegQuality != -1);
 
 #if (BPP == 32)
@@ -201,34 +202,38 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
 #endif
 
   if (forceSolid) {
+    // Subrectangle has already been determined to be solid.
     palNumColors = 1;
-    if (ig->willTransform()) {
-      ig->translatePixels(pixels, &solidColor, 1);
-      pixels = (PIXEL_T *)&solidColor;
-    }
-  }
-  else {
+    ig->translatePixels(rawPixels, &solidColor, 1);
+    pixels = (PIXEL_T *)&solidColor;
+  } else {
+    // Analyze subrectangle's colors to determine best encoding method.
     palMaxColors = r.area() / pconf->idxMaxColorsDivisor;
-    if (jpegQuality != -1) palMaxColors = pconf->palMaxColorsWithJPEG;
-    if (palMaxColors < 2 && r.area() >= pconf->monoMinRectSize) {
+    if (jpegQuality != -1)
+      palMaxColors = pconf->palMaxColorsWithJPEG;
+    if (palMaxColors < 2 && r.area() >= pconf->monoMinRectSize)
       palMaxColors = 2;
-    }
 
     if (clientpf.equal(serverpf) && clientpf.bpp >= 16) {
-      // This is so we can avoid translating the pixels when compressing
-      // with JPEG, since it is unnecessary
+      // Count the colors in the raw buffer, so we can avoid unnecessary pixel
+      // translation when encoding with JPEG.
       if (grayScaleJPEG) palNumColors = 0;
-      else FAST_FILL_PALETTE(pixels, stride, r);
+      else FAST_FILL_PALETTE(rawPixels, stride, r);
+
+      // JPEG can read from the raw buffer, but for the other methods, we need
+      // to translate the raw pixels into an intermediate buffer.
       if(palNumColors != 0 || jpegQuality == -1) {
         pixels = (PIXEL_T *)writer->getImageBuf(r.area());
         stride = r.width();
         ig->getImage(pixels, r);
       }
-    }
-    else {
+    } else {
+      // Pixel translation will be required, so create an intermediate buffer,
+      // translate the raw pixels into it, and count its colors.
       pixels = (PIXEL_T *)writer->getImageBuf(r.area());
       stride = r.width();
       ig->getImage(pixels, r);
+
       if (grayScaleJPEG) palNumColors = 0;
       else FILL_PALETTE(pixels, r.area());
     }
@@ -239,7 +244,10 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
     // Truecolor image
 #if (BPP != 8)
     if (jpegQuality != -1) {
-      ENCODE_JPEG_RECT(pixels, stride, r, os);
+      if (pixels)
+        ENCODE_JPEG_RECT(pixels, stride, r, os);
+      else
+        ENCODE_JPEG_RECT((PIXEL_T *)rawPixels, stride, r, os);
       break;
     }
 #endif
@@ -458,7 +466,7 @@ void FILL_PALETTE (PIXEL_T *data, int count)
   }
 }
 
-void FAST_FILL_PALETTE (PIXEL_T *data, int stride, const Rect& r)
+void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r)
 {
 }
 
@@ -523,12 +531,13 @@ void FILL_PALETTE (PIXEL_T *data, int count)
   paletteInsert (ci, (rdr::U32)ni, BPP);
 }
 
-void FAST_FILL_PALETTE (PIXEL_T *data, int stride, const Rect& r)
+void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r)
 {
   PIXEL_T c0, c1, ci = 0, mask, c0t, c1t, cit;
   int n0, n1, ni;
   int w = r.width(), h = r.height();
-  PIXEL_T *rowptr, *colptr, *rowptr2, *colptr2, *dataend = &data[stride * h];
+  const PIXEL_T *rowptr, *colptr, *rowptr2, *colptr2,
+    *dataend = &data[stride * h];
   bool willTransform = ig->willTransform();
 
   if (willTransform) {
@@ -636,11 +645,12 @@ void FAST_FILL_PALETTE (PIXEL_T *data, int stride, const Rect& r)
 
 bool CHECK_SOLID_TILE(Rect& r, rdr::U32 *colorPtr, bool needSameColor)
 {
-  PIXEL_T *buf, colorValue;
+  const PIXEL_T *buf;
+  PIXEL_T colorValue;
   int w = r.width(), h = r.height();
 
   int stride = w;
-  buf = (PIXEL_T *)ig->getRawPixelsRW(r, &stride);
+  buf = (const PIXEL_T *)ig->getRawPixelsR(r, &stride);
 
   colorValue = *buf;
   if (needSameColor && (rdr::U32)colorValue != *colorPtr)
@@ -648,7 +658,7 @@ bool CHECK_SOLID_TILE(Rect& r, rdr::U32 *colorPtr, bool needSameColor)
 
   int bufPad = stride - w;
   while (h > 0) {
-    PIXEL_T *bufEndOfRow = buf + w;
+    const PIXEL_T *bufEndOfRow = buf + w;
     while (buf < bufEndOfRow) {
       if (colorValue != *(buf++))
         return false;