From cc83b7ce85b3a7959f5af23283e3c62ee76c44fb Mon Sep 17 00:00:00 2001 From: DRC Date: Mon, 13 Feb 2012 03:49:28 +0000 Subject: [PATCH] Prevent the Tight encoder (specifically packPixels*()) from accidentally using the framebuffer as an intermediate buffer. git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/branches/1_2@4851 3789f03b-4d11-0410-bbf8-ca57d06f2519 --- common/rfb/TightEncoder.h | 6 ++-- common/rfb/TransImageGetter.cxx | 6 ++-- common/rfb/TransImageGetter.h | 4 +-- common/rfb/tightEncode.h | 56 +++++++++++++++++++-------------- 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/common/rfb/TightEncoder.h b/common/rfb/TightEncoder.h index 95c2b984..4fff0832 100644 --- a/common/rfb/TightEncoder.h +++ b/common/rfb/TightEncoder.h @@ -100,9 +100,9 @@ namespace rfb { int paletteInsert(rdr::U32 rgb, int numPixels, int bpp); void paletteReset(void); - void fastFillPalette8(rdr::U8 *data, int stride, const Rect &r); - void fastFillPalette16(rdr::U16 *data, int stride, const Rect &r); - void fastFillPalette32(rdr::U32 *data, int stride, const Rect &r); + void fastFillPalette8(const rdr::U8 *data, int stride, const Rect &r); + void fastFillPalette16(const rdr::U16 *data, int stride, const Rect &r); + void fastFillPalette32(const rdr::U32 *data, int stride, const Rect &r); void fillPalette8(rdr::U8 *data, int count); void fillPalette16(rdr::U16 *data, int count); diff --git a/common/rfb/TransImageGetter.cxx b/common/rfb/TransImageGetter.cxx index bf26b44d..c1add1ce 100644 --- a/common/rfb/TransImageGetter.cxx +++ b/common/rfb/TransImageGetter.cxx @@ -56,12 +56,12 @@ void TransImageGetter::setColourMapEntries(int firstCol, int nCols) PixelTransformer::setColourMapEntries(firstCol, nCols); } -rdr::U8 *TransImageGetter::getRawPixelsRW(const Rect &r, int *stride) +const rdr::U8 *TransImageGetter::getRawPixelsR(const Rect &r, int *stride) { if (!offset.equals(Point(0, 0))) - return pb->getPixelsRW(r.translate(offset.negate()), stride); + return pb->getPixelsR(r.translate(offset.negate()), stride); else - return pb->getPixelsRW(r, stride); + return pb->getPixelsR(r, stride); } void TransImageGetter::getImage(void* outPtr, const Rect& r, int outStride) diff --git a/common/rfb/TransImageGetter.h b/common/rfb/TransImageGetter.h index 1ad49b7c..f2b35b45 100644 --- a/common/rfb/TransImageGetter.h +++ b/common/rfb/TransImageGetter.h @@ -72,11 +72,11 @@ namespace rfb { // padding will be outStride-r.width() pixels). void getImage(void* outPtr, const Rect& r, int outStride=0); - // getRawPixelsRW() gets the given rectangle of data directly from the + // getRawPixelsR() gets the given rectangle of data directly from the // underlying PixelBuffer, bypassing the translation logic. Only use // this when doing something that's independent of the client's pixel // format. - rdr::U8 *getRawPixelsRW(const Rect &r, int *stride); + const rdr::U8 *getRawPixelsR(const Rect &r, int *stride); // setPixelBuffer() changes the pixel buffer to be used. The new pixel // buffer MUST have the same pixel format as the old one - if not you diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h index a1d5dbe9..8f900b58 100644 --- a/common/rfb/tightEncode.h +++ b/common/rfb/tightEncode.h @@ -189,9 +189,10 @@ unsigned int PACK_PIXELS (PIXEL_T *buf, unsigned int count) void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid) { - int stride = r.width(); + int stride; rdr::U32 solidColor; - PIXEL_T *pixels = (PIXEL_T *)ig->getRawPixelsRW(r, &stride); + const PIXEL_T *rawPixels = (const PIXEL_T *)ig->getRawPixelsR(r, &stride); + PIXEL_T *pixels = NULL; bool grayScaleJPEG = (jpegSubsampling == SUBSAMP_GRAY && jpegQuality != -1); #if (BPP == 32) @@ -201,34 +202,38 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid) #endif if (forceSolid) { + // Subrectangle has already been determined to be solid. palNumColors = 1; - if (ig->willTransform()) { - ig->translatePixels(pixels, &solidColor, 1); - pixels = (PIXEL_T *)&solidColor; - } - } - else { + ig->translatePixels(rawPixels, &solidColor, 1); + pixels = (PIXEL_T *)&solidColor; + } else { + // Analyze subrectangle's colors to determine best encoding method. palMaxColors = r.area() / pconf->idxMaxColorsDivisor; - if (jpegQuality != -1) palMaxColors = pconf->palMaxColorsWithJPEG; - if (palMaxColors < 2 && r.area() >= pconf->monoMinRectSize) { + if (jpegQuality != -1) + palMaxColors = pconf->palMaxColorsWithJPEG; + if (palMaxColors < 2 && r.area() >= pconf->monoMinRectSize) palMaxColors = 2; - } if (clientpf.equal(serverpf) && clientpf.bpp >= 16) { - // This is so we can avoid translating the pixels when compressing - // with JPEG, since it is unnecessary + // Count the colors in the raw buffer, so we can avoid unnecessary pixel + // translation when encoding with JPEG. if (grayScaleJPEG) palNumColors = 0; - else FAST_FILL_PALETTE(pixels, stride, r); + else FAST_FILL_PALETTE(rawPixels, stride, r); + + // JPEG can read from the raw buffer, but for the other methods, we need + // to translate the raw pixels into an intermediate buffer. if(palNumColors != 0 || jpegQuality == -1) { pixels = (PIXEL_T *)writer->getImageBuf(r.area()); stride = r.width(); ig->getImage(pixels, r); } - } - else { + } else { + // Pixel translation will be required, so create an intermediate buffer, + // translate the raw pixels into it, and count its colors. pixels = (PIXEL_T *)writer->getImageBuf(r.area()); stride = r.width(); ig->getImage(pixels, r); + if (grayScaleJPEG) palNumColors = 0; else FILL_PALETTE(pixels, r.area()); } @@ -239,7 +244,10 @@ void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid) // Truecolor image #if (BPP != 8) if (jpegQuality != -1) { - ENCODE_JPEG_RECT(pixels, stride, r, os); + if (pixels) + ENCODE_JPEG_RECT(pixels, stride, r, os); + else + ENCODE_JPEG_RECT((PIXEL_T *)rawPixels, stride, r, os); break; } #endif @@ -458,7 +466,7 @@ void FILL_PALETTE (PIXEL_T *data, int count) } } -void FAST_FILL_PALETTE (PIXEL_T *data, int stride, const Rect& r) +void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r) { } @@ -523,12 +531,13 @@ void FILL_PALETTE (PIXEL_T *data, int count) paletteInsert (ci, (rdr::U32)ni, BPP); } -void FAST_FILL_PALETTE (PIXEL_T *data, int stride, const Rect& r) +void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r) { PIXEL_T c0, c1, ci = 0, mask, c0t, c1t, cit; int n0, n1, ni; int w = r.width(), h = r.height(); - PIXEL_T *rowptr, *colptr, *rowptr2, *colptr2, *dataend = &data[stride * h]; + const PIXEL_T *rowptr, *colptr, *rowptr2, *colptr2, + *dataend = &data[stride * h]; bool willTransform = ig->willTransform(); if (willTransform) { @@ -636,11 +645,12 @@ void FAST_FILL_PALETTE (PIXEL_T *data, int stride, const Rect& r) bool CHECK_SOLID_TILE(Rect& r, rdr::U32 *colorPtr, bool needSameColor) { - PIXEL_T *buf, colorValue; + const PIXEL_T *buf; + PIXEL_T colorValue; int w = r.width(), h = r.height(); int stride = w; - buf = (PIXEL_T *)ig->getRawPixelsRW(r, &stride); + buf = (const PIXEL_T *)ig->getRawPixelsR(r, &stride); colorValue = *buf; if (needSameColor && (rdr::U32)colorValue != *colorPtr) @@ -648,7 +658,7 @@ bool CHECK_SOLID_TILE(Rect& r, rdr::U32 *colorPtr, bool needSameColor) int bufPad = stride - w; while (h > 0) { - PIXEL_T *bufEndOfRow = buf + w; + const PIXEL_T *bufEndOfRow = buf + w; while (buf < bufEndOfRow) { if (colorValue != *(buf++)) return false; -- 2.39.5