From 4d0bc6e7cac91fedb70c278a6dc36c75a1344a0e Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Wed, 12 Feb 2014 13:12:31 +0100 Subject: [PATCH] Fix some offenders that poke around in the PixelFormat internals --- common/rfb/JpegCompressor.cxx | 51 ++++++++++++++-------------- common/rfb/JpegDecompressor.cxx | 49 +++++++++++++-------------- common/rfb/tightEncode.h | 7 +--- unix/x0vncserver/XPixelBuffer.cxx | 21 ++++++------ win/rfb_win32/DIBSectionBuffer.cxx | 48 +++++++++++++++++---------- win/rfb_win32/DeviceContext.cxx | 53 ++++++++++++++++-------------- 6 files changed, 116 insertions(+), 113 deletions(-) diff --git a/common/rfb/JpegCompressor.cxx b/common/rfb/JpegCompressor.cxx index c1ef3c7d..960bd378 100644 --- a/common/rfb/JpegCompressor.cxx +++ b/common/rfb/JpegCompressor.cxx @@ -1,5 +1,6 @@ /* Copyright (C) 2002-2005 RealVNC Ltd. All Rights Reserved. * Copyright (C) 2011 D. R. Commander. All Rights Reserved. + * Copyright 2014 Pierre Ossman for Cendio AB * * This is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -32,6 +33,15 @@ extern "C" { using namespace rfb; +// +// Special formats that libjpeg can have optimised code paths for +// + +static const PixelFormat pfRGBX(32, 24, false, true, 255, 255, 255, 0, 8, 16); +static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0); +static const PixelFormat pfXRGB(32, 24, false, true, 255, 255, 255, 8, 16, 24); +static const PixelFormat pfXBGR(32, 24, false, true, 255, 255, 255, 24, 16, 8); + // // Error manager implmentation for the JPEG library // @@ -166,33 +176,20 @@ void JpegCompressor::compress(const rdr::U8 *buf, int stride, const Rect& r, pixelsize = 3; #ifdef JCS_EXTENSIONS - // Try to have libjpeg read directly from our native format - if(pf.is888()) { - int redShift, greenShift, blueShift; - - if(pf.bigEndian) { - redShift = 24 - pf.redShift; - greenShift = 24 - pf.greenShift; - blueShift = 24 - pf.blueShift; - } else { - redShift = pf.redShift; - greenShift = pf.greenShift; - blueShift = pf.blueShift; - } - - if(redShift == 0 && greenShift == 8 && blueShift == 16) - cinfo->in_color_space = JCS_EXT_RGBX; - if(redShift == 16 && greenShift == 8 && blueShift == 0) - cinfo->in_color_space = JCS_EXT_BGRX; - if(redShift == 24 && greenShift == 16 && blueShift == 8) - cinfo->in_color_space = JCS_EXT_XBGR; - if(redShift == 8 && greenShift == 16 && blueShift == 24) - cinfo->in_color_space = JCS_EXT_XRGB; - - if (cinfo->in_color_space != JCS_RGB) { - srcBuf = (rdr::U8 *)buf; - pixelsize = 4; - } + // Try to have libjpeg output directly to our native format + // libjpeg can only handle some "standard" formats + if (pfRGBX.equal(pf)) + cinfo->in_color_space = JCS_EXT_RGBX; + else if (pfBGRX.equal(pf)) + cinfo->in_color_space = JCS_EXT_BGRX; + else if (pfXRGB.equal(pf)) + cinfo->in_color_space = JCS_EXT_XRGB; + else if (pfXBGR.equal(pf)) + cinfo->in_color_space = JCS_EXT_XBGR; + + if (cinfo->in_color_space != JCS_RGB) { + srcBuf = (rdr::U8 *)buf; + pixelsize = 4; } #endif diff --git a/common/rfb/JpegDecompressor.cxx b/common/rfb/JpegDecompressor.cxx index 4d230eb8..3f4d2d00 100644 --- a/common/rfb/JpegDecompressor.cxx +++ b/common/rfb/JpegDecompressor.cxx @@ -1,6 +1,7 @@ /* Copyright (C) 2000-2003 Constantin Kaplinsky. All Rights Reserved. * Copyright (C) 2004-2005 Cendio AB. All rights reserved. * Copyright (C) 2011 D. R. Commander. All Rights Reserved. + * Copyright 2014 Pierre Ossman for Cendio AB * * This is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -33,6 +34,14 @@ extern "C" { using namespace rfb; +// +// Special formats that libjpeg can have optimised code paths for +// + +static const PixelFormat pfRGBX(32, 24, false, true, 255, 255, 255, 0, 8, 16); +static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0); +static const PixelFormat pfXRGB(32, 24, false, true, 255, 255, 255, 8, 16, 24); +static const PixelFormat pfXBGR(32, 24, false, true, 255, 255, 255, 24, 16, 8); // // Error manager implmentation for the JPEG library @@ -169,33 +178,19 @@ void JpegDecompressor::decompress(const rdr::U8 *jpegBuf, int jpegBufLen, #ifdef JCS_EXTENSIONS // Try to have libjpeg output directly to our native format - if (pf.is888()) { - int redShift, greenShift, blueShift; - - if(pf.bigEndian) { - redShift = 24 - pf.redShift; - greenShift = 24 - pf.greenShift; - blueShift = 24 - pf.blueShift; - } else { - redShift = pf.redShift; - greenShift = pf.greenShift; - blueShift = pf.blueShift; - } - - // libjpeg can only handle some "standard" formats - if(redShift == 0 && greenShift == 8 && blueShift == 16) - dinfo->out_color_space = JCS_EXT_RGBX; - if(redShift == 16 && greenShift == 8 && blueShift == 0) - dinfo->out_color_space = JCS_EXT_BGRX; - if(redShift == 24 && greenShift == 16 && blueShift == 8) - dinfo->out_color_space = JCS_EXT_XBGR; - if(redShift == 8 && greenShift == 16 && blueShift == 24) - dinfo->out_color_space = JCS_EXT_XRGB; - - if (dinfo->out_color_space != JCS_RGB) { - dstBuf = (rdr::U8 *)buf; - pixelsize = 4; - } + // libjpeg can only handle some "standard" formats + if (pfRGBX.equal(pf)) + dinfo->out_color_space = JCS_EXT_RGBX; + else if (pfBGRX.equal(pf)) + dinfo->out_color_space = JCS_EXT_BGRX; + else if (pfXRGB.equal(pf)) + dinfo->out_color_space = JCS_EXT_XRGB; + else if (pfXBGR.equal(pf)) + dinfo->out_color_space = JCS_EXT_XBGR; + + if (dinfo->out_color_space != JCS_RGB) { + dstBuf = (rdr::U8 *)buf; + pixelsize = 4; } #endif diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h index d5b2c660..c121b7aa 100644 --- a/common/rfb/tightEncode.h +++ b/common/rfb/tightEncode.h @@ -540,12 +540,7 @@ void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r) *dataend = &data[stride * h]; bool willTransform = ig->willTransform(); - if (willTransform) { - mask = serverpf.redMax << serverpf.redShift; - mask |= serverpf.greenMax << serverpf.greenShift; - mask |= serverpf.blueMax << serverpf.blueShift; - } - else mask = ~0; + serverpf.bufferFromPixel((rdr::U8*)&mask, ~0); c0 = data[0] & mask; n0 = 0; diff --git a/unix/x0vncserver/XPixelBuffer.cxx b/unix/x0vncserver/XPixelBuffer.cxx index f691c576..aa52620a 100644 --- a/unix/x0vncserver/XPixelBuffer.cxx +++ b/unix/x0vncserver/XPixelBuffer.cxx @@ -1,4 +1,5 @@ /* Copyright (C) 2007-2008 Constantin Kaplinsky. All Rights Reserved. + * Copyright 2014 Pierre Ossman for Cendio AB * * This is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -38,16 +39,16 @@ XPixelBuffer::XPixelBuffer(Display *dpy, ImageFactory &factory, m_stride(0) { // Fill in the PixelFormat structure of the parent class. - format.bpp = m_image->xim->bits_per_pixel; - format.depth = m_image->xim->depth; - format.bigEndian = (m_image->xim->byte_order == MSBFirst); - format.trueColour = m_image->isTrueColor(); - format.redShift = ffs(m_image->xim->red_mask) - 1; - format.greenShift = ffs(m_image->xim->green_mask) - 1; - format.blueShift = ffs(m_image->xim->blue_mask) - 1; - format.redMax = m_image->xim->red_mask >> format.redShift; - format.greenMax = m_image->xim->green_mask >> format.greenShift; - format.blueMax = m_image->xim->blue_mask >> format.blueShift; + format = PixelFormat(m_image->xim->bits_per_pixel, + m_image->xim->depth, + (m_image->xim->byte_order == MSBFirst), + m_image->isTrueColor(), + m_image->xim->red_mask >> (ffs(m_image->xim->red_mask) - 1), + m_image->xim->green_mask >> (ffs(m_image->xim->green_mask) - 1), + m_image->xim->blue_mask >> (ffs(m_image->xim->blue_mask) - 1), + ffs(m_image->xim->red_mask) - 1, + ffs(m_image->xim->green_mask) - 1, + ffs(m_image->xim->blue_mask) - 1); // Set up the remaining data of the parent class. width_ = rect.width(); diff --git a/win/rfb_win32/DIBSectionBuffer.cxx b/win/rfb_win32/DIBSectionBuffer.cxx index 3c2f5096..0c266b00 100644 --- a/win/rfb_win32/DIBSectionBuffer.cxx +++ b/win/rfb_win32/DIBSectionBuffer.cxx @@ -1,4 +1,5 @@ /* Copyright (C) 2002-2005 RealVNC Ltd. All Rights Reserved. + * Copyright 2014 Pierre Ossman for Cendio AB * * This is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -55,9 +56,11 @@ void DIBSectionBuffer::setPF(const PixelFormat& pf) { if ((pf.bpp <= 8) && pf.trueColour) { vlog.info("creating %d-bit TrueColour palette", pf.depth); for (int i=0; i < (1<<(pf.depth)); i++) { - palette[i].b = ((((i >> pf.blueShift) & pf.blueMax) * 65535) + pf.blueMax/2) / pf.blueMax; - palette[i].g = ((((i >> pf.greenShift) & pf.greenMax) * 65535) + pf.greenMax/2) / pf.greenMax; - palette[i].r = ((((i >> pf.redShift) & pf.redMax) * 65535) + pf.redMax/2) / pf.redMax; + rdr::U16 r, g, b; + pf.rgbFromPixel(i, NULL, &r, &g, &b); + palette[i].r = r; + palette[i].g = g; + palette[i].b = b; } refreshPalette(); } @@ -110,9 +113,9 @@ void DIBSectionBuffer::recreateBuffer() { bi.bmiHeader.biWidth = width_; bi.bmiHeader.biHeight = -height_; bi.bmiHeader.biCompression = (format.bpp > 8) ? BI_BITFIELDS : BI_RGB; - bi.mask.red = format.redMax << format.redShift; - bi.mask.green = format.greenMax << format.greenShift; - bi.mask.blue = format.blueMax << format.blueShift; + bi.mask.red = format.pixelFromRGB((rdr::U16)~0, 0, 0); + bi.mask.green = format.pixelFromRGB(0, (rdr::U16)~0, 0); + bi.mask.blue = format.pixelFromRGB(0, 0, (rdr::U16)~0); // Create a DIBSection to draw into if (device) @@ -160,6 +163,11 @@ void DIBSectionBuffer::recreateBuffer() { } if (new_bitmap) { + int bpp, depth; + bool trueColour; + int redMax, greenMax, blueMax; + int redShift, greenShift, blueShift; + // Set up the new bitmap bitmap = new_bitmap; data = new_data; @@ -180,30 +188,34 @@ void DIBSectionBuffer::recreateBuffer() { } // Calculate the PixelFormat for the DIB - format.bigEndian = 0; - format.bpp = format.depth = ds.dsBm.bmBitsPixel; - format.trueColour = format.trueColour || format.bpp > 8; - if (format.bpp > 8) { + bpp = depth = ds.dsBm.bmBitsPixel; + trueColour = format.trueColour || format.bpp > 8; + if (trueColour) { // Get the truecolour format used by the DIBSection - initMaxAndShift(ds.dsBitfields[0], &format.redMax, &format.redShift); - initMaxAndShift(ds.dsBitfields[1], &format.greenMax, &format.greenShift); - initMaxAndShift(ds.dsBitfields[2], &format.blueMax, &format.blueShift); + initMaxAndShift(ds.dsBitfields[0], &redMax, &redShift); + initMaxAndShift(ds.dsBitfields[1], &greenMax, &greenShift); + initMaxAndShift(ds.dsBitfields[2], &blueMax, &blueShift); // Calculate the effective depth - format.depth = 0; + depth = 0; Pixel bits = ds.dsBitfields[0] | ds.dsBitfields[1] | ds.dsBitfields[2]; while (bits) { - format.depth++; + depth++; bits = bits >> 1; } - if (format.depth > format.bpp) + if (depth > bpp) throw Exception("Bad DIBSection format (depth exceeds bpp)"); - } else { + } + + format = PixelFormat(bpp, depth, false, trueColour, + redMax, greenMax, blueMax, + redShift, greenShift, blueShift); + + if (!trueColour) { // Set the DIBSection's palette refreshPalette(); } - } } diff --git a/win/rfb_win32/DeviceContext.cxx b/win/rfb_win32/DeviceContext.cxx index 3b97e446..26721a60 100644 --- a/win/rfb_win32/DeviceContext.cxx +++ b/win/rfb_win32/DeviceContext.cxx @@ -1,4 +1,5 @@ /* Copyright (C) 2002-2005 RealVNC Ltd. All Rights Reserved. + * Copyright 2014 Pierre Ossman for Cendio AB * * This is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -33,7 +34,11 @@ PixelFormat DeviceContext::getPF() const { } PixelFormat DeviceContext::getPF(HDC dc) { - PixelFormat format; + bool trueColour, bigEndian; + int bpp, depth; + int redMax, greenMax, blueMax; + int redShift, greenShift, blueShift; + CompatibleBitmap bitmap(dc, 1, 1); // -=- Get the bitmap format information @@ -49,11 +54,11 @@ PixelFormat DeviceContext::getPF(HDC dc) { } // Set the initial format information - format.trueColour = bi.bmiHeader.biBitCount > 8; - format.bigEndian = 0; - format.bpp = bi.bmiHeader.biBitCount; + trueColour = bi.bmiHeader.biBitCount > 8; + bigEndian = 0; + bpp = bi.bmiHeader.biBitCount; - if (format.trueColour) { + if (trueColour) { DWORD rMask=0, gMask=0, bMask=0; // Which true colour format is the DIB section using? @@ -92,44 +97,42 @@ PixelFormat DeviceContext::getPF(HDC dc) { }; // Convert the data we just retrieved - initMaxAndShift(rMask, &format.redMax, &format.redShift); - initMaxAndShift(gMask, &format.greenMax, &format.greenShift); - initMaxAndShift(bMask, &format.blueMax, &format.blueShift); + initMaxAndShift(rMask, &redMax, &redShift); + initMaxAndShift(gMask, &greenMax, &greenShift); + initMaxAndShift(bMask, &blueMax, &blueShift); // Calculate the depth from the colour shifts - format.depth = 0; + depth = 0; Pixel bits = rMask | gMask | bMask; while (bits) { - format.depth++; + depth++; bits = bits >> 1; } // Check that the depth & bpp are valid - if (format.depth > format.bpp) { + if (depth > bpp) { vlog.error("depth exceeds bits per pixel!"); - format.bpp = format.depth; + bpp = depth; } // Correct the bits-per-pixel to something we're happy with - if (format.bpp <= 16) - format.bpp = 16; - else if (format.bpp <= 32) - format.bpp = 32; + if (bpp <= 16) + bpp = 16; + else if (bpp <= 32) + bpp = 32; } else { // Palettised format - depth reflects number of colours, // but bits-per-pixel is ALWAYS 8 - format.depth = format.bpp; - if (format.bpp < 8) - format.bpp = 8; - vlog.info("%d-colour palettised", 1<