From 4a6266f5a7db89e70977c62da1d519a00f9a550d Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Mon, 5 Nov 2018 16:28:18 +0100 Subject: [PATCH] Add support for VMware cursors --- common/rfb/CConnection.cxx | 1 + common/rfb/CMsgReader.cxx | 91 +++++++++++++++++++++++++++++++++++++ common/rfb/CMsgReader.h | 1 + common/rfb/ClientParams.cxx | 2 + common/rfb/SMsgWriter.cxx | 29 +++++++++++- common/rfb/SMsgWriter.h | 3 ++ common/rfb/encodings.h | 3 ++ 7 files changed, 129 insertions(+), 1 deletion(-) diff --git a/common/rfb/CConnection.cxx b/common/rfb/CConnection.cxx index 3ea217fa..3a6b7448 100644 --- a/common/rfb/CConnection.cxx +++ b/common/rfb/CConnection.cxx @@ -596,6 +596,7 @@ void CConnection::updateEncodings() if (supportsLocalCursor) { encodings.push_back(pseudoEncodingCursorWithAlpha); + encodings.push_back(pseudoEncodingVMwareCursor); encodings.push_back(pseudoEncodingCursor); encodings.push_back(pseudoEncodingXCursor); } diff --git a/common/rfb/CMsgReader.cxx b/common/rfb/CMsgReader.cxx index 17152ab7..55191663 100644 --- a/common/rfb/CMsgReader.cxx +++ b/common/rfb/CMsgReader.cxx @@ -97,6 +97,9 @@ void CMsgReader::readMsg() case pseudoEncodingCursorWithAlpha: readSetCursorWithAlpha(w, h, Point(x,y)); break; + case pseudoEncodingVMwareCursor: + readSetVMwareCursor(w, h, Point(x,y)); + break; case pseudoEncodingDesktopName: readSetDesktopName(x, y, w, h); break; @@ -349,6 +352,94 @@ void CMsgReader::readSetCursorWithAlpha(int width, int height, const Point& hots pb.getBuffer(pb.getRect(), &stride)); } +void CMsgReader::readSetVMwareCursor(int width, int height, const Point& hotspot) +{ + if (width > maxCursorSize || height > maxCursorSize) + throw Exception("Too big cursor"); + + rdr::U8 type; + + type = is->readU8(); + is->skip(1); + + if (type == 0) { + int len = width * height * (handler->server.pf().bpp/8); + rdr::U8Array andMask(len); + rdr::U8Array xorMask(len); + + rdr::U8Array data(width*height*4); + + rdr::U8* andIn; + rdr::U8* xorIn; + rdr::U8* out; + int Bpp; + + is->readBytes(andMask.buf, len); + is->readBytes(xorMask.buf, len); + + andIn = andMask.buf; + xorIn = xorMask.buf; + out = data.buf; + Bpp = handler->server.pf().bpp/8; + for (int y = 0;y < height;y++) { + for (int x = 0;x < width;x++) { + Pixel andPixel, xorPixel; + + andPixel = handler->server.pf().pixelFromBuffer(andIn); + xorPixel = handler->server.pf().pixelFromBuffer(xorIn); + andIn += Bpp; + xorIn += Bpp; + + if (andPixel == 0) { + rdr::U8 r, g, b; + + // Opaque pixel + + handler->server.pf().rgbFromPixel(xorPixel, &r, &g, &b); + *out++ = r; + *out++ = g; + *out++ = b; + *out++ = 0xff; + } else if (xorPixel == 0) { + // Fully transparent pixel + *out++ = 0; + *out++ = 0; + *out++ = 0; + *out++ = 0; + } else if (andPixel == xorPixel) { + // Inverted pixel + + // We don't really support this, so just turn the pixel black + // FIXME: Do an outline like WinVNC does? + *out++ = 0; + *out++ = 0; + *out++ = 0; + *out++ = 0xff; + } else { + // Partially transparent/inverted pixel + + // We _really_ can't handle this, just make it black + *out++ = 0; + *out++ = 0; + *out++ = 0; + *out++ = 0xff; + } + } + } + + handler->setCursor(width, height, hotspot, data.buf); + } else if (type == 1) { + rdr::U8Array data(width*height*4); + + // FIXME: Is alpha premultiplied? + is->readBytes(data.buf, width*height*4); + + handler->setCursor(width, height, hotspot, data.buf); + } else { + throw Exception("Unknown cursor type"); + } +} + void CMsgReader::readSetDesktopName(int x, int y, int w, int h) { char* name = is->readString(); diff --git a/common/rfb/CMsgReader.h b/common/rfb/CMsgReader.h index 99638276..029e95c1 100644 --- a/common/rfb/CMsgReader.h +++ b/common/rfb/CMsgReader.h @@ -63,6 +63,7 @@ namespace rfb { void readSetXCursor(int width, int height, const Point& hotspot); void readSetCursor(int width, int height, const Point& hotspot); void readSetCursorWithAlpha(int width, int height, const Point& hotspot); + void readSetVMwareCursor(int width, int height, const Point& hotspot); void readSetDesktopName(int x, int y, int w, int h); void readExtendedDesktopSize(int x, int y, int w, int h); void readLEDState(); diff --git a/common/rfb/ClientParams.cxx b/common/rfb/ClientParams.cxx index 2f8783bb..bf2caa47 100644 --- a/common/rfb/ClientParams.cxx +++ b/common/rfb/ClientParams.cxx @@ -140,6 +140,8 @@ bool ClientParams::supportsLocalCursor() const { if (supportsEncoding(pseudoEncodingCursorWithAlpha)) return true; + if (supportsEncoding(pseudoEncodingVMwareCursor)) + return true; if (supportsEncoding(pseudoEncodingCursor)) return true; if (supportsEncoding(pseudoEncodingXCursor)) diff --git a/common/rfb/SMsgWriter.cxx b/common/rfb/SMsgWriter.cxx index a12a6d40..2d59dde3 100644 --- a/common/rfb/SMsgWriter.cxx +++ b/common/rfb/SMsgWriter.cxx @@ -144,7 +144,8 @@ void SMsgWriter::writeCursor() { if (!client->supportsEncoding(pseudoEncodingCursor) && !client->supportsEncoding(pseudoEncodingXCursor) && - !client->supportsEncoding(pseudoEncodingCursorWithAlpha)) + !client->supportsEncoding(pseudoEncodingCursorWithAlpha) && + !client->supportsEncoding(pseudoEncodingVMwareCursor)) throw Exception("Client does not support local cursor"); needCursor = true; @@ -299,6 +300,10 @@ void SMsgWriter::writePseudoRects() writeSetCursorWithAlphaRect(cursor.width(), cursor.height(), cursor.hotspot().x, cursor.hotspot().y, cursor.getBuffer()); + } else if (client->supportsEncoding(pseudoEncodingVMwareCursor)) { + writeSetVMwareCursorRect(cursor.width(), cursor.height(), + cursor.hotspot().x, cursor.hotspot().y, + cursor.getBuffer()); } else if (client->supportsEncoding(pseudoEncodingCursor)) { rdr::U8Array data(cursor.width()*cursor.height() * (client->pf().bpp/8)); rdr::U8Array mask(cursor.getMask()); @@ -503,6 +508,28 @@ void SMsgWriter::writeSetCursorWithAlphaRect(int width, int height, } } +void SMsgWriter::writeSetVMwareCursorRect(int width, int height, + int hotspotX, int hotspotY, + const rdr::U8* data) +{ + if (!client->supportsEncoding(pseudoEncodingVMwareCursor)) + throw Exception("Client does not support local cursors"); + if (++nRectsInUpdate > nRectsInHeader && nRectsInHeader) + throw Exception("SMsgWriter::writeSetVMwareCursorRect: nRects out of sync"); + + os->writeS16(hotspotX); + os->writeS16(hotspotY); + os->writeU16(width); + os->writeU16(height); + os->writeU32(pseudoEncodingVMwareCursor); + + os->writeU8(1); // Alpha cursor + os->pad(1); + + // FIXME: Should alpha be premultiplied? + os->writeBytes(data, width*height*4); +} + void SMsgWriter::writeLEDStateRect(rdr::U8 state) { if (!client->supportsEncoding(pseudoEncodingLEDState)) diff --git a/common/rfb/SMsgWriter.h b/common/rfb/SMsgWriter.h index 80f6de91..4f4c9cc0 100644 --- a/common/rfb/SMsgWriter.h +++ b/common/rfb/SMsgWriter.h @@ -130,6 +130,9 @@ namespace rfb { void writeSetCursorWithAlphaRect(int width, int height, int hotspotX, int hotspotY, const rdr::U8* data); + void writeSetVMwareCursorRect(int width, int height, + int hotspotX, int hotspotY, + const rdr::U8* data); void writeLEDStateRect(rdr::U8 state); void writeQEMUKeyEventRect(); diff --git a/common/rfb/encodings.h b/common/rfb/encodings.h index 122afe7f..315ff54c 100644 --- a/common/rfb/encodings.h +++ b/common/rfb/encodings.h @@ -59,6 +59,9 @@ namespace rfb { const int pseudoEncodingSubsamp8X = -764; const int pseudoEncodingSubsamp16X = -763; + // VMware-specific + const int pseudoEncodingVMwareCursor = 0x574d5664; + int encodingNum(const char* name); const char* encodingName(int num); } -- 2.39.5