Use UTF-8 in clipboard API

In prepartion for better clipboard extensions that can send Unicode
data between the client and server.

common/rfb/CConnection.cxx

@@ -470,7 +470,7 @@ void CConnection::serverCutText(const char* str)
   strFree(serverClipboard);
   serverClipboard = NULL;
 
-  serverClipboard = strDup(str);
+  serverClipboard = latin1ToUTF8(str);
 
   handleClipboardAnnounce(true);
 }
@@ -516,7 +516,9 @@ void CConnection::announceClipboard(bool available)
 
 void CConnection::sendClipboardData(const char* data)
 {
-  writer()->writeClientCutText(data);
+  CharArray latin1(utf8ToLatin1(data));
+
+  writer()->writeClientCutText(latin1.buf);
 }
 
 void CConnection::refreshFramebuffer()

common/rfb/SConnection.cxx

@@ -306,7 +306,7 @@ void SConnection::clientCutText(const char* str)
   strFree(clientClipboard);
   clientClipboard = NULL;
 
-  clientClipboard = strDup(str);
+  clientClipboard = latin1ToUTF8(str);
 
   handleClipboardAnnounce(true);
 }
@@ -450,7 +450,9 @@ void SConnection::announceClipboard(bool available)
 
 void SConnection::sendClipboardData(const char* data)
 {
-  writer()->writeServerCutText(data);
+  CharArray latin1(utf8ToLatin1(data));
+
+  writer()->writeServerCutText(latin1.buf);
 }
 
 void SConnection::writeFakeColourMap(void)

common/rfb/util.cxx

@@ -64,6 +64,10 @@ namespace rfb {
     delete [] s;
   }
 
+  void strFree(wchar_t* s) {
+    delete [] s;
+  }
+
 
   bool strSplit(const char* src, const char limiter, char** out1, char** out2, bool fromEnd) {
     CharArray out1old, out2old;
@@ -163,6 +167,67 @@ namespace rfb {
     return buffer;
   }
 
+  char* convertCRLF(const char* src, size_t bytes)
+  {
+    char* buffer;
+    size_t sz;
+
+    char* out;
+    const char* in;
+    size_t in_len;
+
+    // Always include space for a NULL
+    sz = 1;
+
+    // Compute output size
+    in = src;
+    in_len = bytes;
+    while ((*in != '\0') && (in_len > 0)) {
+      sz++;
+
+      if (*in == '\r') {
+        if ((in_len == 0) || (*(in+1) != '\n'))
+          sz++;
+      } else if (*in == '\n') {
+        if ((in == src) || (*(in-1) != '\r'))
+          sz++;
+      }
+
+      in++;
+      in_len--;
+    }
+
+    // Alloc
+    buffer = new char[sz];
+    memset(buffer, 0, sz);
+
+    // And convert
+    out = buffer;
+    in = src;
+    in_len = bytes;
+    while ((*in != '\0') && (in_len > 0)) {
+      if (*in == '\n') {
+        if ((in == src) || (*(in-1) != '\r'))
+          *out++ = '\r';
+      }
+
+      *out = *in;
+
+      if (*in == '\r') {
+        if ((in_len == 0) || (*(in+1) != '\n')) {
+          out++;
+          *out = '\n';
+        }
+      }
+
+      out++;
+      in++;
+      in_len--;
+    }
+
+    return buffer;
+  }
+
   size_t ucs4ToUTF8(unsigned src, char* dst) {
     if (src < 0x80) {
       *dst++ = src;
@@ -242,6 +307,61 @@ namespace rfb {
     return consumed;
   }
 
+  size_t ucs4ToUTF16(unsigned src, wchar_t* dst) {
+    if ((src < 0xd800) || ((src >= 0xe000) && (src < 0x10000))) {
+      *dst++ = src;
+      *dst++ = L'\0';
+      return 1;
+    } else if (src < 0x110000) {
+      *dst++ = 0xd800 | ((src >> 10) & 0x07ff);
+      *dst++ = 0xdc00 | (src & 0x07ff);
+      *dst++ = L'\0';
+      return 2;
+    } else {
+      return ucs4ToUTF16(0xfffd, dst);
+    }
+  }
+
+  size_t utf16ToUCS4(const wchar_t* src, size_t max, unsigned* dst) {
+    *dst = 0xfffd;
+
+    if (max == 0)
+      return 0;
+
+    if ((*src < 0xd800) || (*src >= 0xe000)) {
+      *dst = *src;
+      return 1;
+    }
+
+    if (*src & 0x0400) {
+      size_t consumed;
+
+      // Invalid sequence, consume all continuation characters
+      consumed = 0;
+      while ((max > 0) && (*src & 0x0400)) {
+        src++;
+        max--;
+        consumed++;
+      }
+
+      return consumed;
+    }
+
+    *dst = *src++;
+    max--;
+
+    // Invalid or truncated sequence?
+    if ((max == 0) || ((*src & 0xfc00) != 0xdc00)) {
+      *dst = 0xfffd;
+      return 1;
+    }
+
+    *dst = 0x10000 | ((*dst & 0x03ff) << 10);
+    *dst |= *src & 0x3ff;
+
+    return 2;
+  }
+
   char* latin1ToUTF8(const char* src, size_t bytes) {
     char* buffer;
     size_t sz;
@@ -329,6 +449,104 @@ namespace rfb {
     return buffer;
   }
 
+  char* utf16ToUTF8(const wchar_t* src, size_t units)
+  {
+    char* buffer;
+    size_t sz;
+
+    char* out;
+    const wchar_t* in;
+    size_t in_len;
+
+    // Always include space for a NULL
+    sz = 1;
+
+    // Compute output size
+    in = src;
+    in_len = units;
+    while ((*in != '\0') && (in_len > 0)) {
+      size_t len;
+      unsigned ucs;
+      char buf[5];
+
+      len = utf16ToUCS4(in, in_len, &ucs);
+      in += len;
+      in_len -= len;
+
+      sz += ucs4ToUTF8(ucs, buf);
+    }
+
+    // Alloc
+    buffer = new char[sz];
+    memset(buffer, 0, sz);
+
+    // And convert
+    out = buffer;
+    in = src;
+    in_len = units;
+    while ((*in != '\0') && (in_len > 0)) {
+      size_t len;
+      unsigned ucs;
+
+      len = utf16ToUCS4(in, in_len, &ucs);
+      in += len;
+      in_len -= len;
+
+      out += ucs4ToUTF8(ucs, out);
+    }
+
+    return buffer;
+  }
+
+  wchar_t* utf8ToUTF16(const char* src, size_t bytes)
+  {
+    wchar_t* buffer;
+    size_t sz;
+
+    wchar_t* out;
+    const char* in;
+    size_t in_len;
+
+    // Always include space for a NULL
+    sz = 1;
+
+    // Compute output size
+    in = src;
+    in_len = bytes;
+    while ((*in != '\0') && (in_len > 0)) {
+      size_t len;
+      unsigned ucs;
+      wchar_t buf[3];
+
+      len = utf8ToUCS4(in, in_len, &ucs);
+      in += len;
+      in_len -= len;
+
+      sz += ucs4ToUTF16(ucs, buf);
+    }
+
+    // Alloc
+    buffer = new wchar_t[sz];
+    memset(buffer, 0, sz);
+
+    // And convert
+    out = buffer;
+    in = src;
+    in_len = bytes;
+    while ((*in != '\0') && (in_len > 0)) {
+      size_t len;
+      unsigned ucs;
+
+      len = utf8ToUCS4(in, in_len, &ucs);
+      in += len;
+      in_len -= len;
+
+      out += ucs4ToUTF16(ucs, out);
+    }
+
+    return buffer;
+  }
+
   unsigned msBetween(const struct timeval *first,
                      const struct timeval *second)
   {

common/rfb/util.h

@@ -68,6 +68,7 @@ namespace rfb {
 
   char* strDup(const char* s);
   void strFree(char* s);
+  void strFree(wchar_t* s);
 
   // Returns true if split successful.  Returns false otherwise.
   // ALWAYS *copies* first part of string to out1 buffer.
@@ -87,6 +88,7 @@ namespace rfb {
   // Makes sure line endings are in a certain format
 
   char* convertLF(const char* src, size_t bytes = (size_t)-1);
+  char* convertCRLF(const char* src, size_t bytes = (size_t)-1);
 
   // Convertions between various Unicode formats. The returned strings are
   // always null terminated and must be freed using strFree().
@@ -94,9 +96,15 @@ namespace rfb {
   size_t ucs4ToUTF8(unsigned src, char* dst);
   size_t utf8ToUCS4(const char* src, size_t max, unsigned* dst);
 
+  size_t ucs4ToUTF16(unsigned src, wchar_t* dst);
+  size_t utf16ToUCS4(const wchar_t* src, size_t max, unsigned* dst);
+
   char* latin1ToUTF8(const char* src, size_t bytes = (size_t)-1);
   char* utf8ToLatin1(const char* src, size_t bytes = (size_t)-1);
 
+  char* utf16ToUTF8(const wchar_t* src, size_t units = (size_t)-1);
+  wchar_t* utf8ToUTF16(const char* src, size_t bytes = (size_t)-1);
+
   // HELPER functions for timeout handling
 
   // soonestTimeout() is a function to help work out the soonest of several

unix/xserver/hw/vnc/vncSelection.c

@@ -362,23 +362,24 @@ static int vncConvertSelection(ClientPtr client, Atom selection,
       return Success;
     } else {
       if ((target == xaSTRING) || (target == xaTEXT)) {
+        char* latin1;
+
+        latin1 = vncUTF8ToLatin1(data, (size_t)-1);
+        if (latin1 == NULL)
+          return BadAlloc;
+
         rc = dixChangeWindowProperty(serverClient, pWin, realProperty,
                                      XA_STRING, 8, PropModeReplace,
-                                     strlen(data), (char*)data,
-                                     TRUE);
+                                     strlen(latin1), latin1, TRUE);
+
+        vncStrFree(latin1);
+
         if (rc != Success)
           return rc;
       } else if (target == xaUTF8_STRING) {
-        char* buffer;
-
-        buffer = vncLatin1ToUTF8(data, (size_t)-1);
-        if (buffer == NULL)
-          return BadAlloc;
-
         rc = dixChangeWindowProperty(serverClient, pWin, realProperty,
                                      xaUTF8_STRING, 8, PropModeReplace,
-                                     strlen(buffer), buffer, TRUE);
-        vncStrFree(buffer);
+                                     strlen(data), data, TRUE);
         if (rc != Success)
           return rc;
       } else {
@@ -515,13 +516,14 @@ static void vncHandleSelection(Atom selection, Atom target,
         vncAnnounceClipboard(TRUE);
       }
     } else {
-      if (vncHasAtom(xaSTRING, (const Atom*)prop->data, prop->size))
-        vncSelectionRequest(selection, xaSTRING);
-      else if (vncHasAtom(xaUTF8_STRING, (const Atom*)prop->data, prop->size))
+      if (vncHasAtom(xaUTF8_STRING, (const Atom*)prop->data, prop->size))
         vncSelectionRequest(selection, xaUTF8_STRING);
+      else if (vncHasAtom(xaSTRING, (const Atom*)prop->data, prop->size))
+        vncSelectionRequest(selection, xaSTRING);
     }
   } else if (target == xaSTRING) {
     char* filtered;
+    char* utf8;
 
     if (prop->format != 8)
       return;
@@ -532,27 +534,26 @@ static void vncHandleSelection(Atom selection, Atom target,
     if (filtered == NULL)
       return;
 
+    utf8 = vncLatin1ToUTF8(filtered, (size_t)-1);
+    vncStrFree(filtered);
+    if (utf8 == NULL)
+      return;
+
     LOG_DEBUG("Sending clipboard to clients (%d bytes)",
-              (int)strlen(filtered));
+              (int)strlen(utf8));
 
-    vncSendClipboardData(filtered);
+    vncSendClipboardData(utf8);
 
-    vncStrFree(filtered);
+    vncStrFree(utf8);
   } else if (target == xaUTF8_STRING) {
     char *filtered;
-    char* buffer;
 
     if (prop->format != 8)
       return;
     if (prop->type != xaUTF8_STRING)
       return;
 
-    buffer = vncUTF8ToLatin1(prop->data, prop->size);
-    if (buffer == NULL)
-      return;
-
-    filtered = vncConvertLF(buffer, (size_t)-1);
-    vncStrFree(buffer);
+    filtered = vncConvertLF(prop->data, prop->size);
     if (filtered == NULL)
       return;
 

vncviewer/Viewport.cxx

@@ -310,14 +310,12 @@ void Viewport::handleClipboardAnnounce(bool available)
 
 void Viewport::handleClipboardData(const char* data)
 {
-  char* buffer;
   size_t len;
 
   if (!hasFocus())
     return;
 
-  buffer = latin1ToUTF8(data);
-  len = strlen(buffer);
+  len = strlen(data);
 
   vlog.debug("Got clipboard data (%d bytes)", (int)len);
 
@@ -325,11 +323,9 @@ void Viewport::handleClipboardData(const char* data)
   // dump the data into both variants.
 #if !defined(WIN32) && !defined(__APPLE__)
   if (setPrimary)
-    Fl::copy(buffer, len, 0);
+    Fl::copy(data, len, 0);
 #endif
-  Fl::copy(buffer, len, 1);
-
-  strFree(buffer);
+  Fl::copy(data, len, 1);
 }
 
 void Viewport::setLEDState(unsigned int state)
@@ -561,15 +557,13 @@ void Viewport::resize(int x, int y, int w, int h)
 
 int Viewport::handle(int event)
 {
-  char *buffer, *filtered;
+  char *filtered;
   int buttonMask, wheelMask;
   DownMap::const_iterator iter;
 
   switch (event) {
   case FL_PASTE:
-    buffer = utf8ToLatin1(Fl::event_text(), Fl::event_length());
-    filtered = convertLF(buffer);
-    strFree(buffer);
+    filtered = convertLF(Fl::event_text(), Fl::event_length());
 
     vlog.debug("Sending clipboard data (%d bytes)", (int)strlen(filtered));
 

win/rfb_win32/Clipboard.cxx

@@ -30,41 +30,6 @@ using namespace rfb::win32;
 
 static LogWriter vlog("Clipboard");
 
-
-//
-// -=- CR/LF handlers
-//
-
-char*
-unix2dos(const char* text) {
-  int len = strlen(text)+1;
-  char* dos = new char[strlen(text)*2+1];
-  int i, j=0;
-  for (i=0; i<len; i++) {
-    if (text[i] == '\x0a')
-      dos[j++] = '\x0d';
-    dos[j++] = text[i];
-  }
-  return dos;
-}
-
-
-//
-// -=- ISO-8859-1 (Latin 1) filter (in-place)
-//
-
-void
-removeNonISOLatin1Chars(char* text) {
-  int len = strlen(text);
-  int i=0, j=0;
-  for (; i<len; i++) {
-    if (((text[i] >= 1) && (text[i] <= 127)) ||
-        ((text[i] >= 160) && (text[i] <= 255)))
-      text[j++] = text[i];
-  }
-  text[j] = 0;
-}
-
 //
 // -=- Clipboard object
 //
@@ -106,7 +71,7 @@ Clipboard::processMessage(UINT msg, WPARAM wParam, LPARAM lParam) {
         if (notifier == NULL)
           vlog.debug("no clipboard notifier registered");
         else
-          notifier->notifyClipboardChanged(IsClipboardFormatAvailable(CF_TEXT));
+          notifier->notifyClipboardChanged(IsClipboardFormatAvailable(CF_UNICODETEXT));
 			}
     }
     if (next_window)
@@ -120,35 +85,34 @@ Clipboard::processMessage(UINT msg, WPARAM wParam, LPARAM lParam) {
 char*
 Clipboard::getClipText() {
   HGLOBAL cliphandle;
-  char* clipdata;
-  char* filtered;
+  wchar_t* clipdata;
+  CharArray utf8;
 
   // Open the clipboard
   if (!OpenClipboard(getHandle()))
     return NULL;
 
   // Get the clipboard data
-  cliphandle = GetClipboardData(CF_TEXT);
+  cliphandle = GetClipboardData(CF_UNICODETEXT);
   if (!cliphandle) {
     CloseClipboard();
     return NULL;
   }
 
-  clipdata = (char*) GlobalLock(cliphandle);
+  clipdata = (wchar_t*) GlobalLock(cliphandle);
   if (!clipdata) {
     CloseClipboard();
     return NULL;
   }
 
-  // Filter out anything unwanted
-  filtered = convertLF(clipdata, strlen(clipdata));
-  removeNonISOLatin1Chars(filtered);
+  // Convert it to UTF-8
+  utf8.replaceBuf(utf16ToUTF8(clipdata));
 
   // Release the buffer and close the clipboard
   GlobalUnlock(cliphandle);
   CloseClipboard();
 
-  return filtered;
+  return convertLF(utf8.buf);
 }
 
 void
@@ -161,26 +125,27 @@ Clipboard::setClipText(const char* text) {
     if (!OpenClipboard(getHandle()))
       throw rdr::SystemException("unable to open Win32 clipboard", GetLastError());
 
-    // - Pre-process the supplied clipboard text into DOS format
-    CharArray dos_text;
-    dos_text.buf = unix2dos(text);
-    removeNonISOLatin1Chars(dos_text.buf);
-    int dos_text_len = strlen(dos_text.buf);
+    // - Convert the supplied clipboard text into UTF-16 format with CRLF
+    CharArray filtered(convertCRLF(text));
+    wchar_t* utf16;
+
+    utf16 = utf8ToUTF16(filtered.buf);
 
     // - Allocate global memory for the data
-    clip_handle = ::GlobalAlloc(GMEM_MOVEABLE, dos_text_len+1);
+    clip_handle = ::GlobalAlloc(GMEM_MOVEABLE, (wcslen(utf16) + 1) * 2);
 
-    char* data = (char*) GlobalLock(clip_handle);
-    memcpy(data, dos_text.buf, dos_text_len+1);
-    data[dos_text_len] = 0;
+    wchar_t* data = (wchar_t*) GlobalLock(clip_handle);
+    wcscpy(data, utf16);
     GlobalUnlock(clip_handle);
 
+    strFree(utf16);
+
     // - Next, we must clear out any existing data
     if (!EmptyClipboard())
       throw rdr::SystemException("unable to empty Win32 clipboard", GetLastError());
 
     // - Set the new clipboard data
-    if (!SetClipboardData(CF_TEXT, clip_handle))
+    if (!SetClipboardData(CF_UNICODETEXT, clip_handle))
       throw rdr::SystemException("unable to set Win32 clipboard", GetLastError());
     clip_handle = 0;