Correctly handle invalid UTF-16 code points

Some code points are reserved for the UTF-16 coding itself and must not
appear as input data to the algorithm.

diff --git a/common/rfb/util.cxx b/common/rfb/util.cxx
index cc5b76905d527875e14f49b61d09e2afdad2d038..ecab8485869f06030d13948e4847e426433377a5 100644 (file)
--- a/common/rfb/util.cxx
+++ b/common/rfb/util.cxx
@@ -314,7 +314,7 @@ namespace rfb {
       *dst++ = src;
       *dst++ = L'\0';
       return 1;
-    } else if (src < 0x110000) {
+    } else if ((src >= 0x10000) && (src < 0x110000)) {
       src -= 0x10000;
       *dst++ = 0xd800 | ((src >> 10) & 0x03ff);
       *dst++ = 0xdc00 | (src & 0x03ff);

diff --git a/tests/unit/unicode.cxx b/tests/unit/unicode.cxx
index bb2525de0c52b0800f718844927e9336b1105522..52548e00012c978a732530e1cb5831035232081f 100644 (file)
--- a/tests/unit/unicode.cxx
+++ b/tests/unit/unicode.cxx
@@ -59,6 +59,7 @@ struct _ucs4utf16 ucs4utf16[] = {
     { 0x2d006, L"\xd874\xdc06" },
     { 0xfffd, L"\xdc40\xdc12" },
     { 0x110200, L"\xfffd" },
+    { 0xd87f, L"\xfffd" },
 };
 
 struct _latin1utf8 latin1utf8[] = {
@@ -75,6 +76,7 @@ struct _utf8utf16 utf8utf16[] = {
     { "\xf0\x9f\x98\xb8\xf0\x9f\x99\x81\xf0\x9f\x99\x82",   L"\xd83d\xde38\xd83d\xde41\xd83d\xde42" },
     { "\xf0\xad\x80\x86\xf0\xad\x80\x88",                   L"\xd874\xdc06\xd874\xdc08" },
     { "\xef\xbf\xbd\xc3\xa5",                               L"\xd840\xe5" },
+    { "\xed\xa1\xbf",                                       L"\xfffd" },
 };
 
 #define ARRAY_SIZE(a) (sizeof(a)/sizeof(*a))