Move hex conversion helpers to util

These are used here and there so let's make them more general rather
than hiding them in the stream classes.

9 files changed, 109 insertions, 88 deletions

diff --git a/common/rdr/HexInStream.cxx b/common/rdr/HexInStream.cxx
index 8a88d337..e23974e1 100644
--- a/common/rdr/HexInStream.cxx
+++ b/common/rdr/HexInStream.cxx
@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2019-2022 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -22,9 +23,7 @@
 
 #include <rdr/HexInStream.h>
 #include <rdr/Exception.h>
-
-#include <stdlib.h>
-#include <ctype.h>
+#include <rfb/util.h>
 
 using namespace rdr;
 
@@ -39,43 +38,6 @@ HexInStream::~HexInStream() {
 }
 
 
-bool HexInStream::readHexAndShift(char c, int* v) {
-  c=tolower(c);
-  if ((c >= '0') && (c <= '9'))
-    *v = (*v << 4) + (c - '0');
-  else if ((c >= 'a') && (c <= 'f'))
-    *v = (*v << 4) + (c - 'a' + 10);
-  else
-    return false;
-  return true;
-}
-
-bool HexInStream::hexStrToBin(const char* s, char** data, size_t* length) {
-  size_t l=strlen(s);
-  if ((l % 2) == 0) {
-    delete [] *data;
-    *data = 0; *length = 0;
-    if (l == 0)
-      return true;
-    *data = new char[l/2];
-    *length = l/2;
-    for(size_t i=0;i<l;i+=2) {
-      int byte = 0;
-      if (!readHexAndShift(s[i], &byte) ||
-        !readHexAndShift(s[i+1], &byte))
-        goto decodeError;
-      (*data)[i/2] = byte;
-    }
-    return true;
-  }
-decodeError:
-  delete [] *data;
-  *data = 0;
-  *length = 0;
-  return false;
-}
-
-
 bool HexInStream::fillBuffer() {
   if (!in_stream.hasData(2))
     return false;
@@ -85,10 +47,8 @@ bool HexInStream::fillBuffer() {
 
   uint8_t* optr = (uint8_t*) end;
   for (size_t i=0; i<length; i++) {
-    int v = 0;
-    readHexAndShift(iptr[i*2], &v);
-    readHexAndShift(iptr[i*2+1], &v);
-    optr[i] = v;
+    if (!rfb::hexToBin((const char*)&iptr[i*2], 2, &optr[i], 1))
+      throw Exception("HexInStream: Invalid input data");
   }
 
   in_stream.setptr(length*2);
diff --git a/common/rdr/HexInStream.h b/common/rdr/HexInStream.h
index 4ce6f7fd..76f91c08 100644
--- a/common/rdr/HexInStream.h
+++ b/common/rdr/HexInStream.h
@@ -29,9 +29,6 @@ namespace rdr {
     HexInStream(InStream& is);
     virtual ~HexInStream();
 
-    static bool readHexAndShift(char c, int* v);
-    static bool hexStrToBin(const char* s, char** data, size_t* length);
-
   private:
     virtual bool fillBuffer();
 
diff --git a/common/rdr/HexOutStream.cxx b/common/rdr/HexOutStream.cxx
index 0cbc312f..b1ab2dc0 100644
--- a/common/rdr/HexOutStream.cxx
+++ b/common/rdr/HexOutStream.cxx
@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2019-2022 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,7 +22,7 @@
 #endif
 
 #include <rdr/HexOutStream.h>
-#include <rdr/Exception.h>
+#include <rfb/util.h>
 
 using namespace rdr;
 
@@ -36,39 +37,14 @@ HexOutStream::~HexOutStream()
 {
 }
 
-char HexOutStream::intToHex(int i) {
-  if ((i>=0) && (i<=9))
-    return '0'+i;
-  else if ((i>=10) && (i<=15))
-    return 'a'+(i-10);
-  else
-    throw rdr::Exception("intToHex failed");
-}
-
-char* HexOutStream::binToHexStr(const char* data, size_t length) {
-  char* buffer = new char[length*2+1];
-  for (size_t i=0; i<length; i++) {
-    buffer[i*2] = intToHex((data[i] >> 4) & 15);
-    buffer[i*2+1] = intToHex((data[i] & 15));
-    if (!buffer[i*2] || !buffer[i*2+1]) {
-      delete [] buffer;
-      return 0;
-    }
-  }
-  buffer[length*2] = 0;
-  return buffer;
-}
-
 bool HexOutStream::flushBuffer()
 {
   while (sentUpTo != ptr) {
     uint8_t* optr = out_stream.getptr(2);
     size_t length = min(ptr-sentUpTo, out_stream.avail()/2);
 
-    for (size_t i=0; i<length; i++) {
-      optr[i*2] = intToHex((sentUpTo[i] >> 4) & 0xf);
-      optr[i*2+1] = intToHex(sentUpTo[i] & 0xf);
-    }
+    for (size_t i=0; i<length; i++)
+      rfb::binToHex(&sentUpTo[i], 1, (char*)&optr[i*2], 2);
 
     out_stream.setptr(length*2);
     sentUpTo += length;
diff --git a/common/rdr/HexOutStream.h b/common/rdr/HexOutStream.h
index 8b1ab9ec..16596bf3 100644
--- a/common/rdr/HexOutStream.h
+++ b/common/rdr/HexOutStream.h
@@ -32,9 +32,6 @@ namespace rdr {
     virtual void flush();
     virtual void cork(bool enable);
 
-    static char intToHex(int i);
-    static char* binToHexStr(const char* data, size_t length);
-
   private:
     virtual bool flushBuffer();
     void writeBuffer();
diff --git a/common/rfb/Configuration.cxx b/common/rfb/Configuration.cxx
index 8848e08d..350582f0 100644
--- a/common/rfb/Configuration.cxx
+++ b/common/rfb/Configuration.cxx
@@ -440,7 +440,13 @@ bool BinaryParameter::setParam(const char* v) {
   LOCK_CONFIG;
   if (immutable) return true;
   vlog.debug("set %s(Binary) to %s", getName(), v);
-  return rdr::HexInStream::hexStrToBin(v, &value, &length);
+  delete [] value;
+  length = 0;
+  value = (char*)hexToBin(v, strlen(v));
+  if (value == NULL)
+    return false;
+  length = strlen(v)/2;
+  return true;
 }
 
 void BinaryParameter::setParam(const void* v, size_t len) {
@@ -456,12 +462,12 @@ void BinaryParameter::setParam(const void* v, size_t len) {
 }
 
 char* BinaryParameter::getDefaultStr() const {
-  return rdr::HexOutStream::binToHexStr(def_value, def_length);
+  return binToHex((const uint8_t*)def_value, def_length);
 }
 
 char* BinaryParameter::getValueStr() const {
   LOCK_CONFIG;
-  return rdr::HexOutStream::binToHexStr(value, length);
+  return binToHex((const uint8_t*)value, length);
 }
 
 void BinaryParameter::getData(void** data_, size_t* length_) const {
diff --git a/common/rfb/util.cxx b/common/rfb/util.cxx
index 649eb0ba..32be6fdd 100644
--- a/common/rfb/util.cxx
+++ b/common/rfb/util.cxx
@@ -1,5 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
- * Copyright 2011-2019 Pierre Ossman for Cendio AB
+ * Copyright 2011-2022 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,6 +21,8 @@
 #include <config.h>
 #endif
 
+#include <assert.h>
+#include <ctype.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <sys/time.h>
@@ -112,6 +114,79 @@ namespace rfb {
     dest[src ? destlen-1 : 0] = 0;
   }
 
+  static char intToHex(uint8_t i) {
+    if (i<=9)
+      return '0'+i;
+    else if ((i>=10) && (i<=15))
+      return 'a'+(i-10);
+    assert(false);
+    return '\0';
+  }
+
+  void binToHex(const uint8_t* in, size_t inlen,
+                char* out, size_t outlen) {
+    if (inlen > outlen/2)
+      inlen = outlen/2;
+
+    if (inlen > 0) {
+      assert(in);
+      assert(out);
+    }
+
+    for (size_t i=0; i<inlen; i++) {
+      out[i*2] = intToHex((in[i] >> 4) & 15);
+      out[i*2+1] = intToHex((in[i] & 15));
+    }
+  }
+
+  char* binToHex(const uint8_t* in, size_t inlen) {
+    char* out = new char[inlen*2+1]();
+    binToHex(in, inlen, out, inlen*2);
+    return out;
+  }
+
+  static bool readHexAndShift(char c, uint8_t* v) {
+    c=tolower(c);
+    if ((c >= '0') && (c <= '9'))
+      *v = (*v << 4) + (c - '0');
+    else if ((c >= 'a') && (c <= 'f'))
+      *v = (*v << 4) + (c - 'a' + 10);
+    else
+      return false;
+    return true;
+  }
+
+  bool hexToBin(const char* in, size_t inlen,
+                uint8_t* out, size_t outlen) {
+    assert(in);
+    assert(out);
+
+    if (inlen & 1)
+      return false;
+
+    if (inlen > outlen*2)
+      inlen = outlen*2;
+
+    for(size_t i=0; i<inlen; i+=2) {
+      uint8_t byte = 0;
+      if (!readHexAndShift(in[i], &byte) ||
+          !readHexAndShift(in[i+1], &byte))
+        return false;
+      out[i/2] = byte;
+    }
+
+    return true;
+  }
+
+  uint8_t* hexToBin(const char* in, size_t inlen) {
+    uint8_t* out = new uint8_t[inlen/2];
+    if (!hexToBin(in, inlen, out, inlen/2)) {
+      delete [] out;
+      return NULL;
+    }
+    return out;
+  }
+
   char* convertLF(const char* src, size_t bytes)
   {
     char* buffer;
diff --git a/common/rfb/util.h b/common/rfb/util.h
index 99d350e3..bac30f3f 100644
--- a/common/rfb/util.h
+++ b/common/rfb/util.h
@@ -1,5 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
- * Copyright 2011-2019 Pierre Ossman for Cendio AB
+ * Copyright 2011-2022 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -26,6 +26,7 @@
 
 #include <limits.h>
 #include <string.h>
+#include <stdint.h>
 
 struct timeval;
 
@@ -73,6 +74,13 @@ namespace rfb {
   // Copies src to dest, up to specified length-1, and guarantees termination
   void strCopy(char* dest, const char* src, int destlen);
 
+  // Conversion to and from a hex string
+
+  void binToHex(const uint8_t* in, size_t inlen, char* out, size_t outlen);
+  char* binToHex(const uint8_t* in, size_t inlen);
+  bool hexToBin(const char* in, size_t inlen, uint8_t* out, size_t outlen);
+  uint8_t* hexToBin(const char* in, size_t inlen);
+
   // Makes sure line endings are in a certain format
 
   char* convertLF(const char* src, size_t bytes = (size_t)-1);
diff --git a/win/rfb_win32/Registry.cxx b/win/rfb_win32/Registry.cxx
index f046188f..cb4e6688 100644
--- a/win/rfb_win32/Registry.cxx
+++ b/win/rfb_win32/Registry.cxx
@@ -175,7 +175,9 @@ TCHAR* RegKey::getString(const TCHAR* valname, const TCHAR* def) const {
 
 void RegKey::getBinary(const TCHAR* valname, void** data, size_t* length) const {
   TCharArray hex(getRepresentation(valname));
-  if (!rdr::HexInStream::hexStrToBin(CStr(hex.buf), (char**)data, length))
+  *data = hexToBin(CStr(hex.buf), strlen(CStr(hex.buf)));
+  *length = strlen(CStr(hex.buf))/2;
+  if (*data == NULL)
     throw rdr::Exception("getBinary failed");
 }
 void RegKey::getBinary(const TCHAR* valname, void** data, size_t* length, void* def, size_t deflen) const {
@@ -233,7 +235,7 @@ TCHAR* RegKey::getRepresentation(const TCHAR* valname) const {
   switch (type) {
   case REG_BINARY:
     {
-      TCharArray hex(rdr::HexOutStream::binToHexStr(data.buf, length));
+      TCharArray hex(binToHex((const uint8_t*)data.buf, length));
       return hex.takeBuf();
     }
   case REG_SZ:
diff --git a/win/rfb_win32/Win32Util.cxx b/win/rfb_win32/Win32Util.cxx
index fc66274f..2fbce0dd 100644
--- a/win/rfb_win32/Win32Util.cxx
+++ b/win/rfb_win32/Win32Util.cxx
@@ -61,13 +61,13 @@ FileVersionInfo::FileVersionInfo(const TCHAR* filename) {
 }
 
 const TCHAR* FileVersionInfo::getVerString(const TCHAR* name, DWORD langId) {
-  char langIdBuf[sizeof(langId)];
+  uint8_t langIdBuf[sizeof(langId)];
   for (int i=sizeof(langIdBuf)-1; i>=0; i--) {
-    langIdBuf[i] = (char) (langId & 0xff);
+    langIdBuf[i] = (langId & 0xff);
     langId = langId >> 8;
   }
 
-  TCharArray langIdStr(rdr::HexOutStream::binToHexStr(langIdBuf, sizeof(langId)));
+  TCharArray langIdStr(binToHex(langIdBuf, sizeof(langId)));
   TCharArray infoName(_tcslen(_T("StringFileInfo")) + 4 + _tcslen(name) + _tcslen(langIdStr.buf));
   _stprintf(infoName.buf, _T("\\StringFileInfo\\%s\\%s"), langIdStr.buf, name);