Move hex conversion helpers to util

These are used here and there so let's make them more general rather
than hiding them in the stream classes.

3 files changed, 94 insertions, 5 deletions

diff --git a/common/rfb/Configuration.cxx b/common/rfb/Configuration.cxx
index 8848e08d..350582f0 100644
--- a/common/rfb/Configuration.cxx
+++ b/common/rfb/Configuration.cxx
@@ -440,7 +440,13 @@ bool BinaryParameter::setParam(const char* v) {
   LOCK_CONFIG;
   if (immutable) return true;
   vlog.debug("set %s(Binary) to %s", getName(), v);
-  return rdr::HexInStream::hexStrToBin(v, &value, &length);
+  delete [] value;
+  length = 0;
+  value = (char*)hexToBin(v, strlen(v));
+  if (value == NULL)
+    return false;
+  length = strlen(v)/2;
+  return true;
 }
 
 void BinaryParameter::setParam(const void* v, size_t len) {
@@ -456,12 +462,12 @@ void BinaryParameter::setParam(const void* v, size_t len) {
 }
 
 char* BinaryParameter::getDefaultStr() const {
-  return rdr::HexOutStream::binToHexStr(def_value, def_length);
+  return binToHex((const uint8_t*)def_value, def_length);
 }
 
 char* BinaryParameter::getValueStr() const {
   LOCK_CONFIG;
-  return rdr::HexOutStream::binToHexStr(value, length);
+  return binToHex((const uint8_t*)value, length);
 }
 
 void BinaryParameter::getData(void** data_, size_t* length_) const {
diff --git a/common/rfb/util.cxx b/common/rfb/util.cxx
index 649eb0ba..32be6fdd 100644
--- a/common/rfb/util.cxx
+++ b/common/rfb/util.cxx
@@ -1,5 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
- * Copyright 2011-2019 Pierre Ossman for Cendio AB
+ * Copyright 2011-2022 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,6 +21,8 @@
 #include <config.h>
 #endif
 
+#include <assert.h>
+#include <ctype.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <sys/time.h>
@@ -112,6 +114,79 @@ namespace rfb {
     dest[src ? destlen-1 : 0] = 0;
   }
 
+  static char intToHex(uint8_t i) {
+    if (i<=9)
+      return '0'+i;
+    else if ((i>=10) && (i<=15))
+      return 'a'+(i-10);
+    assert(false);
+    return '\0';
+  }
+
+  void binToHex(const uint8_t* in, size_t inlen,
+                char* out, size_t outlen) {
+    if (inlen > outlen/2)
+      inlen = outlen/2;
+
+    if (inlen > 0) {
+      assert(in);
+      assert(out);
+    }
+
+    for (size_t i=0; i<inlen; i++) {
+      out[i*2] = intToHex((in[i] >> 4) & 15);
+      out[i*2+1] = intToHex((in[i] & 15));
+    }
+  }
+
+  char* binToHex(const uint8_t* in, size_t inlen) {
+    char* out = new char[inlen*2+1]();
+    binToHex(in, inlen, out, inlen*2);
+    return out;
+  }
+
+  static bool readHexAndShift(char c, uint8_t* v) {
+    c=tolower(c);
+    if ((c >= '0') && (c <= '9'))
+      *v = (*v << 4) + (c - '0');
+    else if ((c >= 'a') && (c <= 'f'))
+      *v = (*v << 4) + (c - 'a' + 10);
+    else
+      return false;
+    return true;
+  }
+
+  bool hexToBin(const char* in, size_t inlen,
+                uint8_t* out, size_t outlen) {
+    assert(in);
+    assert(out);
+
+    if (inlen & 1)
+      return false;
+
+    if (inlen > outlen*2)
+      inlen = outlen*2;
+
+    for(size_t i=0; i<inlen; i+=2) {
+      uint8_t byte = 0;
+      if (!readHexAndShift(in[i], &byte) ||
+          !readHexAndShift(in[i+1], &byte))
+        return false;
+      out[i/2] = byte;
+    }
+
+    return true;
+  }
+
+  uint8_t* hexToBin(const char* in, size_t inlen) {
+    uint8_t* out = new uint8_t[inlen/2];
+    if (!hexToBin(in, inlen, out, inlen/2)) {
+      delete [] out;
+      return NULL;
+    }
+    return out;
+  }
+
   char* convertLF(const char* src, size_t bytes)
   {
     char* buffer;
diff --git a/common/rfb/util.h b/common/rfb/util.h
index 99d350e3..bac30f3f 100644
--- a/common/rfb/util.h
+++ b/common/rfb/util.h
@@ -1,5 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
- * Copyright 2011-2019 Pierre Ossman for Cendio AB
+ * Copyright 2011-2022 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -26,6 +26,7 @@
 
 #include <limits.h>
 #include <string.h>
+#include <stdint.h>
 
 struct timeval;
 
@@ -73,6 +74,13 @@ namespace rfb {
   // Copies src to dest, up to specified length-1, and guarantees termination
   void strCopy(char* dest, const char* src, int destlen);
 
+  // Conversion to and from a hex string
+
+  void binToHex(const uint8_t* in, size_t inlen, char* out, size_t outlen);
+  char* binToHex(const uint8_t* in, size_t inlen);
+  bool hexToBin(const char* in, size_t inlen, uint8_t* out, size_t outlen);
+  uint8_t* hexToBin(const char* in, size_t inlen);
+
   // Makes sure line endings are in a certain format
 
   char* convertLF(const char* src, size_t bytes = (size_t)-1);