Remove variable length arrays

These are not allowed in C++, and have been made optional in C11.
So let's just get rid of them and any issues they may cause.

CMakeLists.txt

 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wformat=2")
 # Make sure we catch these issues whilst developing
 IF(CMAKE_BUILD_TYPE MATCHES Debug)
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=vla")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=vla")
 ENDIF()
 
 option(ENABLE_ASAN "Enable address sanitizer support" OFF)

common/rfb/CMsgReader.cxx

   if (width > maxCursorSize || height > maxCursorSize)
     throw Exception("Too big cursor");
 
-  rdr::U8 buf[width*height*4];
+  rdr::U8Array rgba(width*height*4);
 
   if (width * height > 0) {
     rdr::U8 pr, pg, pb;
     is->readBytes(mask.buf, mask_len);
 
     int maskBytesPerRow = (width+7)/8;
-    out = buf;
+    out = rgba.buf;
     for (y = 0;y < height;y++) {
       for (x = 0;x < width;x++) {
         int byte = y * maskBytesPerRow + x / 8;
     }
   }
 
-  handler->setCursor(width, height, hotspot, buf);
+  handler->setCursor(width, height, hotspot, rgba.buf);
 }
 
 void CMsgReader::readSetCursor(int width, int height, const Point& hotspot)
   rdr::U8Array mask(mask_len);
 
   int x, y;
-  rdr::U8 buf[width*height*4];
+  rdr::U8Array rgba(width*height*4);
   rdr::U8* in;
   rdr::U8* out;
 
 
   int maskBytesPerRow = (width+7)/8;
   in = data.buf;
-  out = buf;
+  out = rgba.buf;
   for (y = 0;y < height;y++) {
     for (x = 0;x < width;x++) {
       int byte = y * maskBytesPerRow + x / 8;
     }
   }
 
-  handler->setCursor(width, height, hotspot, buf);
+  handler->setCursor(width, height, hotspot, rgba.buf);
 }
 
 void CMsgReader::readSetCursorWithAlpha(int width, int height, const Point& hotspot)

common/rfb/Cursor.cxx

 }
 
 // Floyd-Steinberg dithering
-static void dither(int width, int height, int* data)
+static void dither(int width, int height, rdr::U16* data)
 {
   for (int y = 0; y < height; y++) {
     for (int x_ = 0; x_ < width; x_++) {
 rdr::U8* Cursor::getBitmap() const
 {
   // First step is converting to luminance
-  int luminance[width()*height()];
-  int *lum_ptr = luminance;
+  rdr::U16Array luminance(width()*height());
+  rdr::U16 *lum_ptr = luminance.buf;
   const rdr::U8 *data_ptr = data;
   for (int y = 0; y < height(); y++) {
     for (int x = 0; x < width(); x++) {
+      rdr::U32 lum;
+
       // Use BT.709 coefficients for grayscale
-      *lum_ptr = 0;
-      *lum_ptr += (int)srgb_to_lin(data_ptr[0]) * 6947;  // 0.2126
-      *lum_ptr += (int)srgb_to_lin(data_ptr[1]) * 23436; // 0.7152
-      *lum_ptr += (int)srgb_to_lin(data_ptr[2]) * 2366;  // 0.0722
-      *lum_ptr /= 32768;
+      lum = 0;
+      lum += (rdr::U32)srgb_to_lin(data_ptr[0]) * 6947;  // 0.2126
+      lum += (rdr::U32)srgb_to_lin(data_ptr[1]) * 23436; // 0.7152
+      lum += (rdr::U32)srgb_to_lin(data_ptr[2]) * 2366;  // 0.0722
+      lum /= 32768;
 
-      lum_ptr++;
+      *lum_ptr++ = lum;
       data_ptr += 4;
     }
   }
 
   // Then diterhing
-  dither(width(), height(), luminance);
+  dither(width(), height(), luminance.buf);
 
   // Then conversion to a bit mask
   rdr::U8Array source((width()+7)/8*height());
   memset(source.buf, 0, (width()+7)/8*height());
   int maskBytesPerRow = (width() + 7) / 8;
-  lum_ptr = luminance;
+  lum_ptr = luminance.buf;
   data_ptr = data;
   for (int y = 0; y < height(); y++) {
     for (int x = 0; x < width(); x++) {
 rdr::U8* Cursor::getMask() const
 {
   // First step is converting to integer array
-  int alpha[width()*height()];
-  int *alpha_ptr = alpha;
+  rdr::U16Array alpha(width()*height());
+  rdr::U16 *alpha_ptr = alpha.buf;
   const rdr::U8 *data_ptr = data;
   for (int y = 0; y < height(); y++) {
     for (int x = 0; x < width(); x++) {
-      *alpha_ptr = (int)data_ptr[3] * 65535 / 255;
-      alpha_ptr++;
+      *alpha_ptr++ = (rdr::U32)data_ptr[3] * 65535 / 255;
       data_ptr += 4;
     }
   }
 
   // Then diterhing
-  dither(width(), height(), alpha);
+  dither(width(), height(), alpha.buf);
 
   // Then conversion to a bit mask
   rdr::U8Array mask((width()+7)/8*height());
   memset(mask.buf, 0, (width()+7)/8*height());
   int maskBytesPerRow = (width() + 7) / 8;
-  alpha_ptr = alpha;
+  alpha_ptr = alpha.buf;
   data_ptr = data;
   for (int y = 0; y < height(); y++) {
     for (int x = 0; x < width(); x++) {

common/rfb/TightDecoder.cxx

       buflen -= 1;
 
       if (pf.is888()) {
-        rdr::U8 tightPalette[palSize * 3];
+        size_t len = palSize * 3;
+        rdr::U8Array tightPalette(len);
 
-        assert(buflen >= sizeof(tightPalette));
+        assert(buflen >= sizeof(len));
 
-        memcpy(tightPalette, bufptr, sizeof(tightPalette));
-        bufptr += sizeof(tightPalette);
-        buflen -= sizeof(tightPalette);
+        memcpy(tightPalette.buf, bufptr, len);
+        bufptr += len;
+        buflen -= len;
 
-        pf.bufferFromRGB(palette, tightPalette, palSize);
+        pf.bufferFromRGB(palette, tightPalette.buf, palSize);
       } else {
         size_t len;
 

tests/encperf.cxx

   }
 
   int runCount = count;
-  struct stats runs[runCount];
-  double values[runCount], dev[runCount];
+  struct stats *runs = new struct stats[runCount];
+  double *values = new double[runCount];
+  double *dev = new double[runCount];
   double median, meddev;
 
   if (fn == NULL) {

vncviewer/gettext.h

 
 #include <string.h>
 
-#if (((__GNUC__ >= 3 || __GNUG__ >= 2) && !defined __STRICT_ANSI__) \
+#if (((__GNUC__ >= 3 || __GNUG__ >= 2) && !defined __STRICT_ANSI__ && !defined __cplusplus) \
      /* || __STDC_VERSION__ == 199901L
         || (__STDC_VERSION__ >= 201112L && !defined __STDC_NO_VLA__) */ )
 # define _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS 1

vncviewer/parameters.cxx

 
 static bool getKeyString(const char* _name, char* dest, size_t destSize, HKEY* hKey) {
   
-  DWORD buffersize = 256;
-  WCHAR value[destSize];
+  const DWORD buffersize = 256;
   wchar_t name[buffersize];
+  WCHAR* value;
+  DWORD valuesize;
 
   unsigned size = fl_utf8towc(_name, strlen(_name)+1, name, buffersize);
   if (size >= buffersize) {
     return false;
   }
 
-  LONG res = RegQueryValueExW(*hKey, name, 0, NULL, (LPBYTE)value, &buffersize);
+  value = new WCHAR[destSize];
+  valuesize = destSize;
+  LONG res = RegQueryValueExW(*hKey, name, 0, NULL, (LPBYTE)value, &valuesize);
   if (res != ERROR_SUCCESS){
+    delete [] value;
     if (res == ERROR_FILE_NOT_FOUND) {
       // The value does not exist, defaults will be used.
     } else {
     return false;
   }
   
-  char utf8val[destSize];
-  size = fl_utf8fromwc(utf8val, sizeof(utf8val), value, wcslen(value)+1);
-  if (size >= sizeof(utf8val)) {
+  char* utf8val = new char[destSize];
+  size = fl_utf8fromwc(utf8val, destSize, value, wcslen(value)+1);
+  delete [] value;
+  if (size >= destSize) {
+    delete [] utf8val;
     vlog.error(_("The parameter %s was too large to read from the registry"), _name);
     return false;
   }
-  const char *ret = utf8val;
   
-  if(decodeValue(ret, dest, destSize))
-    return true;
-  else 
-    return false;
+  bool ret = decodeValue(utf8val, dest, destSize);
+  delete [] utf8val;
+
+  return ret;
 }