H264 Decoder

Linux implementation using ffmpeg

.github/workflows/build.yml

@@ -12,6 +12,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y libgnutls28-dev libfltk1.3-dev fluid gettext
           sudo apt-get install -y libxtst-dev libxdamage-dev libxfixes-dev libxrandr-dev libpam-dev
+          sudo apt-get install -y libavcodec-dev libavutil-dev libswscale-dev
       - name: Configure
         run: cmake -DCMAKE_BUILD_TYPE=Debug -S . -B build
       - name: Build
@@ -50,7 +51,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Install dependencies
         run: |
-          brew install fltk pixman
+          brew install fltk pixman ffmpeg
       - name: Configure
         run: cmake -DCMAKE_BUILD_TYPE=Debug -S . -B build
       - name: Build

BUILDING.txt

@@ -40,6 +40,8 @@ Build Requirements (Unix)
    * Xorg server source code, 1.16 or never
    * All build requirements Xorg imposes (see its documentation)
 
+-- Optional ffmpeg support (libav)
+
 ============================
 Build Requirements (Windows)
 ============================

CMakeLists.txt

@@ -171,6 +171,36 @@ if(ENABLE_NLS)
   endif()
 endif()
 
+option(ENABLE_H264 "Enable H.264 RFB encoding" ON)
+if(ENABLE_H264)
+  if(WIN32)
+    add_definitions("-DHAVE_H264")
+    set(H264_LIBS "NONE")  # may be LIBAV in the future
+    message(WARNING "NO H264 LIBs are supported on Windows")
+  else()
+    check_include_files(libavcodec/avcodec.h HAVE_AVCODEC_H)
+    check_include_files(libavutil/avutil.h HAVE_AVUTIL_H)
+    check_include_files(libswscale/swscale.h HAVE_SWSCALE_H)
+    if (HAVE_AVCODEC_H AND HAVE_AVUTIL_H AND HAVE_SWSCALE_H)
+      find_path(AVCODEC_INCLUDE_DIR libavcodec/avcodec.h)
+      find_library(AVCODEC_LIBRARY avcodec)
+      find_path(AVUTIL_INCLUDE_DIR libavutil/avutil.h)
+      find_library(AVUTIL_LIBRARY avutil)
+      find_path(SWSCALE_INCLUDE_DIR libswscale/swscale.h)
+      find_library(SWSCALE_LIBRARY swscale)
+      include_directories(${AVCODEC_INCLUDE_DIR} ${AVUTIL_INCLUDE_DIR} ${SWSCALE_INCLUDE_DIR})
+      set(H264_LIBRARIES ${AVCODEC_LIBRARY} ${AVUTIL_LIBRARY} ${SWSCALE_LIBRARY})
+      add_definitions("-D__STDC_CONSTANT_MACROS")
+      add_definitions("-DHAVE_H264")
+      set(H264_LIBS "LIBAV")
+    else()
+      set(H264_LIBS "NONE")
+      message(WARNING "FFMPEG support can't be found")
+    endif()
+  endif()
+  add_definitions("-DH264_${H264_LIBS}")
+endif()
+
 # Check for libjpeg
 find_package(JPEG REQUIRED)
 

common/rfb/CMakeLists.txt

@@ -64,6 +64,13 @@ set(RFB_SOURCES
   encodings.cxx
   util.cxx)
 
+if(ENABLE_H264 AND NOT H264_LIBS STREQUAL "NONE")
+  set(RFB_SOURCES ${RFB_SOURCES} H264Decoder.cxx H264DecoderContext.cxx)
+  if(H264_LIBS STREQUAL "LIBAV")
+    set(RFB_SOURCES ${RFB_SOURCES} H264LibavDecoderContext.cxx)
+  endif()
+endif()
+
 if(UNIX)
   set(RFB_SOURCES ${RFB_SOURCES} Logger_syslog.cxx)
 endif()

common/rfb/Decoder.cxx

@@ -31,6 +31,9 @@
 #include <rfb/HextileDecoder.h>
 #include <rfb/ZRLEDecoder.h>
 #include <rfb/TightDecoder.h>
+#ifdef HAVE_H264
+#include <rfb/H264Decoder.h>
+#endif
 
 using namespace rfb;
 
@@ -66,6 +69,9 @@ bool Decoder::supported(int encoding)
   case encodingHextile:
   case encodingZRLE:
   case encodingTight:
+#ifdef HAVE_H264
+  case encodingH264:
+#endif
     return true;
   default:
     return false;
@@ -87,6 +93,10 @@ Decoder* Decoder::createDecoder(int encoding)
     return new ZRLEDecoder();
   case encodingTight:
     return new TightDecoder();
+#ifdef HAVE_H264
+  case encodingH264:
+    return new H264Decoder();
+#endif
   default:
     return NULL;
   }

common/rfb/H264Decoder.cxx

@@ -0,0 +1,138 @@
+/* Copyright (C) 2021 Vladimir Sukhonosov <xornet@xornet.org>
+ * Copyright (C) 2021 Martins Mozeiko <martins.mozeiko@gmail.com>
+ * All Rights Reserved.
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#define MAX_H264_INSTANCES 64
+
+#include <deque>
+
+#include <rdr/MemInStream.h>
+#include <rdr/InStream.h>
+#include <rdr/OutStream.h>
+#include <rfb/LogWriter.h>
+#include <rfb/Exception.h>
+#include <rfb/H264Decoder.h>
+#include <rfb/H264DecoderContext.h>
+
+using namespace rfb;
+
+static LogWriter vlog("H264Decoder");
+
+enum rectFlags {
+  resetContext       = 0x1,
+  resetAllContexts   = 0x2,
+};
+
+H264Decoder::H264Decoder() : Decoder(DecoderOrdered)
+{
+}
+
+H264Decoder::~H264Decoder()
+{
+  resetContexts();
+}
+
+void H264Decoder::resetContexts()
+{
+  os::AutoMutex lock(&mutex);
+  for (std::deque<H264DecoderContext*>::iterator it = contexts.begin(); it != contexts.end(); it++)
+    delete *it;
+  contexts.clear();
+}
+
+H264DecoderContext* H264Decoder::findContext(const Rect& r)
+{
+  os::AutoMutex m(&mutex);
+  for (std::deque<H264DecoderContext*>::iterator it = contexts.begin(); it != contexts.end(); it++)
+    if ((*it)->isEqualRect(r))
+      return *it;
+  return NULL;
+}
+
+bool H264Decoder::readRect(const Rect& r, rdr::InStream* is,
+                           const ServerParams& server, rdr::OutStream* os)
+{
+  rdr::U32 len;
+
+  if (!is->hasData(8))
+    return false;
+
+  is->setRestorePoint();
+
+  len = is->readU32();
+  os->writeU32(len);
+  rdr::U32 flags = is->readU32();
+
+  os->writeU32(flags);
+
+  if (!is->hasDataOrRestore(len))
+    return false;
+
+  is->clearRestorePoint();
+
+  os->copyBytes(is, len);
+
+  return true;
+}
+
+void H264Decoder::decodeRect(const Rect& r, const void* buffer,
+                             size_t buflen, const ServerParams& server,
+                             ModifiablePixelBuffer* pb)
+{
+  rdr::MemInStream is(buffer, buflen);
+  rdr::U32 len = is.readU32();
+  rdr::U32 flags = is.readU32();
+
+  H264DecoderContext* ctx = NULL;
+  if (flags & resetAllContexts)
+  {
+    resetContexts();
+    if (!len)
+      return;
+    flags &= ~(resetContext | resetAllContexts);
+  } else {
+    ctx = findContext(r);
+  }
+
+  if (!ctx)
+  {
+    os::AutoMutex lock(&mutex);
+    if (contexts.size() >= MAX_H264_INSTANCES)
+    {
+      auto excecc_ctx = contexts.front();
+      delete excecc_ctx;
+      contexts.pop_front();
+    }
+    ctx = H264DecoderContext::createContext(r);
+    if (!ctx)
+      throw Exception("H264Decoder: Context not be created");
+    contexts.push_back(ctx);
+  }
+
+  if (!ctx->isReady())
+    throw Exception("H264Decoder: Context is not ready");
+
+  if (flags & resetContext)
+    ctx->reset();
+
+  if (!len)
+    return;
+
+  ctx->decode(is.getptr(len), len, flags, pb);
+}

common/rfb/H264Decoder.h

@@ -0,0 +1,51 @@
+/* Copyright (C) 2021 Vladimir Sukhonosov <xornet@xornet.org>
+ * Copyright (C) 2021 Martins Mozeiko <martins.mozeiko@gmail.com>
+ * All Rights Reserved.
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#ifndef __RFB_H264DECODER_H__
+#define __RFB_H264DECODER_H__
+
+#include <deque>
+
+#include <os/Mutex.h>
+#include <rfb/Decoder.h>
+
+namespace rfb {
+  class H264DecoderContext;
+
+  class H264Decoder : public Decoder {
+  public:
+    H264Decoder();
+    virtual ~H264Decoder();
+    virtual bool readRect(const Rect& r, rdr::InStream* is,
+                          const ServerParams& server, rdr::OutStream* os);
+    virtual void decodeRect(const Rect& r, const void* buffer,
+                            size_t buflen, const ServerParams& server,
+                            ModifiablePixelBuffer* pb);
+
+  private:
+    void resetContexts();
+    H264DecoderContext* findContext(const Rect& r);
+
+    os::Mutex mutex;
+    std::deque<H264DecoderContext*> contexts;
+  };
+}
+
+#endif

common/rfb/H264DecoderContext.cxx

@@ -0,0 +1,64 @@
+/* Copyright (C) 2021 Vladimir Sukhonosov <xornet@xornet.org>
+ * Copyright (C) 2021 Martins Mozeiko <martins.mozeiko@gmail.com>
+ * All Rights Reserved.
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#include <os/Mutex.h>
+#include <rfb/Exception.h>
+#include <rfb/LogWriter.h>
+
+#include <rfb/H264DecoderContext.h>
+
+#ifdef H264_LIBAV
+#include <rfb/H264LibavDecoderContext.h>
+#define H264DecoderContextType H264LibavDecoderContext
+#elif H264_WIN
+#include <rfb/H264WinDecoderContext.h>
+#define H264DecoderContextType H264WinDecoderContext
+#endif
+
+using namespace rfb;
+
+static LogWriter vlog("H264DecoderContext");
+
+H264DecoderContext *H264DecoderContext::createContext(const Rect &r)
+{
+  H264DecoderContext *ret = new H264DecoderContextType(r);
+  if (!ret->initCodec())
+  {
+    throw Exception("H264DecoderContext: Unable to create context");
+  }
+
+  return ret;
+}
+
+H264DecoderContext::~H264DecoderContext()
+{
+}
+
+bool H264DecoderContext::isReady()
+{
+  os::AutoMutex lock(&mutex);
+  return initialized;
+}
+
+void H264DecoderContext::reset()
+{
+  freeCodec();
+  initCodec();
+}

common/rfb/H264DecoderContext.h

@@ -0,0 +1,54 @@
+/* Copyright (C) 2021 Vladimir Sukhonosov <xornet@xornet.org>
+ * Copyright (C) 2021 Martins Mozeiko <martins.mozeiko@gmail.com>
+ * All Rights Reserved.
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#ifndef __RFB_H264DECODERCONTEXT_H__
+#define __RFB_H264DECODERCONTEXT_H__
+
+#include <os/Mutex.h>
+#include <rdr/types.h>
+#include <rfb/Rect.h>
+#include <rfb/Decoder.h>
+
+namespace rfb {
+  class H264DecoderContext {
+    public:
+      static H264DecoderContext *createContext(const Rect &r);
+
+      virtual ~H264DecoderContext() = 0;
+
+      virtual void decode(const rdr::U8* h264_buffer, rdr::U32 len, rdr::U32 flags, ModifiablePixelBuffer* pb) {}
+      void reset();
+
+      inline bool isEqualRect(const Rect &r) const { return r.equals(rect); }
+      bool isReady();
+
+    protected:
+      os::Mutex mutex;
+      rfb::Rect rect;
+      bool initialized = false;
+
+      H264DecoderContext(const Rect &r) : rect(r) { initialized = false; }
+
+      virtual bool initCodec() { return false; }
+      virtual void freeCodec() {}
+  };
+}
+
+#endif

common/rfb/H264LibavDecoderContext.cxx

@@ -0,0 +1,228 @@
+/* Copyright (C) 2021 Vladimir Sukhonosov <xornet@xornet.org>
+ * Copyright (C) 2021 Martins Mozeiko <martins.mozeiko@gmail.com>
+ * All Rights Reserved.
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+
+extern "C" {
+#include <libavutil/imgutils.h>
+#include <libavcodec/version.h>
+}
+#if LIBAVCODEC_VERSION_MAJOR > 57 || LIBAVCODEC_VERSION_MAJOR == 57 && LIBAVCODEC_VERSION_MINOR >= 37
+#define FFMPEG_DECODE_VIDEO2_DEPRECATED
+#endif
+#if LIBAVCODEC_VERSION_MAJOR >= 58
+#define FFMPEG_INIT_PACKET_DEPRECATED
+#endif
+
+#include <rfb/Exception.h>
+#include <rfb/LogWriter.h>
+#include <rfb/PixelBuffer.h>
+#include <rfb/H264LibavDecoderContext.h>
+
+using namespace rfb;
+
+static LogWriter vlog("H264LibavDecoderContext");
+
+bool H264LibavDecoderContext::initCodec() {
+  os::AutoMutex lock(&mutex);
+
+  sws = NULL;
+  swsBuffer = NULL;
+  h264WorkBuffer = NULL;
+  h264WorkBufferLength = 0;
+
+  AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264);
+  if (!codec)
+  {
+    vlog.error("Codec not found");
+    return false;
+  }
+
+  parser = av_parser_init(codec->id);
+  if (!parser)
+  {
+    vlog.error("Could not create H264 parser");
+    return false;
+  }
+
+  avctx = avcodec_alloc_context3(codec);
+  if (!avctx)
+  {
+    av_parser_close(parser);
+    vlog.error("Could not allocate video codec context");
+    return false;
+  }
+
+  frame = av_frame_alloc();
+  if (!frame)
+  {
+    av_parser_close(parser);
+    avcodec_free_context(&avctx);
+    vlog.error("Could not allocate video frame");
+    return false;
+  }
+
+  if (avcodec_open2(avctx, codec, NULL) < 0)
+  {
+    av_parser_close(parser);
+    avcodec_free_context(&avctx);
+    av_frame_free(&frame);
+    vlog.error("Could not open codec");
+    return false;
+  }
+
+  int numBytes = av_image_get_buffer_size(AV_PIX_FMT_RGB32, rect.width(), rect.height(), 1);
+  swsBuffer = new uint8_t[numBytes];
+
+  initialized = true;
+  return true;
+}
+
+void H264LibavDecoderContext::freeCodec() {
+  os::AutoMutex lock(&mutex);
+
+  if (!initialized)
+    return;
+  av_parser_close(parser);
+  avcodec_free_context(&avctx);
+  av_frame_free(&frame);
+  delete[] swsBuffer;
+  free(h264WorkBuffer);
+  initialized = false;
+}
+
+// We need to reallocate buffer because AVPacket uses non-const pointer.
+// We don't want to const_cast our buffer somewhere. So we would rather to maintain context's own buffer
+// Also avcodec requires a right padded buffer
+rdr::U8* H264LibavDecoderContext::makeH264WorkBuffer(const rdr::U8* buffer, rdr::U32 len)
+{
+  rdr::U32 reserve_len = len + len % AV_INPUT_BUFFER_PADDING_SIZE;
+
+  if (!h264WorkBuffer || reserve_len > h264WorkBufferLength)
+  {
+    h264WorkBuffer = (rdr::U8*)realloc(h264WorkBuffer, reserve_len);
+    if (h264WorkBuffer == NULL) {
+      throw Exception("H264LibavDecoderContext: Unable to allocate memory");
+    }
+    h264WorkBufferLength = reserve_len;
+  }
+
+  memcpy(h264WorkBuffer, buffer, len);
+  memset(h264WorkBuffer + len, 0, h264WorkBufferLength - len);
+  return h264WorkBuffer;
+}
+
+void H264LibavDecoderContext::decode(const rdr::U8* h264_in_buffer, rdr::U32 len, rdr::U32 flags, ModifiablePixelBuffer* pb) {
+  os::AutoMutex lock(&mutex);
+  if (!initialized)
+    return;
+  rdr::U8* h264_work_buffer = makeH264WorkBuffer(h264_in_buffer, len);
+
+#ifdef FFMPEG_INIT_PACKET_DEPRECATED
+  AVPacket *packet = av_packet_alloc();
+#else
+  AVPacket *packet = new AVPacket();
+  av_init_packet(packet);
+#endif
+
+  int ret;
+  int frames_received = 0;
+  while (len)
+  {
+    ret = av_parser_parse2(parser, avctx, &packet->data, &packet->size, h264_work_buffer, len, AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0);
+    if (ret < 0)
+    {
+      vlog.error("Error while parsing");
+      break;
+    }
+    // We need to slap on tv to make it work here (don't ask me why)
+    if (!packet->size && len == static_cast<rdr::U32>(ret))
+      ret = av_parser_parse2(parser, avctx, &packet->data, &packet->size, h264_work_buffer, len, AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0);
+    if (ret < 0)
+    {
+      vlog.error("Error while parsing");
+      break;
+    }
+    h264_work_buffer += ret;
+    len -= ret;
+
+    if (!ret)
+    {
+      packet->size = len;
+      packet->data = h264_work_buffer;
+      len = 0;
+    }
+
+    if (!packet->size)
+      continue;
+
+#ifndef FFMPEG_DECODE_VIDEO2_DEPRECATED
+    int got_frame;
+    ret = avcodec_decode_video2(avctx, frame, &got_frame, packet);
+    if (ret < 0 || !got_frame)
+    {
+      vlog.error("Error during decoding");
+      break;
+    }
+#else
+    ret = avcodec_send_packet(avctx, packet);
+    if (ret < 0)
+    {
+      vlog.error("Error sending a packet to decoding");
+      break;
+    }
+
+    ret = avcodec_receive_frame(avctx, frame);
+    if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
+      break;
+    else if (ret < 0)
+    {
+      vlog.error("Error during decoding");
+      break;
+    }
+#endif
+    frames_received++;
+  }
+
+#ifdef FFMPEG_INIT_PACKET_DEPRECATED
+  packet->size = 0;
+  packet->data = NULL;
+  av_packet_free(&packet);
+#else
+  delete packet;
+#endif
+
+  if (!frames_received)
+    return;
+
+  if (!frame->height)
+    return;
+
+  sws = sws_getCachedContext(sws, frame->width, frame->height, avctx->pix_fmt,
+                             frame->width, frame->height, AV_PIX_FMT_RGB32,
+                             0, NULL, NULL, NULL);
+
+  int stride;
+  pb->getBuffer(rect, &stride);
+  int dst_linesize = stride * pb->getPF().bpp/8;  // stride is in pixels, linesize is in bytes (stride x4). We need bytes
+
+  sws_scale(sws, frame->data, frame->linesize, 0, frame->height, &swsBuffer, &dst_linesize);
+
+  pb->imageRect(rect, swsBuffer, stride);
+}

common/rfb/H264LibavDecoderContext.h

@@ -0,0 +1,56 @@
+/* Copyright (C) 2021 Vladimir Sukhonosov <xornet@xornet.org>
+ * Copyright (C) 2021 Martins Mozeiko <martins.mozeiko@gmail.com>
+ * All Rights Reserved.
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#ifndef __RFB_H264LIBAVDECODER_H__
+#define __RFB_H264LIBAVDECODER_H__
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libswscale/swscale.h>
+}
+
+#include <rfb/H264DecoderContext.h>
+
+namespace rfb {
+  class H264LibavDecoderContext : public H264DecoderContext {
+    public:
+      H264LibavDecoderContext(const Rect &r) : H264DecoderContext(r) {}
+      ~H264LibavDecoderContext() { freeCodec(); }
+
+      virtual void decode(const rdr::U8* h264_buffer, rdr::U32 len, rdr::U32 flags, ModifiablePixelBuffer* pb);
+
+    protected:
+      virtual bool initCodec();
+      virtual void freeCodec();
+
+    private:
+      rdr::U8* makeH264WorkBuffer(const rdr::U8* buffer, rdr::U32 len);
+
+      AVCodecContext *avctx;
+      AVCodecParserContext *parser;
+      AVFrame* frame;
+      SwsContext* sws;
+      uint8_t* swsBuffer;
+      rdr::U8* h264WorkBuffer;
+      rdr::U32 h264WorkBufferLength;
+  };
+}
+
+#endif

common/rfb/encodings.cxx

@@ -33,6 +33,9 @@ int rfb::encodingNum(const char* name)
   if (strcasecmp(name, "hextile") == 0)  return encodingHextile;
   if (strcasecmp(name, "ZRLE") == 0)     return encodingZRLE;
   if (strcasecmp(name, "Tight") == 0)    return encodingTight;
+#ifdef HAVE_H264
+  if (strcasecmp(name, "H.264") == 0)    return encodingH264;
+#endif
   return -1;
 }
 
@@ -46,6 +49,9 @@ const char* rfb::encodingName(int num)
   case encodingHextile:  return "hextile";
   case encodingZRLE:     return "ZRLE";
   case encodingTight:    return "Tight";
+#ifdef HAVE_H264
+  case encodingH264:     return "H.264";
+#endif
   default:               return "[unknown encoding]";
   }
 }

common/rfb/encodings.h

@@ -28,6 +28,9 @@ namespace rfb {
   const int encodingHextile = 5;
   const int encodingTight = 7;
   const int encodingZRLE = 16;
+#ifdef HAVE_H264
+  const int encodingH264 = 50;
+#endif
 
   const int encodingMax = 255;
 

contrib/packages/deb/ubuntu-bionic/debian/control

@@ -3,7 +3,7 @@ Section: x11
 Priority: optional
 Maintainer: Brian P. Hinz <bphinz@users.sourceforge.net>
 Standards-Version: 3.8.4
-Build-Depends: debhelper (>= 9),dh-autoreconf,po-debconf,dpkg-dev (>= 1.16.1),quilt,lsb-release,pkg-config,bison,flex,xauth,xutils-dev (>= 1:7.6+4),xfonts-utils (>= 1:7.5+1),x11proto-bigreqs-dev (>= 1:1.1.0),x11proto-composite-dev (>= 1:0.4),x11proto-core-dev (>= 7.0.31),x11proto-damage-dev (>= 1.1),x11proto-fixes-dev (>= 1:5.0),x11proto-fonts-dev (>= 2.1.3),x11proto-kb-dev (>= 1.0.3),x11proto-xinerama-dev,x11proto-randr-dev (>= 1.5.0),x11proto-record-dev (>= 1.13.99.1),x11proto-render-dev (>= 2:0.11),x11proto-resource-dev (>= 1.2.0),x11proto-scrnsaver-dev,x11proto-video-dev,x11proto-xcmisc-dev (>= 1.2.0),x11proto-xext-dev (>= 7.2.99.901),x11proto-xf86bigfont-dev (>= 1.2.0),x11proto-xf86dga-dev (>= 2.0.99.1),x11proto-xf86vidmode-dev (>= 2.2.99.1),x11proto-present-dev,x11proto-dri3-dev,xtrans-dev (>= 1.3.5),libxau-dev (>= 1:1.0.5-2),x11proto-input-dev (>= 2.3),x11proto-dri2-dev (>= 2.8),libxdmcp-dev (>= 1:0.99.1),libxfont-dev (>= 1:2.0.1),libxkbfile-dev (>= 1:0.99.1),libpixman-1-dev (>= 0.27.2),libpciaccess-dev (>= 0.12.901),libgcrypt-dev,nettle-dev,libudev-dev (>= 151-3) [linux-any],libselinux1-dev (>= 2.0.80) [linux-any],libaudit-dev [linux-any],x11proto-xf86dri-dev (>= 2.1.0),libdrm-dev (>= 2.4.46) [!hurd-i386],x11proto-gl-dev (>= 1.4.17),libgl1-mesa-dev (>= 9.2),libxmuu-dev (>= 1:0.99.1),libxext-dev (>= 1:0.99.1),libx11-dev (>= 2:1.6),libxrender-dev (>= 1:0.9.0),libxi-dev (>= 2:1.6.99.1),x11proto-dmx-dev (>= 1:2.2.99.1),libdmx-dev (>= 1:1.0.1),libxpm-dev (>= 1:3.5.3),libxaw7-dev (>= 1:0.99.1),libxt-dev (>= 1:0.99.1),libxmu-dev (>= 1:0.99.1),libxtst-dev (>= 1:0.99.1),libxres-dev (>= 1:0.99.1),libxfixes-dev (>= 1:3.0.0),libxv-dev,libxinerama-dev,libxshmfence-dev (>= 1.1) [!hurd-i386],libepoxy-dev [linux-any kfreebsd-any],libegl1-mesa-dev [linux-any kfreebsd-any],libgbm-dev (>= 10.2) [linux-any kfreebsd-any],libxcb1-dev,libxcb-xkb-dev,libxcb-shape0-dev,libxcb-render0-dev,libxcb-render-util0-dev,libxcb-util0-dev,libxcb-image0-dev,libxcb-icccm4-dev,libxcb-shm0-dev,libxcb-keysyms1-dev,libxcb-randr0-dev,libxcb-xv0-dev,libxcb-glx0-dev,libxcb-xf86dri0-dev (>= 1.6),xkb-data,x11-xkb-utils,libbsd-dev,libwayland-dev [linux-any],wayland-protocols (>= 1.9) [linux-any],libdbus-1-dev (>= 1.0) [linux-any],libsystemd-dev [linux-any],libmirclient-dev (>= 0.13.1) [!powerpc !ppc64el !s390x],mir-client-platform-mesa-dev,zlib1g-dev,libjpeg-turbo8-dev,perl-modules,cmake,gnutls-dev,libpam0g-dev,libpng-dev,automake,autoconf,libtool,pkg-config,openjdk-8-jdk,xorg-server-source,libfltk1.3-dev,fluid,curl,bzip2,quilt,libosmesa6-dev,libgl1-mesa-dri,libgl1-mesa-glx,libxft-dev,xfonts-base,libexpat1-dev,libfontconfig1-dev,libglu1-mesa-dev,libxcursor-dev,libxrandr-dev,libcairo2-dev
+Build-Depends: debhelper (>= 9),dh-autoreconf,po-debconf,dpkg-dev (>= 1.16.1),quilt,lsb-release,pkg-config,bison,flex,xauth,xutils-dev (>= 1:7.6+4),xfonts-utils (>= 1:7.5+1),x11proto-bigreqs-dev (>= 1:1.1.0),x11proto-composite-dev (>= 1:0.4),x11proto-core-dev (>= 7.0.31),x11proto-damage-dev (>= 1.1),x11proto-fixes-dev (>= 1:5.0),x11proto-fonts-dev (>= 2.1.3),x11proto-kb-dev (>= 1.0.3),x11proto-xinerama-dev,x11proto-randr-dev (>= 1.5.0),x11proto-record-dev (>= 1.13.99.1),x11proto-render-dev (>= 2:0.11),x11proto-resource-dev (>= 1.2.0),x11proto-scrnsaver-dev,x11proto-video-dev,x11proto-xcmisc-dev (>= 1.2.0),x11proto-xext-dev (>= 7.2.99.901),x11proto-xf86bigfont-dev (>= 1.2.0),x11proto-xf86dga-dev (>= 2.0.99.1),x11proto-xf86vidmode-dev (>= 2.2.99.1),x11proto-present-dev,x11proto-dri3-dev,xtrans-dev (>= 1.3.5),libxau-dev (>= 1:1.0.5-2),x11proto-input-dev (>= 2.3),x11proto-dri2-dev (>= 2.8),libxdmcp-dev (>= 1:0.99.1),libxfont-dev (>= 1:2.0.1),libxkbfile-dev (>= 1:0.99.1),libpixman-1-dev (>= 0.27.2),libpciaccess-dev (>= 0.12.901),libgcrypt-dev,nettle-dev,libudev-dev (>= 151-3) [linux-any],libselinux1-dev (>= 2.0.80) [linux-any],libaudit-dev [linux-any],x11proto-xf86dri-dev (>= 2.1.0),libdrm-dev (>= 2.4.46) [!hurd-i386],x11proto-gl-dev (>= 1.4.17),libgl1-mesa-dev (>= 9.2),libxmuu-dev (>= 1:0.99.1),libxext-dev (>= 1:0.99.1),libx11-dev (>= 2:1.6),libxrender-dev (>= 1:0.9.0),libxi-dev (>= 2:1.6.99.1),x11proto-dmx-dev (>= 1:2.2.99.1),libdmx-dev (>= 1:1.0.1),libxpm-dev (>= 1:3.5.3),libxaw7-dev (>= 1:0.99.1),libxt-dev (>= 1:0.99.1),libxmu-dev (>= 1:0.99.1),libxtst-dev (>= 1:0.99.1),libxres-dev (>= 1:0.99.1),libxfixes-dev (>= 1:3.0.0),libxv-dev,libxinerama-dev,libxshmfence-dev (>= 1.1) [!hurd-i386],libepoxy-dev [linux-any kfreebsd-any],libegl1-mesa-dev [linux-any kfreebsd-any],libgbm-dev (>= 10.2) [linux-any kfreebsd-any],libxcb1-dev,libxcb-xkb-dev,libxcb-shape0-dev,libxcb-render0-dev,libxcb-render-util0-dev,libxcb-util0-dev,libxcb-image0-dev,libxcb-icccm4-dev,libxcb-shm0-dev,libxcb-keysyms1-dev,libxcb-randr0-dev,libxcb-xv0-dev,libxcb-glx0-dev,libxcb-xf86dri0-dev (>= 1.6),xkb-data,x11-xkb-utils,libbsd-dev,libwayland-dev [linux-any],wayland-protocols (>= 1.9) [linux-any],libdbus-1-dev (>= 1.0) [linux-any],libsystemd-dev [linux-any],libmirclient-dev (>= 0.13.1) [!powerpc !ppc64el !s390x],mir-client-platform-mesa-dev,zlib1g-dev,libjpeg-turbo8-dev,perl-modules,cmake,gnutls-dev,libpam0g-dev,libpng-dev,automake,autoconf,libtool,pkg-config,openjdk-8-jdk,xorg-server-source,libfltk1.3-dev,fluid,curl,bzip2,quilt,libosmesa6-dev,libgl1-mesa-dri,libgl1-mesa-glx,libxft-dev,xfonts-base,libexpat1-dev,libfontconfig1-dev,libglu1-mesa-dev,libxcursor-dev,libxrandr-dev,libcairo2-dev,libavcodec-dev,libavutil-dev,libswscale-dev
 Homepage: http://www.tigervnc.com
 
 Package: tigervncserver
@@ -30,7 +30,7 @@ Description: virtual network computing server software
 Package: xtigervncviewer
 Architecture: any
 Provides: vncviewer, vnc-viewer
-Depends: libc6, libexpat1, libfontconfig1, libfreetype6, libgcc1, libgnutls30, libjpeg-turbo8, libp11-kit0, libpng16-16, libstdc++6, libtasn1-6, libx11-6, libxau6, libxcb1, libxdmcp6, libxext6, libxft2, libxrender1, zlib1g, libglu1-mesa, libxcursor1, libxinerama1, libxfixes3, libfltk1.3, libfltk-images1.3
+Depends: libc6, libexpat1, libfontconfig1, libfreetype6, libgcc1, libgnutls30, libjpeg-turbo8, libp11-kit0, libpng16-16, libstdc++6, libtasn1-6, libx11-6, libxau6, libxcb1, libxdmcp6, libxext6, libxft2, libxrender1, zlib1g, libglu1-mesa, libxcursor1, libxinerama1, libxfixes3, libfltk1.3, libfltk-images1.3, ffmpeg
 Recommends: xfonts-base
 Suggests: tigervncserver, ssh
 Description: virtual network computing client software for X

contrib/packages/deb/ubuntu-focal/debian/control

@@ -3,7 +3,7 @@ Section: x11
 Priority: optional
 Maintainer: Brian P. Hinz <bphinz@users.sourceforge.net>
 Standards-Version: 3.8.4
-Build-Depends: debhelper (>= 9),dh-autoreconf,po-debconf,dpkg-dev (>= 1.16.1),quilt,lsb-release,pkg-config,bison,flex,xauth,xutils-dev (>= 1:7.6+4),xfonts-utils (>= 1:7.5+1),x11proto-bigreqs-dev (>= 1:1.1.0),x11proto-composite-dev (>= 1:0.4),x11proto-core-dev (>= 7.0.31),x11proto-damage-dev (>= 1.1),x11proto-fixes-dev (>= 1:5.0),x11proto-fonts-dev (>= 2.1.3),x11proto-kb-dev (>= 1.0.3),x11proto-xinerama-dev,x11proto-randr-dev (>= 1.5.0),x11proto-record-dev (>= 1.13.99.1),x11proto-render-dev (>= 2:0.11),x11proto-resource-dev (>= 1.2.0),x11proto-scrnsaver-dev,x11proto-video-dev,x11proto-xcmisc-dev (>= 1.2.0),x11proto-xext-dev (>= 7.2.99.901),x11proto-xf86bigfont-dev (>= 1.2.0),x11proto-xf86dga-dev (>= 2.0.99.1),x11proto-xf86vidmode-dev (>= 2.2.99.1),x11proto-present-dev,x11proto-dri3-dev,xtrans-dev (>= 1.3.5),libxau-dev (>= 1:1.0.5-2),x11proto-input-dev (>= 2.3),x11proto-dri2-dev (>= 2.8),libxdmcp-dev (>= 1:0.99.1),libxfont-dev (>= 1:2.0.1),libxkbfile-dev (>= 1:0.99.1),libpixman-1-dev (>= 0.27.2),libpciaccess-dev (>= 0.12.901),libgcrypt-dev,nettle-dev,libudev-dev (>= 151-3) [linux-any],libselinux1-dev (>= 2.0.80) [linux-any],libaudit-dev [linux-any],x11proto-xf86dri-dev (>= 2.1.0),libdrm-dev (>= 2.4.46) [!hurd-i386],x11proto-gl-dev (>= 1.4.17),libgl1-mesa-dev (>= 9.2),libxmuu-dev (>= 1:0.99.1),libxext-dev (>= 1:0.99.1),libx11-dev (>= 2:1.6),libxrender-dev (>= 1:0.9.0),libxi-dev (>= 2:1.6.99.1),x11proto-dmx-dev (>= 1:2.2.99.1),libdmx-dev (>= 1:1.0.1),libxpm-dev (>= 1:3.5.3),libxaw7-dev (>= 1:0.99.1),libxt-dev (>= 1:0.99.1),libxmu-dev (>= 1:0.99.1),libxtst-dev (>= 1:0.99.1),libxres-dev (>= 1:0.99.1),libxfixes-dev (>= 1:3.0.0),libxv-dev,libxinerama-dev,libxshmfence-dev (>= 1.1) [!hurd-i386],libepoxy-dev [linux-any kfreebsd-any],libegl1-mesa-dev [linux-any kfreebsd-any],libgbm-dev (>= 10.2) [linux-any kfreebsd-any],libxcb1-dev,libxcb-xkb-dev,libxcb-shape0-dev,libxcb-render0-dev,libxcb-render-util0-dev,libxcb-util0-dev,libxcb-image0-dev,libxcb-icccm4-dev,libxcb-shm0-dev,libxcb-keysyms1-dev,libxcb-randr0-dev,libxcb-xv0-dev,libxcb-glx0-dev,libxcb-xf86dri0-dev (>= 1.6),xkb-data,x11-xkb-utils,libbsd-dev,libwayland-dev [linux-any],wayland-protocols (>= 1.9) [linux-any],libdbus-1-dev (>= 1.0) [linux-any],libsystemd-dev [linux-any],libmirclient-dev (>= 0.13.1) [!powerpc !ppc64el !s390x],mir-client-platform-mesa-dev,zlib1g-dev,libjpeg-turbo8-dev,perl-modules,cmake,gnutls-dev,libpam0g-dev,libpng-dev,automake,autoconf,libtool,pkg-config,openjdk-8-jdk,xorg-server-source,libfltk1.3-dev,fluid,curl,bzip2,quilt,libosmesa6-dev,libgl1-mesa-dri,libgl1-mesa-glx,libxft-dev,xfonts-base,libexpat1-dev,libfontconfig1-dev,libglu1-mesa-dev,libxcursor-dev,libxrandr-dev,libcairo2-dev,mesa-common-dev
+Build-Depends: debhelper (>= 9),dh-autoreconf,po-debconf,dpkg-dev (>= 1.16.1),quilt,lsb-release,pkg-config,bison,flex,xauth,xutils-dev (>= 1:7.6+4),xfonts-utils (>= 1:7.5+1),x11proto-bigreqs-dev (>= 1:1.1.0),x11proto-composite-dev (>= 1:0.4),x11proto-core-dev (>= 7.0.31),x11proto-damage-dev (>= 1.1),x11proto-fixes-dev (>= 1:5.0),x11proto-fonts-dev (>= 2.1.3),x11proto-kb-dev (>= 1.0.3),x11proto-xinerama-dev,x11proto-randr-dev (>= 1.5.0),x11proto-record-dev (>= 1.13.99.1),x11proto-render-dev (>= 2:0.11),x11proto-resource-dev (>= 1.2.0),x11proto-scrnsaver-dev,x11proto-video-dev,x11proto-xcmisc-dev (>= 1.2.0),x11proto-xext-dev (>= 7.2.99.901),x11proto-xf86bigfont-dev (>= 1.2.0),x11proto-xf86dga-dev (>= 2.0.99.1),x11proto-xf86vidmode-dev (>= 2.2.99.1),x11proto-present-dev,x11proto-dri3-dev,xtrans-dev (>= 1.3.5),libxau-dev (>= 1:1.0.5-2),x11proto-input-dev (>= 2.3),x11proto-dri2-dev (>= 2.8),libxdmcp-dev (>= 1:0.99.1),libxfont-dev (>= 1:2.0.1),libxkbfile-dev (>= 1:0.99.1),libpixman-1-dev (>= 0.27.2),libpciaccess-dev (>= 0.12.901),libgcrypt-dev,nettle-dev,libudev-dev (>= 151-3) [linux-any],libselinux1-dev (>= 2.0.80) [linux-any],libaudit-dev [linux-any],x11proto-xf86dri-dev (>= 2.1.0),libdrm-dev (>= 2.4.46) [!hurd-i386],x11proto-gl-dev (>= 1.4.17),libgl1-mesa-dev (>= 9.2),libxmuu-dev (>= 1:0.99.1),libxext-dev (>= 1:0.99.1),libx11-dev (>= 2:1.6),libxrender-dev (>= 1:0.9.0),libxi-dev (>= 2:1.6.99.1),x11proto-dmx-dev (>= 1:2.2.99.1),libdmx-dev (>= 1:1.0.1),libxpm-dev (>= 1:3.5.3),libxaw7-dev (>= 1:0.99.1),libxt-dev (>= 1:0.99.1),libxmu-dev (>= 1:0.99.1),libxtst-dev (>= 1:0.99.1),libxres-dev (>= 1:0.99.1),libxfixes-dev (>= 1:3.0.0),libxv-dev,libxinerama-dev,libxshmfence-dev (>= 1.1) [!hurd-i386],libepoxy-dev [linux-any kfreebsd-any],libegl1-mesa-dev [linux-any kfreebsd-any],libgbm-dev (>= 10.2) [linux-any kfreebsd-any],libxcb1-dev,libxcb-xkb-dev,libxcb-shape0-dev,libxcb-render0-dev,libxcb-render-util0-dev,libxcb-util0-dev,libxcb-image0-dev,libxcb-icccm4-dev,libxcb-shm0-dev,libxcb-keysyms1-dev,libxcb-randr0-dev,libxcb-xv0-dev,libxcb-glx0-dev,libxcb-xf86dri0-dev (>= 1.6),xkb-data,x11-xkb-utils,libbsd-dev,libwayland-dev [linux-any],wayland-protocols (>= 1.9) [linux-any],libdbus-1-dev (>= 1.0) [linux-any],libsystemd-dev [linux-any],libmirclient-dev (>= 0.13.1) [!powerpc !ppc64el !s390x],mir-client-platform-mesa-dev,zlib1g-dev,libjpeg-turbo8-dev,perl-modules,cmake,gnutls-dev,libpam0g-dev,libpng-dev,automake,autoconf,libtool,pkg-config,openjdk-8-jdk,xorg-server-source,libfltk1.3-dev,fluid,curl,bzip2,quilt,libosmesa6-dev,libgl1-mesa-dri,libgl1-mesa-glx,libxft-dev,xfonts-base,libexpat1-dev,libfontconfig1-dev,libglu1-mesa-dev,libxcursor-dev,libxrandr-dev,libcairo2-dev,mesa-common-dev,libavcodec-dev,libavutil-dev,libswscale-dev
 Homepage: http://www.tigervnc.com
 
 Package: tigervncserver
@@ -30,7 +30,7 @@ Description: virtual network computing server software
 Package: xtigervncviewer
 Architecture: any
 Provides: vncviewer, vnc-viewer
-Depends: libc6, libexpat1, libfontconfig1, libfreetype6, libgcc1, libgnutls30, libjpeg-turbo8, libp11-kit0, libpng16-16, libstdc++6, libtasn1-6, libx11-6, libxau6, libxcb1, libxdmcp6, libxext6, libxft2, libxrender1, zlib1g, libglu1-mesa, libxcursor1, libxinerama1, libxfixes3, libfltk1.3, libfltk-images1.3
+Depends: libc6, libexpat1, libfontconfig1, libfreetype6, libgcc1, libgnutls30, libjpeg-turbo8, libp11-kit0, libpng16-16, libstdc++6, libtasn1-6, libx11-6, libxau6, libxcb1, libxdmcp6, libxext6, libxft2, libxrender1, zlib1g, libglu1-mesa, libxcursor1, libxinerama1, libxfixes3, libfltk1.3, libfltk-images1.3, ffmpeg
 Recommends: xfonts-base
 Suggests: tigervncserver, ssh
 Description: virtual network computing client software for X

contrib/packages/rpm/el7/SPECS/tigervnc.spec

@@ -160,7 +160,7 @@ export CPPFLAGS="$CXXFLAGS"
 
 export CMAKE_EXE_LINKER_FLAGS=$LDFLAGS
 
-%{cmake3} -G"Unix Makefiles" -DBUILD_STATIC=off
+%{cmake3} -G"Unix Makefiles" -DBUILD_STATIC=off -DENABLE_H264=off
 make %{?_smp_mflags}
 
 pushd unix/xserver

contrib/packages/rpm/el8/SPECS/tigervnc.spec

@@ -161,7 +161,7 @@ export CPPFLAGS="$CXXFLAGS"
 
 export CMAKE_EXE_LINKER_FLAGS=$LDFLAGS
 
-%{cmake} -G"Unix Makefiles" -DBUILD_STATIC=off
+%{cmake} -G"Unix Makefiles" -DBUILD_STATIC=off -DENABLE_H264=off
 make %{?_smp_mflags}
 
 pushd unix/xserver

tests/perf/CMakeLists.txt

@@ -3,6 +3,7 @@ include_directories(${GETTEXT_INCLUDE_DIR})
 include_directories(${CMAKE_SOURCE_DIR}/common)
 
 add_library(test_util STATIC util.cxx)
+target_link_libraries(test_util ${H264_LIBRARIES})
 
 add_executable(convperf convperf.cxx)
 target_link_libraries(convperf test_util rfb)

vncviewer/CMakeLists.txt

@@ -53,7 +53,7 @@ else()
   add_executable(vncviewer ${VNCVIEWER_SOURCES})
 endif()
 
-target_link_libraries(vncviewer rfb network rdr os ${FLTK_LIBRARIES} ${GETTEXT_LIBRARIES})
+target_link_libraries(vncviewer rfb network rdr os ${FLTK_LIBRARIES} ${GETTEXT_LIBRARIES} ${H264_LIBRARIES})
 
 if(WIN32)
   target_link_libraries(vncviewer msimg32)

vncviewer/OptionsDialog.cxx

@@ -164,6 +164,11 @@ void OptionsDialog::loadOptions(void)
   case encodingHextile:
     hextileButton->setonly();
     break;
+#ifdef HAVE_H264
+  case encodingH264:
+    h264Button->setonly();
+    break;
+#endif
   case encodingRaw:
     rawButton->setonly();
     break;
@@ -327,6 +332,10 @@ void OptionsDialog::storeOptions(void)
     preferredEncoding.setParam(encodingName(encodingZRLE));
   else if (hextileButton->value())
     preferredEncoding.setParam(encodingName(encodingHextile));
+#ifdef HAVE_H264
+  else if (h264Button->value())
+    preferredEncoding.setParam(encodingName(encodingH264));
+#endif
   else if (rawButton->value())
     preferredEncoding.setParam(encodingName(encodingRaw));
 
@@ -436,6 +445,7 @@ void OptionsDialog::createCompressionPage(int tx, int ty, int tw, int th)
   Fl_Group *group = new Fl_Group(tx, ty, tw, th, _("Compression"));
 
   int orig_tx, orig_ty;
+  int col1_ty, col2_ty;
   int half_width, full_width;
   int height;
 
@@ -459,7 +469,7 @@ void OptionsDialog::createCompressionPage(int tx, int ty, int tw, int th)
 
   /* VNC encoding box */
   ty += GROUP_LABEL_OFFSET;
-  height = GROUP_MARGIN * 2 + TIGHT_MARGIN * 3 + RADIO_HEIGHT * 4;
+  height = GROUP_MARGIN * 2 + TIGHT_MARGIN * 4 + RADIO_HEIGHT * 5;
   encodingGroup = new Fl_Group(tx, ty, half_width, height,
                                 _("Preferred encoding"));
   encodingGroup->box(FL_ENGRAVED_BOX);
@@ -490,6 +500,15 @@ void OptionsDialog::createCompressionPage(int tx, int ty, int tw, int th)
     hextileButton->type(FL_RADIO_BUTTON);
     ty += RADIO_HEIGHT + TIGHT_MARGIN;
 
+#ifdef HAVE_H264
+    h264Button = new Fl_Round_Button(LBLRIGHT(tx, ty,
+                                             RADIO_MIN_WIDTH,
+                                             RADIO_HEIGHT,
+                                             "H.264"));
+    h264Button->type(FL_RADIO_BUTTON);
+    ty += RADIO_HEIGHT + TIGHT_MARGIN;
+#endif
+
     rawButton = new Fl_Round_Button(LBLRIGHT(tx, ty,
                                              RADIO_MIN_WIDTH,
                                              RADIO_HEIGHT,
@@ -501,6 +520,7 @@ void OptionsDialog::createCompressionPage(int tx, int ty, int tw, int th)
   ty += GROUP_MARGIN - TIGHT_MARGIN;
 
   encodingGroup->end();
+  col1_ty = ty;
 
   /* Second column */
   tx = orig_tx + half_width + INNER_MARGIN;
@@ -549,10 +569,11 @@ void OptionsDialog::createCompressionPage(int tx, int ty, int tw, int th)
   ty += GROUP_MARGIN - TIGHT_MARGIN;
 
   colorlevelGroup->end();
+  col2_ty = ty;
 
   /* Back to normal */
   tx = orig_tx;
-  ty += INNER_MARGIN;
+  ty = (col1_ty > col2_ty ? col1_ty : col2_ty) + INNER_MARGIN;
 
   /* Checkboxes */
   compressionCheckbox = new Fl_Check_Button(LBLRIGHT(tx, ty,

vncviewer/OptionsDialog.h

@@ -80,6 +80,9 @@ protected:
   Fl_Round_Button *tightButton;
   Fl_Round_Button *zrleButton;
   Fl_Round_Button *hextileButton;
+#ifdef HAVE_H264
+  Fl_Round_Button *h264Button;
+#endif
   Fl_Round_Button *rawButton;
 
   Fl_Group *colorlevelGroup;