support H264 decoding with MediaFoundation on Windows

CMakeLists.txt

@@ -175,8 +175,8 @@ option(ENABLE_H264 "Enable H.264 RFB encoding" ON)
 if(ENABLE_H264)
   if(WIN32)
     add_definitions("-DHAVE_H264")
-    set(H264_LIBS "NONE")  # may be LIBAV in the future
-    message(WARNING "NO H264 LIBs are supported on Windows")
+    set(H264_LIBS "WIN")  # may be LIBAV in the future
+    set(H264_LIBRARIES ole32 mfplat mfuuid wmcodecdspuuid)
   else()
     check_include_files(libavcodec/avcodec.h HAVE_AVCODEC_H)
     check_include_files(libavutil/avutil.h HAVE_AVUTIL_H)

common/rfb/CMakeLists.txt

@@ -68,6 +68,8 @@ if(ENABLE_H264 AND NOT H264_LIBS STREQUAL "NONE")
   set(RFB_SOURCES ${RFB_SOURCES} H264Decoder.cxx H264DecoderContext.cxx)
   if(H264_LIBS STREQUAL "LIBAV")
     set(RFB_SOURCES ${RFB_SOURCES} H264LibavDecoderContext.cxx)
+  elseif(H264_LIBS STREQUAL "WIN")
+    set(RFB_SOURCES ${RFB_SOURCES} H264WinDecoderContext.cxx)
   endif()
 endif()
 

common/rfb/H264WinDecoderContext.cxx

@@ -0,0 +1,325 @@
+/* Copyright (C) 2021 Vladimir Sukhonosov <xornet@xornet.org>
+ * Copyright (C) 2021 Martins Mozeiko <martins.mozeiko@gmail.com>
+ * All Rights Reserved.
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#include <mfapi.h>
+#include <mferror.h>
+#include <wmcodecdsp.h>
+#define SAFE_RELEASE(obj) if (obj) { obj->Release(); obj = NULL; }
+
+#include <os/Mutex.h>
+#include <rfb/LogWriter.h>
+#include <rfb/PixelBuffer.h>
+#include <rfb/H264WinDecoderContext.h>
+
+using namespace rfb;
+
+static LogWriter vlog("H264WinDecoderContext");
+
+bool H264WinDecoderContext::initCodec() {
+  os::AutoMutex lock(&mutex);
+
+  if (FAILED(MFStartup(MF_VERSION, MFSTARTUP_LITE)))
+  {
+    vlog.error("Could not initialize MediaFoundation");
+    return false;
+  }
+
+  if (FAILED(CoCreateInstance(CLSID_CMSH264DecoderMFT, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, (LPVOID*)&decoder)))
+  {
+    vlog.error("MediaFoundation H264 codec not found");
+    return false;
+  }
+
+  GUID CLSID_VideoProcessorMFT = { 0x88753b26, 0x5b24, 0x49bd, { 0xb2, 0xe7, 0xc, 0x44, 0x5c, 0x78, 0xc9, 0x82 } };
+  if (FAILED(CoCreateInstance(CLSID_VideoProcessorMFT, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, (LPVOID*)&converter)))
+  {
+    vlog.error("Cannot create MediaFoundation Video Processor (available only on Windows 8+). Trying ColorConvert DMO.");
+    if (FAILED(CoCreateInstance(CLSID_CColorConvertDMO, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, (LPVOID*)&converter)))
+    {
+      decoder->Release();
+      vlog.error("ColorConvert DMO not found");
+      return false;
+    }
+  }
+
+  // if possible, enable low-latency decoding (Windows 8 and up)
+  IMFAttributes* attributes;
+  if (SUCCEEDED(decoder->GetAttributes(&attributes)))
+  {
+    GUID MF_LOW_LATENCY = { 0x9c27891a, 0xed7a, 0x40e1, { 0x88, 0xe8, 0xb2, 0x27, 0x27, 0xa0, 0x24, 0xee } };
+    if (SUCCEEDED(attributes->SetUINT32(MF_LOW_LATENCY, TRUE)))
+    {
+      vlog.info("Enabled low latency mode");
+    }
+    attributes->Release();
+  }
+
+  // set decoder input type
+  IMFMediaType* input_type;
+  if (FAILED(MFCreateMediaType(&input_type)))
+  {
+    decoder->Release();
+    converter->Release();
+    vlog.error("Could not create MF MediaType");
+    return false;
+  }
+  input_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
+  input_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
+  decoder->SetInputType(0, input_type, 0);
+  input_type->Release();
+
+  // set decoder output type (NV12)
+  DWORD output_index = 0;
+  IMFMediaType* output_type = NULL;
+  while (SUCCEEDED(decoder->GetOutputAvailableType(0, output_index++, &output_type)))
+  {
+    GUID subtype;
+    if (SUCCEEDED(output_type->GetGUID(MF_MT_SUBTYPE, &subtype)) && subtype == MFVideoFormat_NV12)
+    {
+      decoder->SetOutputType(0, output_type, 0);
+      output_type->Release();
+      break;
+    }
+    output_type->Release();
+  }
+
+  if (FAILED(decoder->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0)))
+  {
+    decoder->Release();
+    converter->Release();
+    input_type->Release();
+    vlog.error("Could not start H264 decoder");
+    return false;
+  }
+
+  MFT_OUTPUT_STREAM_INFO info;
+  decoder->GetOutputStreamInfo(0, &info);
+
+  if (FAILED(MFCreateSample(&input_sample)) ||
+      FAILED(MFCreateSample(&decoded_sample)) ||
+      FAILED(MFCreateSample(&converted_sample)) ||
+      FAILED(MFCreateMemoryBuffer(4 * 1024 * 1024, &input_buffer)) ||
+      FAILED(MFCreateMemoryBuffer(info.cbSize, &decoded_buffer)))
+  {
+    decoder->Release();
+    converter->Release();
+    input_type->Release();
+    SAFE_RELEASE(input_sample);
+    SAFE_RELEASE(decoded_sample);
+    SAFE_RELEASE(converted_sample);
+    SAFE_RELEASE(input_buffer);
+    SAFE_RELEASE(decoded_buffer);
+    vlog.error("Could not allocate media samples/buffers");
+    return false;
+  }
+
+  input_sample->AddBuffer(input_buffer);
+  decoded_sample->AddBuffer(decoded_buffer);
+
+  initialized = true;
+  return true;
+}
+
+void H264WinDecoderContext::freeCodec() {
+  os::AutoMutex lock(&mutex);
+
+  if (!initialized)
+    return;
+  SAFE_RELEASE(decoder)
+  SAFE_RELEASE(converter)
+  SAFE_RELEASE(input_sample)
+  SAFE_RELEASE(decoded_sample)
+  SAFE_RELEASE(converted_sample)
+  SAFE_RELEASE(input_buffer)
+  SAFE_RELEASE(decoded_buffer)
+  SAFE_RELEASE(converted_buffer)
+  MFShutdown();
+  initialized = false;
+}
+
+void H264WinDecoderContext::decode(const rdr::U8* h264_buffer, rdr::U32 len, rdr::U32 flags, ModifiablePixelBuffer* pb) {  
+  os::AutoMutex lock(&mutex);
+  if (!initialized)
+    return;
+
+  if (FAILED(input_buffer->SetCurrentLength(len)))
+  {
+    input_buffer->Release();
+    if (FAILED(MFCreateMemoryBuffer(len, &input_buffer)))
+    {
+      vlog.error("Could not allocate media buffer");
+      return;
+    }
+    input_buffer->SetCurrentLength(len);
+    input_sample->RemoveAllBuffers();
+    input_sample->AddBuffer(input_buffer);
+  }
+
+  BYTE* locked;
+  input_buffer->Lock(&locked, NULL, NULL);
+  memcpy(locked, h264_buffer, len);
+  input_buffer->Unlock();
+
+  vlog.debug("Received %u bytes, decoding", len);
+
+  if (FAILED(decoder->ProcessInput(0, input_sample, 0)))
+  {
+    vlog.error("Error sending a packet to decoding");
+    return;
+  }
+
+  bool decoded = false;
+
+  // try to retrieve all decoded output, as input can submit multiple h264 packets in one buffer
+  for (;;)
+  {
+    DWORD curlen;
+    decoded_buffer->GetCurrentLength(&curlen);
+    decoded_buffer->SetCurrentLength(0);
+
+    MFT_OUTPUT_DATA_BUFFER decoded_data;
+    decoded_data.dwStreamID = 0;
+    decoded_data.pSample = decoded_sample;
+    decoded_data.dwStatus = 0;
+    decoded_data.pEvents = NULL;
+
+    DWORD status;
+    HRESULT hr = decoder->ProcessOutput(0, 1, &decoded_data, &status);
+    SAFE_RELEASE(decoded_data.pEvents)
+
+    if (SUCCEEDED(hr))
+    {
+      vlog.debug("Frame decoded");
+      // successfully decoded next frame
+      // but do not exit loop, try again if there is next frame
+      decoded = true;
+    }
+    else if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT)
+    {
+      // no more frames to decode
+      if (decoded)
+      {
+        // restore previous buffer length for converter
+        decoded_buffer->SetCurrentLength(curlen);
+      }
+      break;
+    }
+    else if (hr == MF_E_TRANSFORM_STREAM_CHANGE)
+    {
+      // something changed (resolution, framerate, h264 properties...)
+      // need to setup output type and try decoding again
+
+      DWORD output_index = 0;
+      IMFMediaType* output_type = NULL;
+      while (SUCCEEDED(decoder->GetOutputAvailableType(0, output_index++, &output_type)))
+      {
+        GUID subtype;
+        if (SUCCEEDED(output_type->GetGUID(MF_MT_SUBTYPE, &subtype)) && subtype == MFVideoFormat_NV12)
+        {
+          decoder->SetOutputType(0, output_type, 0);
+          break;
+        }
+        output_type->Release();
+        output_type = NULL;
+      }
+
+      // reinitialize output type (NV12) that now has correct properties (width/height/framerate)
+      decoder->SetOutputType(0, output_type, 0);
+
+      UINT32 width = 0;
+      UINT32 height = 0;
+      MFGetAttributeSize(output_type, MF_MT_FRAME_SIZE, &width, &height);
+      vlog.debug("Setting up decoded output with %ux%u size", width, height);
+
+      // input type to converter, BGRX pixel format
+      IMFMediaType* converted_type;
+      if (FAILED(MFCreateMediaType(&converted_type)))
+      {
+        vlog.error("Error creating media type");
+      }
+      else
+      {
+        converted_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
+        converted_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32);
+        converted_type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
+        MFSetAttributeSize(converted_type, MF_MT_FRAME_SIZE, width, height);
+        MFGetStrideForBitmapInfoHeader(MFVideoFormat_RGB32.Data1, width, &stride);
+        // bottom-up
+        stride = -stride;
+        converted_type->SetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32)stride);
+
+        // setup NV12 -> BGRX converter
+        converter->SetOutputType(0, converted_type, 0);
+        converter->SetInputType(0, output_type, 0);
+        converted_type->Release();
+
+        // create converter output buffer
+
+        MFT_OUTPUT_STREAM_INFO info;
+        converter->GetOutputStreamInfo(0, &info);
+
+        if (FAILED(MFCreateMemoryBuffer(info.cbSize, &converted_buffer)))
+        {
+          vlog.error("Error creating media buffer");
+        }
+        else
+        {
+          converted_sample->AddBuffer(converted_buffer);
+        }
+      }
+      output_type->Release();
+    }
+  }
+
+  // we care only about final image
+  // we ignore previous images if decoded multiple in a row
+  if (decoded)
+  {
+    if (FAILED(converter->ProcessInput(0, decoded_sample, 0)))
+    {
+      vlog.error("Error sending a packet to converter");
+      return;
+    }
+
+    MFT_OUTPUT_DATA_BUFFER converted_data;
+    converted_data.dwStreamID = 0;
+    converted_data.pSample = converted_sample;
+    converted_data.dwStatus = 0;
+    converted_data.pEvents = NULL;
+
+    DWORD status;
+    HRESULT hr = converter->ProcessOutput(0, 1, &converted_data, &status);
+    SAFE_RELEASE(converted_data.pEvents)
+
+    if (FAILED(hr))
+    {
+      vlog.error("Error converting to RGB");
+    }
+    else
+    {
+      vlog.debug("Frame converted to RGB");
+
+      BYTE* out;
+      converted_buffer->Lock(&out, NULL, NULL);
+      pb->imageRect(rect, out, (int)stride / 4);
+      converted_buffer->Unlock();
+    }
+  }
+}

common/rfb/H264WinDecoderContext.h

@@ -0,0 +1,55 @@
+/* Copyright (C) 2021 Vladimir Sukhonosov <xornet@xornet.org>
+ * Copyright (C) 2021 Martins Mozeiko <martins.mozeiko@gmail.com>
+ * All Rights Reserved.
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#ifndef __RFB_H264WINDECODER_H__
+#define __RFB_H264WINDECODER_H__
+
+#include <windows.h>
+#include <mfidl.h>
+#include <mftransform.h>
+
+#include <rfb/H264DecoderContext.h>
+
+namespace rfb {
+  class H264WinDecoderContext : public H264DecoderContext {
+    public:
+      H264WinDecoderContext(const Rect &r) : H264DecoderContext(r) {};
+      ~H264WinDecoderContext() { freeCodec(); }
+
+      virtual void decode(const rdr::U8* h264_buffer, rdr::U32 len, rdr::U32 flags, ModifiablePixelBuffer* pb);
+
+    protected:
+      virtual bool initCodec();
+      virtual void freeCodec();
+
+    private:
+      LONG stride;
+      IMFTransform *decoder = NULL;
+      IMFTransform *converter = NULL;
+      IMFSample *input_sample = NULL;
+      IMFSample *decoded_sample = NULL;
+      IMFSample *converted_sample = NULL;
+      IMFMediaBuffer *input_buffer = NULL;
+      IMFMediaBuffer *decoded_buffer = NULL;
+      IMFMediaBuffer *converted_buffer = NULL;
+  };
+}
+
+#endif