From: Mārtiņš Možeiko Date: Fri, 21 Jan 2022 05:00:48 +0000 (+0300) Subject: support H264 decoding with MediaFoundation on Windows X-Git-Tag: v1.12.90~57^2 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=refs%2Fpull%2F1194%2Fhead;p=tigervnc.git support H264 decoding with MediaFoundation on Windows --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 404d558e..a3d4c071 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -175,8 +175,8 @@ option(ENABLE_H264 "Enable H.264 RFB encoding" ON) if(ENABLE_H264) if(WIN32) add_definitions("-DHAVE_H264") - set(H264_LIBS "NONE") # may be LIBAV in the future - message(WARNING "NO H264 LIBs are supported on Windows") + set(H264_LIBS "WIN") # may be LIBAV in the future + set(H264_LIBRARIES ole32 mfplat mfuuid wmcodecdspuuid) else() check_include_files(libavcodec/avcodec.h HAVE_AVCODEC_H) check_include_files(libavutil/avutil.h HAVE_AVUTIL_H) diff --git a/common/rfb/CMakeLists.txt b/common/rfb/CMakeLists.txt index 84e0be62..630bf7b6 100644 --- a/common/rfb/CMakeLists.txt +++ b/common/rfb/CMakeLists.txt @@ -68,6 +68,8 @@ if(ENABLE_H264 AND NOT H264_LIBS STREQUAL "NONE") set(RFB_SOURCES ${RFB_SOURCES} H264Decoder.cxx H264DecoderContext.cxx) if(H264_LIBS STREQUAL "LIBAV") set(RFB_SOURCES ${RFB_SOURCES} H264LibavDecoderContext.cxx) + elseif(H264_LIBS STREQUAL "WIN") + set(RFB_SOURCES ${RFB_SOURCES} H264WinDecoderContext.cxx) endif() endif() diff --git a/common/rfb/H264WinDecoderContext.cxx b/common/rfb/H264WinDecoderContext.cxx new file mode 100644 index 00000000..6acebd7c --- /dev/null +++ b/common/rfb/H264WinDecoderContext.cxx @@ -0,0 +1,325 @@ +/* Copyright (C) 2021 Vladimir Sukhonosov + * Copyright (C) 2021 Martins Mozeiko + * All Rights Reserved. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this software; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, + * USA. + */ + +#include +#include +#include +#define SAFE_RELEASE(obj) if (obj) { obj->Release(); obj = NULL; } + +#include +#include +#include +#include + +using namespace rfb; + +static LogWriter vlog("H264WinDecoderContext"); + +bool H264WinDecoderContext::initCodec() { + os::AutoMutex lock(&mutex); + + if (FAILED(MFStartup(MF_VERSION, MFSTARTUP_LITE))) + { + vlog.error("Could not initialize MediaFoundation"); + return false; + } + + if (FAILED(CoCreateInstance(CLSID_CMSH264DecoderMFT, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, (LPVOID*)&decoder))) + { + vlog.error("MediaFoundation H264 codec not found"); + return false; + } + + GUID CLSID_VideoProcessorMFT = { 0x88753b26, 0x5b24, 0x49bd, { 0xb2, 0xe7, 0xc, 0x44, 0x5c, 0x78, 0xc9, 0x82 } }; + if (FAILED(CoCreateInstance(CLSID_VideoProcessorMFT, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, (LPVOID*)&converter))) + { + vlog.error("Cannot create MediaFoundation Video Processor (available only on Windows 8+). Trying ColorConvert DMO."); + if (FAILED(CoCreateInstance(CLSID_CColorConvertDMO, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, (LPVOID*)&converter))) + { + decoder->Release(); + vlog.error("ColorConvert DMO not found"); + return false; + } + } + + // if possible, enable low-latency decoding (Windows 8 and up) + IMFAttributes* attributes; + if (SUCCEEDED(decoder->GetAttributes(&attributes))) + { + GUID MF_LOW_LATENCY = { 0x9c27891a, 0xed7a, 0x40e1, { 0x88, 0xe8, 0xb2, 0x27, 0x27, 0xa0, 0x24, 0xee } }; + if (SUCCEEDED(attributes->SetUINT32(MF_LOW_LATENCY, TRUE))) + { + vlog.info("Enabled low latency mode"); + } + attributes->Release(); + } + + // set decoder input type + IMFMediaType* input_type; + if (FAILED(MFCreateMediaType(&input_type))) + { + decoder->Release(); + converter->Release(); + vlog.error("Could not create MF MediaType"); + return false; + } + input_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + input_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + decoder->SetInputType(0, input_type, 0); + input_type->Release(); + + // set decoder output type (NV12) + DWORD output_index = 0; + IMFMediaType* output_type = NULL; + while (SUCCEEDED(decoder->GetOutputAvailableType(0, output_index++, &output_type))) + { + GUID subtype; + if (SUCCEEDED(output_type->GetGUID(MF_MT_SUBTYPE, &subtype)) && subtype == MFVideoFormat_NV12) + { + decoder->SetOutputType(0, output_type, 0); + output_type->Release(); + break; + } + output_type->Release(); + } + + if (FAILED(decoder->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0))) + { + decoder->Release(); + converter->Release(); + input_type->Release(); + vlog.error("Could not start H264 decoder"); + return false; + } + + MFT_OUTPUT_STREAM_INFO info; + decoder->GetOutputStreamInfo(0, &info); + + if (FAILED(MFCreateSample(&input_sample)) || + FAILED(MFCreateSample(&decoded_sample)) || + FAILED(MFCreateSample(&converted_sample)) || + FAILED(MFCreateMemoryBuffer(4 * 1024 * 1024, &input_buffer)) || + FAILED(MFCreateMemoryBuffer(info.cbSize, &decoded_buffer))) + { + decoder->Release(); + converter->Release(); + input_type->Release(); + SAFE_RELEASE(input_sample); + SAFE_RELEASE(decoded_sample); + SAFE_RELEASE(converted_sample); + SAFE_RELEASE(input_buffer); + SAFE_RELEASE(decoded_buffer); + vlog.error("Could not allocate media samples/buffers"); + return false; + } + + input_sample->AddBuffer(input_buffer); + decoded_sample->AddBuffer(decoded_buffer); + + initialized = true; + return true; +} + +void H264WinDecoderContext::freeCodec() { + os::AutoMutex lock(&mutex); + + if (!initialized) + return; + SAFE_RELEASE(decoder) + SAFE_RELEASE(converter) + SAFE_RELEASE(input_sample) + SAFE_RELEASE(decoded_sample) + SAFE_RELEASE(converted_sample) + SAFE_RELEASE(input_buffer) + SAFE_RELEASE(decoded_buffer) + SAFE_RELEASE(converted_buffer) + MFShutdown(); + initialized = false; +} + +void H264WinDecoderContext::decode(const rdr::U8* h264_buffer, rdr::U32 len, rdr::U32 flags, ModifiablePixelBuffer* pb) { + os::AutoMutex lock(&mutex); + if (!initialized) + return; + + if (FAILED(input_buffer->SetCurrentLength(len))) + { + input_buffer->Release(); + if (FAILED(MFCreateMemoryBuffer(len, &input_buffer))) + { + vlog.error("Could not allocate media buffer"); + return; + } + input_buffer->SetCurrentLength(len); + input_sample->RemoveAllBuffers(); + input_sample->AddBuffer(input_buffer); + } + + BYTE* locked; + input_buffer->Lock(&locked, NULL, NULL); + memcpy(locked, h264_buffer, len); + input_buffer->Unlock(); + + vlog.debug("Received %u bytes, decoding", len); + + if (FAILED(decoder->ProcessInput(0, input_sample, 0))) + { + vlog.error("Error sending a packet to decoding"); + return; + } + + bool decoded = false; + + // try to retrieve all decoded output, as input can submit multiple h264 packets in one buffer + for (;;) + { + DWORD curlen; + decoded_buffer->GetCurrentLength(&curlen); + decoded_buffer->SetCurrentLength(0); + + MFT_OUTPUT_DATA_BUFFER decoded_data; + decoded_data.dwStreamID = 0; + decoded_data.pSample = decoded_sample; + decoded_data.dwStatus = 0; + decoded_data.pEvents = NULL; + + DWORD status; + HRESULT hr = decoder->ProcessOutput(0, 1, &decoded_data, &status); + SAFE_RELEASE(decoded_data.pEvents) + + if (SUCCEEDED(hr)) + { + vlog.debug("Frame decoded"); + // successfully decoded next frame + // but do not exit loop, try again if there is next frame + decoded = true; + } + else if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) + { + // no more frames to decode + if (decoded) + { + // restore previous buffer length for converter + decoded_buffer->SetCurrentLength(curlen); + } + break; + } + else if (hr == MF_E_TRANSFORM_STREAM_CHANGE) + { + // something changed (resolution, framerate, h264 properties...) + // need to setup output type and try decoding again + + DWORD output_index = 0; + IMFMediaType* output_type = NULL; + while (SUCCEEDED(decoder->GetOutputAvailableType(0, output_index++, &output_type))) + { + GUID subtype; + if (SUCCEEDED(output_type->GetGUID(MF_MT_SUBTYPE, &subtype)) && subtype == MFVideoFormat_NV12) + { + decoder->SetOutputType(0, output_type, 0); + break; + } + output_type->Release(); + output_type = NULL; + } + + // reinitialize output type (NV12) that now has correct properties (width/height/framerate) + decoder->SetOutputType(0, output_type, 0); + + UINT32 width = 0; + UINT32 height = 0; + MFGetAttributeSize(output_type, MF_MT_FRAME_SIZE, &width, &height); + vlog.debug("Setting up decoded output with %ux%u size", width, height); + + // input type to converter, BGRX pixel format + IMFMediaType* converted_type; + if (FAILED(MFCreateMediaType(&converted_type))) + { + vlog.error("Error creating media type"); + } + else + { + converted_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + converted_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); + converted_type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + MFSetAttributeSize(converted_type, MF_MT_FRAME_SIZE, width, height); + MFGetStrideForBitmapInfoHeader(MFVideoFormat_RGB32.Data1, width, &stride); + // bottom-up + stride = -stride; + converted_type->SetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32)stride); + + // setup NV12 -> BGRX converter + converter->SetOutputType(0, converted_type, 0); + converter->SetInputType(0, output_type, 0); + converted_type->Release(); + + // create converter output buffer + + MFT_OUTPUT_STREAM_INFO info; + converter->GetOutputStreamInfo(0, &info); + + if (FAILED(MFCreateMemoryBuffer(info.cbSize, &converted_buffer))) + { + vlog.error("Error creating media buffer"); + } + else + { + converted_sample->AddBuffer(converted_buffer); + } + } + output_type->Release(); + } + } + + // we care only about final image + // we ignore previous images if decoded multiple in a row + if (decoded) + { + if (FAILED(converter->ProcessInput(0, decoded_sample, 0))) + { + vlog.error("Error sending a packet to converter"); + return; + } + + MFT_OUTPUT_DATA_BUFFER converted_data; + converted_data.dwStreamID = 0; + converted_data.pSample = converted_sample; + converted_data.dwStatus = 0; + converted_data.pEvents = NULL; + + DWORD status; + HRESULT hr = converter->ProcessOutput(0, 1, &converted_data, &status); + SAFE_RELEASE(converted_data.pEvents) + + if (FAILED(hr)) + { + vlog.error("Error converting to RGB"); + } + else + { + vlog.debug("Frame converted to RGB"); + + BYTE* out; + converted_buffer->Lock(&out, NULL, NULL); + pb->imageRect(rect, out, (int)stride / 4); + converted_buffer->Unlock(); + } + } +} diff --git a/common/rfb/H264WinDecoderContext.h b/common/rfb/H264WinDecoderContext.h new file mode 100644 index 00000000..8da58819 --- /dev/null +++ b/common/rfb/H264WinDecoderContext.h @@ -0,0 +1,55 @@ +/* Copyright (C) 2021 Vladimir Sukhonosov + * Copyright (C) 2021 Martins Mozeiko + * All Rights Reserved. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this software; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, + * USA. + */ + +#ifndef __RFB_H264WINDECODER_H__ +#define __RFB_H264WINDECODER_H__ + +#include +#include +#include + +#include + +namespace rfb { + class H264WinDecoderContext : public H264DecoderContext { + public: + H264WinDecoderContext(const Rect &r) : H264DecoderContext(r) {}; + ~H264WinDecoderContext() { freeCodec(); } + + virtual void decode(const rdr::U8* h264_buffer, rdr::U32 len, rdr::U32 flags, ModifiablePixelBuffer* pb); + + protected: + virtual bool initCodec(); + virtual void freeCodec(); + + private: + LONG stride; + IMFTransform *decoder = NULL; + IMFTransform *converter = NULL; + IMFSample *input_sample = NULL; + IMFSample *decoded_sample = NULL; + IMFSample *converted_sample = NULL; + IMFMediaBuffer *input_buffer = NULL; + IMFMediaBuffer *decoded_buffer = NULL; + IMFMediaBuffer *converted_buffer = NULL; + }; +} + +#endif