From: Andri Yngvason <andri@yngvason.is>
Date: Mon, 19 Aug 2024 22:34:37 +0000 (+0000)
Subject: H264Libav: Handle colour space conversion
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=28cc17e309fc6117370b38ec11d803dc36e8ef01;p=tigervnc.git

H264Libav: Handle colour space conversion

The scaler is now informed of the colour space encoded into the stream
so that it may do the proper conversion.
---

diff --git a/common/rfb/H264LibavDecoderContext.cxx b/common/rfb/H264LibavDecoderContext.cxx
index 38210205..c3153fd1 100644
--- a/common/rfb/H264LibavDecoderContext.cxx
+++ b/common/rfb/H264LibavDecoderContext.cxx
@@ -217,6 +217,21 @@ void H264LibavDecoderContext::decode(const uint8_t* h264_in_buffer,
                              frame->width, frame->height, AV_PIX_FMT_RGB32,
                              0, nullptr, nullptr, nullptr);
 
+  int inFull, outFull, brightness, contrast, saturation;
+  const int* inTable;
+  const int* outTable;
+
+  sws_getColorspaceDetails(sws, (int**)&inTable, &inFull, (int**)&outTable,
+      &outFull, &brightness, &contrast, &saturation);
+  if (frame->colorspace != AVCOL_SPC_UNSPECIFIED) {
+    inTable = sws_getCoefficients(frame->colorspace);
+  }
+  if (frame->color_range != AVCOL_RANGE_UNSPECIFIED) {
+    inFull = frame->color_range == AVCOL_RANGE_JPEG;
+  }
+  sws_setColorspaceDetails(sws, inTable, inFull, outTable, outFull, brightness,
+      contrast, saturation);
+
   if (rgbFrame && (rgbFrame->width != frame->width || rgbFrame->height != frame->height)) {
     av_frame_free(&rgbFrame);