From: Andri Yngvason <andri@yngvason.is>
Date: Mon, 19 Aug 2024 22:34:37 +0000 (+0000)
Subject: H264Libav: Handle colour space conversion
X-Git-Tag: v1.14.1~14
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=2c4b0385a1b6138e8c28ebf1d656bfd5e8c05aa3;p=tigervnc.git

H264Libav: Handle colour space conversion

The scaler is now informed of the colour space encoded into the stream
so that it may do the proper conversion.

(cherry picked from commit 28cc17e309fc6117370b38ec11d803dc36e8ef01)
---

diff --git a/common/rfb/H264LibavDecoderContext.cxx b/common/rfb/H264LibavDecoderContext.cxx
index f1599f4c..a7c817bf 100644
--- a/common/rfb/H264LibavDecoderContext.cxx
+++ b/common/rfb/H264LibavDecoderContext.cxx
@@ -217,6 +217,21 @@ void H264LibavDecoderContext::decode(const uint8_t* h264_in_buffer,
                              frame->width, frame->height, AV_PIX_FMT_RGB32,
                              0, NULL, NULL, NULL);
 
+  int inFull, outFull, brightness, contrast, saturation;
+  const int* inTable;
+  const int* outTable;
+
+  sws_getColorspaceDetails(sws, (int**)&inTable, &inFull, (int**)&outTable,
+      &outFull, &brightness, &contrast, &saturation);
+  if (frame->colorspace != AVCOL_SPC_UNSPECIFIED) {
+    inTable = sws_getCoefficients(frame->colorspace);
+  }
+  if (frame->color_range != AVCOL_RANGE_UNSPECIFIED) {
+    inFull = frame->color_range == AVCOL_RANGE_JPEG;
+  }
+  sws_setColorspaceDetails(sws, inTable, inFull, outTable, outFull, brightness,
+      contrast, saturation);
+
   if (rgbFrame && (rgbFrame->width != frame->width || rgbFrame->height != frame->height)) {
     av_frame_free(&rgbFrame);