Allow conditional dependencies between rects when decoding

Some encodings only cause dependencies between rects some of the
time. Make sure we can allow parallel decoding of those rect that
aren't dependent on each other.

common/rfb/DecodeManager.cxx

@@ -268,6 +268,23 @@ DecodeManager::QueueEntry* DecodeManager::DecodeThread::findEntry()
       }
     }
 
+    // For a partially ordered decoder we must ask the decoder for each
+    // pair of rectangles.
+    if (entry->decoder->flags & DecoderPartiallyOrdered) {
+      for (iter2 = manager->workQueue.begin(); iter2 != iter; ++iter2) {
+        if (entry->encoding != (*iter2)->encoding)
+          continue;
+        if (entry->decoder->doRectsConflict(entry->rect,
+                                            entry->bufferStream->data(),
+                                            entry->bufferStream->length(),
+                                            (*iter2)->rect,
+                                            (*iter2)->bufferStream->data(),
+                                            (*iter2)->bufferStream->length(),
+                                            *entry->cp))
+          goto next;
+      }
+    }
+
     // Check overlap with earlier rectangles
     if (!lockedRegion.intersect(entry->affectedRegion).is_empty())
       goto next;

common/rfb/Decoder.cxx

@@ -44,6 +44,14 @@ void Decoder::getAffectedRegion(const Rect& rect, const void* buffer,
   region->reset(rect);
 }
 
+bool Decoder::doRectsConflict(const Rect& rectA, const void* bufferA,
+                              size_t buflenA, const Rect& rectB,
+                              const void* bufferB, size_t buflenB,
+                              const ConnParams& cp)
+{
+  return false;
+}
+
 bool Decoder::supported(int encoding)
 {
   switch (encoding) {

common/rfb/Decoder.h

@@ -35,6 +35,9 @@ namespace rfb {
     DecoderPlain = 0,
     // All rects for this decoder must be handled in order
     DecoderOrdered = 1 << 0,
+    // Only some of the rects must be handled in order,
+    // see doesRectsConflict()
+    DecoderPartiallyOrdered = 1 << 1,
   };
 
   class Decoder {
@@ -62,6 +65,17 @@ namespace rfb {
                                    size_t buflen, const ConnParams& cp,
                                    Region* region);
 
+    // doesRectsConflict() determines if two rectangles must be decoded
+    // in the order they were received. This will only be called if the
+    // DecoderPartiallyOrdered flag has been set.
+    virtual bool doRectsConflict(const Rect& rectA,
+                                 const void* bufferA,
+                                 size_t buflenA,
+                                 const Rect& rectB,
+                                 const void* bufferB,
+                                 size_t buflenB,
+                                 const ConnParams& cp);
+
     // decodeRect() decodes the given rectangle with data from the
     // given buffer, onto the ModifiablePixelBuffer. The PixelFormat of
     // the PixelBuffer might not match the ConnParams and it is up to

common/rfb/TightDecoder.cxx

@@ -46,7 +46,7 @@ static const int TIGHT_MIN_TO_COMPRESS = 12;
 #include <rfb/tightDecode.h>
 #undef BPP
 
-TightDecoder::TightDecoder() : Decoder(DecoderOrdered)
+TightDecoder::TightDecoder() : Decoder(DecoderPartiallyOrdered)
 {
 }
 
@@ -148,6 +148,39 @@ void TightDecoder::readRect(const Rect& r, rdr::InStream* is,
   }
 }
 
+bool TightDecoder::doRectsConflict(const Rect& rectA,
+                                   const void* bufferA,
+                                   size_t buflenA,
+                                   const Rect& rectB,
+                                   const void* bufferB,
+                                   size_t buflenB,
+                                   const ConnParams& cp)
+{
+  rdr::U8 comp_ctl_a, comp_ctl_b;
+
+  assert(buflenA >= 1);
+  assert(buflenB >= 1);
+
+  comp_ctl_a = *(const rdr::U8*)bufferA;
+  comp_ctl_b = *(const rdr::U8*)bufferB;
+
+  // Resets or use of zlib pose the same problem, so merge them
+  if ((comp_ctl_a & 0x80) == 0x00)
+    comp_ctl_a |= 1 << ((comp_ctl_a >> 4) & 0x03);
+  if ((comp_ctl_b & 0x80) == 0x00)
+    comp_ctl_b |= 1 << ((comp_ctl_b >> 4) & 0x03);
+
+  if (((comp_ctl_a & 0x0f) & (comp_ctl_b & 0x0f)) != 0)
+    return true;
+
+  // We have a shared JpegDecompressor, so one at a time
+  if (((comp_ctl_a >> 4) == tightJpeg) &&
+      ((comp_ctl_b >> 4) == tightJpeg))
+    return true;
+
+  return false;
+}
+
 void TightDecoder::decodeRect(const Rect& r, const void* buffer,
                               size_t buflen, const ConnParams& cp,
                               ModifiablePixelBuffer* pb)

common/rfb/TightDecoder.h

@@ -33,6 +33,13 @@ namespace rfb {
     virtual ~TightDecoder();
     virtual void readRect(const Rect& r, rdr::InStream* is,
                           const ConnParams& cp, rdr::OutStream* os);
+    virtual bool doRectsConflict(const Rect& rectA,
+                                 const void* bufferA,
+                                 size_t buflenA,
+                                 const Rect& rectB,
+                                 const void* bufferB,
+                                 size_t buflenB,
+                                 const ConnParams& cp);
     virtual void decodeRect(const Rect& r, const void* buffer,
                             size_t buflen, const ConnParams& cp,
                             ModifiablePixelBuffer* pb);