Move image encoding logic into a central EncodeManager class

This allows us to apply a lot more server logic
independently of which encoder is in use.

Most of this class are things moved over from the
Tight encoder.

common/rfb/CMakeLists.txt

@@ -17,6 +17,7 @@ set(RFB_SOURCES
   Cursor.cxx
   Decoder.cxx
   d3des.c
+  EncodeManager.cxx
   Encoder.cxx
   HTTPServer.cxx
   HextileDecoder.cxx
@@ -52,6 +53,7 @@ set(RFB_SOURCES
   Timer.cxx
   TightDecoder.cxx
   TightEncoder.cxx
+  TightJPEGEncoder.cxx
   UpdateTracker.cxx
   VNCSConnectionST.cxx
   VNCServerST.cxx

common/rfb/ConnParams.cxx

@@ -22,7 +22,7 @@
 #include <rdr/OutStream.h>
 #include <rfb/Exception.h>
 #include <rfb/encodings.h>
-#include <rfb/Encoder.h>
+#include <rfb/EncodeManager.h>
 #include <rfb/ConnParams.h>
 #include <rfb/util.h>
 
@@ -38,7 +38,7 @@ ConnParams::ConnParams()
     supportsContinuousUpdates(false),
     compressLevel(2), qualityLevel(-1), fineQualityLevel(-1),
     subsampling(subsampleUndefined), name_(0),
-    currentEncoding_(encodingRaw), verStrPos(0)
+    preferredEncoding_(encodingRaw), verStrPos(0)
 {
   setName("");
 }
@@ -101,6 +101,11 @@ void ConnParams::setCursor(const Cursor& other)
   memcpy(cursor_.mask.buf, other.mask.buf, cursor_.maskLen());
 }
 
+bool ConnParams::supportsEncoding(rdr::S32 encoding)
+{
+  return encodings_.count(encoding) != 0;
+}
+
 void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
 {
   useCopyRect = false;
@@ -113,7 +118,10 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
   qualityLevel = -1;
   fineQualityLevel = -1;
   subsampling = subsampleUndefined;
-  currentEncoding_ = encodingRaw;
+  preferredEncoding_ = encodingRaw;
+
+  encodings_.clear();
+  encodings_.insert(encodingRaw);
 
   for (int i = nEncodings-1; i >= 0; i--) {
     switch (encodings[i]) {
@@ -176,7 +184,10 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
         encodings[i] <= pseudoEncodingFineQualityLevel100)
       fineQualityLevel = encodings[i] - pseudoEncodingFineQualityLevel0;
 
-    if (Encoder::supported(encodings[i]))
-      currentEncoding_ = encodings[i];
+    if (EncodeManager::supported(encodings[i]))
+      preferredEncoding_ = encodings[i];
+
+    if (encodings[i] > 0)
+      encodings_.insert(encodings[i]);
   }
 }

common/rfb/ConnParams.h

@@ -23,6 +23,8 @@
 #ifndef __RFB_CONNPARAMS_H__
 #define __RFB_CONNPARAMS_H__
 
+#include <set>
+
 #include <rdr/types.h>
 #include <rfb/Cursor.h>
 #include <rfb/PixelFormat.h>
@@ -78,7 +80,8 @@ namespace rfb {
     const Cursor& cursor() { return cursor_; }
     void setCursor(const Cursor& cursor);
 
-    rdr::S32 currentEncoding() { return currentEncoding_; }
+    rdr::S32 preferredEncoding() { return preferredEncoding_; }
+    bool supportsEncoding(rdr::S32 encoding);
 
     void setEncodings(int nEncodings, const rdr::S32* encodings);
 
@@ -105,7 +108,8 @@ namespace rfb {
     PixelFormat pf_;
     char* name_;
     Cursor cursor_;
-    int currentEncoding_;
+    rdr::S32 preferredEncoding_;
+    std::set<rdr::S32> encodings_;
     char verStr[13];
     int verStrPos;
   };

common/rfb/EncodeManager.cxx

@@ -0,0 +1,707 @@
+/* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+#include <rfb/EncodeManager.h>
+#include <rfb/Encoder.h>
+#include <rfb/Palette.h>
+#include <rfb/SConnection.h>
+#include <rfb/SMsgWriter.h>
+#include <rfb/UpdateTracker.h>
+
+#include <rfb/RawEncoder.h>
+#include <rfb/RREEncoder.h>
+#include <rfb/HextileEncoder.h>
+#include <rfb/ZRLEEncoder.h>
+#include <rfb/TightEncoder.h>
+#include <rfb/TightJPEGEncoder.h>
+
+using namespace rfb;
+
+// Split each rectangle into smaller ones no larger than this area,
+// and no wider than this width.
+static const int SubRectMaxArea = 65536;
+static const int SubRectMaxWidth = 2048;
+
+// The size in pixels of either side of each block tested when looking
+// for solid blocks.
+static const int SolidSearchBlock = 16;
+// Don't bother with blocks smaller than this
+static const int SolidBlockMinArea = 2048;
+
+namespace rfb {
+
+enum EncoderClass {
+  encoderRaw,
+  encoderRRE,
+  encoderHextile,
+  encoderTight,
+  encoderTightJPEG,
+  encoderZRLE,
+  encoderClassMax,
+};
+
+enum EncoderType {
+  encoderSolid,
+  encoderBitmap,
+  encoderBitmapRLE,
+  encoderIndexed,
+  encoderIndexedRLE,
+  encoderFullColour,
+  encoderTypeMax,
+};
+
+struct RectInfo {
+  int rleRuns;
+  Palette palette;
+};
+
+};
+
+EncodeManager::EncodeManager(SConnection* conn_) : conn(conn_)
+{
+  encoders.resize(encoderClassMax, NULL);
+  activeEncoders.resize(encoderTypeMax, encoderRaw);
+
+  encoders[encoderRaw] = new RawEncoder(conn);
+  encoders[encoderRRE] = new RREEncoder(conn);
+  encoders[encoderHextile] = new HextileEncoder(conn);
+  encoders[encoderTight] = new TightEncoder(conn);
+  encoders[encoderTightJPEG] = new TightJPEGEncoder(conn);
+  encoders[encoderZRLE] = new ZRLEEncoder(conn);
+}
+
+EncodeManager::~EncodeManager()
+{
+  std::vector<Encoder*>::iterator iter;
+
+  for (iter = encoders.begin();iter != encoders.end();iter++)
+    delete *iter;
+}
+
+bool EncodeManager::supported(int encoding)
+{
+  switch (encoding) {
+  case encodingRaw:
+  case encodingRRE:
+  case encodingHextile:
+  case encodingZRLE:
+  case encodingTight:
+    return true;
+  default:
+    return false;
+  }
+}
+
+void EncodeManager::writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
+                                const RenderedCursor* renderedCursor)
+{
+    int nRects;
+    Region changed;
+
+    prepareEncoders();
+
+    if (conn->cp.supportsLastRect)
+      nRects = 0xFFFF;
+    else {
+      nRects = ui.copied.numRects();
+      nRects += computeNumRects(ui.changed);
+
+      if (renderedCursor != NULL)
+        nRects += 1;
+    }
+
+    conn->writer()->writeFramebufferUpdateStart(nRects);
+
+    writeCopyRects(ui);
+
+    /*
+     * We start by searching for solid rects, which are then removed
+     * from the changed region.
+     */
+    changed.copyFrom(ui.changed);
+
+    if (conn->cp.supportsLastRect)
+      writeSolidRects(&changed, pb);
+
+    writeRects(changed, pb);
+
+    if (renderedCursor != NULL) {
+      Rect renderedCursorRect;
+
+      renderedCursorRect = renderedCursor->getEffectiveRect();
+      writeSubRect(renderedCursorRect, renderedCursor);
+    }
+
+    conn->writer()->writeFramebufferUpdateEnd();
+}
+
+void EncodeManager::prepareEncoders()
+{
+  enum EncoderClass solid, bitmap, bitmapRLE;
+  enum EncoderClass indexed, indexedRLE, fullColour;
+
+  rdr::S32 preferred;
+
+  std::vector<int>::iterator iter;
+
+  solid = bitmap = bitmapRLE = encoderRaw;
+  indexed = indexedRLE = fullColour = encoderRaw;
+
+  // Try to respect the client's wishes
+  preferred = conn->cp.preferredEncoding();
+  switch (preferred) {
+  case encodingRRE:
+    // Horrible for anything high frequency and/or lots of colours
+    bitmapRLE = indexedRLE = encoderRRE;
+    break;
+  case encodingHextile:
+    // Slightly less horrible
+    bitmapRLE = indexedRLE = fullColour = encoderHextile;
+    break;
+  case encodingTight:
+    if (encoders[encoderTightJPEG]->isSupported() &&
+        (conn->cp.pf().bpp >= 16))
+      fullColour = encoderTightJPEG;
+    else
+      fullColour = encoderTight;
+    indexed = indexedRLE = encoderTight;
+    bitmap = bitmapRLE = encoderTight;
+    break;
+  case encodingZRLE:
+    fullColour = encoderZRLE;
+    bitmapRLE = indexedRLE = encoderZRLE;
+    bitmap = indexed = encoderZRLE;
+    break;
+  }
+
+  // Any encoders still unassigned?
+
+  if (fullColour == encoderRaw) {
+    if (encoders[encoderTightJPEG]->isSupported() &&
+        (conn->cp.pf().bpp >= 16))
+      fullColour = encoderTightJPEG;
+    else if (encoders[encoderZRLE]->isSupported())
+      fullColour = encoderZRLE;
+    else if (encoders[encoderTight]->isSupported())
+      fullColour = encoderTight;
+    else if (encoders[encoderHextile]->isSupported())
+      fullColour = encoderHextile;
+  }
+
+  if (indexed == encoderRaw) {
+    if (encoders[encoderZRLE]->isSupported())
+      indexed = encoderZRLE;
+    else if (encoders[encoderTight]->isSupported())
+      indexed = encoderTight;
+    else if (encoders[encoderHextile]->isSupported())
+      indexed = encoderHextile;
+  }
+
+  if (indexedRLE == encoderRaw)
+    indexedRLE = indexed;
+
+  if (bitmap == encoderRaw)
+    bitmap = indexed;
+  if (bitmapRLE == encoderRaw)
+    bitmapRLE = bitmap;
+
+  if (solid == encoderRaw) {
+    if (encoders[encoderTight]->isSupported())
+      solid = encoderTight;
+    else if (encoders[encoderRRE]->isSupported())
+      solid = encoderRRE;
+    else if (encoders[encoderZRLE]->isSupported())
+      solid = encoderZRLE;
+    else if (encoders[encoderHextile]->isSupported())
+      solid = encoderHextile;
+  }
+
+  // JPEG is the only encoder that can reduce things to grayscale
+  if ((conn->cp.subsampling == subsampleGray) &&
+      encoders[encoderTightJPEG]->isSupported()) {
+    solid = bitmap = bitmapRLE = encoderTightJPEG;
+    indexed = indexedRLE = fullColour = encoderTightJPEG;
+  }
+
+  activeEncoders[encoderSolid] = solid;
+  activeEncoders[encoderBitmap] = bitmap;
+  activeEncoders[encoderBitmapRLE] = bitmapRLE;
+  activeEncoders[encoderIndexed] = indexed;
+  activeEncoders[encoderIndexedRLE] = indexedRLE;
+  activeEncoders[encoderFullColour] = fullColour;
+
+  for (iter = activeEncoders.begin(); iter != activeEncoders.end(); ++iter) {
+    Encoder *encoder;
+
+    encoder = encoders[*iter];
+
+    encoder->setCompressLevel(conn->cp.compressLevel);
+    encoder->setQualityLevel(conn->cp.qualityLevel);
+    encoder->setFineQualityLevel(conn->cp.fineQualityLevel,
+                                 conn->cp.subsampling);
+  }
+}
+
+int EncodeManager::computeNumRects(const Region& changed)
+{
+  int numRects;
+  std::vector<Rect> rects;
+  std::vector<Rect>::const_iterator rect;
+
+  numRects = 0;
+  changed.get_rects(&rects);
+  for (rect = rects.begin(); rect != rects.end(); ++rect) {
+    int w, h, sw, sh;
+
+    w = rect->width();
+    h = rect->height();
+
+    // No split necessary?
+    if (((w*h) < SubRectMaxArea) && (w < SubRectMaxWidth)) {
+      numRects += 1;
+      continue;
+    }
+
+    if (w <= SubRectMaxWidth)
+      sw = w;
+    else
+      sw = SubRectMaxWidth;
+
+    sh = SubRectMaxArea / sw;
+
+    // ceil(w/sw) * ceil(h/sh)
+    numRects += (((w - 1)/sw) + 1) * (((h - 1)/sh) + 1);
+  }
+
+  return numRects;
+}
+
+void EncodeManager::writeCopyRects(const UpdateInfo& ui)
+{
+  std::vector<Rect> rects;
+  std::vector<Rect>::const_iterator rect;
+
+  ui.copied.get_rects(&rects, ui.copy_delta.x <= 0, ui.copy_delta.y <= 0);
+  for (rect = rects.begin(); rect != rects.end(); ++rect) {
+    conn->writer()->writeCopyRect(*rect, rect->tl.x - ui.copy_delta.x,
+                                   rect->tl.y - ui.copy_delta.y);
+  }
+}
+
+void EncodeManager::writeSolidRects(Region *changed, const PixelBuffer* pb)
+{
+  std::vector<Rect> rects;
+  std::vector<Rect>::const_iterator rect;
+
+  // FIXME: This gives up after the first rect it finds. A large update
+  //        (like a whole screen refresh) might have lots of large solid
+  //        areas.
+
+  changed->get_rects(&rects);
+  for (rect = rects.begin(); rect != rects.end(); ++rect) {
+    Rect sr;
+    int dx, dy, dw, dh;
+
+    // We start by finding a solid 16x16 block
+    for (dy = rect->tl.y; dy < rect->br.y; dy += SolidSearchBlock) {
+
+      dh = SolidSearchBlock;
+      if (dy + dh > rect->br.y)
+        dh = rect->br.y - dy;
+
+      for (dx = rect->tl.x; dx < rect->br.x; dx += SolidSearchBlock) {
+        // We define it like this to guarantee alignment
+        rdr::U32 _buffer;
+        rdr::U8* colourValue = (rdr::U8*)&_buffer;
+
+        dw = SolidSearchBlock;
+        if (dx + dw > rect->br.x)
+          dw = rect->br.x - dx;
+
+        pb->getImage(colourValue, Rect(dx, dy, dx+1, dy+1));
+
+        sr.setXYWH(dx, dy, dw, dh);
+        if (checkSolidTile(sr, colourValue, pb)) {
+          Rect erb, erp;
+
+          Encoder *encoder;
+
+          // We then try extending the area by adding more blocks
+          // in both directions and pick the combination that gives
+          // the largest area.
+          sr.setXYWH(dx, dy, rect->br.x - dx, rect->br.y - dy);
+          extendSolidAreaByBlock(sr, colourValue, pb, &erb);
+
+          // Did we end up getting the entire rectangle?
+          if (erb.equals(*rect))
+            erp = erb;
+          else {
+            // Don't bother with sending tiny rectangles
+            if (erb.area() < SolidBlockMinArea)
+              continue;
+
+            // Extend the area again, but this time one pixel
+            // row/column at a time.
+            extendSolidAreaByPixel(*rect, erb, colourValue, pb, &erp);
+          }
+
+          // Send solid-color rectangle.
+          encoder = encoders[activeEncoders[encoderSolid]];
+          conn->writer()->startRect(erp, encoder->encoding);
+          if (encoder->flags & EncoderUseNativePF) {
+            encoder->writeSolidRect(erp.width(), erp.height(),
+                                    pb->getPF(), colourValue);
+          } else {
+            rdr::U32 _buffer2;
+            rdr::U8* converted = (rdr::U8*)&_buffer2;
+
+            conn->cp.pf().bufferFromBuffer(converted, pb->getPF(),
+                                           colourValue, 1);
+
+            encoder->writeSolidRect(erp.width(), erp.height(),
+                                    conn->cp.pf(), converted);
+          }
+          conn->writer()->endRect();
+
+          changed->assign_subtract(Region(erp));
+
+          break;
+        }
+      }
+
+      if (dx < rect->br.x)
+        break;
+    }
+  }
+}
+
+void EncodeManager::writeRects(const Region& changed, const PixelBuffer* pb)
+{
+  std::vector<Rect> rects;
+  std::vector<Rect>::const_iterator rect;
+
+  changed.get_rects(&rects);
+  for (rect = rects.begin(); rect != rects.end(); ++rect) {
+    int w, h, sw, sh;
+    Rect sr;
+
+    w = rect->width();
+    h = rect->height();
+
+    // No split necessary?
+    if (((w*h) < SubRectMaxArea) && (w < SubRectMaxWidth)) {
+      writeSubRect(*rect, pb);
+      continue;
+    }
+
+    if (w <= SubRectMaxWidth)
+      sw = w;
+    else
+      sw = SubRectMaxWidth;
+
+    sh = SubRectMaxArea / sw;
+
+    for (sr.tl.y = rect->tl.y; sr.tl.y < rect->br.y; sr.tl.y += sh) {
+      sr.br.y = sr.tl.y + sh;
+      if (sr.br.y > rect->br.y)
+        sr.br.y = rect->br.y;
+
+      for (sr.tl.x = rect->tl.x; sr.tl.x < rect->br.x; sr.tl.x += sw) {
+        sr.br.x = sr.tl.x + sw;
+        if (sr.br.x > rect->br.x)
+          sr.br.x = rect->br.x;
+
+        writeSubRect(sr, pb);
+      }
+    }
+  }
+}
+
+void EncodeManager::writeSubRect(const Rect& rect, const PixelBuffer *pb)
+{
+  PixelBuffer *ppb;
+
+  Encoder *encoder;
+
+  struct RectInfo info;
+  int divisor, maxColours;
+
+  bool useRLE;
+  EncoderType type;
+
+  // FIXME: This is roughly the algorithm previously used by the Tight
+  //        encoder. It seems a bit backwards though, that higher
+  //        compression setting means spending less effort in building
+  //        a palette. It might be that they figured the increase in
+  //        zlib setting compensated for the loss.
+  if (conn->cp.compressLevel == -1)
+    divisor = 2 * 8;
+  else
+    divisor = conn->cp.compressLevel * 8;
+  if (divisor < 4)
+    divisor = 4;
+
+  maxColours = rect.area()/divisor;
+
+  // Special exception inherited from the Tight encoder
+  if (activeEncoders[encoderFullColour] == encoderTightJPEG) {
+    if (conn->cp.compressLevel < 2)
+      maxColours = 24;
+    else
+      maxColours = 96;
+  }
+
+  if (maxColours < 2)
+    maxColours = 2;
+
+  encoder = encoders[activeEncoders[encoderIndexedRLE]];
+  if (maxColours > encoder->maxPaletteSize)
+    maxColours = encoder->maxPaletteSize;
+  encoder = encoders[activeEncoders[encoderIndexed]];
+  if (maxColours > encoder->maxPaletteSize)
+    maxColours = encoder->maxPaletteSize;
+
+  ppb = preparePixelBuffer(rect, pb, true);
+
+  if (!analyseRect(ppb, &info, maxColours))
+    info.palette.clear();
+
+  // Different encoders might have different RLE overhead, but
+  // here we do a guess at RLE being the better choice if reduces
+  // the pixel count by 50%.
+  useRLE = info.rleRuns <= (rect.area() * 2);
+
+  switch (info.palette.size()) {
+  case 0:
+    type = encoderFullColour;
+    break;
+  case 1:
+    type = encoderSolid;
+    break;
+  case 2:
+    if (useRLE)
+      type = encoderBitmapRLE;
+    else
+      type = encoderBitmap;
+    break;
+  default:
+    if (useRLE)
+      type = encoderIndexedRLE;
+    else
+      type = encoderIndexed;
+  }
+
+  encoder = encoders[activeEncoders[type]];
+
+  if (encoder->flags & EncoderUseNativePF)
+    ppb = preparePixelBuffer(rect, pb, false);
+
+  conn->writer()->startRect(rect, encoder->encoding);
+  encoder->writeRect(ppb, info.palette);
+  conn->writer()->endRect();
+}
+
+bool EncodeManager::checkSolidTile(const Rect& r, const rdr::U8* colourValue,
+                                   const PixelBuffer *pb)
+{
+  switch (pb->getPF().bpp) {
+  case 32:
+    return checkSolidTile(r, *(const rdr::U32*)colourValue, pb);
+  case 16:
+    return checkSolidTile(r, *(const rdr::U16*)colourValue, pb);
+  default:
+    return checkSolidTile(r, *(const rdr::U8*)colourValue, pb);
+  }
+}
+
+void EncodeManager::extendSolidAreaByBlock(const Rect& r,
+                                           const rdr::U8* colourValue,
+                                           const PixelBuffer *pb, Rect* er)
+{
+  int dx, dy, dw, dh;
+  int w_prev;
+  Rect sr;
+  int w_best = 0, h_best = 0;
+
+  w_prev = r.width();
+
+  // We search width first, back off when we hit a different colour,
+  // and restart with a larger height. We keep track of the
+  // width/height combination that gives us the largest area.
+  for (dy = r.tl.y; dy < r.br.y; dy += SolidSearchBlock) {
+
+    dh = SolidSearchBlock;
+    if (dy + dh > r.br.y)
+      dh = r.br.y - dy;
+
+    // We test one block here outside the x loop in order to break
+    // the y loop right away.
+    dw = SolidSearchBlock;
+    if (dw > w_prev)
+      dw = w_prev;
+
+    sr.setXYWH(r.tl.x, dy, dw, dh);
+    if (!checkSolidTile(sr, colourValue, pb))
+      break;
+
+    for (dx = r.tl.x + dw; dx < r.tl.x + w_prev;) {
+
+      dw = SolidSearchBlock;
+      if (dx + dw > r.tl.x + w_prev)
+        dw = r.tl.x + w_prev - dx;
+
+      sr.setXYWH(dx, dy, dw, dh);
+      if (!checkSolidTile(sr, colourValue, pb))
+        break;
+
+      dx += dw;
+    }
+
+    w_prev = dx - r.tl.x;
+    if (w_prev * (dy + dh - r.tl.y) > w_best * h_best) {
+      w_best = w_prev;
+      h_best = dy + dh - r.tl.y;
+    }
+  }
+
+  er->tl.x = r.tl.x;
+  er->tl.y = r.tl.y;
+  er->br.x = er->tl.x + w_best;
+  er->br.y = er->tl.y + h_best;
+}
+
+void EncodeManager::extendSolidAreaByPixel(const Rect& r, const Rect& sr,
+                                           const rdr::U8* colourValue,
+                                           const PixelBuffer *pb, Rect* er)
+{
+  int cx, cy;
+  Rect tr;
+
+  // Try to extend the area upwards.
+  for (cy = sr.tl.y - 1; cy >= r.tl.y; cy--) {
+    tr.setXYWH(sr.tl.x, cy, sr.width(), 1);
+    if (!checkSolidTile(tr, colourValue, pb))
+      break;
+  }
+  er->tl.y = cy + 1;
+
+  // ... downwards.
+  for (cy = sr.br.y; cy < r.br.y; cy++) {
+    tr.setXYWH(sr.tl.x, cy, sr.width(), 1);
+    if (!checkSolidTile(tr, colourValue, pb))
+      break;
+  }
+  er->br.y = cy;
+
+  // ... to the left.
+  for (cx = sr.tl.x - 1; cx >= r.tl.x; cx--) {
+    tr.setXYWH(cx, er->tl.y, 1, er->height());
+    if (!checkSolidTile(tr, colourValue, pb))
+      break;
+  }
+  er->tl.x = cx + 1;
+
+  // ... to the right.
+  for (cx = sr.br.x; cx < r.br.x; cx++) {
+    tr.setXYWH(cx, er->tl.y, 1, er->height());
+    if (!checkSolidTile(tr, colourValue, pb))
+      break;
+  }
+  er->br.x = cx;
+}
+
+PixelBuffer* EncodeManager::preparePixelBuffer(const Rect& rect,
+                                               const PixelBuffer *pb,
+                                               bool convert)
+{
+  const rdr::U8* buffer;
+  int stride;
+
+  // Do wo need to convert the data?
+  if (convert && !conn->cp.pf().equal(pb->getPF())) {
+    convertedPixelBuffer.setPF(conn->cp.pf());
+    convertedPixelBuffer.setSize(rect.width(), rect.height());
+
+    buffer = pb->getBuffer(rect, &stride);
+    convertedPixelBuffer.imageRect(pb->getPF(),
+                                   convertedPixelBuffer.getRect(),
+                                   buffer, stride);
+
+    return &convertedPixelBuffer;
+  }
+
+  // Otherwise we still need to shift the coordinates. We have our own
+  // abusive subclass of FullFramePixelBuffer for this.
+
+  buffer = pb->getBuffer(rect, &stride);
+
+  offsetPixelBuffer.update(pb->getPF(), rect.width(), rect.height(),
+                           buffer, stride);
+
+  return &offsetPixelBuffer;
+}
+
+bool EncodeManager::analyseRect(const PixelBuffer *pb,
+                                struct RectInfo *info, int maxColours)
+{
+  const rdr::U8* buffer;
+  int stride;
+
+  buffer = pb->getBuffer(pb->getRect(), &stride);
+
+  switch (pb->getPF().bpp) {
+  case 32:
+    return analyseRect(pb->width(), pb->height(),
+                       (const rdr::U32*)buffer, stride,
+                       info, maxColours);
+  case 16:
+    return analyseRect(pb->width(), pb->height(),
+                       (const rdr::U16*)buffer, stride,
+                       info, maxColours);
+  default:
+    return analyseRect(pb->width(), pb->height(),
+                       (const rdr::U8*)buffer, stride,
+                       info, maxColours);
+  }
+}
+
+void EncodeManager::OffsetPixelBuffer::update(const PixelFormat& pf,
+                                              int width, int height,
+                                              const rdr::U8* data_,
+                                              int stride_)
+{
+  format = pf;
+  width_ = width;
+  height_ = height;
+  // Forced cast. We never write anything though, so it should be safe.
+  data = (rdr::U8*)data_;
+  stride = stride_;
+}
+
+// Preprocessor generated, optimised methods
+
+#define BPP 8
+#include "EncodeManagerBPP.cxx"
+#undef BPP
+#define BPP 16
+#include "EncodeManagerBPP.cxx"
+#undef BPP
+#define BPP 32
+#include "EncodeManagerBPP.cxx"
+#undef BPP

common/rfb/EncodeManager.h

@@ -0,0 +1,114 @@
+/* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+#ifndef __RFB_ENCODEMANAGER_H__
+#define __RFB_ENCODEMANAGER_H__
+
+#include <vector>
+
+#include <rdr/types.h>
+#include <rfb/PixelBuffer.h>
+
+namespace rfb {
+  class SConnection;
+  class Encoder;
+  class UpdateInfo;
+  class PixelBuffer;
+  class RenderedCursor;
+  class Region;
+  class Rect;
+
+  struct RectInfo;
+
+  class EncodeManager {
+  public:
+    EncodeManager(SConnection* conn);
+    ~EncodeManager();
+
+    // Hack to let ConnParams calculate the client's preferred encoding
+    static bool supported(int encoding);
+
+    void writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
+                     const RenderedCursor* renderedCursor);
+
+  protected:
+    void prepareEncoders();
+
+    int computeNumRects(const Region& changed);
+
+    void writeCopyRects(const UpdateInfo& ui);
+    void writeSolidRects(Region *changed, const PixelBuffer* pb);
+    void writeRects(const Region& changed, const PixelBuffer* pb);
+
+    void writeSubRect(const Rect& rect, const PixelBuffer *pb);
+
+    bool checkSolidTile(const Rect& r, const rdr::U8* colourValue,
+                        const PixelBuffer *pb);
+    void extendSolidAreaByBlock(const Rect& r, const rdr::U8* colourValue,
+                                const PixelBuffer *pb, Rect* er);
+    void extendSolidAreaByPixel(const Rect& r, const Rect& sr,
+                                const rdr::U8* colourValue,
+                                const PixelBuffer *pb, Rect* er);
+
+    PixelBuffer* preparePixelBuffer(const Rect& rect,
+                                    const PixelBuffer *pb, bool convert);
+
+    bool analyseRect(const PixelBuffer *pb,
+                     struct RectInfo *info, int maxColours);
+
+  protected:
+    // Preprocessor generated, optimised methods
+    inline bool checkSolidTile(const Rect& r, rdr::U8 colourValue,
+                               const PixelBuffer *pb);
+    inline bool checkSolidTile(const Rect& r, rdr::U16 colourValue,
+                               const PixelBuffer *pb);
+    inline bool checkSolidTile(const Rect& r, rdr::U32 colourValue,
+                               const PixelBuffer *pb);
+
+    inline bool analyseRect(int width, int height,
+                            const rdr::U8* buffer, int stride,
+                            struct RectInfo *info, int maxColours);
+    inline bool analyseRect(int width, int height,
+                            const rdr::U16* buffer, int stride,
+                            struct RectInfo *info, int maxColours);
+    inline bool analyseRect(int width, int height,
+                            const rdr::U32* buffer, int stride,
+                            struct RectInfo *info, int maxColours);
+
+  protected:
+    SConnection *conn;
+
+    std::vector<Encoder*> encoders;
+    std::vector<int> activeEncoders;
+
+    class OffsetPixelBuffer : public FullFramePixelBuffer {
+    public:
+      OffsetPixelBuffer() {}
+      virtual ~OffsetPixelBuffer() {}
+
+      void update(const PixelFormat& pf, int width, int height,
+                  const rdr::U8* data_, int stride);
+    };
+
+    OffsetPixelBuffer offsetPixelBuffer;
+    ManagedPixelBuffer convertedPixelBuffer;
+  };
+}
+
+#endif

common/rfb/EncodeManagerBPP.cxx

@@ -0,0 +1,101 @@
+/* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#define CONCAT2(a,b) a##b
+#define CONCAT2E(a,b) CONCAT2(a,b)
+
+#define UBPP CONCAT2E(U,BPP)
+
+inline bool EncodeManager::checkSolidTile(const Rect& r,
+                                          rdr::UBPP colourValue,
+                                          const PixelBuffer *pb)
+{
+  int w, h;
+  const rdr::UBPP* buffer;
+  int stride, pad;
+
+  w = r.width();
+  h = r.height();
+
+  buffer = (const rdr::UBPP*)pb->getBuffer(r, &stride);
+  pad = stride - w;
+
+  while (h--) {
+    int w_ = w;
+    while (w_--) {
+      if (*buffer != colourValue)
+        return false;
+      buffer++;
+    }
+    buffer += pad;
+  }
+
+  return true;
+}
+
+inline bool EncodeManager::analyseRect(int width, int height,
+                                       const rdr::UBPP* buffer, int stride,
+                                       struct RectInfo *info, int maxColours)
+{
+  int pad;
+
+  rdr::UBPP colour;
+  int count;
+
+  rdr::UBPP c0, c1, ci = 0;
+  int i, n0, n1, ni;
+
+  info->rleRuns = 0;
+  info->palette.clear();
+
+  pad = stride - width;
+
+  // For efficiency, we only update the palette on changes in colour
+  colour = buffer[0];
+  count = 0;
+  while (height--) {
+    int w_ = width;
+    while (w_--) {
+      if (*buffer != colour) {
+        if (!info->palette.insert(colour, count))
+          return false;
+        if (info->palette.size() > maxColours)
+          return false;
+
+        // FIXME: This doesn't account for switching lines
+        info->rleRuns++;
+
+        colour = *buffer;
+        count = 0;
+      }
+      buffer++;
+      count++;
+    }
+    buffer += pad;
+  }
+
+  // Make sure the final pixels also get counted
+  if (!info->palette.insert(colour, count))
+    return false;
+  if (info->palette.size() > maxColours)
+    return false;
+
+  return true;
+}

common/rfb/Encoder.cxx

@@ -16,20 +16,17 @@
  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
  * USA.
  */
-#include <stdio.h>
-#include <rfb/encodings.h>
-#include <rfb/Exception.h>
+
 #include <rfb/Encoder.h>
-#include <rfb/RawEncoder.h>
-#include <rfb/RREEncoder.h>
-#include <rfb/HextileEncoder.h>
-#include <rfb/ZRLEEncoder.h>
-#include <rfb/TightEncoder.h>
-#include <rfb/SConnection.h>
+#include <rfb/PixelBuffer.h>
+#include <rfb/Palette.h>
 
 using namespace rfb;
 
-Encoder::Encoder(SConnection *conn_) : conn(conn_)
+Encoder::Encoder(SConnection *conn_, int encoding_,
+                 enum EncoderFlags flags_, unsigned int maxPaletteSize_) :
+  conn(conn_), encoding(encoding_), flags(flags_),
+  maxPaletteSize(maxPaletteSize_)
 {
 }
 
@@ -37,34 +34,50 @@ Encoder::~Encoder()
 {
 }
 
-bool Encoder::supported(int encoding)
+void Encoder::writeSolidRect(int width, int height,
+                             const PixelFormat& pf, const rdr::U8* colour)
 {
-  switch (encoding) {
-  case encodingRaw:
-  case encodingRRE:
-  case encodingHextile:
-  case encodingZRLE:
-  case encodingTight:
-    return true;
-  default:
-    return false;
-  }
+  ManagedPixelBuffer buffer(pf, width, height);
+  Pixel pixel;
+
+  Palette palette;
+  rdr::U32 palcol;
+
+  pixel = pf.pixelFromBuffer(colour);
+  buffer.fillRect(buffer.getRect(), pixel);
+
+  palcol = 0;
+  memcpy(&palcol, colour, pf.bpp/8);
+  palette.insert(palcol, 1);
+
+  writeRect(&buffer, palette);
 }
 
-Encoder* Encoder::createEncoder(int encoding, SConnection* conn)
+void Encoder::writeSolidRect(const PixelBuffer* pb, const Palette& palette)
 {
-  switch (encoding) {
-  case encodingRaw:
-    return new RawEncoder(conn);
-  case encodingRRE:
-    return new RREEncoder(conn);
-  case encodingHextile:
-    return new HextileEncoder(conn);
-  case encodingZRLE:
-    return new ZRLEEncoder(conn);
-  case encodingTight:
-    return new TightEncoder(conn);
+  rdr::U32 col32;
+  rdr::U16 col16;
+  rdr::U8 col8;
+
+  rdr::U8* buffer;
+
+  assert(palette.size() == 1);
+
+  // The Palette relies on implicit up and down conversion
+  switch (pb->getPF().bpp) {
+  case 32:
+    col32 = (rdr::U32)palette.getColour(0);
+    buffer = (rdr::U8*)&col32;
+    break;
+  case 16:
+    col16 = (rdr::U16)palette.getColour(0);
+    buffer = (rdr::U8*)&col16;
+    break;
   default:
-    return NULL;
+    col8 = (rdr::U8)palette.getColour(0);
+    buffer = (rdr::U8*)&col8;
+    break;
   }
+
+  writeSolidRect(pb->width(), pb->height(), pb->getPF(), buffer);
 }

common/rfb/Encoder.h

@@ -20,31 +20,75 @@
 #ifndef __RFB_ENCODER_H__
 #define __RFB_ENCODER_H__
 
+#include <rdr/types.h>
 #include <rfb/Rect.h>
 
 namespace rfb {
   class SConnection;
   class PixelBuffer;
+  class Palette;
+  class PixelFormat;
+
+  enum EncoderFlags {
+    // A constant for encoders that don't need anything special
+    EncoderPlain = 0,
+    // Give us the raw frame buffer, and not something converted to
+    // the what the client is asking for.
+    EncoderUseNativePF = 1 << 0,
+  };
 
   class Encoder {
   public:
-    Encoder(SConnection* conn);
+    Encoder(SConnection* conn, int encoding,
+            enum EncoderFlags flags, unsigned int maxPaletteSize);
     virtual ~Encoder();
 
+    // isSupported() should return a boolean indiciating if this encoder
+    // is okay to use with the current connection. This usually involves
+    // checking the list of encodings in the connection parameters.
+    virtual bool isSupported()=0;
+
     virtual void setCompressLevel(int level) {};
     virtual void setQualityLevel(int level) {};
     virtual void setFineQualityLevel(int quality, int subsampling) {};
-    virtual int getNumRects(const Rect &r) { return 1; }
 
     // writeRect() is the main interface that encodes the given rectangle
     // with data from the PixelBuffer onto the SConnection given at
-    // encoder creation. The PixelFormat of the PixelBuffer might not
-    // match the ConnParams and it is up ot the encoder to do
-    // any necessary conversion.
-    virtual void writeRect(const Rect& r, PixelBuffer* pb)=0;
+    // encoder creation.
+    //
+    // The PixelBuffer will be in the PixelFormat specified in ConnParams
+    // unless the flag UseNativePF is specified. In that case the
+    // PixelBuffer will remain in its native format and encoder will have
+    // to handle any conversion itself.
+    //
+    // The Palette will always be in the PixelFormat specified in
+    // ConnParams. An empty palette indicates a large number of colours,
+    // but could still be less than maxPaletteSize.
+    virtual void writeRect(const PixelBuffer* pb, const Palette& palette)=0;
+
+    // writeSolidRect() is a short cut in order to encode single colour
+    // rectangles efficiently without having to create a fake single
+    // colour PixelBuffer. The colour argument follows the same semantics
+    // as the PixelBuffer for writeRect().
+    //
+    // Note that there is a default implementation that can be called
+    // using Encoder::writeSolidRect() in the event that there is no
+    // efficient short cut.
+    virtual void writeSolidRect(int width, int height,
+                                const PixelFormat& pf,
+                                const rdr::U8* colour)=0;
+
+  protected:
+    // Helper method for redirecting a single colour palette to the
+    // short cut method.
+    void writeSolidRect(const PixelBuffer* pb, const Palette& palette);
+
+  public:
+    const int encoding;
+    const enum EncoderFlags flags;
 
-    static bool supported(int encoding);
-    static Encoder* createEncoder(int encoding, SConnection* conn);
+    // Maximum size of the palette per rect
+    const unsigned int maxPaletteSize;
 
   protected:
     SConnection* conn;

common/rfb/HextileEncoder.cxx

@@ -1,5 +1,6 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
  * Copyright (C) 2005 Constantin Kaplinsky.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -17,10 +18,8 @@
  * USA.
  */
 #include <rfb/encodings.h>
-#include <rfb/SMsgWriter.h>
 #include <rfb/SConnection.h>
 #include <rfb/HextileEncoder.h>
-#include <rfb/PixelFormat.h>
 #include <rfb/PixelBuffer.h>
 #include <rfb/Configuration.h>
 
@@ -45,7 +44,8 @@ BoolParameter improvedHextile("ImprovedHextile",
 #include <rfb/hextileEncodeBetter.h>
 #undef BPP
 
-HextileEncoder::HextileEncoder(SConnection* conn) : Encoder(conn)
+HextileEncoder::HextileEncoder(SConnection* conn) :
+  Encoder(conn, encodingHextile, EncoderPlain, -1)
 {
 }
 
@@ -53,33 +53,54 @@ HextileEncoder::~HextileEncoder()
 {
 }
 
-void HextileEncoder::writeRect(const Rect& r, PixelBuffer* pb)
+bool HextileEncoder::isSupported()
+{
+  return conn->cp.supportsEncoding(encodingHextile);
+}
+
+void HextileEncoder::writeRect(const PixelBuffer* pb, const Palette& palette)
 {
-  conn->writer()->startRect(r, encodingHextile);
   rdr::OutStream* os = conn->getOutStream();
-  const PixelFormat& pf = conn->cp.pf();
-  switch (pf.bpp) {
+  switch (pb->getPF().bpp) {
   case 8:
     if (improvedHextile) {
-      hextileEncodeBetter8(r, os, pf, pb);
+      hextileEncodeBetter8(os, pb);
     } else {
-      hextileEncode8(r, os, pf, pb);
+      hextileEncode8(os, pb);
     }
     break;
   case 16:
     if (improvedHextile) {
-      hextileEncodeBetter16(r, os, pf, pb);
+      hextileEncodeBetter16(os, pb);
     } else {
-      hextileEncode16(r, os, pf, pb);
+      hextileEncode16(os, pb);
     }
     break;
   case 32:
     if (improvedHextile) {
-      hextileEncodeBetter32(r, os, pf, pb);
+      hextileEncodeBetter32(os, pb);
     } else {
-      hextileEncode32(r, os, pf, pb);
+      hextileEncode32(os, pb);
     }
     break;
   }
-  conn->writer()->endRect();
+}
+
+void HextileEncoder::writeSolidRect(int width, int height,
+                                    const PixelFormat& pf,
+                                    const rdr::U8* colour)
+{
+  rdr::OutStream* os;
+  int tiles;
+
+  os = conn->getOutStream();
+
+  tiles = ((width + 15)/16) * ((height + 15)/16);
+
+  os->writeU8(hextileBgSpecified);
+  os->writeBytes(colour, pf.bpp/8);
+  tiles--;
+
+  while (tiles--)
+      os->writeU8(0);
 }

common/rfb/HextileEncoder.h

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -26,7 +27,11 @@ namespace rfb {
   public:
     HextileEncoder(SConnection* conn);
     virtual ~HextileEncoder();
-    virtual void writeRect(const Rect& r, PixelBuffer* pb);
+    virtual bool isSupported();
+    virtual void writeRect(const PixelBuffer* pb, const Palette& palette);
+    virtual void writeSolidRect(int width, int height,
+                                const PixelFormat& pf,
+                                const rdr::U8* colour);
   };
 }
 #endif

common/rfb/RREEncoder.cxx

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -17,10 +18,10 @@
  */
 #include <rdr/OutStream.h>
 #include <rfb/encodings.h>
-#include <rfb/SMsgWriter.h>
 #include <rfb/SConnection.h>
 #include <rfb/PixelFormat.h>
 #include <rfb/PixelBuffer.h>
+#include <rfb/Palette.h>
 #include <rfb/RREEncoder.h>
 
 using namespace rfb;
@@ -35,7 +36,8 @@ using namespace rfb;
 #include <rfb/rreEncode.h>
 #undef BPP
 
-RREEncoder::RREEncoder(SConnection* conn) : RawEncoder(conn)
+RREEncoder::RREEncoder(SConnection* conn) :
+  Encoder(conn, encodingRRE, EncoderPlain, -1)
 {
 }
 
@@ -43,30 +45,71 @@ RREEncoder::~RREEncoder()
 {
 }
 
-void RREEncoder::writeRect(const Rect& r, PixelBuffer* pb)
+bool RREEncoder::isSupported()
 {
-  int w = r.width();
-  int h = r.height();
-  rdr::U8* imageBuf = conn->writer()->getImageBuf(w*h);
-  pb->getImage(conn->cp.pf(), imageBuf, r);
+  return conn->cp.supportsEncoding(encodingRRE);
+}
 
-  mos.clear();
+void RREEncoder::writeRect(const PixelBuffer* pb, const Palette& palette)
+{
+  rdr::U8* imageBuf;
+  int stride;
+  rdr::U32 bg;
 
-  int nSubrects = -1;
-  switch (conn->cp.pf().bpp) {
-  case 8:  nSubrects = rreEncode8(imageBuf, w, h, &mos);  break;
-  case 16: nSubrects = rreEncode16(imageBuf, w, h, &mos); break;
-  case 32: nSubrects = rreEncode32(imageBuf, w, h, &mos); break;
-  }
-  
-  if (nSubrects < 0) {
-    RawEncoder::writeRect(r, pb);
+  int w = pb->width();
+  int h = pb->height();
+
+  if (palette.size() == 1) {
+    Encoder::writeSolidRect(pb, palette);
     return;
   }
 
-  conn->writer()->startRect(r, encodingRRE);
+  // We have to have our own copy of the data as we modify it as
+  // we find subrects.
+  bufferCopy.setPF(pb->getPF());
+  bufferCopy.setSize(w, h);
+
+  imageBuf = bufferCopy.getBufferRW(pb->getRect(), &stride);
+  pb->getImage(imageBuf, pb->getRect());
+
+  if (palette.size() > 0)
+    bg = palette.getColour(0);
+  else {
+    // Some crazy person is using this encoder for high colour
+    // data. Just pick the first pixel as the background colour.
+    bg = 0;
+    memcpy(&bg, imageBuf, pb->getPF().bpp/8);
+  }
+
+  int nSubrects = -1;
+  switch (pb->getPF().bpp) {
+  case 8:
+    nSubrects = rreEncode8((rdr::U8*)imageBuf, w, h, &mos, bg);
+    break;
+  case 16:
+    nSubrects = rreEncode16((rdr::U16*)imageBuf, w, h, &mos, bg);
+    break;
+  case 32:
+    nSubrects = rreEncode32((rdr::U32*)imageBuf, w, h, &mos, bg);
+    break;
+  }
+
+  bufferCopy.commitBufferRW(pb->getRect());
+
   rdr::OutStream* os = conn->getOutStream();
   os->writeU32(nSubrects);
   os->writeBytes(mos.data(), mos.length());
-  conn->writer()->endRect();
+  mos.clear();
+}
+
+void RREEncoder::writeSolidRect(int width, int height,
+                                const PixelFormat& pf,
+                                const rdr::U8* colour)
+{
+  rdr::OutStream* os;
+
+  os = conn->getOutStream();
+
+  os->writeU32(0);
+  os->writeBytes(colour, pf.bpp/8);
 }

common/rfb/RREEncoder.h

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -19,17 +20,23 @@
 #define __RFB_RREENCODER_H__
 
 #include <rdr/MemOutStream.h>
-#include <rfb/RawEncoder.h>
+#include <rfb/Encoder.h>
+#include <rfb/PixelBuffer.h>
 
 namespace rfb {
 
-  class RREEncoder : public RawEncoder {
+  class RREEncoder : public Encoder {
   public:
     RREEncoder(SConnection* conn);
     virtual ~RREEncoder();
-    virtual void writeRect(const Rect& r, PixelBuffer* pb);
+    virtual bool isSupported();
+    virtual void writeRect(const PixelBuffer* pb, const Palette& palette);
+    virtual void writeSolidRect(int width, int height,
+                                const PixelFormat& pf,
+                                const rdr::U8* colour);
   private:
     rdr::MemOutStream mos;
+    ManagedPixelBuffer bufferCopy;
   };
 }
 #endif

common/rfb/RawEncoder.cxx

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -17,14 +18,14 @@
  */
 #include <rdr/OutStream.h>
 #include <rfb/encodings.h>
-#include <rfb/SMsgWriter.h>
 #include <rfb/SConnection.h>
 #include <rfb/PixelBuffer.h>
 #include <rfb/RawEncoder.h>
 
 using namespace rfb;
 
-RawEncoder::RawEncoder(SConnection* conn) : Encoder(conn)
+RawEncoder::RawEncoder(SConnection* conn) :
+  Encoder(conn, encodingRaw, EncoderPlain, -1)
 {
 }
 
@@ -32,13 +33,44 @@ RawEncoder::~RawEncoder()
 {
 }
 
-void RawEncoder::writeRect(const Rect& r, PixelBuffer* pb)
+bool RawEncoder::isSupported()
 {
-  rdr::U8* buf = conn->writer()->getImageBuf(r.area());
+  // Implicitly required;
+  return true;
+}
+
+void RawEncoder::writeRect(const PixelBuffer* pb, const Palette& palette)
+{
+  const rdr::U8* buffer;
+  int stride;
+
+  rdr::OutStream* os;
+  int h, line_bytes, stride_bytes;
+
+  buffer = pb->getBuffer(pb->getRect(), &stride);
+
+  os = conn->getOutStream();
+
+  h = pb->height();
+  line_bytes = pb->width() * pb->getPF().bpp/8;
+  stride_bytes = stride * pb->getPF().bpp/8;
+  while (h--) {
+    os->writeBytes(buffer, line_bytes);
+    buffer += stride_bytes;
+  }
+}
+
+void RawEncoder::writeSolidRect(int width, int height,
+                                const PixelFormat& pf,
+                                const rdr::U8* colour)
+{
+  rdr::OutStream* os;
+  int pixels, pixel_size;
 
-  pb->getImage(conn->cp.pf(), buf, r);
+  os = conn->getOutStream();
 
-  conn->writer()->startRect(r, encodingRaw);
-  conn->getOutStream()->writeBytes(buf, r.area() * conn->cp.pf().bpp/8);
-  conn->writer()->endRect();
+  pixels = width*height;
+  pixel_size = pf.bpp/8;
+  while (pixels--)
+    os->writeBytes(colour, pixel_size);
 }

common/rfb/RawEncoder.h

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -26,7 +27,11 @@ namespace rfb {
   public:
     RawEncoder(SConnection* conn);
     virtual ~RawEncoder();
-    virtual void writeRect(const Rect& r, PixelBuffer* pb);
+    virtual bool isSupported();
+    virtual void writeRect(const PixelBuffer* pb, const Palette& palette);
+    virtual void writeSolidRect(int width, int height,
+                                const PixelFormat& pf,
+                                const rdr::U8* colour);
   };
 }
 #endif

common/rfb/SMsgWriter.cxx

@@ -34,12 +34,11 @@ using namespace rfb;
 static LogWriter vlog("SMsgWriter");
 
 SMsgWriter::SMsgWriter(ConnParams* cp_, rdr::OutStream* os_)
-  : imageBufIdealSize(0), cp(cp_), os(os_), currentEncoding(0),
+  : cp(cp_), os(os_), currentEncoding(0),
     nRectsInUpdate(0), nRectsInHeader(0),
     needSetDesktopSize(false), needExtendedDesktopSize(false),
     needSetDesktopName(false), needSetCursor(false), needSetXCursor(false),
-    lenBeforeRect(0), updatesSent(0), rawBytesEquivalent(0),
-    imageBuf(0), imageBufSize(0)
+    lenBeforeRect(0), updatesSent(0), rawBytesEquivalent(0)
 {
   for (int i = 0; i <= encodingMax; i++) {
     bytesSent[i] = 0;
@@ -60,7 +59,6 @@ SMsgWriter::~SMsgWriter()
   }
   vlog.info("  raw bytes equivalent %llu, compression ratio %f",
           rawBytesEquivalent, (double)rawBytesEquivalent / bytes);
-  delete [] imageBuf;
 }
 
 void SMsgWriter::writeServerInit()
@@ -316,26 +314,6 @@ void SMsgWriter::endRect()
   }
 }
 
-rdr::U8* SMsgWriter::getImageBuf(int required, int requested, int* nPixels)
-{
-  int requiredBytes = required * (cp->pf().bpp / 8);
-  int requestedBytes = requested * (cp->pf().bpp / 8);
-  int size = requestedBytes;
-  if (size > imageBufIdealSize) size = imageBufIdealSize;
-
-  if (size < requiredBytes)
-    size = requiredBytes;
-
-  if (imageBufSize < size) {
-    imageBufSize = size;
-    delete [] imageBuf;
-    imageBuf = new rdr::U8[imageBufSize];
-  }
-  if (nPixels)
-    *nPixels = imageBufSize / (cp->pf().bpp / 8);
-  return imageBuf;
-}
-
 void SMsgWriter::startMsg(int type)
 {
   os->writeU8(type);

common/rfb/SMsgWriter.h

@@ -109,15 +109,11 @@ namespace rfb {
     void startRect(const Rect& r, int enc);
     void endRect();
 
-    rdr::U8* getImageBuf(int required, int requested=0, int* nPixels=0);
-
     int getUpdatesSent()           { return updatesSent; }
     int getRectsSent(int encoding) { return rectsSent[encoding]; }
     int getBytesSent(int encoding) { return bytesSent[encoding]; }
     rdr::U64 getRawBytesEquivalent()    { return rawBytesEquivalent; }
 
-    int imageBufIdealSize;
-
   protected:
     void startMsg(int type);
     void endMsg();
@@ -159,9 +155,6 @@ namespace rfb {
     int rectsSent[encodingMax+1];
     rdr::U64 rawBytesEquivalent;
 
-    rdr::U8* imageBuf;
-    int imageBufSize;
-
     typedef struct {
       rdr::U16 reason, result;
       int fb_width, fb_height;

common/rfb/TightConstants.h

@@ -0,0 +1,32 @@
+/* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
+ *    
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+#ifndef __RFB_TIGHTCONSTANTS_H__
+#define __RFB_TIGHTCONSTANTS_H__
+namespace rfb {
+  // Compression control 
+  const unsigned int tightExplicitFilter = 0x04;
+  const unsigned int tightFill = 0x08;
+  const unsigned int tightJpeg = 0x09;
+  const unsigned int tightMaxSubencoding = 0x09;
+
+  // Filters to improve compression efficiency
+  const unsigned int tightFilterCopy = 0x00;
+  const unsigned int tightFilterPalette = 0x01;
+  const unsigned int tightFilterGradient = 0x02;
+}
+#endif

common/rfb/TightDecoder.h

@@ -64,17 +64,6 @@ namespace rfb {
     PixelFormat serverpf;
     bool directDecode;
   };
-
-  // Compression control 
-  const unsigned int rfbTightExplicitFilter = 0x04;
-  const unsigned int rfbTightFill = 0x08;
-  const unsigned int rfbTightJpeg = 0x09;
-  const unsigned int rfbTightMaxSubencoding = 0x09;
-
-  // Filters to improve compression efficiency
-  const unsigned int rfbTightFilterCopy = 0x00;
-  const unsigned int rfbTightFilterPalette = 0x01;
-  const unsigned int rfbTightFilterGradient = 0x02;
 }
 
 #endif

common/rfb/TightEncoder.cxx

@@ -1,5 +1,6 @@
 /* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  *    
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -16,399 +17,202 @@
  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
  * USA.
  */
+#include <assert.h>
+
 #include <rdr/OutStream.h>
 #include <rfb/PixelBuffer.h>
+#include <rfb/Palette.h>
 #include <rfb/encodings.h>
 #include <rfb/ConnParams.h>
-#include <rfb/SMsgWriter.h>
 #include <rfb/SConnection.h>
 #include <rfb/TightEncoder.h>
+#include <rfb/TightConstants.h>
 
 using namespace rfb;
 
-// Minimum amount of data to be compressed. This value should not be
-// changed, doing so will break compatibility with existing clients.
-#define TIGHT_MIN_TO_COMPRESS 12
-
-// Adjustable parameters.
-// FIXME: Get rid of #defines
-#define TIGHT_MAX_SPLIT_TILE_SIZE      16
-#define TIGHT_MIN_SPLIT_RECT_SIZE    4096
-#define TIGHT_MIN_SOLID_SUBRECT_SIZE 2048
+struct TightConf {
+  int idxZlibLevel, monoZlibLevel, rawZlibLevel;
+};
 
 //
-// Compression level stuff. The following array contains various
-// encoder parameters for each of 10 compression levels (0..9).
-// Last three parameters correspond to JPEG quality levels (0..9).
+// Compression level stuff. The following array contains zlib
+// settings for each of 10 compression levels (0..9).
 //
 // NOTE: The parameters used in this encoder are the result of painstaking
 // research by The VirtualGL Project using RFB session captures from a variety
 // of both 2D and 3D applications.  See http://www.VirtualGL.org for the full
 // reports.
 
-// NOTE:  The JPEG quality and subsampling levels below were obtained
-// experimentally by the VirtualGL Project.  They represent the approximate
-// average compression ratios listed below, as measured across the set of
-// every 10th frame in the SPECviewperf 9 benchmark suite.
-//
-// 9 = JPEG quality 100, no subsampling (ratio ~= 10:1)
-//     [this should be lossless, except for round-off error]
-// 8 = JPEG quality 92,  no subsampling (ratio ~= 20:1)
-//     [this should be perceptually lossless, based on current research]
-// 7 = JPEG quality 86,  no subsampling (ratio ~= 25:1)
-// 6 = JPEG quality 79,  no subsampling (ratio ~= 30:1)
-// 5 = JPEG quality 77,  4:2:2 subsampling (ratio ~= 40:1)
-// 4 = JPEG quality 62,  4:2:2 subsampling (ratio ~= 50:1)
-// 3 = JPEG quality 42,  4:2:2 subsampling (ratio ~= 60:1)
-// 2 = JPEG quality 41,  4:2:0 subsampling (ratio ~= 70:1)
-// 1 = JPEG quality 29,  4:2:0 subsampling (ratio ~= 80:1)
-// 0 = JPEG quality 15,  4:2:0 subsampling (ratio ~= 100:1)
-
-const TIGHT_CONF TightEncoder::conf[10] = {
-  { 65536, 2048,   6, 0, 0, 0,   4, 24, 15, subsample4X }, // 0
-  { 65536, 2048,   6, 1, 1, 1,   8, 24, 29, subsample4X }, // 1
-  { 65536, 2048,   8, 3, 3, 2,  24, 96, 41, subsample4X }, // 2
-  { 65536, 2048,  12, 5, 5, 2,  32, 96, 42, subsample2X }, // 3
-  { 65536, 2048,  12, 6, 7, 3,  32, 96, 62, subsample2X }, // 4
-  { 65536, 2048,  12, 7, 8, 4,  32, 96, 77, subsample2X }, // 5
-  { 65536, 2048,  16, 7, 8, 5,  32, 96, 79, subsampleNone }, // 6
-  { 65536, 2048,  16, 8, 9, 6,  64, 96, 86, subsampleNone }, // 7
-  { 65536, 2048,  24, 9, 9, 7,  64, 96, 92, subsampleNone }, // 8
-  { 65536, 2048,  32, 9, 9, 9,  96, 96,100, subsampleNone }  // 9
+static const TightConf conf[10] = {
+  { 0, 0, 0 }, // 0
+  { 1, 1, 1 }, // 1
+  { 3, 3, 2 }, // 2
+  { 5, 5, 2 }, // 3
+  { 6, 7, 3 }, // 4
+  { 7, 8, 4 }, // 5
+  { 7, 8, 5 }, // 6
+  { 8, 9, 6 }, // 7
+  { 9, 9, 7 }, // 8
+  { 9, 9, 9 }  // 9
 };
 
-const int TightEncoder::defaultCompressLevel = 2;
-
-//
-// Including BPP-dependent implementation of the encoder.
-//
-
-#define BPP 8
-#include <rfb/tightEncode.h>
-#undef BPP
-#define BPP 16
-#include <rfb/tightEncode.h>
-#undef BPP
-#define BPP 32
-#include <rfb/tightEncode.h>
-#undef BPP
-
-TightEncoder::TightEncoder(SConnection* conn) : Encoder(conn)
+TightEncoder::TightEncoder(SConnection* conn) :
+  Encoder(conn, encodingTight, EncoderPlain, 256)
 {
-  setCompressLevel(defaultCompressLevel);
-  setQualityLevel(-1);
+  setCompressLevel(-1);
 }
 
 TightEncoder::~TightEncoder()
 {
 }
 
+bool TightEncoder::isSupported()
+{
+  return conn->cp.supportsEncoding(encodingTight);
+}
+
 void TightEncoder::setCompressLevel(int level)
 {
-  if (level >= 0 && level <= 9) {
-    pconf = &conf[level];
-  } else {
-    pconf = &conf[defaultCompressLevel];
-  }
+  if (level < 0 || level > 9)
+    level = 2;
+
+  idxZlibLevel = conf[level].idxZlibLevel;
+  monoZlibLevel = conf[level].idxZlibLevel;
+  rawZlibLevel = conf[level].rawZlibLevel;
 }
 
-void TightEncoder::setQualityLevel(int level)
+void TightEncoder::writeRect(const PixelBuffer* pb, const Palette& palette)
 {
-  if (level >= 0 && level <= 9) {
-    jpegQuality = conf[level].jpegQuality;
-    jpegSubsampling = conf[level].jpegSubsampling;
-  } else {
-    jpegQuality = -1;
-    jpegSubsampling = subsampleUndefined;
+  switch (palette.size()) {
+  case 0:
+    writeFullColourRect(pb, palette);
+    break;
+  case 1:
+    Encoder::writeSolidRect(pb, palette);
+    break;
+  case 2:
+    writeMonoRect(pb, palette);
+    break;
+  default:
+    writeIndexedRect(pb, palette);
   }
 }
 
-void TightEncoder::setFineQualityLevel(int quality, int subsampling)
+void TightEncoder::writeSolidRect(int width, int height,
+                                  const PixelFormat& pf,
+                                  const rdr::U8* colour)
 {
-  jpegQuality = quality;
-  jpegSubsampling = subsampling;
+  rdr::OutStream* os;
+
+  os = conn->getOutStream();
+
+  os->writeU8(tightFill << 4);
+  writePixels(colour, pf, 1, os);
 }
 
-bool TightEncoder::checkSolidTile(Rect& r, rdr::U32* colorPtr,
-                                  bool needSameColor)
+void TightEncoder::writeMonoRect(const PixelBuffer* pb, const Palette& palette)
 {
-  switch (serverpf.bpp) {
+  const rdr::U8* buffer;
+  int stride;
+
+  buffer = pb->getBuffer(pb->getRect(), &stride);
+
+  switch (pb->getPF().bpp) {
   case 32:
-    return checkSolidTile32(r, colorPtr, needSameColor);
+    writeMonoRect(pb->width(), pb->height(), (rdr::U32*)buffer, stride,
+                  pb->getPF(), palette);
+    break;
   case 16:
-    return checkSolidTile16(r, colorPtr, needSameColor);
+    writeMonoRect(pb->width(), pb->height(), (rdr::U16*)buffer, stride,
+                  pb->getPF(), palette);
+    break;
   default:
-    return checkSolidTile8(r, colorPtr, needSameColor);
+    writeMonoRect(pb->width(), pb->height(), (rdr::U8*)buffer, stride,
+                  pb->getPF(), palette);
   }
 }
 
-void TightEncoder::findBestSolidArea(Rect& r, rdr::U32 colorValue, Rect& bestr)
+void TightEncoder::writeIndexedRect(const PixelBuffer* pb, const Palette& palette)
 {
-  int dx, dy, dw, dh;
-  int w_prev;
-  Rect sr;
-  int w_best = 0, h_best = 0;
-
-  bestr.tl.x = bestr.br.x = r.tl.x;
-  bestr.tl.y = bestr.br.y = r.tl.y;
-
-  w_prev = r.width();
-
-  for (dy = r.tl.y; dy < r.br.y; dy += TIGHT_MAX_SPLIT_TILE_SIZE) {
-
-    dh = (dy + TIGHT_MAX_SPLIT_TILE_SIZE <= r.br.y) ?
-      TIGHT_MAX_SPLIT_TILE_SIZE : (r.br.y - dy);
-    dw = (w_prev > TIGHT_MAX_SPLIT_TILE_SIZE) ?
-      TIGHT_MAX_SPLIT_TILE_SIZE : w_prev;
-
-    sr.setXYWH(r.tl.x, dy, dw, dh);
-    if (!checkSolidTile(sr, &colorValue, true))
-      break;
-
-    for (dx = r.tl.x + dw; dx < r.tl.x + w_prev;) {
-      dw = (dx + TIGHT_MAX_SPLIT_TILE_SIZE <= r.tl.x + w_prev) ?
-        TIGHT_MAX_SPLIT_TILE_SIZE : (r.tl.x + w_prev - dx);
-      sr.setXYWH(dx, dy, dw, dh);
-      if (!checkSolidTile(sr, &colorValue, true))
-        break;
-      dx += dw;
-    }
+  const rdr::U8* buffer;
+  int stride;
 
-    w_prev = dx - r.tl.x;
-    if (w_prev * (dy + dh - r.tl.y) > w_best * h_best) {
-      w_best = w_prev;
-      h_best = dy + dh - r.tl.y;
-    }
-  }
+  buffer = pb->getBuffer(pb->getRect(), &stride);
 
-  bestr.br.x = bestr.tl.x + w_best;
-  bestr.br.y = bestr.tl.y + h_best;
+  switch (pb->getPF().bpp) {
+  case 32:
+    writeIndexedRect(pb->width(), pb->height(), (rdr::U32*)buffer, stride,
+                     pb->getPF(), palette);
+    break;
+  case 16:
+    writeIndexedRect(pb->width(), pb->height(), (rdr::U16*)buffer, stride,
+                     pb->getPF(), palette);
+    break;
+  default:
+    // It's more efficient to just do raw pixels
+    writeFullColourRect(pb, palette);
+  }
 }
 
-void TightEncoder::extendSolidArea(const Rect& r, rdr::U32 colorValue,
-                                   Rect& er)
+void TightEncoder::writeFullColourRect(const PixelBuffer* pb, const Palette& palette)
 {
-  int cx, cy;
-  Rect sr;
-
-  // Try to extend the area upwards.
-  for (cy = er.tl.y - 1; ; cy--) {
-    sr.setXYWH(er.tl.x, cy, er.width(), 1);
-    if (cy < r.tl.y || !checkSolidTile(sr, &colorValue, true))
-      break;
-  }
-  er.tl.y = cy + 1;
+  const int streamId = 0;
 
-  // ... downwards.
-  for (cy = er.br.y; ; cy++) {
-    sr.setXYWH(er.tl.x, cy, er.width(), 1);
-    if (cy >= r.br.y || !checkSolidTile(sr, &colorValue, true))
-      break;
-  }
-  er.br.y = cy;
+  rdr::OutStream* os;
+  rdr::OutStream* zos;
+  int length;
 
-  // ... to the left.
-  for (cx = er.tl.x - 1; ; cx--) {
-    sr.setXYWH(cx, er.tl.y, 1, er.height());
-    if (cx < r.tl.x || !checkSolidTile(sr, &colorValue, true))
-      break;
-  }
-  er.tl.x = cx + 1;
+  const rdr::U8* buffer;
+  int stride, w, h;
 
-  // ... to the right.
-  for (cx = er.br.x; ; cx++) {
-    sr.setXYWH(cx, er.tl.y, 1, er.height());
-    if (cx >= r.br.x || !checkSolidTile(sr, &colorValue, true))
-      break;
-  }
-  er.br.x = cx;
-}
+  os = conn->getOutStream();
 
-int TightEncoder::getNumRects(const Rect &r)
-{
-  ConnParams* cp = &conn->cp;
-  const unsigned int w = r.width();
-  const unsigned int h = r.height();
-
-  // If last rect. encoding is enabled, we can use the higher-performance
-  // code that pre-computes solid rectangles.  In that case, we don't care
-  // about the rectangle count.
-  if (cp->supportsLastRect && w * h >= TIGHT_MIN_SPLIT_RECT_SIZE)
-    return 0;
-
-  // Will this rectangle split into subrects?
-  bool rectTooBig = w > pconf->maxRectWidth || w * h > pconf->maxRectSize;
-  if (!rectTooBig)
-    return 1;
-
-  // Compute max sub-rectangle size.
-  const unsigned int subrectMaxWidth =
-    (w > pconf->maxRectWidth) ? pconf->maxRectWidth : w;
-  const unsigned int subrectMaxHeight =
-    pconf->maxRectSize / subrectMaxWidth;
-
-  // Return the number of subrects.
-  return (((w - 1) / pconf->maxRectWidth + 1) *
-          ((h - 1) / subrectMaxHeight + 1));
-}
+  os->writeU8(streamId << 4);
 
-void TightEncoder::sendRectSimple(const Rect& r)
-{
-  // Shortcuts to rectangle coordinates and dimensions.
-  const int x = r.tl.x;
-  const int y = r.tl.y;
-  const unsigned int w = r.width();
-  const unsigned int h = r.height();
-
-  // Encode small rects as is.
-  bool rectTooBig = w > pconf->maxRectWidth || w * h > pconf->maxRectSize;
-  if (!rectTooBig) {
-    writeSubrect(r);
-    return;
-  }
+  // Set up compression
+  if ((pb->getPF().bpp != 32) || !pb->getPF().is888())
+    length = pb->getRect().area() * pb->getPF().bpp/8;
+  else
+    length = pb->getRect().area() * 3;
 
-  // Compute max sub-rectangle size.
-  const unsigned int subrectMaxWidth =
-    (w > pconf->maxRectWidth) ? pconf->maxRectWidth : w;
-  const unsigned int subrectMaxHeight =
-    pconf->maxRectSize / subrectMaxWidth;
-
-  // Split big rects into separately encoded subrects.
-  Rect sr;
-  unsigned int dx, dy, sw, sh;
-  for (dy = 0; dy < h; dy += subrectMaxHeight) {
-    for (dx = 0; dx < w; dx += pconf->maxRectWidth) {
-      sw = (dx + pconf->maxRectWidth < w) ? pconf->maxRectWidth : w - dx;
-      sh = (dy + subrectMaxHeight < h) ? subrectMaxHeight : h - dy;
-      sr.setXYWH(x + dx, y + dy, sw, sh);
-      writeSubrect(sr);
-    }
+  zos = getZlibOutStream(streamId, rawZlibLevel, length);
+
+  // And then just dump all the raw pixels
+  buffer = pb->getBuffer(pb->getRect(), &stride);
+  h = pb->height();
+
+  while (h--) {
+    writePixels(buffer, pb->getPF(), pb->width(), zos);
+    buffer += stride * pb->getPF().bpp/8;
   }
+
+  // Finish the zlib stream
+  flushZlibOutStream(zos);
 }
 
-void TightEncoder::writeRect(const Rect& _r, PixelBuffer* _pb)
+void TightEncoder::writePixels(const rdr::U8* buffer, const PixelFormat& pf,
+                               unsigned int count, rdr::OutStream* os)
 {
-  pb = _pb;
-  serverpf = pb->getPF();
-  ConnParams* cp = &conn->cp;
-  clientpf = cp->pf();
-
-  // Shortcuts to rectangle coordinates and dimensions.
-  Rect r = _r;
-  int x = r.tl.x;
-  int y = r.tl.y;
-  int w = r.width();
-  int h = r.height();
-
-  // Encode small rects as is.
-  if (!cp->supportsLastRect || w * h < TIGHT_MIN_SPLIT_RECT_SIZE) {
-    sendRectSimple(r);
+  rdr::U8 rgb[2048];
+
+  if ((pf.bpp != 32) || !pf.is888()) {
+    os->writeBytes(buffer, count * pf.bpp/8);
     return;
   }
 
-  // Split big rects into separately encoded subrects.
-  Rect sr, bestr;
-  int dx, dy, dw, dh;
-  rdr::U32 colorValue;
-  int maxRectWidth = pconf->maxRectWidth;
-  int nMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
-  int nMaxRows = pconf->maxRectSize / nMaxWidth;
-
-  // Try to find large solid-color areas and send them separately.
-  for (dy = y; dy < y + h; dy += TIGHT_MAX_SPLIT_TILE_SIZE) {
-
-    // If a rectangle becomes too large, send its upper part now.
-    if (dy - y >= nMaxRows) {
-      sr.setXYWH(x, y, w, nMaxRows);
-      sendRectSimple(sr);
-      r.tl.y += nMaxRows;
-      y = r.tl.y;
-      h = r.height();
-    }
+  while (count) {
+    int iter_count;
 
-    dh = (dy + TIGHT_MAX_SPLIT_TILE_SIZE <= y + h) ?
-      TIGHT_MAX_SPLIT_TILE_SIZE : (y + h - dy);
-
-    for (dx = x; dx < x + w; dx += TIGHT_MAX_SPLIT_TILE_SIZE) {
-
-      dw = (dx + TIGHT_MAX_SPLIT_TILE_SIZE <= x + w) ?
-        TIGHT_MAX_SPLIT_TILE_SIZE : (x + w - dx);
- 
-      sr.setXYWH(dx, dy, dw, dh);
-      if (checkSolidTile(sr, &colorValue, false)) {
-
-         if (jpegSubsampling == subsampleGray && jpegQuality != -1) {
-           rdr::U16 r, g, b;
-           serverpf.rgbFromPixel(colorValue, &r, &g, &b);
-           rdr::U32 lum = ((257 * r) + (504 * g) + (98 * b)
-                           + 16500) / 1000;
-           colorValue = lum + (lum << 8) + (lum << 16);
-         }
-
-        // Get dimensions of solid-color area.
-        sr.setXYWH(dx, dy, r.br.x - dx, r.br.y - dy);
-        findBestSolidArea(sr, colorValue, bestr);
-
-        // Make sure a solid rectangle is large enough
-        // (or the whole rectangle is of the same color).
-        if (bestr.area() != r.area()
-          && bestr.area() < TIGHT_MIN_SOLID_SUBRECT_SIZE)
-          continue;
-
-        // Try to extend solid rectangle to maximum size.
-        extendSolidArea(r, colorValue, bestr);
- 
-        // Send rectangles at top and left to solid-color area.
-        if (bestr.tl.y != y) {
-          sr.setXYWH(x, y, w, bestr.tl.y - y);
-          sendRectSimple(sr);
-        }
-        if (bestr.tl.x != x) {
-          sr.setXYWH(x, bestr.tl.y, bestr.tl.x - x, bestr.height());
-          writeRect(sr, _pb);
-        }
-
-        // Send solid-color rectangle.
-        writeSubrect(bestr, true);
-
-        // Send remaining rectangles (at right and bottom).
-        if (bestr.br.x != r.br.x) {
-          sr.setXYWH(bestr.br.x, bestr.tl.y, r.br.x - bestr.br.x,
-            bestr.height());
-          writeRect(sr, _pb);
-        }
-        if (bestr.br.y != r.br.y) {
-          sr.setXYWH(x, bestr.br.y, w, r.br.y - bestr.br.y);
-          writeRect(sr, _pb);
-        }
-
-        return;
-      }
-    }
-  }
+    iter_count = sizeof(rgb)/3;
+    if (iter_count > count)
+      iter_count = count;
 
-  // No suitable solid-color rectangles found.
-  sendRectSimple(r);
-  return;
-}
-
-void TightEncoder::writeSubrect(const Rect& r, bool forceSolid)
-{
-  mos.clear();
+    pf.rgbFromBuffer(rgb, buffer, iter_count);
+    os->writeBytes(rgb, iter_count * 3);
 
-  switch (clientpf.bpp) {
-  case 8:
-    tightEncode8(r, &mos, forceSolid);  break;
-  case 16:
-    tightEncode16(r, &mos, forceSolid); break;
-  case 32:
-    tightEncode32(r, &mos, forceSolid); break;
+    buffer += iter_count * pf.bpp/8;
+    count -= iter_count;
   }
-
-  conn->writer()->startRect(r, encodingTight);
-  rdr::OutStream* os = conn->getOutStream();
-  os->writeBytes(mos.data(), mos.length());
-  conn->writer()->endRect();
 }
 
 void TightEncoder::writeCompact(rdr::OutStream* os, rdr::U32 value)
@@ -428,3 +232,52 @@ void TightEncoder::writeCompact(rdr::OutStream* os, rdr::U32 value)
     }
   }
 }
+
+rdr::OutStream* TightEncoder::getZlibOutStream(int streamId, int level, size_t length)
+{
+  // Minimum amount of data to be compressed. This value should not be
+  // changed, doing so will break compatibility with existing clients.
+  if (length < 12)
+    return conn->getOutStream();
+
+  assert(streamId >= 0);
+  assert(streamId < 4);
+
+  zlibStreams[streamId].setUnderlying(&memStream);
+  zlibStreams[streamId].setCompressionLevel(level);
+
+  return &zlibStreams[streamId];
+}
+
+void TightEncoder::flushZlibOutStream(rdr::OutStream* os_)
+{
+  rdr::OutStream* os;
+  rdr::ZlibOutStream* zos;
+
+  zos = dynamic_cast<rdr::ZlibOutStream*>(os_);
+  if (zos == NULL)
+    return;
+
+  zos->flush();
+  zos->setUnderlying(NULL);
+
+  os = conn->getOutStream();
+
+  writeCompact(os, memStream.length());
+  os->writeBytes(memStream.data(), memStream.length());
+  memStream.clear();
+}
+
+//
+// Including BPP-dependent implementation of the encoder.
+//
+
+#define BPP 8
+#include <rfb/TightEncoderBPP.cxx>
+#undef BPP
+#define BPP 16
+#include <rfb/TightEncoderBPP.cxx>
+#undef BPP
+#define BPP 32
+#include <rfb/TightEncoderBPP.cxx>
+#undef BPP

common/rfb/TightEncoder.h

@@ -1,5 +1,6 @@
 /* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
  * Copyright (C) 2011 D. R. Commander
+ * Copyright 2014 Pierre Ossman for Cendio AB
  *    
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -22,120 +23,59 @@
 #include <rdr/MemOutStream.h>
 #include <rdr/ZlibOutStream.h>
 #include <rfb/Encoder.h>
-#include <rfb/JpegCompressor.h>
-#include <rfb/Palette.h>
-
-// FIXME: Check if specifying extern "C" is really necessary.
-#include <stdio.h>
-extern "C" {
-#include <jpeglib.h>
-}
 
 namespace rfb {
 
-  class PixelBuffer;
-
-  struct TIGHT_CONF {
-    unsigned int maxRectSize, maxRectWidth;
-    unsigned int monoMinRectSize;
-    int idxZlibLevel, monoZlibLevel, rawZlibLevel;
-    int idxMaxColorsDivisor;
-    int palMaxColorsWithJPEG;
-    int jpegQuality;
-    int jpegSubsampling;
-  };
-
-  //
-  // Compression level stuff. The following array contains various
-  // encoder parameters for each of 10 compression levels (0..9).
-  // Last three parameters correspond to JPEG quality levels (0..9).
-  //
-  // NOTE: s_conf[9].maxRectSize should be >= s_conf[i].maxRectSize,
-  // where i in [0..8]. RequiredBuffSize() method depends on this.
-  // FIXME: Is this comment obsolete?
-  //
-
-  
   class TightEncoder : public Encoder {
   public:
     TightEncoder(SConnection* conn);
     virtual ~TightEncoder();
 
-    virtual void setCompressLevel(int level);
-    virtual void setQualityLevel(int level);
-    virtual void setFineQualityLevel(int quality, int subsampling);
-    virtual int getNumRects(const Rect &r);
-    virtual void writeRect(const Rect& r, PixelBuffer* pb);
-
-  private:
-    bool checkSolidTile(Rect& r, rdr::U32* colorPtr, bool needSameColor);
-    void extendSolidArea(const Rect& r, rdr::U32 colorValue, Rect& er);
-    void findBestSolidArea(Rect& r, rdr::U32 colorValue, Rect& bestr);
-    void sendRectSimple(const Rect& r);
-    void writeSubrect(const Rect& r, bool forceSolid = false);
-
-    void writeCompact(rdr::OutStream* os, rdr::U32 value);
-
-    void compressData(const void *buf, unsigned int length,
-                      rdr::ZlibOutStream *zos, int zlibLevel,
-                      rdr::OutStream *os);
+    virtual bool isSupported();
 
-    void fastFillPalette8(const rdr::U8 *data, int stride, const Rect &r);
-    void fastFillPalette16(const rdr::U16 *data, int stride, const Rect &r);
-    void fastFillPalette32(const rdr::U32 *data, int stride, const Rect &r);
-
-    void fillPalette8(rdr::U8 *data, int count);
-    void fillPalette16(rdr::U16 *data, int count);
-    void fillPalette32(rdr::U32 *data, int count);
-
-    unsigned int packPixels8(rdr::U8 *buf, unsigned int count);
-    unsigned int packPixels16(rdr::U16 *buf, unsigned int count);
-    unsigned int packPixels32(rdr::U32 *buf, unsigned int count);
-
-    void tightEncode8(const Rect& r, rdr::OutStream *os, bool forceSolid);
-    void tightEncode16(const Rect& r, rdr::OutStream *os, bool forceSolid);
-    void tightEncode32(const Rect& r, rdr::OutStream *os, bool forceSolid);
-
-    bool checkSolidTile8(Rect& r, rdr::U32 *colorPtr, bool needSameColor);
-    bool checkSolidTile16(Rect& r, rdr::U32 *colorPtr, bool needSameColor);
-    bool checkSolidTile32(Rect& r, rdr::U32 *colorPtr, bool needSameColor);
-
-    void encodeSolidRect8(rdr::U8 *buf, rdr::OutStream *os);
-    void encodeSolidRect16(rdr::U16 *buf, rdr::OutStream *os);
-    void encodeSolidRect32(rdr::U32 *buf, rdr::OutStream *os);
-
-    void encodeFullColorRect8(rdr::U8 *buf, const Rect& r, rdr::OutStream *os);
-    void encodeFullColorRect16(rdr::U16 *buf, const Rect& r, rdr::OutStream *os);
-    void encodeFullColorRect32(rdr::U32 *buf, const Rect& r, rdr::OutStream *os);
-
-    void encodeMonoRect8(rdr::U8 *buf, const Rect& r, rdr::OutStream *os);
-    void encodeMonoRect16(rdr::U16 *buf, const Rect& r, rdr::OutStream *os);
-    void encodeMonoRect32(rdr::U32 *buf, const Rect& r, rdr::OutStream *os);
+    virtual void setCompressLevel(int level);
 
-    void encodeIndexedRect16(rdr::U16 *buf, const Rect& r, rdr::OutStream *os);
-    void encodeIndexedRect32(rdr::U32 *buf, const Rect& r, rdr::OutStream *os);
+    virtual void writeRect(const PixelBuffer* pb, const Palette& palette);
+    virtual void writeSolidRect(int width, int height,
+                                const PixelFormat& pf,
+                                const rdr::U8* colour);
 
-    void encodeJpegRect16(rdr::U16 *buf, int stride, const Rect& r,
-                          rdr::OutStream *os);
-    void encodeJpegRect32(rdr::U32 *buf, int stride, const Rect& r,
-                          rdr::OutStream *os);
+  protected:
+    void writeMonoRect(const PixelBuffer* pb, const Palette& palette);
+    void writeIndexedRect(const PixelBuffer* pb, const Palette& palette);
+    void writeFullColourRect(const PixelBuffer* pb, const Palette& palette);
 
-    rdr::MemOutStream mos;
-    rdr::ZlibOutStream zos[4];
-    JpegCompressor jc;
-    PixelBuffer *pb;
-    PixelFormat serverpf, clientpf;
+    void writePixels(const rdr::U8* buffer, const PixelFormat& pf,
+                     unsigned int count, rdr::OutStream* os);
 
-    bool pack24;
-    int palMaxColors;
-    Palette palette;
+    void writeCompact(rdr::OutStream* os, rdr::U32 value);
 
-    static const int defaultCompressLevel;
-    static const TIGHT_CONF conf[];
+    rdr::OutStream* getZlibOutStream(int streamId, int level, size_t length);
+    void flushZlibOutStream(rdr::OutStream* os);
+
+  protected:
+    // Preprocessor generated, optimised methods
+    void writeMonoRect(int width, int height,
+                       const rdr::U8* buffer, int stride,
+                       const PixelFormat& pf, const Palette& palette);
+    void writeMonoRect(int width, int height,
+                       const rdr::U16* buffer, int stride,
+                       const PixelFormat& pf, const Palette& palette);
+    void writeMonoRect(int width, int height,
+                       const rdr::U32* buffer, int stride,
+                       const PixelFormat& pf, const Palette& palette);
+
+    void writeIndexedRect(int width, int height,
+                          const rdr::U16* buffer, int stride,
+                          const PixelFormat& pf, const Palette& palette);
+    void writeIndexedRect(int width, int height,
+                          const rdr::U32* buffer, int stride,
+                          const PixelFormat& pf, const Palette& palette);
+
+    rdr::ZlibOutStream zlibStreams[4];
+    rdr::MemOutStream memStream;
 
-    const TIGHT_CONF* pconf;
-    int jpegQuality;
-    int jpegSubsampling;
+    int idxZlibLevel, monoZlibLevel, rawZlibLevel;
   };
 
 }

common/rfb/TightEncoderBPP.cxx

@@ -0,0 +1,165 @@
+/* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#define CONCAT2(a,b) a##b
+#define CONCAT2E(a,b) CONCAT2(a,b)
+
+#define UBPP CONCAT2E(U,BPP)
+
+void TightEncoder::writeMonoRect(int width, int height,
+                                 const rdr::UBPP* buffer, int stride,
+                                 const PixelFormat& pf,
+                                 const Palette& palette)
+{
+  rdr::OutStream* os;
+
+  const int streamId = 1;
+  rdr::UBPP pal[2];
+
+  int length;
+  rdr::OutStream* zos;
+
+  assert(palette.size() == 2);
+
+  os = conn->getOutStream();
+
+  os->writeU8((streamId | tightExplicitFilter) << 4);
+  os->writeU8(tightFilterPalette);
+
+  // Write the palette
+  pal[0] = (rdr::UBPP)palette.getColour(0);
+  pal[1] = (rdr::UBPP)palette.getColour(1);
+
+  os->writeU8(1);
+  writePixels((rdr::U8*)pal, pf, 2, os);
+
+  // Set up compression
+  length = (width + 7)/8 * height;
+  zos = getZlibOutStream(streamId, monoZlibLevel, length);
+
+  // Encode the data
+  rdr::UBPP bg;
+  unsigned int value, mask;
+  int pad, aligned_width;
+  int x, y, bg_bits;
+
+  bg = pal[0];
+  aligned_width = width - width % 8;
+  pad = stride - width;
+
+  for (y = 0; y < height; y++) {
+    for (x = 0; x < aligned_width; x += 8) {
+      for (bg_bits = 0; bg_bits < 8; bg_bits++) {
+        if (*buffer++ != bg)
+          break;
+      }
+      if (bg_bits == 8) {
+        zos->writeU8(0);
+        continue;
+      }
+      mask = 0x80 >> bg_bits;
+      value = mask;
+      for (bg_bits++; bg_bits < 8; bg_bits++) {
+        mask >>= 1;
+        if (*buffer++ != bg) {
+          value |= mask;
+        }
+      }
+      zos->writeU8(value);
+    }
+
+    if (x < width) {
+      mask = 0x80;
+      value = 0;
+
+      for (; x < width; x++) {
+        if (*buffer++ != bg) {
+          value |= mask;
+        }
+        mask >>= 1;
+      }
+      zos->writeU8(value);
+    }
+
+    buffer += pad;
+  }
+
+  // Finish the zlib stream
+  flushZlibOutStream(zos);
+}
+
+#if (BPP != 8)
+void TightEncoder::writeIndexedRect(int width, int height,
+                                    const rdr::UBPP* buffer, int stride,
+                                    const PixelFormat& pf,
+                                    const Palette& palette)
+{
+  rdr::OutStream* os;
+
+  const int streamId = 2;
+  rdr::UBPP pal[256];
+
+  rdr::OutStream* zos;
+
+  int pad;
+  rdr::UBPP prevColour;
+  unsigned char idx;
+
+  assert(palette.size() > 0);
+  assert(palette.size() <= 256);
+
+  os = conn->getOutStream();
+
+  os->writeU8((streamId | tightExplicitFilter) << 4);
+  os->writeU8(tightFilterPalette);
+
+  // Write the palette
+  for (int i = 0; i < palette.size(); i++)
+    pal[i] = (rdr::UBPP)palette.getColour(i);
+
+  os->writeU8(palette.size() - 1);
+  writePixels((rdr::U8*)pal, pf, palette.size(), os);
+
+  // Set up compression
+  zos = getZlibOutStream(streamId, idxZlibLevel, width * height);
+
+  // Encode the data
+  pad = stride - width;
+
+  prevColour = *buffer;
+  idx = palette.lookup(*buffer);
+
+  while (height--) {
+    int w = width;
+    while (w--) {
+      if (*buffer != prevColour) {
+        prevColour = *buffer;
+        idx = palette.lookup(*buffer);
+      }
+      zos->writeU8(idx);
+      buffer++;
+    }
+    buffer += pad;
+  }
+
+  // Finish the zlib stream
+  flushZlibOutStream(zos);
+}
+#endif  // #if (BPP != 8)

common/rfb/TightJPEGEncoder.cxx

@@ -0,0 +1,168 @@
+/* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+#include <rdr/OutStream.h>
+#include <rfb/encodings.h>
+#include <rfb/SConnection.h>
+#include <rfb/PixelBuffer.h>
+#include <rfb/TightJPEGEncoder.h>
+#include <rfb/TightConstants.h>
+
+using namespace rfb;
+
+struct TightJPEGConfiguration {
+    int quality;
+    int subsampling;
+};
+
+// NOTE:  The JPEG quality and subsampling levels below were obtained
+// experimentally by the VirtualGL Project.  They represent the approximate
+// average compression ratios listed below, as measured across the set of
+// every 10th frame in the SPECviewperf 9 benchmark suite.
+//
+// 9 = JPEG quality 100, no subsampling (ratio ~= 10:1)
+//     [this should be lossless, except for round-off error]
+// 8 = JPEG quality 92,  no subsampling (ratio ~= 20:1)
+//     [this should be perceptually lossless, based on current research]
+// 7 = JPEG quality 86,  no subsampling (ratio ~= 25:1)
+// 6 = JPEG quality 79,  no subsampling (ratio ~= 30:1)
+// 5 = JPEG quality 77,  4:2:2 subsampling (ratio ~= 40:1)
+// 4 = JPEG quality 62,  4:2:2 subsampling (ratio ~= 50:1)
+// 3 = JPEG quality 42,  4:2:2 subsampling (ratio ~= 60:1)
+// 2 = JPEG quality 41,  4:2:0 subsampling (ratio ~= 70:1)
+// 1 = JPEG quality 29,  4:2:0 subsampling (ratio ~= 80:1)
+// 0 = JPEG quality 15,  4:2:0 subsampling (ratio ~= 100:1)
+
+static const struct TightJPEGConfiguration conf[10] = {
+  {  15, subsample4X }, // 0
+  {  29, subsample4X }, // 1
+  {  41, subsample4X }, // 2
+  {  42, subsample2X }, // 3
+  {  62, subsample2X }, // 4
+  {  77, subsample2X }, // 5
+  {  79, subsampleNone }, // 6
+  {  86, subsampleNone }, // 7
+  {  92, subsampleNone }, // 8
+  { 100, subsampleNone }  // 9
+};
+
+
+TightJPEGEncoder::TightJPEGEncoder(SConnection* conn) :
+  Encoder(conn, encodingTight, EncoderUseNativePF, -1),
+  qualityLevel(-1), fineQuality(-1), fineSubsampling(subsampleUndefined)
+{
+}
+
+TightJPEGEncoder::~TightJPEGEncoder()
+{
+}
+
+bool TightJPEGEncoder::isSupported()
+{
+  if (!conn->cp.supportsEncoding(encodingTight))
+    return false;
+
+  // Any one of these indicates support for JPEG
+  if (conn->cp.qualityLevel != -1)
+    return true;
+  if (conn->cp.fineQualityLevel != -1)
+    return true;
+  if (conn->cp.subsampling != -1)
+    return true;
+
+  // Tight support, but not JPEG
+  return false;
+}
+
+void TightJPEGEncoder::setQualityLevel(int level)
+{
+  qualityLevel = level;
+}
+
+void TightJPEGEncoder::setFineQualityLevel(int quality, int subsampling)
+{
+  fineQuality = quality;
+  fineSubsampling = subsampling;
+}
+
+void TightJPEGEncoder::writeRect(const PixelBuffer* pb, const Palette& palette)
+{
+  const rdr::U8* buffer;
+  int stride;
+
+  int quality, subsampling;
+
+  rdr::OutStream* os;
+
+  buffer = pb->getBuffer(pb->getRect(), &stride);
+
+  if (qualityLevel >= 0 && qualityLevel <= 9) {
+    quality = conf[qualityLevel].quality;
+    subsampling = conf[qualityLevel].subsampling;
+  } else {
+    quality = -1;
+    subsampling = subsampleUndefined;
+  }
+
+  // Fine settings trump level
+  if (fineQuality != -1)
+    quality = fineQuality;
+  if (fineSubsampling != subsampleUndefined)
+    subsampling = fineSubsampling;
+
+  jc.clear();
+  jc.compress(buffer, stride, pb->getRect(),
+              pb->getPF(), quality, subsampling);
+
+  os = conn->getOutStream();
+
+  os->writeU8(tightJpeg << 4);
+
+  writeCompact(jc.length(), os);
+  os->writeBytes(jc.data(), jc.length());
+}
+
+void TightJPEGEncoder::writeSolidRect(int width, int height,
+                                      const PixelFormat& pf,
+                                      const rdr::U8* colour)
+{
+  // FIXME: Add a shortcut in the JPEG compressor to handle this case
+  //        without having to use the default fallback which is very slow.
+  Encoder::writeSolidRect(width, height, pf, colour);
+}
+
+void TightJPEGEncoder::writeCompact(rdr::U32 value, rdr::OutStream* os)
+{
+  // Copied from TightEncoder as it's overkill to inherit just for this
+  rdr::U8 b;
+
+  b = value & 0x7F;
+  if (value <= 0x7F) {
+    os->writeU8(b);
+  } else {
+    os->writeU8(b | 0x80);
+    b = value >> 7 & 0x7F;
+    if (value <= 0x3FFF) {
+      os->writeU8(b);
+    } else {
+      os->writeU8(b | 0x80);
+      os->writeU8(value >> 14 & 0xFF);
+    }
+  }
+}

common/rfb/TightJPEGEncoder.h

@@ -0,0 +1,54 @@
+/* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander
+ * Copyright 2014 Pierre Ossman for Cendio AB
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+#ifndef __RFB_TIGHTJPEGENCODER_H__
+#define __RFB_TIGHTJPEGENCODER_H__
+
+#include <rfb/Encoder.h>
+#include <rfb/JpegCompressor.h>
+
+namespace rfb {
+
+  class TightJPEGEncoder : public Encoder {
+  public:
+    TightJPEGEncoder(SConnection* conn);
+    virtual ~TightJPEGEncoder();
+
+    virtual bool isSupported();
+
+    virtual void setQualityLevel(int level);
+    virtual void setFineQualityLevel(int quality, int subsampling);
+
+    virtual void writeRect(const PixelBuffer* pb, const Palette& palette);
+    virtual void writeSolidRect(int width, int height,
+                                const PixelFormat& pf,
+                                const rdr::U8* colour);
+
+  protected:
+    void writeCompact(rdr::U32 value, rdr::OutStream* os);
+
+  protected:
+    JpegCompressor jc;
+
+    int qualityLevel;
+    int fineQuality;
+    int fineSubsampling;
+  };
+}
+#endif

common/rfb/VNCSConnectionST.cxx

@@ -73,7 +73,7 @@ VNCSConnectionST::VNCSConnectionST(VNCServerST* server_, network::Socket *s,
     ackedOffset(0), sentOffset(0), congWindow(0), congestionTimer(this),
     server(server_), updates(false),
     drawRenderedCursor(false), removeRenderedCursor(false),
-    continuousUpdates(false),
+    continuousUpdates(false), encodeManager(this),
     updateTimer(this), pointerEventTime(0),
     accessRights(AccessDefault), startTime(time(0))
 {
@@ -81,8 +81,6 @@ VNCSConnectionST::VNCSConnectionST(VNCServerST* server_, network::Socket *s,
   peerEndpoint.buf = sock->getPeerEndpoint();
   VNCServerST::connectionsLog.write(1,"accepted: %s", peerEndpoint.buf);
 
-  memset(encoders, 0, sizeof(encoders));
-
   // Configure the socket
   setSocketTimeouts();
   lastEventTime = time(0);
@@ -108,9 +106,6 @@ VNCSConnectionST::~VNCSConnectionST()
   // Remove this client from the server
   server->clients.remove(this);
 
-  for (int i = 0; i <= encodingMax; i++)
-    delete encoders[i];
-
   delete [] fenceData;
 }
 
@@ -1027,72 +1022,19 @@ void VNCSConnectionST::writeFramebufferUpdate()
   }
 
   if (!ui.is_empty() || writer()->needFakeUpdate() || drawRenderedCursor) {
-    std::vector<Rect> rects;
-    std::vector<Rect>::const_iterator i;
-    int encoding;
-
-    Encoder* encoder;
-    PixelBuffer* pb;
-
-    // Make sure the encoder has the latest settings
-    encoding = cp.currentEncoding();
-
-    if (!encoders[encoding])
-      encoders[encoding] = Encoder::createEncoder(encoding, this);
-
-    encoder = encoders[encoding];
-
-    encoder->setCompressLevel(cp.compressLevel);
-    encoder->setQualityLevel(cp.qualityLevel);
-    encoder->setFineQualityLevel(cp.fineQualityLevel, cp.subsampling);
-
-    // Compute the number of rectangles. Tight encoder makes the things more
-    // complicated as compared to the original VNC4.
-    int nRects = (ui.copied.numRects() +
-                  (drawRenderedCursor ? 1 : 0));
-
-    ui.changed.get_rects(&rects);
-    for (i = rects.begin(); i != rects.end(); i++) {
-      if (i->width() && i->height()) {
-        int nUpdateRects = encoder->getNumRects(*i);
-        if (nUpdateRects == 0 && cp.currentEncoding() == encodingTight) {
-          // With Tight encoding and LastRect support, the client does not
-          // care about the number of rectangles in the update - it will
-          // stop parsing when it encounters a LastRect "rectangle".
-          // In this case, pretend to send 65535 rectangles.
-          nRects = 0xFFFF;  break;
-        }
-        else
-          nRects += nUpdateRects;
-      }
-    }
-
-    writeRTTPing();
-    
-    writer()->writeFramebufferUpdateStart(nRects);
+    RenderedCursor *cursor;
 
-    ui.copied.get_rects(&rects);
-    for (i = rects.begin(); i != rects.end(); i++)
-      writer()->writeCopyRect(*i, i->tl.x - ui.copy_delta.x,
-                              i->tl.y - ui.copy_delta.y);
+    cursor = NULL;
+    if (drawRenderedCursor)
+      cursor = &server->renderedCursor;
 
-    pb = server->getPixelBuffer();
-
-    ui.changed.get_rects(&rects);
-    for (i = rects.begin(); i != rects.end(); i++)
-      encoder->writeRect(*i, pb);
-
-    if (drawRenderedCursor) {
-      renderedCursorRect = server->renderedCursor.getEffectiveRect();
-      encoder->writeRect(renderedCursorRect, &server->renderedCursor);
-
-      drawRenderedCursor = false;
-    }
+    writeRTTPing();
 
-    writer()->writeFramebufferUpdateEnd();
+    encodeManager.writeUpdate(ui, server->getPixelBuffer(), cursor);
 
     writeRTTPing();
 
+    drawRenderedCursor = false;
     requested.clear();
     updates.clear();
   }

common/rfb/VNCSConnectionST.h

@@ -32,11 +32,11 @@
 #include <rfb/SMsgWriter.h>
 #include <rfb/VNCServerST.h>
 #include <rfb/Timer.h>
+#include <rfb/EncodeManager.h>
 
 struct RTTInfo;
 
 namespace rfb {
-  class Encoder;
 
   class VNCSConnectionST : public SConnection,
                            public Timer::Callback {
@@ -197,7 +197,7 @@ namespace rfb {
     Rect renderedCursorRect;
     bool continuousUpdates;
     Region cuRegion;
-    Encoder* encoders[encodingMax+1];
+    EncodeManager encodeManager;
 
     Timer updateTimer;
 

common/rfb/ZRLEEncoder.cxx

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -19,7 +20,7 @@
 #include <rfb/Exception.h>
 #include <rfb/encodings.h>
 #include <rfb/ConnParams.h>
-#include <rfb/SMsgWriter.h>
+#include <rfb/Palette.h>
 #include <rfb/SConnection.h>
 #include <rfb/ZRLEEncoder.h>
 #include <rfb/Configuration.h>
@@ -28,87 +29,223 @@ using namespace rfb;
 
 IntParameter zlibLevel("ZlibLevel","Zlib compression level",-1);
 
-static inline void writeOpaque24A(rdr::OutStream* os, rdr::U32 u)
+ZRLEEncoder::ZRLEEncoder(SConnection* conn)
+  : Encoder(conn, encodingZRLE, EncoderPlain, 127),
+  zos(0,0,zlibLevel), mos(129*1024)
 {
-  os->check(3);
-  os->writeU8(((rdr::U8*)&u)[0]);
-  os->writeU8(((rdr::U8*)&u)[1]);
-  os->writeU8(((rdr::U8*)&u)[2]);
+  zos.setUnderlying(&mos);
 }
-static inline void writeOpaque24B(rdr::OutStream* os, rdr::U32 u)
+
+ZRLEEncoder::~ZRLEEncoder()
 {
-  os->check(3);
-  os->writeU8(((rdr::U8*)&u)[1]);
-  os->writeU8(((rdr::U8*)&u)[2]);
-  os->writeU8(((rdr::U8*)&u)[3]);
+  zos.setUnderlying(NULL);
 }
 
-#define BPP 8
-#include <rfb/zrleEncode.h>
-#undef BPP
-#define BPP 16
-#include <rfb/zrleEncode.h>
-#undef BPP
-#define BPP 32
-#include <rfb/zrleEncode.h>
-#define CPIXEL 24A
-#include <rfb/zrleEncode.h>
-#undef CPIXEL
-#define CPIXEL 24B
-#include <rfb/zrleEncode.h>
-#undef CPIXEL
-#undef BPP
-
-ZRLEEncoder::ZRLEEncoder(SConnection* conn)
-  : Encoder(conn), zos(0,0,zlibLevel), mos(129*1024)
+bool ZRLEEncoder::isSupported()
 {
+  return conn->cp.supportsEncoding(encodingZRLE);
 }
 
-ZRLEEncoder::~ZRLEEncoder()
+void ZRLEEncoder::writeRect(const PixelBuffer* pb, const Palette& palette)
 {
+  int x, y;
+  Rect tile;
+
+  rdr::OutStream* os;
+
+  // A bit of a special case
+  if (palette.size() == 1) {
+    Encoder::writeSolidRect(pb, palette);
+    return;
+  }
+
+  for (y = 0;y < pb->height();y += 64) {
+    tile.tl.y = y;
+    tile.br.y = y + 64;
+    if (tile.br.y > pb->height())
+      tile.br.y = pb->height();
+
+    for (x = 0;x < pb->width();x += 64) {
+      tile.tl.x = x;
+      tile.br.x = x + 64;
+      if (tile.br.x > pb->width())
+        tile.br.x = pb->width();
+
+      if (palette.size() == 0)
+        writeRawTile(tile, pb, palette);
+      else if (palette.size() <= 16)
+        writePaletteTile(tile, pb, palette);
+      else
+        writePaletteRLETile(tile, pb, palette);
+    }
+  }
+
+  zos.flush();
+
+  os = conn->getOutStream();
+
+  os->writeU32(mos.length());
+  os->writeBytes(mos.data(), mos.length());
+
+  mos.clear();
 }
 
-void ZRLEEncoder::writeRect(const Rect& r, PixelBuffer* pb)
+void ZRLEEncoder::writeSolidRect(int width, int height,
+                                 const PixelFormat& pf,
+                                 const rdr::U8* colour)
 {
-  const PixelFormat& pf = conn->cp.pf();
+  int tiles;
+
+  rdr::OutStream* os;
+
+  tiles = ((width + 63)/64) * ((height + 63)/64);
+
+  while (tiles--) {
+    zos.writeU8(1);
+    writePixels(colour, pf, 1);
+  }
+
+  zos.flush();
+
+  os = conn->getOutStream();
+
+  os->writeU32(mos.length());
+  os->writeBytes(mos.data(), mos.length());
 
-  rdr::U8* imageBuf = conn->writer()->getImageBuf(64 * 64 * 4 + 4);
   mos.clear();
+}
 
-  switch (pf.bpp) {
-  case 8:
-    zrleEncode8(r, &mos, &zos, imageBuf, pf, pb);
+void ZRLEEncoder::writePaletteTile(const Rect& tile, const PixelBuffer* pb,
+                                   const Palette& palette)
+{
+  const rdr::U8* buffer;
+  int stride;
+
+  buffer = pb->getBuffer(tile, &stride);
+
+  switch (pb->getPF().bpp) {
+  case 32:
+    writePaletteTile(tile.width(), tile.height(),
+                     (rdr::U32*)buffer, stride,
+                     pb->getPF(), palette);
     break;
   case 16:
-    zrleEncode16(r, &mos, &zos, imageBuf, pf, pb);
+    writePaletteTile(tile.width(), tile.height(),
+                     (rdr::U16*)buffer, stride,
+                     pb->getPF(), palette);
     break;
+  default:
+    writePaletteTile(tile.width(), tile.height(),
+                     (rdr::U8*)buffer, stride,
+                     pb->getPF(), palette);
+  }
+}
+
+void ZRLEEncoder::writePaletteRLETile(const Rect& tile, const PixelBuffer* pb,
+                                      const Palette& palette)
+{
+  const rdr::U8* buffer;
+  int stride;
+
+  buffer = pb->getBuffer(tile, &stride);
+
+  switch (pb->getPF().bpp) {
   case 32:
-    {
-      Pixel maxPixel = pf.pixelFromRGB((rdr::U16)-1, (rdr::U16)-1, (rdr::U16)-1);
-      bool fitsInLS3Bytes = maxPixel < (1<<24);
-      bool fitsInMS3Bytes = (maxPixel & 0xff) == 0;
-
-      if ((fitsInLS3Bytes && pf.isLittleEndian()) ||
-          (fitsInMS3Bytes && pf.isBigEndian()))
-      {
-        zrleEncode24A(r, &mos, &zos, imageBuf, pf, pb);
-      }
-      else if ((fitsInLS3Bytes && pf.isBigEndian()) ||
-               (fitsInMS3Bytes && pf.isLittleEndian()))
-      {
-        zrleEncode24B(r, &mos, &zos, imageBuf, pf, pb);
-      }
-      else
-      {
-        zrleEncode32(r, &mos, &zos, imageBuf, pf, pb);
-      }
-      break;
-    }
+    writePaletteRLETile(tile.width(), tile.height(),
+                        (rdr::U32*)buffer, stride,
+                        pb->getPF(), palette);
+    break;
+  case 16:
+    writePaletteRLETile(tile.width(), tile.height(),
+                        (rdr::U16*)buffer, stride,
+                        pb->getPF(), palette);
+    break;
+  default:
+    writePaletteRLETile(tile.width(), tile.height(),
+                        (rdr::U8*)buffer, stride,
+                        pb->getPF(), palette);
   }
+}
 
-  conn->writer()->startRect(r, encodingZRLE);
-  rdr::OutStream* os = conn->getOutStream();
-  os->writeU32(mos.length());
-  os->writeBytes(mos.data(), mos.length());
-  conn->writer()->endRect();
+void ZRLEEncoder::writeRawTile(const Rect& tile, const PixelBuffer* pb,
+                               const Palette& palette)
+{
+  const rdr::U8* buffer;
+  int stride;
+
+  int w, h, stride_bytes;
+
+  buffer = pb->getBuffer(tile, &stride);
+
+  zos.writeU8(0); // Empty palette (i.e. raw pixels)
+
+  w = tile.width();
+  h = tile.height();
+  stride_bytes = stride * pb->getPF().bpp/8;
+  while (h--) {
+    writePixels(buffer, pb->getPF(), w);
+    buffer += stride_bytes;
+  }
+}
+
+void ZRLEEncoder::writePalette(const PixelFormat& pf, const Palette& palette)
+{
+  rdr::U8 buffer[256*4];
+  int i;
+
+  if (pf.bpp == 32) {
+    rdr::U32* buf;
+    buf = (rdr::U32*)buffer;
+    for (i = 0;i < palette.size();i++)
+      *buf++ = palette.getColour(i);
+  } else if (pf.bpp == 16) {
+    rdr::U16* buf;
+    buf = (rdr::U16*)buffer;
+    for (i = 0;i < palette.size();i++)
+      *buf++ = palette.getColour(i);
+  } else {
+    rdr::U8* buf;
+    buf = (rdr::U8*)buffer;
+    for (i = 0;i < palette.size();i++)
+      *buf++ = palette.getColour(i);
+  }
+
+  writePixels(buffer, pf, palette.size());
 }
+
+void ZRLEEncoder::writePixels(const rdr::U8* buffer, const PixelFormat& pf,
+                              unsigned int count)
+{
+  Pixel maxPixel;
+  rdr::U8 pixBuf[4];
+
+  maxPixel = pf.pixelFromRGB((rdr::U16)-1, (rdr::U16)-1, (rdr::U16)-1);
+  pf.bufferFromPixel(pixBuf, maxPixel);
+
+  if ((pf.bpp != 32) || ((pixBuf[0] != 0) && (pixBuf[3] != 0))) {
+    zos.writeBytes(buffer, count * pf.bpp/8);
+    return;
+  }
+
+  if (pixBuf[0] == 0)
+    buffer++;
+
+  while (count--) {
+    zos.writeBytes(buffer, 3);
+    buffer += 4;
+  }
+}
+
+//
+// Including BPP-dependent implementation of the encoder.
+//
+
+#define BPP 8
+#include <rfb/ZRLEEncoderBPP.cxx>
+#undef BPP
+#define BPP 16
+#include <rfb/ZRLEEncoderBPP.cxx>
+#undef BPP
+#define BPP 32
+#include <rfb/ZRLEEncoderBPP.cxx>
+#undef BPP

common/rfb/ZRLEEncoder.h

@@ -1,4 +1,5 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -28,8 +29,51 @@ namespace rfb {
   public:
     ZRLEEncoder(SConnection* conn);
     virtual ~ZRLEEncoder();
-    virtual void writeRect(const Rect& r, PixelBuffer* pb);
-  private:
+
+    virtual bool isSupported();
+
+    virtual void writeRect(const PixelBuffer* pb, const Palette& palette);
+    virtual void writeSolidRect(int width, int height,
+                                const PixelFormat& pf,
+                                const rdr::U8* colour);
+
+  protected:
+    void writePaletteTile(const Rect& tile, const PixelBuffer* pb,
+                          const Palette& palette);
+    void writePaletteRLETile(const Rect& tile, const PixelBuffer* pb,
+                             const Palette& palette);
+    void writeRawTile(const Rect& tile, const PixelBuffer* pb,
+                      const Palette& palette);
+
+    void writePalette(const PixelFormat& pf, const Palette& palette);
+
+    void writePixels(const rdr::U8* buffer, const PixelFormat& pf,
+                     unsigned int count);
+
+  protected:
+    // Preprocessor generated, optimised methods
+
+    void writePaletteTile(int width, int height,
+                          const rdr::U8* buffer, int stride,
+                          const PixelFormat& pf, const Palette& palette);
+    void writePaletteTile(int width, int height,
+                          const rdr::U16* buffer, int stride,
+                          const PixelFormat& pf, const Palette& palette);
+    void writePaletteTile(int width, int height,
+                          const rdr::U32* buffer, int stride,
+                          const PixelFormat& pf, const Palette& palette);
+
+    void writePaletteRLETile(int width, int height,
+                             const rdr::U8* buffer, int stride,
+                             const PixelFormat& pf, const Palette& palette);
+    void writePaletteRLETile(int width, int height,
+                             const rdr::U16* buffer, int stride,
+                             const PixelFormat& pf, const Palette& palette);
+    void writePaletteRLETile(int width, int height,
+                             const rdr::U32* buffer, int stride,
+                             const PixelFormat& pf, const Palette& palette);
+
+  protected:
     rdr::ZlibOutStream zos;
     rdr::MemOutStream mos;
   };

common/rfb/ZRLEEncoderBPP.cxx

@@ -0,0 +1,128 @@
+/* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
+ * 
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ * USA.
+ */
+
+#define CONCAT2(a,b) a##b
+#define CONCAT2E(a,b) CONCAT2(a,b)
+
+#define UBPP CONCAT2E(U,BPP)
+
+void ZRLEEncoder::writePaletteTile(int width, int height,
+                                   const rdr::UBPP* buffer, int stride,
+                                   const PixelFormat& pf,
+                                   const Palette& palette)
+{
+  const int bitsPerPackedPixel[] = {
+    0, 1, 2, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
+  };
+
+  int bppp;
+  int pad;
+
+  assert(palette.size() > 1);
+  assert(palette.size() <= 16);
+
+  zos.writeU8(palette.size());
+  writePalette(pf, palette);
+
+  bppp = bitsPerPackedPixel[palette.size()-1];
+  pad = stride - width;
+
+  for (int i = 0; i < height; i++) {
+    int w;
+
+    rdr::U8 nbits = 0;
+    rdr::U8 byte = 0;
+
+    w = width;
+    while (w--) {
+      rdr::UBPP pix = *buffer++;
+      rdr::U8 index = palette.lookup(pix);
+      byte = (byte << bppp) | index;
+      nbits += bppp;
+      if (nbits >= 8) {
+        zos.writeU8(byte);
+        nbits = 0;
+      }
+    }
+    if (nbits > 0) {
+      byte <<= 8 - nbits;
+      zos.writeU8(byte);
+    }
+
+    buffer += pad;
+  }
+}
+
+void ZRLEEncoder::writePaletteRLETile(int width, int height,
+                                      const rdr::UBPP* buffer, int stride,
+                                      const PixelFormat& pf,
+                                      const Palette& palette)
+{
+  int pad;
+
+  rdr::UBPP prevColour;
+  int runLength;
+
+  assert(palette.size() > 1);
+  assert(palette.size() <= 127);
+
+  zos.writeU8(palette.size() | 0x80);
+  writePalette(pf, palette);
+
+  pad = stride - width;
+
+  prevColour = *buffer;
+  runLength = 0;
+
+  while (height--) {
+    int w = width;
+    while (w--) {
+      if (prevColour != *buffer) {
+        if (runLength == 1)
+          zos.writeU8(palette.lookup(prevColour));
+        else {
+          zos.writeU8(palette.lookup(prevColour) | 0x80);
+
+          while (runLength > 255) {
+            zos.writeU8(255);
+            runLength -= 255;
+          }
+          zos.writeU8(runLength - 1);
+        }
+
+        prevColour = *buffer;
+        runLength = 0;
+      }
+
+      runLength++;
+      buffer++;
+    }
+    buffer += pad;
+  }
+  if (runLength == 1)
+    zos.writeU8(palette.lookup(prevColour));
+  else {
+    zos.writeU8(palette.lookup(prevColour) | 0x80);
+
+    while (runLength > 255) {
+      zos.writeU8(255);
+      runLength -= 255;
+    }
+    zos.writeU8(runLength - 1);
+  }
+}

common/rfb/hextileEncode.h

@@ -44,8 +44,7 @@ int TEST_TILE_TYPE (PIXEL_T* data, int w, int h, PIXEL_T* bg, PIXEL_T* fg);
 int HEXTILE_ENCODE_TILE (PIXEL_T* data, int w, int h, int tileType,
                          rdr::U8* encoded, PIXEL_T bg);
 
-void HEXTILE_ENCODE(const Rect& r, rdr::OutStream* os,
-                    const PixelFormat& pf, PixelBuffer* pb)
+void HEXTILE_ENCODE(rdr::OutStream* os, const PixelBuffer* pb)
 {
   Rect t;
   PIXEL_T buf[256];
@@ -54,15 +53,15 @@ void HEXTILE_ENCODE(const Rect& r, rdr::OutStream* os,
   bool oldFgValid = false;
   rdr::U8 encoded[256*(BPP/8)];
 
-  for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 16) {
+  for (t.tl.y = 0; t.tl.y < pb->height(); t.tl.y += 16) {
 
-    t.br.y = __rfbmin(r.br.y, t.tl.y + 16);
+    t.br.y = __rfbmin(pb->height(), t.tl.y + 16);
 
-    for (t.tl.x = r.tl.x; t.tl.x < r.br.x; t.tl.x += 16) {
+    for (t.tl.x = 0; t.tl.x < pb->width(); t.tl.x += 16) {
 
-      t.br.x = __rfbmin(r.br.x, t.tl.x + 16);
+      t.br.x = __rfbmin(pb->width(), t.tl.x + 16);
 
-      pb->getImage(pf, buf, t);
+      pb->getImage(buf, t);
 
       PIXEL_T bg = 0, fg = 0;
       int tileType = TEST_TILE_TYPE(buf, t.width(), t.height(), &bg, &fg);
@@ -91,7 +90,7 @@ void HEXTILE_ENCODE(const Rect& r, rdr::OutStream* os,
                                          encoded, bg);
 
         if (encodedLen < 0) {
-          pb->getImage(pf, buf, t);
+          pb->getImage(buf, t);
           os->writeU8(hextileRaw);
           os->writeBytes(buf, t.width() * t.height() * (BPP/8));
           oldBgValid = oldFgValid = false;

common/rfb/hextileEncodeBetter.h

@@ -275,8 +275,7 @@ void HEXTILE_TILE::encode(rdr::U8 *dst) const
 // Main encoding function.
 //
 
-void HEXTILE_ENCODE(const Rect& r, rdr::OutStream* os,
-                    const PixelFormat& pf, PixelBuffer* pb)
+void HEXTILE_ENCODE(rdr::OutStream* os, const PixelBuffer* pb)
 {
   Rect t;
   PIXEL_T buf[256];
@@ -287,15 +286,15 @@ void HEXTILE_ENCODE(const Rect& r, rdr::OutStream* os,
 
   HEXTILE_TILE tile;
 
-  for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 16) {
+  for (t.tl.y = 0; t.tl.y < pb->height(); t.tl.y += 16) {
 
-    t.br.y = __rfbmin(r.br.y, t.tl.y + 16);
+    t.br.y = __rfbmin(pb->height(), t.tl.y + 16);
 
-    for (t.tl.x = r.tl.x; t.tl.x < r.br.x; t.tl.x += 16) {
+    for (t.tl.x = 0; t.tl.x < pb->width(); t.tl.x += 16) {
 
-      t.br.x = __rfbmin(r.br.x, t.tl.x + 16);
+      t.br.x = __rfbmin(pb->width(), t.tl.x + 16);
 
-      pb->getImage(pf, buf, t);
+      pb->getImage(buf, t);
 
       tile.newTile(buf, t.width(), t.height());
       int tileType = tile.getFlags();

common/rfb/rreEncode.h

@@ -41,47 +41,8 @@ namespace rfb {
 #define WRITE_PIXEL CONCAT2E(writeOpaque,BPP)
 #define RRE_ENCODE CONCAT2E(rreEncode,BPP)
 
-int RRE_ENCODE (PIXEL_T* data, int w, int h, rdr::OutStream* os, PIXEL_T bg);
-
-int RRE_ENCODE (void* data, int w, int h, rdr::OutStream* os)
-{
-  // Find the background colour - count occurrences of up to 4 different pixel
-  // values, and choose the one which occurs most often.
-
-  const int nCols = 4;
-  PIXEL_T pix[nCols];
-  int count[nCols] = { 0, };
-  PIXEL_T* ptr = (PIXEL_T*)data;
-  PIXEL_T* end = ptr + w*h;
-
-  while (ptr < end) {
-    int i;
-    for (i = 0; i < nCols; i++) {
-      if (count[i] == 0)
-        pix[i] = *ptr;
-
-      if (pix[i] == *ptr) {
-        count[i]++;
-        break;
-      }
-    }
-
-    if (i == nCols) break;
-    ptr++;
-  }
-  
-  int bg = 0;
-  for (int i = 1; i < nCols; i++)
-    if (count[i] > count[bg]) bg = i;
-
-  // Now call the function to do the encoding.
-
-  return RRE_ENCODE ((PIXEL_T*)data, w, h, os, pix[bg]);
-}
-
 int RRE_ENCODE (PIXEL_T* data, int w, int h, rdr::OutStream* os, PIXEL_T bg)
 {
-  int oldLen = os->length();
   os->WRITE_PIXEL(bg);
 
   int nSubrects = 0;
@@ -141,7 +102,6 @@ int RRE_ENCODE (PIXEL_T* data, int w, int h, rdr::OutStream* os, PIXEL_T bg)
       os->writeU16(y);
       os->writeU16(sw);
       os->writeU16(sh);
-      if (os->length() > oldLen + w*h) return -1;
 
       ptr = data+w;
       PIXEL_T* eor = data+w*sh;

common/rfb/tightDecode.h

@@ -27,6 +27,7 @@
 #include <rdr/InStream.h>
 #include <rdr/ZlibInStream.h>
 #include <rfb/Exception.h>
+#include <rfb/TightConstants.h>
 #include <assert.h>
 
 namespace rfb {
@@ -68,7 +69,7 @@ void TIGHT_DECODE (const Rect& r)
   }
 
   // "Fill" compression type.
-  if (comp_ctl == rfbTightFill) {
+  if (comp_ctl == tightFill) {
     PIXEL_T pix;
     if (cutZeros) {
       rdr::U8 bytebuf[3];
@@ -82,13 +83,13 @@ void TIGHT_DECODE (const Rect& r)
   }
 
   // "JPEG" compression type.
-  if (comp_ctl == rfbTightJpeg) {
+  if (comp_ctl == tightJpeg) {
     DECOMPRESS_JPEG_RECT(r);
     return;
   }
 
   // Quit on unsupported compression type.
-  if (comp_ctl > rfbTightMaxSubencoding) {
+  if (comp_ctl > tightMaxSubencoding) {
     throw Exception("TightDecoder: bad subencoding value received");
     return;
   }
@@ -98,11 +99,11 @@ void TIGHT_DECODE (const Rect& r)
   static PIXEL_T palette[256];
   bool useGradient = false;
 
-  if ((comp_ctl & rfbTightExplicitFilter) != 0) {
+  if ((comp_ctl & tightExplicitFilter) != 0) {
     rdr::U8 filterId = is->readU8();
 
     switch (filterId) {
-    case rfbTightFilterPalette: 
+    case tightFilterPalette:
       palSize = is->readU8() + 1;
       if (cutZeros) {
         rdr::U8 tightPalette[256 * 3];
@@ -112,10 +113,10 @@ void TIGHT_DECODE (const Rect& r)
         is->readBytes(palette, palSize * sizeof(PIXEL_T));
       }
       break;
-    case rfbTightFilterGradient: 
+    case tightFilterGradient:
       useGradient = true;
       break;
-    case rfbTightFilterCopy:
+    case tightFilterCopy:
       break;
     default:
       throw Exception("TightDecoder: unknown filter code received");

common/rfb/tightEncode.h

@@ -1,580 +0,0 @@
-/* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
- * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
- * Copyright 2014 Pierre Ossman for Cendio AB.
- * 
- * This is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- * 
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with this software; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
- * USA.
- */
-
-//
-// tightEncode.h - Tight encoding function.
-//
-// This file is #included after having set the following macro:
-// BPP                - 8, 16 or 32
-//
-
-#include <assert.h>
-
-namespace rfb {
-
-// CONCAT2E concatenates its arguments, expanding them if they are macros
-
-#ifndef CONCAT2E
-#define CONCAT2(a,b) a##b
-#define CONCAT2E(a,b) CONCAT2(a,b)
-#endif
-
-#define PIXEL_T rdr::CONCAT2E(U,BPP)
-#define TIGHT_ENCODE TightEncoder::CONCAT2E(tightEncode,BPP)
-#define HASH_FUNCTION CONCAT2E(HASH_FUNC,BPP)
-#define PACK_PIXELS TightEncoder::CONCAT2E(packPixels,BPP)
-#define ENCODE_SOLID_RECT TightEncoder::CONCAT2E(encodeSolidRect,BPP)
-#define ENCODE_FULLCOLOR_RECT TightEncoder::CONCAT2E(encodeFullColorRect,BPP)
-#define ENCODE_MONO_RECT TightEncoder::CONCAT2E(encodeMonoRect,BPP)
-#define ENCODE_INDEXED_RECT TightEncoder::CONCAT2E(encodeIndexedRect,BPP)
-#define ENCODE_JPEG_RECT TightEncoder::CONCAT2E(encodeJpegRect,BPP)
-#define FAST_FILL_PALETTE TightEncoder::CONCAT2E(fastFillPalette,BPP)
-#define FILL_PALETTE TightEncoder::CONCAT2E(fillPalette,BPP)
-#define CHECK_SOLID_TILE TightEncoder::CONCAT2E(checkSolidTile,BPP)
-
-#ifndef TIGHT_ONCE
-#define TIGHT_ONCE
-
-//
-// Compress the data (but do not perform actual compression if the data
-// size is less than TIGHT_MIN_TO_COMPRESS bytes.
-//
-
-void TightEncoder::compressData(const void *buf, unsigned int length,
-                                rdr::ZlibOutStream *zos, int zlibLevel,
-                                rdr::OutStream *os)
-{
-  if (length < TIGHT_MIN_TO_COMPRESS) {
-    os->writeBytes(buf, length);
-  } else {
-    // FIXME: Using a temporary MemOutStream may be not efficient.
-    //        Maybe use the same static object used in the JPEG coder?
-    int maxBeforeSize = pconf->maxRectSize * (clientpf.bpp / 8);
-    int maxAfterSize = maxBeforeSize + (maxBeforeSize + 99) / 100 + 12;
-    rdr::MemOutStream mem_os(maxAfterSize);
-    zos->setUnderlying(&mem_os);
-    zos->setCompressionLevel(zlibLevel);
-    zos->writeBytes(buf, length);
-    zos->flush();
-    zos->setUnderlying(NULL);
-    writeCompact(os, mem_os.length());
-    os->writeBytes(mem_os.data(), mem_os.length());
-  }
-}
-
-#endif  // #ifndef TIGHT_ONCE
-
-//
-// Convert 32-bit color samples into 24-bit colors, in place.
-// Performs packing only when redMax, greenMax and blueMax are all 255.
-// Color components are assumed to be byte-aligned.
-//
-
-unsigned int PACK_PIXELS (PIXEL_T *buf, unsigned int count)
-{
-#if (BPP != 32)
-  return count * sizeof(PIXEL_T);
-#else
-  if (!pack24)
-    return count * sizeof(PIXEL_T);
-
-  rdr::U32 pix;
-  rdr::U8 *dst = (rdr::U8 *)buf;
-  for (unsigned int i = 0; i < count; i++) {
-    pix = *buf++;
-    clientpf.rgbFromBuffer(dst, (rdr::U8*)&pix, 1);
-    dst += 3;
-  }
-  return count * 3;
-#endif
-}
-
-//
-// Main function of the Tight encoder
-//
-
-void TIGHT_ENCODE (const Rect& r, rdr::OutStream *os, bool forceSolid)
-{
-  int stride;
-  rdr::U32 solidColor;
-  const PIXEL_T *rawPixels = (const PIXEL_T *)pb->getBuffer(r, &stride);
-  PIXEL_T *pixels = NULL;
-  bool grayScaleJPEG = (jpegSubsampling == subsampleGray && jpegQuality != -1);
-
-#if (BPP == 32)
-  // Check if it's necessary to pack 24-bit pixels, and
-  // compute appropriate shift values if necessary.
-  pack24 = clientpf.is888();
-#endif
-
-  if (forceSolid) {
-    // Subrectangle has already been determined to be solid.
-    clientpf.bufferFromBuffer((rdr::U8*)&solidColor, serverpf,
-                              (const rdr::U8*)rawPixels, 1);
-    pixels = (PIXEL_T *)&solidColor;
-    palette.clear();
-    palette.insert(solidColor, 1);
-  } else {
-    // Analyze subrectangle's colors to determine best encoding method.
-    palMaxColors = r.area() / pconf->idxMaxColorsDivisor;
-    if (jpegQuality != -1)
-      palMaxColors = pconf->palMaxColorsWithJPEG;
-    if (palMaxColors < 2 && r.area() >= pconf->monoMinRectSize)
-      palMaxColors = 2;
-
-    if (clientpf.equal(serverpf) && clientpf.bpp >= 16) {
-      // Count the colors in the raw buffer, so we can avoid unnecessary pixel
-      // translation when encoding with JPEG.
-      if (grayScaleJPEG) palette.clear();
-      else FAST_FILL_PALETTE(rawPixels, stride, r);
-
-      // JPEG can read from the raw buffer, but for the other methods, we need
-      // to translate the raw pixels into an intermediate buffer.
-      if(palette.size() != 0 || jpegQuality == -1) {
-        pixels = (PIXEL_T *)conn->writer()->getImageBuf(r.area());
-        stride = r.width();
-        pb->getImage(clientpf, pixels, r);
-      }
-    } else {
-      // Pixel translation will be required, so create an intermediate buffer,
-      // translate the raw pixels into it, and count its colors.
-      pixels = (PIXEL_T *)conn->writer()->getImageBuf(r.area());
-      stride = r.width();
-      pb->getImage(clientpf, pixels, r);
-
-      if (grayScaleJPEG) palette.clear();
-      else FILL_PALETTE(pixels, r.area());
-    }
-  }
-
-  switch (palette.size()) {
-  case 0:
-    // Truecolor image
-#if (BPP != 8)
-    if (jpegQuality != -1) {
-      if (pixels)
-        ENCODE_JPEG_RECT(pixels, stride, r, os);
-      else
-        ENCODE_JPEG_RECT((PIXEL_T *)rawPixels, stride, r, os);
-      break;
-    }
-#endif
-    ENCODE_FULLCOLOR_RECT(pixels, r, os);
-    break;
-  case 1:
-    // Solid rectangle
-    ENCODE_SOLID_RECT(pixels, os);
-    break;
-  case 2:
-    // Two-color rectangle
-    ENCODE_MONO_RECT(pixels, r, os);
-    break;
-#if (BPP != 8)
-  default:
-    // Up to 256 different colors
-    ENCODE_INDEXED_RECT(pixels, r, os);
-#endif
-  }
-}
-
-//
-// Subencoding implementations.
-//
-
-void ENCODE_SOLID_RECT (PIXEL_T *buf, rdr::OutStream *os)
-{
-  os->writeU8(0x08 << 4);
-
-  int length = PACK_PIXELS(buf, 1);
-  os->writeBytes(buf, length);
-}
-
-void ENCODE_FULLCOLOR_RECT (PIXEL_T *buf, const Rect& r, rdr::OutStream *os)
-{
-  const int streamId = 0;
-  os->writeU8(streamId << 4);
-
-  int length = PACK_PIXELS(buf, r.area());
-  compressData(buf, length, &zos[streamId], pconf->rawZlibLevel, os);
-}
-
-void ENCODE_MONO_RECT (PIXEL_T *buf, const Rect& r, rdr::OutStream *os)
-{
-  const int streamId = 1;
-  os->writeU8((streamId | 0x04) << 4);
-  os->writeU8(0x01);
-
-  // Write the palette
-  PIXEL_T pal[2] = { (PIXEL_T)palette.getColour(0),
-                     (PIXEL_T)palette.getColour(1) };
-  os->writeU8(1);
-  os->writeBytes(pal, PACK_PIXELS(pal, 2));
-
-  // Encode the data in-place
-  PIXEL_T *src = buf;
-  rdr::U8 *dst = (rdr::U8 *)buf;
-  int w = r.width();
-  int h = r.height();
-  PIXEL_T bg;
-  unsigned int value, mask;
-  int aligned_width;
-  int x, y, bg_bits;
-
-  bg = (PIXEL_T) pal[0];
-  aligned_width = w - w % 8;
-
-  for (y = 0; y < h; y++) {
-    for (x = 0; x < aligned_width; x += 8) {
-      for (bg_bits = 0; bg_bits < 8; bg_bits++) {
-        if (*src++ != bg)
-          break;
-      }
-      if (bg_bits == 8) {
-        *dst++ = 0;
-        continue;
-      }
-      mask = 0x80 >> bg_bits;
-      value = mask;
-      for (bg_bits++; bg_bits < 8; bg_bits++) {
-        mask >>= 1;
-        if (*src++ != bg) {
-          value |= mask;
-        }
-      }
-      *dst++ = (rdr::U8)value;
-    }
-
-    mask = 0x80;
-    value = 0;
-    if (x >= w)
-      continue;
-
-    for (; x < w; x++) {
-      if (*src++ != bg) {
-        value |= mask;
-      }
-      mask >>= 1;
-    }
-    *dst++ = (rdr::U8)value;
-  }
-
-  // Write the data
-  int length = (w + 7) / 8;
-  length *= h;
-  compressData(buf, length, &zos[streamId], pconf->monoZlibLevel, os);
-}
-
-#if (BPP != 8)
-void ENCODE_INDEXED_RECT (PIXEL_T *buf, const Rect& r, rdr::OutStream *os)
-{
-  const int streamId = 2;
-  os->writeU8((streamId | 0x04) << 4);
-  os->writeU8(0x01);
-
-  // Write the palette
-  {
-    PIXEL_T pal[256];
-    for (int i = 0; i < palette.size(); i++)
-      pal[i] = (PIXEL_T)palette.getColour(i);
-    os->writeU8((rdr::U8)(palette.size() - 1));
-    os->writeBytes(pal, PACK_PIXELS(pal, palette.size()));
-  }
-
-  // Encode data in-place
-  PIXEL_T *src = buf;
-  rdr::U8 *dst = (rdr::U8 *)buf;
-  int count = r.area();
-  PIXEL_T rgb;
-  int rep = 0;
-  unsigned char idx;
-
-  while (count--) {
-    rgb = *src++;
-    while (count && *src == rgb) {
-      rep++, src++, count--;
-    }
-    idx = palette.lookup(rgb);
-    *dst++ = idx;
-    while (rep) {
-      *dst++ = idx;
-      rep--;
-    }
-  }
-
-  // Write the data
-  compressData(buf, r.area(), &zos[streamId], pconf->idxZlibLevel, os);
-}
-#endif  // #if (BPP != 8)
-
-//
-// JPEG compression.
-//
-
-#if (BPP != 8)
-void ENCODE_JPEG_RECT (PIXEL_T *buf, int stride, const Rect& r,
-                       rdr::OutStream *os)
-{
-  jc.clear();
-  jc.compress((rdr::U8 *)buf, stride, r, clientpf,
-    jpegQuality, jpegSubsampling);
-  os->writeU8(0x09 << 4);
-  writeCompact(os, jc.length());
-  os->writeBytes(jc.data(), jc.length());
-}
-#endif  // #if (BPP != 8)
-
-//
-// Determine the number of colors in the rectangle, and fill in the palette.
-//
-
-#if (BPP == 8)
-
-void FILL_PALETTE (PIXEL_T *data, int count)
-{
-  PIXEL_T c0, c1;
-  int i, n0, n1;
-
-  palette.clear();
-
-  c0 = data[0];
-  for (i = 1; i < count && data[i] == c0; i++);
-  if (i == count) {
-    palette.insert(c0, i);
-    return;                       // Solid rectangle
-  }
-
-  if (palMaxColors < 2)
-    return;
-
-  n0 = i;
-  c1 = data[i];
-  n1 = 0;
-  for (i++; i < count; i++) {
-    if (data[i] == c0) {
-      n0++;
-    } else if (data[i] == c1) {
-      n1++;
-    } else
-      break;
-  }
-  if (i == count) {
-    palette.insert(c0, n0);     // Two colors
-    palette.insert(c1, n1);
-  }
-}
-
-void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r)
-{
-}
-
-#else   // (BPP != 8)
-
-void FILL_PALETTE (PIXEL_T *data, int count)
-{
-  PIXEL_T c0, c1, ci = 0;
-  int i, n0, n1, ni;
-
-  palette.clear();
-
-  c0 = data[0];
-  for (i = 1; i < count && data[i] == c0; i++);
-  if (i >= count) {
-    palette.insert(c0, i);      // Solid rectangle
-    return;
-  }
-
-  if (palMaxColors < 2)
-    return;                     // Full-color format preferred
-
-  n0 = i;
-  c1 = data[i];
-  n1 = 0;
-  for (i++; i < count; i++) {
-    ci = data[i];
-    if (ci == c0) {
-      n0++;
-    } else if (ci == c1) {
-      n1++;
-    } else
-      break;
-  }
-  palette.insert(c0, n0);
-  palette.insert(c1, n1);
-  if (i >= count)
-    return;                     // Two colors
-
-  ni = 1;
-  for (i++; i < count; i++) {
-    if (data[i] == ci) {
-      ni++;
-    } else {
-      if (!palette.insert (ci, ni) || (palette.size() > palMaxColors)) {
-        palette.clear();
-        return;
-      }
-      ci = data[i];
-      ni = 1;
-    }
-  }
-  if (!palette.insert (ci, ni) || (palette.size() > palMaxColors))
-    palette.clear();
-}
-
-void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r)
-{
-  PIXEL_T c0, c1, ci = 0, mask, c0t, c1t, cit;
-  int n0, n1, ni;
-  int w = r.width(), h = r.height();
-  const PIXEL_T *rowptr, *colptr, *rowptr2, *colptr2,
-    *dataend = &data[stride * h];
-  bool willTransform = !serverpf.equal(clientpf);
-
-  serverpf.bufferFromPixel((rdr::U8*)&mask, ~0);
-
-  palette.clear();
-
-  c0 = data[0] & mask;
-  n0 = 0;
-  for (rowptr = data; rowptr < dataend; rowptr += stride) {
-    for (colptr = rowptr; colptr < &rowptr[w]; colptr++) {
-      if (((*colptr) & mask) != c0)
-        goto soliddone;
-      n0++;
-    }
-  }
-
-  soliddone:
-  if (rowptr >= dataend) {
-    palette.insert(c0, 1);      // Solid rectangle
-    return;
-  }
-  if (palMaxColors < 2)
-    return;                     // Full-color format preferred
-
-  c1 = *colptr & mask;
-  n1 = 0;
-  colptr++;
-  if (colptr >= &rowptr[w]) {
-    rowptr += stride;  colptr = rowptr;
-  }
-  colptr2 = colptr;
-  for (rowptr2 = rowptr; rowptr2 < dataend;) {
-    for (; colptr2 < &rowptr2[w]; colptr2++) {
-      ci = (*colptr2) & mask;
-      if (ci == c0) {
-        n0++;
-      } else if (ci == c1) {
-        n1++;
-      } else
-        goto monodone;
-    }
-    rowptr2 += stride;
-    colptr2 = rowptr2;
-  }
-
-  monodone:
-  if (willTransform) {
-    clientpf.bufferFromBuffer((rdr::U8*)&c0t, serverpf, (rdr::U8*)&c0, 1);
-    clientpf.bufferFromBuffer((rdr::U8*)&c1t, serverpf, (rdr::U8*)&c1, 1);
-  }
-  else {
-    c0t = c0;  c1t = c1;
-  }
-
-  palette.insert(c0t, n0);
-  palette.insert(c1t, n1);
-
-  if (colptr2 >= dataend)
-    return;                      // Two colors
-
-  ni = 1;
-  colptr2++;
-  if (colptr2 >= &rowptr2[w]) {
-    rowptr2 += stride;  colptr2 = rowptr2;
-  }
-  colptr = colptr2;
-  for (rowptr = rowptr2; rowptr < dataend;) {
-    for (; colptr < &rowptr[w]; colptr++) {
-      if (((*colptr) & mask) == ci) {
-        ni++;
-      } else {
-        if (willTransform)
-          clientpf.bufferFromBuffer((rdr::U8*)&cit, serverpf, (rdr::U8*)&ci, 1);
-        else
-          cit = ci;
-        if (!palette.insert (cit, ni) || (palette.size() > palMaxColors)) {
-          palette.clear();
-          return;
-        }
-        ci = (*colptr) & mask;
-        ni = 1;
-      }
-    }
-    rowptr += stride;
-    colptr = rowptr;
-  }
-  clientpf.bufferFromBuffer((rdr::U8*)&cit, serverpf, (rdr::U8*)&ci, 1);
-  if (!palette.insert (cit, ni) || (palette.size() > palMaxColors))
-    palette.clear();
-}
-
-#endif  // #if (BPP == 8)
-
-bool CHECK_SOLID_TILE(Rect& r, rdr::U32 *colorPtr, bool needSameColor)
-{
-  const PIXEL_T *buf;
-  PIXEL_T colorValue;
-  int w = r.width(), h = r.height();
-
-  int stride = w;
-  buf = (const PIXEL_T *)pb->getBuffer(r, &stride);
-
-  colorValue = *buf;
-  if (needSameColor && (rdr::U32)colorValue != *colorPtr)
-    return false;
-
-  int bufPad = stride - w;
-  while (h > 0) {
-    const PIXEL_T *bufEndOfRow = buf + w;
-    while (buf < bufEndOfRow) {
-      if (colorValue != *(buf++))
-        return false;
-    }
-    buf += bufPad;
-    h--;
-  }
-
-  *colorPtr = (rdr::U32)colorValue;
-  return true;
-}
-
-#undef PIXEL_T
-#undef TIGHT_ENCODE
-#undef HASH_FUNCTION
-#undef PACK_PIXELS
-#undef ENCODE_SOLID_RECT
-#undef ENCODE_FULLCOLOR_RECT
-#undef ENCODE_MONO_RECT
-#undef ENCODE_INDEXED_RECT
-#undef ENCODE_JPEG_RECT
-#undef FAST_FILL_PALETTE
-#undef FILL_PALETTE
-#undef CHECK_SOLID_TILE
-}

common/rfb/zrleEncode.h

@@ -1,266 +0,0 @@
-/* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
- * 
- * This is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- * 
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with this software; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
- * USA.
- */
-
-//
-// zrleEncode.h - zrle encoding function.
-//
-// This file is #included after having set the following macro:
-// BPP                - 8, 16 or 32
-//
-// Note that the buf argument to ZRLE_ENCODE needs to be at least one pixel
-// bigger than the largest tile of pixel data, since the ZRLE encoding
-// algorithm writes to the position one past the end of the pixel data.
-//
-
-#include <rdr/OutStream.h>
-#include <rdr/ZlibOutStream.h>
-#include <rfb/Palette.h>
-#include <rfb/PixelBuffer.h>
-#include <assert.h>
-
-namespace rfb {
-
-// CONCAT2E concatenates its arguments, expanding them if they are macros
-
-#ifndef CONCAT2E
-#define CONCAT2(a,b) a##b
-#define CONCAT2E(a,b) CONCAT2(a,b)
-#endif
-
-#ifdef CPIXEL
-#define PIXEL_T rdr::CONCAT2E(U,BPP)
-#define WRITE_PIXEL(os, u) CONCAT2E(writeOpaque,CPIXEL)(os, u)
-#define ZRLE_ENCODE CONCAT2E(zrleEncode,CPIXEL)
-#define ZRLE_ENCODE_TILE CONCAT2E(zrleEncodeTile,CPIXEL)
-#define BPPOUT 24
-#else
-#define PIXEL_T rdr::CONCAT2E(U,BPP)
-#define WRITE_PIXEL(os, u) os->CONCAT2E(writeOpaque,BPP)(u)
-#define ZRLE_ENCODE CONCAT2E(zrleEncode,BPP)
-#define ZRLE_ENCODE_TILE CONCAT2E(zrleEncodeTile,BPP)
-#define BPPOUT BPP
-#endif
-
-#ifndef ZRLE_ONCE
-#define ZRLE_ONCE
-static const int bitsPerPackedPixel[] = {
-  0, 1, 2, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
-};
-#endif
-
-void ZRLE_ENCODE_TILE (PIXEL_T* data, int w, int h, rdr::OutStream* os);
-
-void ZRLE_ENCODE (const Rect& r, rdr::OutStream* os,
-                  rdr::ZlibOutStream* zos, void* buf,
-                  const PixelFormat& pf, PixelBuffer* pb)
-{
-  zos->setUnderlying(os);
-  // RLE overhead is at worst 1 byte per 64x64 (4Kpixel) block
-  int worstCaseLine = r.width() * 64 * (BPPOUT/8) + 1 + r.width() / 64;
-  // Zlib overhead is at worst 6 bytes plus 5 bytes per 32Kbyte block.
-  worstCaseLine += 11 + 5 * (worstCaseLine >> 15);
-  Rect t;
-
-  for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 64) {
-
-    t.br.y = __rfbmin(r.br.y, t.tl.y + 64);
-
-    for (t.tl.x = r.tl.x; t.tl.x < r.br.x; t.tl.x += 64) {
-
-      t.br.x = __rfbmin(r.br.x, t.tl.x + 64);
-
-      pb->getImage(pf, buf, t);
-
-      ZRLE_ENCODE_TILE((PIXEL_T*)buf, t.width(), t.height(), zos);
-    }
-
-    zos->flush();
-  }
-}
-
-
-void ZRLE_ENCODE_TILE (PIXEL_T* data, int w, int h, rdr::OutStream* os)
-{
-  // First find the palette and the number of runs
-
-  Palette palette;
-
-  int runs = 0;
-  int singlePixels = 0;
-
-  PIXEL_T* ptr = data;
-  PIXEL_T* end = ptr + h * w;
-  *end = ~*(end-1); // one past the end is different so the while loop ends
-
-  while (ptr < end) {
-    PIXEL_T pix = *ptr;
-    if (*++ptr != pix) {
-      singlePixels++;
-    } else {
-      while (*++ptr == pix) ;
-      runs++;
-    }
-    palette.insert(pix, 1);
-  }
-
-  //fprintf(stderr,"runs %d, single pixels %d, paletteSize %d\n",
-  //        runs, singlePixels, ph.size);
-
-  // Solid tile is a special case
-
-  if (palette.size() == 1) {
-    os->writeU8(1);
-    WRITE_PIXEL(os, palette.getColour(0));
-    return;
-  }
-
-  // Try to work out whether to use RLE and/or a palette.  We do this by
-  // estimating the number of bytes which will be generated and picking the
-  // method which results in the fewest bytes.  Of course this may not result
-  // in the fewest bytes after compression...
-
-  bool useRle = false;
-  bool usePalette = false;
-
-  int estimatedBytes = w * h * (BPPOUT/8); // start assuming raw
-
-  int plainRleBytes = ((BPPOUT/8)+1) * (runs + singlePixels);
-
-  if (plainRleBytes < estimatedBytes) {
-    useRle = true;
-    estimatedBytes = plainRleBytes;
-  }
-
-  if (palette.size() < 128) {
-    int paletteRleBytes = (BPPOUT/8) * palette.size() + 2 * runs + singlePixels;
-
-    if (paletteRleBytes < estimatedBytes) {
-      useRle = true;
-      usePalette = true;
-      estimatedBytes = paletteRleBytes;
-    }
-
-    if (palette.size() < 17) {
-      int packedBytes = ((BPPOUT/8) * palette.size() +
-                         w * h * bitsPerPackedPixel[palette.size()-1] / 8);
-
-      if (packedBytes < estimatedBytes) {
-        useRle = false;
-        usePalette = true;
-        estimatedBytes = packedBytes;
-      }
-    }
-  }
-
-  if (!usePalette) palette.clear();
-
-  os->writeU8((useRle ? 128 : 0) | palette.size());
-
-  for (int i = 0; i < palette.size(); i++) {
-    WRITE_PIXEL(os, palette.getColour(i));
-  }
-
-  if (useRle) {
-
-    PIXEL_T* ptr = data;
-    PIXEL_T* end = ptr + w * h;
-    PIXEL_T* runStart;
-    PIXEL_T pix;
-    while (ptr < end) {
-      runStart = ptr;
-      pix = *ptr++;
-      while (*ptr == pix && ptr < end)
-        ptr++;
-      int len = ptr - runStart;
-      if (len <= 2 && usePalette) {
-        int index = palette.lookup(pix);
-        if (len == 2)
-          os->writeU8(index);
-        os->writeU8(index);
-        continue;
-      }
-      if (usePalette) {
-        int index = palette.lookup(pix);
-        os->writeU8(index | 128);
-      } else {
-        WRITE_PIXEL(os, pix);
-      }
-      len -= 1;
-      while (len >= 255) {
-        os->writeU8(255);
-        len -= 255;
-      }
-      os->writeU8(len);
-    }
-
-  } else {
-
-    // no RLE
-
-    if (usePalette) {
-
-      // packed pixels
-
-      assert (palette.size() < 17);
-
-      int bppp = bitsPerPackedPixel[palette.size()-1];
-
-      PIXEL_T* ptr = data;
-
-      for (int i = 0; i < h; i++) {
-        rdr::U8 nbits = 0;
-        rdr::U8 byte = 0;
-
-        PIXEL_T* eol = ptr + w;
-
-        while (ptr < eol) {
-          PIXEL_T pix = *ptr++;
-          rdr::U8 index = palette.lookup(pix);
-          byte = (byte << bppp) | index;
-          nbits += bppp;
-          if (nbits >= 8) {
-            os->writeU8(byte);
-            nbits = 0;
-          }
-        }
-        if (nbits > 0) {
-          byte <<= 8 - nbits;
-          os->writeU8(byte);
-        }
-      }
-    } else {
-
-      // raw
-
-#ifdef CPIXEL
-      for (PIXEL_T* ptr = data; ptr < data+w*h; ptr++) {
-        WRITE_PIXEL(os, *ptr);
-      }
-#else
-      os->writeBytes(data, w*h*(BPP/8));
-#endif
-    }
-  }
-}
-
-#undef PIXEL_T
-#undef WRITE_PIXEL
-#undef ZRLE_ENCODE
-#undef ZRLE_ENCODE_TILE
-#undef BPPOUT
-}