Add trle decoder

2 files changed, 359 insertions, 0 deletions

diff --git a/libvncclient/rfbproto.c b/libvncclient/rfbproto.c
index df8b6d0..242fd52 100644
--- a/libvncclient/rfbproto.c
+++ b/libvncclient/rfbproto.c
@@ -161,6 +161,13 @@ static rfbBool HandleUltra32(rfbClient* client, int rx, int ry, int rw, int rh);
 static rfbBool HandleUltraZip8(rfbClient* client, int rx, int ry, int rw, int rh);
 static rfbBool HandleUltraZip16(rfbClient* client, int rx, int ry, int rw, int rh);
 static rfbBool HandleUltraZip32(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleTRLE8(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleTRLE15(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleTRLE16(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleTRLE24(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleTRLE24Up(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleTRLE24Down(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleTRLE32(rfbClient* client, int rx, int ry, int rw, int rh);
 #ifdef LIBVNCSERVER_HAVE_LIBZ
 static rfbBool HandleZlib8(rfbClient* client, int rx, int ry, int rw, int rh);
 static rfbBool HandleZlib16(rfbClient* client, int rx, int ry, int rw, int rh);
@@ -1295,6 +1302,8 @@ SetFormatAndEncodings(rfbClient* client)
 	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZlibHex);
 	if (client->appData.compressLevel >= 0 && client->appData.compressLevel <= 9)
 	  requestCompressLevel = TRUE;
+      } else if (strncasecmp(encStr,"trle",encStrLen) == 0) {
+	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingTRLE);
       } else if (strncasecmp(encStr,"zrle",encStrLen) == 0) {
 	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZRLE);
       } else if (strncasecmp(encStr,"zywrle",encStrLen) == 0) {
@@ -2010,6 +2019,47 @@ HandleRFBServerMessage(rfbClient* client)
         break;
       }
 
+      case rfbEncodingTRLE:
+	  {
+        switch (client->format.bitsPerPixel) {
+        case 8:
+          if (!HandleTRLE8(client, rect.r.x, rect.r.y, rect.r.w, rect.r.h))
+            return FALSE;
+          break;
+        case 16:
+          if (client->si.format.greenMax > 0x1F) {
+            if (!HandleTRLE16(client, rect.r.x, rect.r.y, rect.r.w, rect.r.h))
+              return FALSE;
+          } else {
+            if (!HandleTRLE15(client, rect.r.x, rect.r.y, rect.r.w, rect.r.h))
+              return FALSE;
+          }
+          break;
+        case 32: {
+          uint32_t maxColor =
+              (client->format.redMax << client->format.redShift) |
+              (client->format.greenMax << client->format.greenShift) |
+              (client->format.blueMax << client->format.blueShift);
+          if ((client->format.bigEndian && (maxColor & 0xff) == 0) ||
+              (!client->format.bigEndian && (maxColor & 0xff000000) == 0)) {
+            if (!HandleTRLE24(client, rect.r.x, rect.r.y, rect.r.w, rect.r.h))
+              return FALSE;
+          } else if (!client->format.bigEndian && (maxColor & 0xff) == 0) {
+            if (!HandleTRLE24Up(client, rect.r.x, rect.r.y, rect.r.w, rect.r.h))
+              return FALSE;
+          } else if (client->format.bigEndian && (maxColor & 0xff000000) == 0) {
+            if (!HandleTRLE24Down(client, rect.r.x, rect.r.y, rect.r.w,
+                                  rect.r.h))
+              return FALSE;
+          } else if (!HandleTRLE32(client, rect.r.x, rect.r.y, rect.r.w,
+                                   rect.r.h))
+            return FALSE;
+          break;
+        }
+        }
+        break;
+      }
+
 #ifdef LIBVNCSERVER_HAVE_LIBZ
       case rfbEncodingZlib:
       {
@@ -2300,6 +2350,7 @@ HandleRFBServerMessage(rfbClient* client)
 #include "ultra.c"
 #include "zlib.c"
 #include "tight.c"
+#include "trle.c"
 #include "zrle.c"
 #undef BPP
 #define BPP 16
@@ -2309,8 +2360,11 @@ HandleRFBServerMessage(rfbClient* client)
 #include "ultra.c"
 #include "zlib.c"
 #include "tight.c"
+#include "trle.c"
 #include "zrle.c"
 #define REALBPP 15
+#include "trle.c"
+#define REALBPP 15
 #include "zrle.c"
 #undef BPP
 #define BPP 32
@@ -2320,14 +2374,23 @@ HandleRFBServerMessage(rfbClient* client)
 #include "ultra.c"
 #include "zlib.c"
 #include "tight.c"
+#include "trle.c"
 #include "zrle.c"
 #define REALBPP 24
+#include "trle.c"
+#define REALBPP 24
 #include "zrle.c"
 #define REALBPP 24
 #define UNCOMP 8
+#include "trle.c"
+#define REALBPP 24
+#define UNCOMP 8
 #include "zrle.c"
 #define REALBPP 24
 #define UNCOMP -8
+#include "trle.c"
+#define REALBPP 24
+#define UNCOMP -8
 #include "zrle.c"
 #undef BPP
 
diff --git a/libvncclient/trle.c b/libvncclient/trle.c
new file mode 100644
index 0000000..b8d6e5c
--- /dev/null
+++ b/libvncclient/trle.c
@@ -0,0 +1,296 @@
+/*
+ *  Copyright (C) 2017 Wiki Wang <wikiwang@live.com>.  All Rights Reserved.
+ *
+ *  This is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This software is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this software; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ *  USA.
+ */
+
+/*
+ * trle.c - handle trle encoding.
+ *
+ * This file shouldn't be compiled directly.  It is included multiple times by
+ * rfbproto.c, each time with a different definition of the macro BPP.  For
+ * each value of BPP, this file defines a function which handles a trle
+ * encoded rectangle with BPP bits per pixel.
+ */
+
+#ifndef REALBPP
+#define REALBPP BPP
+#endif
+
+#if !defined(UNCOMP) || UNCOMP == 0
+#define HandleTRLE CONCAT2E(HandleTRLE, REALBPP)
+#elif UNCOMP > 0
+#define HandleTRLE CONCAT3E(HandleTRLE, REALBPP, Down)
+#else
+#define HandleTRLE CONCAT3E(HandleTRLE, REALBPP, Up)
+#endif
+#define CARDBPP CONCAT3E(uint, BPP, _t)
+#define CARDREALBPP CONCAT3E(uint, REALBPP, _t)
+
+#if REALBPP != BPP && defined(UNCOMP) && UNCOMP != 0
+#if UNCOMP > 0
+#define UncompressCPixel(pointer) ((*(CARDBPP *)pointer) >> UNCOMP)
+#else
+#define UncompressCPixel(pointer) ((*(CARDBPP *)pointer) << (-(UNCOMP)))
+#endif
+#else
+#define UncompressCPixel(pointer) (*(CARDBPP *)pointer)
+#endif
+
+static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) {
+  int x, y, w, h;
+  uint8_t type, last_type;
+  int min_buffer_size = 16 * 16 * (REALBPP / 8) * 2;
+  uint8_t *buffer;
+  CARDBPP palette[128];
+  int bpp, mask, divider;
+  CARDBPP color;
+
+  /* First make sure we have a large enough raw buffer to hold the
+   * decompressed data.  In practice, with a fixed REALBPP, fixed frame
+   * buffer size and the first update containing the entire frame
+   * buffer, this buffer allocation should only happen once, on the
+   * first update.
+   */
+  if (client->raw_buffer_size < min_buffer_size) {
+
+    if (client->raw_buffer != NULL) {
+
+      free(client->raw_buffer);
+    }
+
+    client->raw_buffer_size = min_buffer_size;
+    client->raw_buffer = (char *)malloc(client->raw_buffer_size);
+  }
+
+  rfbClientLog("Update %d %d %d %d\n", rx, ry, rw, rh);
+
+  for (y = ry; y < ry + rh; y += 16) {
+    for (x = rx; x < rx + rw; x += 16) {
+      w = h = 16;
+      if (rx + rw - x < 16)
+        w = rx + rw - x;
+      if (ry + rh - y < 16)
+        h = ry + rh - y;
+
+      if (!ReadFromRFBServer(client, &type, 1))
+        return FALSE;
+
+      buffer = client->raw_buffer;
+
+      switch (type) {
+      case_0:
+      case 0: {
+        if (!ReadFromRFBServer(client, buffer, w * h * REALBPP / 8))
+          return FALSE;
+#if REALBPP != BPP
+        int i, j;
+
+        for (j = y * client->width; j < (y + h) * client->width;
+             j += client->width)
+          for (i = x; i < x + w; i++, buffer += REALBPP / 8)
+            ((CARDBPP *)client->frameBuffer)[j + i] = UncompressCPixel(buffer);
+#else
+        client->GotBitmap(client, buffer, x, y, w, h);
+#endif
+        type = last_type;
+        break;
+      }
+      case 1: {
+        if (!ReadFromRFBServer(client, buffer, REALBPP / 8))
+          return FALSE;
+
+        color = UncompressCPixel(buffer);
+
+        client->GotFillRect(client, x, y, w, h, color);
+
+        last_type = type;
+        break;
+      }
+      case_127:
+      case 127:
+        switch (last_type) {
+        case 0:
+          return FALSE;
+        case 1:
+          client->GotFillRect(client, x, y, w, h, color);
+          type = last_type;
+          break;
+        case 128:
+          return FALSE;
+        default:
+          if (last_type >= 130) {
+            last_type = last_type & 0x7f;
+
+            bpp = (last_type > 4 ? (last_type > 16 ? 8 : 4)
+                                 : (last_type > 2 ? 2 : 1)),
+            mask = (1 << bpp) - 1, divider = (8 / bpp);
+          }
+          if (last_type <= 16) {
+            int i, j, shift;
+
+            if (!ReadFromRFBServer(client, buffer,
+                                   (w + divider - 1) / divider * h))
+              return FALSE;
+
+            /* read palettized pixels */
+            for (j = y * client->width; j < (y + h) * client->width;
+                 j += client->width) {
+              for (i = x, shift = 8 - bpp; i < x + w; i++) {
+                ((CARDBPP *)client->frameBuffer)[j + i] =
+                    palette[((*buffer) >> shift) & mask];
+                shift -= bpp;
+                if (shift < 0) {
+                  shift = 8 - bpp;
+                  buffer++;
+                }
+              }
+              if (shift < 8 - bpp)
+                buffer++;
+
+              type = last_type;
+            }
+          } else
+            return FALSE;
+        }
+        break;
+      case 128: {
+        int i = 0, j = 0;
+        while (j < h) {
+          int color, length;
+          /* read color */
+          if (!ReadFromRFBServer(client, buffer, REALBPP / 8 + 1))
+            return FALSE;
+          color = UncompressCPixel(buffer);
+          buffer += REALBPP / 8;
+          /* read run length */
+          length = 1;
+          while (*buffer == 0xff) {
+            if (!ReadFromRFBServer(client, buffer + 1, 1))
+              return FALSE;
+            length += *buffer;
+            buffer++;
+          }
+          length += *buffer;
+          buffer++;
+          while (j < h && length > 0) {
+            ((CARDBPP *)client->frameBuffer)[(y + j) * client->width + x + i] =
+                color;
+            length--;
+            i++;
+            if (i >= w) {
+              i = 0;
+              j++;
+            }
+          }
+          if (length > 0)
+            rfbClientLog("Warning: possible TRLE corruption\n");
+        }
+
+        type = last_type;
+
+        break;
+      }
+      case_129:
+      case 129: {
+        int i, j;
+        /* read palettized pixels */
+        i = j = 0;
+        while (j < h) {
+          int color, length;
+          /* read color */
+          if (!ReadFromRFBServer(client, buffer, 1))
+            return FALSE;
+          color = palette[(*buffer) & 0x7f];
+          length = 1;
+          if (*buffer & 0x80) {
+            if (!ReadFromRFBServer(client, buffer + 1, 1))
+              return FALSE;
+            buffer++;
+            /* read run length */
+            while (*buffer == 0xff) {
+              if (!ReadFromRFBServer(client, buffer + 1, 1))
+                return FALSE;
+              length += *buffer;
+              buffer++;
+            }
+            length += *buffer;
+          }
+          buffer++;
+          while (j < h && length > 0) {
+            ((CARDBPP *)client->frameBuffer)[(y + j) * client->width + x + i] =
+                color;
+            length--;
+            i++;
+            if (i >= w) {
+              i = 0;
+              j++;
+            }
+          }
+          if (length > 0)
+            rfbClientLog("Warning: possible TRLE corruption\n");
+        }
+
+        if (type == 129) {
+          type = last_type;
+        }
+
+        break;
+      }
+      default:
+        if (type <= 16) {
+          int i;
+
+          bpp = (type > 4 ? (type > 16 ? 8 : 4) : (type > 2 ? 2 : 1)),
+          mask = (1 << bpp) - 1, divider = (8 / bpp);
+
+          if (!ReadFromRFBServer(client, buffer, type * REALBPP / 8))
+            return FALSE;
+
+          /* read palette */
+          for (i = 0; i < type; i++, buffer += REALBPP / 8)
+            palette[i] = UncompressCPixel(buffer);
+
+          last_type = type;
+          goto case_127;
+        } else if (type >= 130) {
+          int i;
+
+          if (!ReadFromRFBServer(client, buffer, (type - 128) * REALBPP / 8))
+            return FALSE;
+
+          /* read palette */
+          for (i = 0; i < type - 128; i++, buffer += REALBPP / 8)
+            palette[i] = UncompressCPixel(buffer);
+
+          last_type = type;
+          goto case_129;
+        } else
+          return FALSE;
+      }
+      last_type = type;
+    }
+  }
+
+  return TRUE;
+}
+
+#undef CARDBPP
+#undef CARDREALBPP
+#undef HandleTRLE
+#undef UncompressCPixel
+#undef REALBPP
+#undef UNCOMP