ZYWRLE patch for libvncclient (thanks Noriaki Yamazaki)

Highlite:
 * use qualityLevel/zlib_buffer. No new variable is needed.
 * Change coding style to recursive fashion.
 * Change meaning of qualityLevel== 9 for easy calc zywrle_level:
     old:zywrle_level== 1
     new:disable ZYWRLE(same as ZRLE)
   so, we should not use this value for compatible reason.
 * Color mode handling isn't complete.
   I provided and checked 16 bit colors(RGB555,RGB565) and
   some color mode of 32 bit colors for little endian mode.
   we must make and check 24 bit colors and big endian mode.

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

1 files changed, 23 insertions, 3 deletions

diff --git a/libvncclient/rfbproto.c b/libvncclient/rfbproto.c
index 6e87f69..07584f6 100644
--- a/libvncclient/rfbproto.c
+++ b/libvncclient/rfbproto.c
@@ -27,9 +27,13 @@
 #define _BSD_SOURCE
 #define _POSIX_SOURCE
 #endif
+#ifndef WIN32
 #include <unistd.h>
+#else
+#define strncasecmp _strnicmp
+#endif
 #include <errno.h>
-#ifndef __MINGW32__
+#ifndef WIN32
 #include <pwd.h>
 #endif
 #include <rfb/rfbclient.h>
@@ -237,6 +241,7 @@ static void JpegSetSrcManager(j_decompress_ptr cinfo, uint8_t *compressedData,
                               int compressedLen);
 #endif
 static rfbBool HandleZRLE8(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleZRLE15(rfbClient* client, int rx, int ry, int rw, int rh);
 static rfbBool HandleZRLE16(rfbClient* client, int rx, int ry, int rw, int rh);
 static rfbBool HandleZRLE24(rfbClient* client, int rx, int ry, int rw, int rh);
 static rfbBool HandleZRLE24Up(rfbClient* client, int rx, int ry, int rw, int rh);
@@ -718,6 +723,9 @@ SetFormatAndEncodings(rfbClient* client)
 	  requestCompressLevel = TRUE;
       } else if (strncasecmp(encStr,"zrle",encStrLen) == 0) {
 	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZRLE);
+      } else if (strncasecmp(encStr,"zywrle",encStrLen) == 0) {
+	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZYWRLE);
+	requestQualityLevel = TRUE;
 #endif
       } else if ((strncasecmp(encStr,"ultra",encStrLen) == 0) || (strncasecmp(encStr,"ultrazip",encStrLen) == 0)) {
         /* There are 2 encodings used in 'ultra' */
@@ -771,6 +779,7 @@ SetFormatAndEncodings(rfbClient* client)
 #ifdef LIBVNCSERVER_HAVE_LIBZ
     encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZlib);
     encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZRLE);
+    encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZYWRLE);
 #endif
     encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingUltra);
     encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingUltraZip);
@@ -1430,6 +1439,10 @@ HandleRFBServerMessage(rfbClient* client)
       }
 #endif
       case rfbEncodingZRLE:
+	/* Fail safe for ZYWRLE unsupport VNC server. */
+	client->appData.qualityLevel = 9;
+	/* fall through */
+      case rfbEncodingZYWRLE:
       {
 	switch (client->format.bitsPerPixel) {
 	case 8:
@@ -1437,8 +1450,13 @@ HandleRFBServerMessage(rfbClient* client)
 	    return FALSE;
 	  break;
 	case 16:
-	  if (!HandleZRLE16(client, rect.r.x,rect.r.y,rect.r.w,rect.r.h))
-	    return FALSE;
+	  if (client->si.format.greenMax > 0x1F) {
+	    if (!HandleZRLE16(client, rect.r.x,rect.r.y,rect.r.w,rect.r.h))
+	      return FALSE;
+	  } else {
+	    if (!HandleZRLE15(client, rect.r.x,rect.r.y,rect.r.w,rect.r.h))
+	      return FALSE;
+	  }
 	  break;
 	case 32:
 	{
@@ -1650,6 +1668,8 @@ HandleRFBServerMessage(rfbClient* client)
 #include "zlib.c"
 #include "tight.c"
 #include "zrle.c"
+#define REALBPP 15
+#include "zrle.c"
 #undef BPP
 #define BPP 32
 #include "rre.c"