libxrdp: changes for planar compression

3 files changed, 216 insertions, 72 deletions

diff --git a/libxrdp/libxrdp.h b/libxrdp/libxrdp.h
index 2ea8b61d..ddfe260b 100644
--- a/libxrdp/libxrdp.h
+++ b/libxrdp/libxrdp.h
@@ -117,9 +117,10 @@ struct xrdp_sec
     void *decrypt_rc4_info;
     void *encrypt_rc4_info;
     char pub_exp[4];
-    char pub_mod[64];
+    char pub_mod[256];
     char pub_sig[64];
-    char pri_exp[64];
+    char pri_exp[256];
+    int rsa_key_bytes; /* 64 or 256 */
     int channel_code;
     int multimon;
     char fips_encrypt_key[24];
@@ -509,7 +510,7 @@ int APP_CC
 xrdp_bitmap32_compress(char *in_data, int width, int height,
                        struct stream *s, int bpp, int byte_limit,
                        int start_line, struct stream *temp_s,
-                       int e);
+                       int e, int flags);
 int APP_CC
 xrdp_jpeg_compress(void *handle, char *in_data, int width, int height,
                    struct stream *s, int bpp, int byte_limit,
diff --git a/libxrdp/xrdp_bitmap32_compress.c b/libxrdp/xrdp_bitmap32_compress.c
index ef20ba0f..d843cff1 100644
--- a/libxrdp/xrdp_bitmap32_compress.c
+++ b/libxrdp/xrdp_bitmap32_compress.c
@@ -36,9 +36,10 @@ http://msdn.microsoft.com/en-us/library/cc241877.aspx
   do { if (_level < LLOG_LEVEL) { g_hexdump _args ; } } while (0)
 
 /*****************************************************************************/
+/* split ARGB */
 static int APP_CC
-fsplit(char *in_data, int start_line, int width, int e,
-       char *alpha_data, char *red_data, char *green_data, char *blue_data)
+fsplit4(char *in_data, int start_line, int width, int e,
+        char *alpha_data, char *red_data, char *green_data, char *blue_data)
 {
 #if defined(L_ENDIAN)
     int alpha;
@@ -106,10 +107,10 @@ fsplit(char *in_data, int start_line, int width, int e,
         }
         for (index = 0; index < e; index++)
         {
-            alpha_data[out_index] = 0;
-            red_data[out_index] = 0;
-            green_data[out_index] = 0;
-            blue_data[out_index] = 0;
+            alpha_data[out_index] = alpha_data[out_index - 1];
+            red_data[out_index] = red_data[out_index - 1];
+            green_data[out_index] = green_data[out_index - 1];
+            blue_data[out_index] = blue_data[out_index - 1];
             out_index++;
         }
         start_line--;
@@ -119,20 +120,99 @@ fsplit(char *in_data, int start_line, int width, int e,
 }
 
 /*****************************************************************************/
+/* split RGB */
 static int APP_CC
-fdelta(char *plane, int cx, int cy)
+fsplit3(char *in_data, int start_line, int width, int e,
+        char *red_data, char *green_data, char *blue_data)
+{
+#if defined(L_ENDIAN)
+    int red;
+    int green;
+    int blue;
+#endif
+    int index;
+    int out_index;
+    int pixel;
+    int cy;
+    int *ptr32;
+
+    cy = 0;
+    out_index = 0;
+    while (start_line >= 0)
+    {
+        ptr32 = (int *) (in_data + start_line * width * 4);
+        index = 0;
+#if defined(L_ENDIAN)
+        while (index + 4 <= width)
+        {
+            pixel = *ptr32;
+            ptr32++;
+            red    = (pixel >> 16) & 0x000000ff;
+            green  = (pixel >>  8) & 0x000000ff;
+            blue   = (pixel >>  0) & 0x000000ff;
+            pixel  = *ptr32;
+            ptr32++;
+            red   |= (pixel >>  8) & 0x0000ff00;
+            green |= (pixel <<  0) & 0x0000ff00;
+            blue  |= (pixel <<  8) & 0x0000ff00;
+            pixel = *ptr32;
+            ptr32++;
+            red   |= (pixel >>  0) & 0x00ff0000;
+            green |= (pixel <<  8) & 0x00ff0000;
+            blue  |= (pixel << 16) & 0x00ff0000;
+            pixel = *ptr32;
+            ptr32++;
+            red   |= (pixel <<  8) & 0xff000000;
+            green |= (pixel << 16) & 0xff000000;
+            blue  |= (pixel << 24) & 0xff000000;
+            *((int*)(red_data + out_index)) = red;
+            *((int*)(green_data + out_index)) = green;
+            *((int*)(blue_data + out_index)) = blue;
+            out_index += 4;
+            index += 4;
+        }
+#endif
+        while (index < width)
+        {
+            pixel = *ptr32;
+            ptr32++;
+            red_data[out_index] = pixel >> 16;
+            green_data[out_index] = pixel >> 8;
+            blue_data[out_index] = pixel >> 0;
+            out_index++;
+            index++;
+        }
+        for (index = 0; index < e; index++)
+        {
+            red_data[out_index] = red_data[out_index - 1];
+            green_data[out_index] = green_data[out_index - 1];
+            blue_data[out_index] = blue_data[out_index - 1];
+            out_index++;
+        }
+        start_line--;
+        cy++;
+    }
+    return cy;
+}
+
+/*****************************************************************************/
+static int APP_CC
+fdelta(char *in_plane, char *out_plane, int cx, int cy)
 {
     char delta;
-    char *ptr8;
+    char *src8;
+    char *dst8;
     int index;
     int jndex;
 
+    g_memcpy(out_plane, in_plane, cx);
     for (jndex = cy - 2; jndex >= 0; jndex--)
     {
-        ptr8 = plane + jndex * cx;
+        src8 = in_plane + jndex * cx;
+        dst8 = out_plane + jndex * cx;
         for (index = 0; index < cx; index++)
         {
-            delta = ptr8[cx] - ptr8[0];
+            delta = src8[cx] - src8[0];
             if (delta & 0x80)
             {
                 delta = (((~delta) + 1) << 1) - 1;
@@ -141,8 +221,9 @@ fdelta(char *plane, int cx, int cy)
             {
                 delta = delta << 1;
             }
-            ptr8[cx] = delta;
-            ptr8++;
+            dst8[cx] = delta;
+            src8++;
+            dst8++;
         }
     }
     return 0;
@@ -279,98 +360,158 @@ fpack(char *plane, int cx, int cy, struct stream *s)
 }
 
 /*****************************************************************************/
+static int APP_CC
+foutraw3(struct stream *s, int bytes, int header,
+         char *red_data, char *green_data, char *blue_data)
+{
+    out_uint8(s, header);
+    out_uint8a(s, red_data, bytes);
+    out_uint8a(s, green_data, bytes);
+    out_uint8a(s, blue_data, bytes);
+    /* pad if no RLE */
+    out_uint8(s, 0x00);
+    return 0;
+}
+
+/*****************************************************************************/
+static int APP_CC
+foutraw4(struct stream *s, int bytes, int header,
+         char *alpha_data, char *red_data, char *green_data, char *blue_data)
+{
+    out_uint8(s, header);
+    out_uint8a(s, alpha_data, bytes);
+    out_uint8a(s, red_data, bytes);
+    out_uint8a(s, green_data, bytes);
+    out_uint8a(s, blue_data, bytes);
+    /* pad if no RLE */
+    out_uint8(s, 0x00);
+    return 0;
+}
+
+/*****************************************************************************/
 /* returns the number of lines compressed */
 int APP_CC
 xrdp_bitmap32_compress(char *in_data, int width, int height,
                        struct stream *s, int bpp, int byte_limit,
                        int start_line, struct stream *temp_s,
-                       int e)
+                       int e, int flags)
 {
     char *alpha_data;
     char *red_data;
     char *green_data;
     char *blue_data;
+    char *salpha_data;
+    char *sred_data;
+    char *sgreen_data;
+    char *sblue_data;
     int alpha_bytes;
     int red_bytes;
     int green_bytes;
     int blue_bytes;
     int cx;
     int cy;
-    int header;
     int max_bytes;
+    int total_bytes;
+    int header;
 
     LLOGLN(10, ("xrdp_bitmap32_compress:"));
 
-    //header = FLAGS_NOALPHA | FLAGS_RLE;
-    //header = FLAGS_NOALPHA;
-    header = FLAGS_RLE;
-
+    header = flags & 0xFF;
     cx = width + e;
-    alpha_data = temp_s->data;
+    salpha_data = temp_s->data;
+    sred_data = salpha_data + cx * height;
+    sgreen_data = sred_data + cx * height;
+    sblue_data = sgreen_data + cx * height;
+    alpha_data = sblue_data + cx * height;
     red_data = alpha_data + cx * height;
     green_data = red_data + cx * height;
     blue_data = green_data + cx * height;
 
-    /* split planes */
-    cy = fsplit(in_data, start_line, width, e,
-                alpha_data, red_data, green_data, blue_data);
-
-    if (header & FLAGS_RLE)
+    if (header & FLAGS_NOALPHA)
     {
-        out_uint8(s, header);
-        if (header & FLAGS_NOALPHA)
+        cy = fsplit3(in_data, start_line, width, e,
+                     sred_data, sgreen_data, sblue_data);
+        if (header & FLAGS_RLE)
         {
-            fdelta(red_data, cx, cy);
-            fdelta(green_data, cx, cy);
-            fdelta(blue_data, cx, cy);
-            alpha_bytes = 0;
+            fdelta(sred_data, red_data, cx, cy);
+            fdelta(sgreen_data, green_data, cx, cy);
+            fdelta(sblue_data, blue_data, cx, cy);
+            out_uint8(s, header);
             red_bytes = fpack(red_data, cx, cy, s);
             green_bytes = fpack(green_data, cx, cy, s);
             blue_bytes = fpack(blue_data, cx, cy, s);
+            total_bytes = red_bytes + green_bytes + blue_bytes;
+            if (1 + total_bytes > byte_limit)
+            {
+                /* failed */
+                LLOGLN(0, ("xrdp_bitmap32_compress: too big, rgb "
+                       "bytes %d %d %d total_bytes %d cx %d cy %d byte_limit %d",
+                       red_bytes, green_bytes, blue_bytes,
+                       total_bytes, cx, cy, byte_limit));
+                return 0;
+            }
             max_bytes = cx * cy * 3;
+            if (total_bytes > max_bytes)
+            {
+                /* raw is better */
+                LLOGLN(10, ("xrdp_bitmap32_compress: too big, rgb "
+                       "bytes %d %d %d total_bytes %d cx %d cy %d max_bytes %d",
+                       red_bytes, green_bytes, blue_bytes,
+                       total_bytes, cx, cy, max_bytes));
+                init_stream(s, 0);
+                foutraw3(s, cx * cy, FLAGS_NOALPHA, sred_data,
+                         sgreen_data, sblue_data);
+            }
         }
         else
         {
-            fdelta(alpha_data, cx, cy);
-            fdelta(red_data, cx, cy);
-            fdelta(green_data, cx, cy);
-            fdelta(blue_data, cx, cy);
-            alpha_bytes = fpack(alpha_data, cx, cy, s);
-            red_bytes = fpack(red_data, cx, cy, s);
-            green_bytes = fpack(green_data, cx, cy, s);
-            blue_bytes = fpack(blue_data, cx, cy, s);
-            max_bytes = cx * cy * 4;
-        }
-        if (alpha_bytes + red_bytes + green_bytes + blue_bytes > max_bytes)
-        {
-            LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb "
-                   "bytes %d %d %d %d cx %d cy %d", alpha_bytes, red_bytes,
-                   green_bytes, blue_bytes, cx, cy));
+            foutraw3(s, cx * cy, FLAGS_NOALPHA, sred_data,
+                     sgreen_data, sblue_data);
         }
     }
     else
     {
-        out_uint8(s, header);
-        red_bytes = cx * cy;
-        green_bytes = cx * cy;
-        blue_bytes = cx * cy;
-        if (header & FLAGS_NOALPHA)
+        cy = fsplit4(in_data, start_line, width, e,
+                     salpha_data, sred_data, sgreen_data, sblue_data);
+        if (header & FLAGS_RLE)
         {
-            out_uint8a(s, red_data, red_bytes);
-            out_uint8a(s, green_data, green_bytes);
-            out_uint8a(s, blue_data, blue_bytes);
+            fdelta(salpha_data, alpha_data, cx, cy);
+            fdelta(sred_data, red_data, cx, cy);
+            fdelta(sgreen_data, green_data, cx, cy);
+            fdelta(sblue_data, blue_data, cx, cy);
+            out_uint8(s, header);
+            alpha_bytes = fpack(alpha_data, cx, cy, s);
+            red_bytes = fpack(red_data, cx, cy, s);
+            green_bytes = fpack(green_data, cx, cy, s);
+            blue_bytes = fpack(blue_data, cx, cy, s);
+            max_bytes = cx * cy * 4;
+            total_bytes = alpha_bytes + red_bytes + green_bytes + blue_bytes;
+            if (1 + total_bytes > byte_limit)
+            {
+                /* failed */
+                LLOGLN(0, ("xrdp_bitmap32_compress: too big, argb "
+                       "bytes %d %d %d %d total_bytes %d cx %d cy %d byte_limit %d",
+                       alpha_bytes, red_bytes, green_bytes, blue_bytes,
+                       total_bytes, cx, cy, byte_limit));
+                return 0;
+            }
+            if (total_bytes > max_bytes)
+            {
+                /* raw is better */
+                LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb "
+                       "bytes %d %d %d %d total_bytes %d cx %d cy %d max_bytes %d",
+                       alpha_bytes, red_bytes, green_bytes, blue_bytes,
+                       total_bytes, cx, cy, max_bytes));
+                init_stream(s, 0);
+                foutraw4(s, cx * cy, 0, salpha_data, sred_data,
+                         sgreen_data, sblue_data);
+            }
         }
         else
         {
-            alpha_bytes = cx * cy;
-            out_uint8a(s, alpha_data, alpha_bytes);
-            out_uint8a(s, red_data, red_bytes);
-            out_uint8a(s, green_data, green_bytes);
-            out_uint8a(s, blue_data, blue_bytes);
+            foutraw4(s, cx * cy, 0, salpha_data, sred_data,
+                     sgreen_data, sblue_data);
         }
-        /* pad if no RLE */
-        out_uint8(s, 0x00);
     }
-
     return cy;
 }
diff --git a/libxrdp/xrdp_orders.c b/libxrdp/xrdp_orders.c
index 1412386e..88ffda8e 100644
--- a/libxrdp/xrdp_orders.c
+++ b/libxrdp/xrdp_orders.c
@@ -2322,15 +2322,16 @@ xrdp_orders_send_bitmap(struct xrdp_orders *self,
     }
 
     make_stream(s);
-    init_stream(s, 16384);
+    init_stream(s, 16384 * 2);
     make_stream(temp_s);
-    init_stream(temp_s, 16384);
+    init_stream(temp_s, 16384 * 2);
     p = s->p;
     i = height;
     if (bpp > 24)
     {
-        lines_sending = xrdp_bitmap32_compress(data, width, height, s, bpp, 16384,
-                                               i - 1, temp_s, e);
+        lines_sending = xrdp_bitmap32_compress(data, width, height, s,
+                                               bpp, 16384,
+                                               i - 1, temp_s, e, 0x30);
     }
     else
     {
@@ -2590,15 +2591,16 @@ xrdp_orders_send_bitmap2(struct xrdp_orders *self,
     }
 
     make_stream(s);
-    init_stream(s, 16384);
+    init_stream(s, 16384 * 2);
     make_stream(temp_s);
-    init_stream(temp_s, 16384);
+    init_stream(temp_s, 16384 * 2);
     p = s->p;
     i = height;
     if (bpp > 24)
     {
-        lines_sending = xrdp_bitmap32_compress(data, width, height, s, bpp, 16384,
-                                               i - 1, temp_s, e);
+        lines_sending = xrdp_bitmap32_compress(data, width, height, s,
+                                               bpp, 16384,
+                                               i - 1, temp_s, e, 0x30);
     }
     else
     {