libxrdp: 32 bpp compression changes / fixes

2 files changed, 105 insertions, 58 deletions

diff --git a/libxrdp/xrdp_bitmap32_compress.c b/libxrdp/xrdp_bitmap32_compress.c
index c3afff90..1bcf5db9 100644
--- a/libxrdp/xrdp_bitmap32_compress.c
+++ b/libxrdp/xrdp_bitmap32_compress.c
@@ -37,6 +37,58 @@ http://msdn.microsoft.com/en-us/library/cc241877.aspx
 
 /*****************************************************************************/
 static int APP_CC
+fsplit(char *in_data, int start_line, int width, int e,
+       char *alpha_data, char *red_data, char *green_data, char *blue_data)
+{
+    int index;
+    int pixel;
+    int cy;
+    int alpha_bytes;
+    int red_bytes;
+    int green_bytes;
+    int blue_bytes;
+    int *ptr32;
+
+    cy = 0;
+    alpha_bytes = 0;
+    red_bytes = 0;
+    green_bytes = 0;
+    blue_bytes = 0;
+    while (start_line >= 0)
+    {
+        ptr32 = (int *) (in_data + start_line * width * 4);
+        for (index = 0; index < width; index++)
+        {
+            pixel = *ptr32;
+            ptr32++;
+            alpha_data[alpha_bytes] = pixel >> 24;
+            alpha_bytes++;
+            red_data[red_bytes] = pixel >> 16;
+            red_bytes++;
+            green_data[green_bytes] = pixel >> 8;
+            green_bytes++;
+            blue_data[blue_bytes] = pixel >> 0;
+            blue_bytes++;
+        }
+        for (index = 0; index < e; index++)
+        {
+            alpha_data[alpha_bytes] = 0;
+            alpha_bytes++;
+            red_data[red_bytes] = 0;
+            red_bytes++;
+            green_data[green_bytes] = 0;
+            green_bytes++;
+            blue_data[blue_bytes] = 0;
+            blue_bytes++;
+        }
+        start_line--;
+        cy++;
+    }
+    return cy;
+}
+
+/*****************************************************************************/
+static int APP_CC
 fdelta(char *plane, int cx, int cy)
 {
     char delta;
@@ -135,11 +187,13 @@ fpack(char *plane, int cx, int cy, struct stream *s)
     char *ptr8;
     char *colptr;
     char *lend;
+    char *holdp;
     int jndex;
     int collen;
     int replen;
 
     LLOGLN(10, ("fpack:"));
+    holdp = s->p;
     for (jndex = 0; jndex < cy; jndex++)
     {
         LLOGLN(10, ("line start line %d cx %d cy %d", jndex, cx, cy));
@@ -190,7 +244,7 @@ fpack(char *plane, int cx, int cy, struct stream *s)
         /* end of line */
         fout(collen, replen, colptr, s);
     }
-    return 0;
+    return (int) (s->p - holdp);
 }
 
 /*****************************************************************************/
@@ -201,8 +255,6 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
                        int start_line, struct stream *temp_s,
                        int e)
 {
-    int pixel;
-    int *ptr32;
     char *alpha_data;
     char *red_data;
     char *green_data;
@@ -211,10 +263,10 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
     int red_bytes;
     int green_bytes;
     int blue_bytes;
-    int index;
     int cx;
     int cy;
     int header;
+    int max_bytes;
 
     LLOGLN(10, ("xrdp_bitmap32_compress:"));
 
@@ -223,80 +275,67 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
     header = FLAGS_RLE;
 
     cx = width + e;
-    cy = 0;
     alpha_data = temp_s->data;
     red_data = alpha_data + cx * height;
     green_data = red_data + cx * height;
     blue_data = green_data + cx * height;
-    alpha_bytes = 0;
-    red_bytes = 0;
-    green_bytes = 0;
-    blue_bytes = 0;
 
     /* split planes */
-    while (start_line >= 0)
-    {
-        ptr32 = (int *) (in_data + start_line * width * 4);
-        for (index = 0; index < width; index++)
-        {
-            pixel = *ptr32;
-            ptr32++;
-            alpha_data[alpha_bytes] = pixel >> 24;
-            alpha_bytes++;
-            red_data[red_bytes] = pixel >> 16;
-            red_bytes++;
-            green_data[green_bytes] = pixel >> 8;
-            green_bytes++;
-            blue_data[blue_bytes] = pixel >> 0;
-            blue_bytes++;
-        }
-        for (index = 0; index < e; index++)
-        {
-            alpha_data[alpha_bytes] = 0;
-            alpha_bytes++;
-            red_data[red_bytes] = 0;
-            red_bytes++;
-            green_data[green_bytes] = 0;
-            green_bytes++;
-            blue_data[blue_bytes] = 0;
-            blue_bytes++;
-        }
-        start_line--;
-        cy++;
-    }
+    cy = fsplit(in_data, start_line, width, e,
+                alpha_data, red_data, green_data, blue_data);
 
     if (header & FLAGS_RLE)
     {
         out_uint8(s, header);
-
-        /* delta, other steps */
-        if ((header & FLAGS_NOALPHA) == 0)
+        if (header & FLAGS_NOALPHA)
+        {
+            fdelta(red_data, cx, cy);
+            fdelta(green_data, cx, cy);
+            fdelta(blue_data, cx, cy);
+            red_bytes = fpack(red_data, cx, cy, s);
+            green_bytes = fpack(green_data, cx, cy, s);
+            blue_bytes = fpack(blue_data, cx, cy, s);
+            max_bytes = cx * cy * 3;
+        }
+        else
         {
             fdelta(alpha_data, cx, cy);
+            fdelta(red_data, cx, cy);
+            fdelta(green_data, cx, cy);
+            fdelta(blue_data, cx, cy);
+            alpha_bytes = fpack(alpha_data, cx, cy, s);
+            red_bytes = fpack(red_data, cx, cy, s);
+            green_bytes = fpack(green_data, cx, cy, s);
+            blue_bytes = fpack(blue_data, cx, cy, s);
+            max_bytes = cx * cy * 4;
         }
-        fdelta(red_data, cx, cy);
-        fdelta(green_data, cx, cy);
-        fdelta(blue_data, cx, cy);
-
-        /* pack */
-        if ((header & FLAGS_NOALPHA) == 0)
+        if (alpha_bytes + red_bytes + green_bytes + blue_bytes > max_bytes)
         {
-            fpack(alpha_data, cx, cy, s);
+            LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb "
+                   "bytes %d %d %d %d cx %d cy %d", alpha_bytes, red_bytes,
+                   green_bytes, blue_bytes, cx, cy));
         }
-        fpack(red_data, cx, cy, s);
-        fpack(green_data, cx, cy, s);
-        fpack(blue_data, cx, cy, s);
     }
     else
     {
         out_uint8(s, header);
-        if ((header & FLAGS_NOALPHA) == 0)
+        red_bytes = cx * cy;
+        green_bytes = cx * cy;
+        blue_bytes = cx * cy;
+        if (header & FLAGS_NOALPHA)
+        {
+            out_uint8a(s, red_data, red_bytes);
+            out_uint8a(s, green_data, green_bytes);
+            out_uint8a(s, blue_data, blue_bytes);
+        }
+        else
         {
+            alpha_bytes = cx * cy;
             out_uint8a(s, alpha_data, alpha_bytes);
+            out_uint8a(s, red_data, red_bytes);
+            out_uint8a(s, green_data, green_bytes);
+            out_uint8a(s, blue_data, blue_bytes);
         }
-        out_uint8a(s, red_data, red_bytes);
-        out_uint8a(s, green_data, green_bytes);
-        out_uint8a(s, blue_data, blue_bytes);
         /* pad if no RLE */
         out_uint8(s, 0x00);
     }
diff --git a/libxrdp/xrdp_orders.c b/libxrdp/xrdp_orders.c
index 7ba6cf01..1412386e 100644
--- a/libxrdp/xrdp_orders.c
+++ b/libxrdp/xrdp_orders.c
@@ -2327,8 +2327,16 @@ xrdp_orders_send_bitmap(struct xrdp_orders *self,
     init_stream(temp_s, 16384);
     p = s->p;
     i = height;
-    lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384,
-                                         i - 1, temp_s, e);
+    if (bpp > 24)
+    {
+        lines_sending = xrdp_bitmap32_compress(data, width, height, s, bpp, 16384,
+                                               i - 1, temp_s, e);
+    }
+    else
+    {
+        lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384,
+                                             i - 1, temp_s, e);
+    }
 
     if (lines_sending != height)
     {