fixes for rle bitmap compress limits and raw bitmaps

3 files changed, 153 insertions, 92 deletions

diff --git a/libxrdp/xrdp_bitmap32_compress.c b/libxrdp/xrdp_bitmap32_compress.c
index 083c4409..daec9f28 100644
--- a/libxrdp/xrdp_bitmap32_compress.c
+++ b/libxrdp/xrdp_bitmap32_compress.c
@@ -107,7 +107,7 @@ fsplit3(char *in_data, int start_line, int width, int e,
         }
         start_line--;
         cy++;
-        if (out_index > 64 * 64)
+        if (out_index + width + e > 64 * 64)
         {
             break;
         }
@@ -195,7 +195,7 @@ fsplit4(char *in_data, int start_line, int width, int e,
         }
         start_line--;
         cy++;
-        if (out_index > 64 * 64)
+        if (out_index + width + e > 64 * 64)
         {
             break;
         }
@@ -422,6 +422,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
     char *sr_data;
     char *sg_data;
     char *sb_data;
+    char *hold_p;
     int a_bytes;
     int r_bytes;
     int g_bytes;
@@ -449,6 +450,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
     r_data = a_data + max_bytes;
     g_data = r_data + max_bytes;
     b_data = g_data + max_bytes;
+    hold_p = s->p;
 
     if (header & FLAGS_NOALPHA)
     {
@@ -459,35 +461,44 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
             fdelta(sr_data, r_data, cx, cy);
             fdelta(sg_data, g_data, cx, cy);
             fdelta(sb_data, b_data, cx, cy);
-            out_uint8(s, header);
-            r_bytes = fpack(r_data, cx, cy, s);
-            g_bytes = fpack(g_data, cx, cy, s);
-            b_bytes = fpack(b_data, cx, cy, s);
-            total_bytes = r_bytes + g_bytes + b_bytes;
-            if (1 + total_bytes > byte_limit)
+            while (cy > 0)
             {
-                /* failed */
-                LLOGLN(0, ("xrdp_bitmap32_compress: too big, rgb "
-                       "bytes %d %d %d total_bytes %d cx %d cy %d "
-                       "byte_limit %d", r_bytes, g_bytes, b_bytes,
-                       total_bytes, cx, cy, byte_limit));
-                return 0;
-            }
-            max_bytes = cx * cy * 3;
-            if (total_bytes > max_bytes)
-            {
-                /* raw is better */
-                LLOGLN(10, ("xrdp_bitmap32_compress: too big, rgb "
-                       "bytes %d %d %d total_bytes %d cx %d cy %d "
-                       "max_bytes %d", r_bytes, g_bytes, b_bytes,
-                       total_bytes, cx, cy, max_bytes));
-                init_stream(s, 0);
-                foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
+                s->p = hold_p;
+                out_uint8(s, header);
+                r_bytes = fpack(r_data, cx, cy, s);
+                g_bytes = fpack(g_data, cx, cy, s);
+                b_bytes = fpack(b_data, cx, cy, s);
+                max_bytes = cx * cy * 3;
+                total_bytes = r_bytes + g_bytes + b_bytes;
+                if (total_bytes > max_bytes)
+                {
+                    if (2 + max_bytes <= byte_limit)
+                    {
+                        s->p = hold_p;
+                        foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
+                        break;
+                    }
+                }
+                if (1 + total_bytes <= byte_limit)
+                {
+                    break;
+                }
+                cy--;
             }
         }
         else
         {
-            foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
+            while (cy > 0)
+            {
+                max_bytes = cx * cy * 3;
+                if (2 + max_bytes <= byte_limit)
+                {
+                    s->p = hold_p;
+                    foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
+                    break;
+                }
+                cy--;
+            }
         }
     }
     else
@@ -500,36 +511,45 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
             fdelta(sr_data, r_data, cx, cy);
             fdelta(sg_data, g_data, cx, cy);
             fdelta(sb_data, b_data, cx, cy);
-            out_uint8(s, header);
-            a_bytes = fpack(a_data, cx, cy, s);
-            r_bytes = fpack(r_data, cx, cy, s);
-            g_bytes = fpack(g_data, cx, cy, s);
-            b_bytes = fpack(b_data, cx, cy, s);
-            max_bytes = cx * cy * 4;
-            total_bytes = a_bytes + r_bytes + g_bytes + b_bytes;
-            if (1 + total_bytes > byte_limit)
+            while (cy > 0)
             {
-                /* failed */
-                LLOGLN(0, ("xrdp_bitmap32_compress: too big, argb "
-                       "bytes %d %d %d %d total_bytes %d cx %d cy %d "
-                       "byte_limit %d", a_bytes, r_bytes, g_bytes, b_bytes,
-                       total_bytes, cx, cy, byte_limit));
-                return 0;
-            }
-            if (total_bytes > max_bytes)
-            {
-                /* raw is better */
-                LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb "
-                       "bytes %d %d %d %d total_bytes %d cx %d cy %d "
-                       "max_bytes %d", a_bytes, r_bytes, g_bytes, b_bytes,
-                       total_bytes, cx, cy, max_bytes));
-                init_stream(s, 0);
-                foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
+                s->p = hold_p;
+                out_uint8(s, header);
+                a_bytes = fpack(a_data, cx, cy, s);
+                r_bytes = fpack(r_data, cx, cy, s);
+                g_bytes = fpack(g_data, cx, cy, s);
+                b_bytes = fpack(b_data, cx, cy, s);
+                max_bytes = cx * cy * 4;
+                total_bytes = a_bytes + r_bytes + g_bytes + b_bytes;
+                if (total_bytes > max_bytes)
+                {
+                    if (2 + max_bytes <= byte_limit)
+                    {
+                        s->p = hold_p;
+                        foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
+                        break;
+                    }
+                }
+                if (1 + total_bytes <= byte_limit)
+                {
+                    break;
+                }
+                cy--;
             }
         }
         else
         {
-            foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
+            while (cy > 0)
+            {
+                max_bytes = cx * cy * 4;
+                if (2 + max_bytes <= byte_limit)
+                {
+                    s->p = hold_p;
+                    foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
+                    break;
+                }
+                cy--;
+            }
         }
     }
     return cy;
diff --git a/libxrdp/xrdp_bitmap_compress.c b/libxrdp/xrdp_bitmap_compress.c
index 03c56f10..56898776 100644
--- a/libxrdp/xrdp_bitmap_compress.c
+++ b/libxrdp/xrdp_bitmap_compress.c
@@ -22,6 +22,8 @@
 
 #include "libxrdp.h"
 
+#define BC_MAX_BYTES (16 * 1024)
+
 /*****************************************************************************/
 #define IN_PIXEL8(in_ptr, in_x, in_y, in_w, in_last_pixel, in_pixel); \
     do { \
@@ -695,7 +697,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
         out_count = end;
         line = in_data + width * start_line;
 
-        while (start_line >= 0 && out_count < 32768)
+        while (start_line >= 0 && out_count <= BC_MAX_BYTES)
         {
             i = (s->p - s->data) + count;
 
@@ -987,7 +989,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
         out_count = end * 2;
         line = in_data + width * start_line * 2;
 
-        while (start_line >= 0 && out_count < 32768)
+        while (start_line >= 0 && out_count <= BC_MAX_BYTES)
         {
             i = (s->p - s->data) + count * 2;
 
@@ -1279,7 +1281,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
         out_count = end * 3;
         line = in_data + width * start_line * 4;
 
-        while (start_line >= 0 && out_count < 32768)
+        while (start_line >= 0 && out_count <= BC_MAX_BYTES)
         {
             i = (s->p - s->data) + count * 3;
 
diff --git a/libxrdp/xrdp_orders.c b/libxrdp/xrdp_orders.c
index 18a5ad38..b0c28ed8 100644
--- a/libxrdp/xrdp_orders.c
+++ b/libxrdp/xrdp_orders.c
@@ -2235,6 +2235,11 @@ xrdp_orders_send_raw_bitmap(struct xrdp_orders *self,
 
     Bpp = (bpp + 7) / 8;
     bufsize = (width + e) * height * Bpp;
+    while (bufsize + 16 > MAX_ORDERS_SIZE)
+    {
+        height--;
+        bufsize = (width + e) * height * Bpp;
+    }
     if (xrdp_orders_check(self, bufsize + 16) != 0)
     {
         return 1;
@@ -2254,33 +2259,58 @@ xrdp_orders_send_raw_bitmap(struct xrdp_orders *self,
     out_uint16_le(self->out_s, bufsize);
     out_uint16_le(self->out_s, cache_idx);
 
-    for (i = height - 1; i >= 0; i--)
+    if (Bpp == 4)
     {
-        for (j = 0; j < width; j++)
+        for (i = height - 1; i >= 0; i--)
         {
-            if (Bpp == 3)
+            for (j = 0; j < width; j++)
             {
                 pixel = GETPIXEL32(data, j, i, width);
                 out_uint8(self->out_s, pixel);
                 out_uint8(self->out_s, pixel >> 8);
                 out_uint8(self->out_s, pixel >> 16);
+                out_uint8(self->out_s, pixel >> 24);
             }
-            else if (Bpp == 2)
+            out_uint8s(self->out_s, Bpp * e);
+        }
+    }
+    else if (Bpp == 3)
+    {
+        for (i = height - 1; i >= 0; i--)
+        {
+            for (j = 0; j < width; j++)
             {
-                pixel = GETPIXEL16(data, j, i, width);
+                pixel = GETPIXEL32(data, j, i, width);
                 out_uint8(self->out_s, pixel);
                 out_uint8(self->out_s, pixel >> 8);
+                out_uint8(self->out_s, pixel >> 16);
             }
-            else if (Bpp == 1)
+            out_uint8s(self->out_s, Bpp * e);
+        }
+    }
+    else if (Bpp == 2)
+    {
+        for (i = height - 1; i >= 0; i--)
+        {
+            for (j = 0; j < width; j++)
             {
-                pixel = GETPIXEL8(data, j, i, width);
+                pixel = GETPIXEL16(data, j, i, width);
                 out_uint8(self->out_s, pixel);
+                out_uint8(self->out_s, pixel >> 8);
             }
+            out_uint8s(self->out_s, Bpp * e);
         }
-
-        for (j = 0; j < e; j++)
+    }
+    else if (Bpp == 1)
+    {
+        for (i = height - 1; i >= 0; i--)
         {
-            out_uint8s(self->out_s, Bpp);
+            for (j = 0; j < width; j++)
+            {
+                pixel = GETPIXEL8(data, j, i, width);
+                out_uint8(self->out_s, pixel);
+            }
+            out_uint8s(self->out_s, Bpp * e);
         }
     }
 
@@ -2334,20 +2364,19 @@ xrdp_orders_send_bitmap(struct xrdp_orders *self,
     if (bpp > 24)
     {
         lines_sending = xrdp_bitmap32_compress(data, width, height, s,
-                                               bpp, 16384,
+                                               bpp, MAX_ORDERS_SIZE,
                                                i - 1, temp_s, e, 0x10);
     }
     else
     {
-        lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384,
+        lines_sending = xrdp_bitmap_compress(data, width, height, s,
+                                             bpp, MAX_ORDERS_SIZE,
                                              i - 1, temp_s, e);
     }
 
     if (lines_sending != height)
     {
-        g_writeln("error in xrdp_orders_send_bitmap, lines_sending(%d) != \
-height(%d)", lines_sending, height);
-        return 1;
+        height = lines_sending;
     }
 
     bufsize = (int)(s->p - p);
@@ -2458,6 +2487,7 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
     int pixel = 0;
     int e = 0;
 
+    g_writeln("xrdp_orders_send_raw_bitmap2:");
     if (width > 64)
     {
         g_writeln("error, width > 64");
@@ -2479,6 +2509,11 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
 
     Bpp = (bpp + 7) / 8;
     bufsize = (width + e) * height * Bpp;
+    while (bufsize + 14 > MAX_ORDERS_SIZE)
+    {
+        height--;
+        bufsize = (width + e) * height * Bpp;
+    }
     if (xrdp_orders_check(self, bufsize + 14) != 0)
     {
         return 1;
@@ -2499,7 +2534,7 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
     i = cache_idx & 0xff;
     out_uint8(self->out_s, i);
 
-    if (1 && Bpp == 3)
+    if (Bpp == 4)
     {
         for (i = height - 1; i >= 0; i--)
         {
@@ -2509,44 +2544,49 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
                 out_uint8(self->out_s, pixel);
                 out_uint8(self->out_s, pixel >> 8);
                 out_uint8(self->out_s, pixel >> 16);
+                out_uint8(self->out_s, pixel >> 24);
             }
-            for (j = 0; j < e; j++)
-            {
-                out_uint8s(self->out_s, Bpp);
-            }
+            out_uint8s(self->out_s, Bpp * e);
         }
     }
-    else
-    {
-    for (i = height - 1; i >= 0; i--)
+    else if (Bpp == 3)
     {
-        for (j = 0; j < width; j++)
+        for (i = height - 1; i >= 0; i--)
         {
-            if (Bpp == 3)
+            for (j = 0; j < width; j++)
             {
                 pixel = GETPIXEL32(data, j, i, width);
                 out_uint8(self->out_s, pixel);
                 out_uint8(self->out_s, pixel >> 8);
                 out_uint8(self->out_s, pixel >> 16);
             }
-            else if (Bpp == 2)
+            out_uint8s(self->out_s, Bpp * e);
+        }
+    }
+    else if (Bpp == 2)
+    {
+        for (i = height - 1; i >= 0; i--)
+        {
+            for (j = 0; j < width; j++)
             {
                 pixel = GETPIXEL16(data, j, i, width);
                 out_uint8(self->out_s, pixel);
                 out_uint8(self->out_s, pixel >> 8);
             }
-            else if (Bpp == 1)
+            out_uint8s(self->out_s, Bpp * e);
+        }
+    }
+    else if (Bpp == 1)
+    {
+        for (i = height - 1; i >= 0; i--)
+        {
+            for (j = 0; j < width; j++)
             {
                 pixel = GETPIXEL8(data, j, i, width);
                 out_uint8(self->out_s, pixel);
             }
+            out_uint8s(self->out_s, Bpp * e);
         }
-
-        for (j = 0; j < e; j++)
-        {
-            out_uint8s(self->out_s, Bpp);
-        }
-    }
     }
 
     return 0;
@@ -2599,20 +2639,19 @@ xrdp_orders_send_bitmap2(struct xrdp_orders *self,
     if (bpp > 24)
     {
         lines_sending = xrdp_bitmap32_compress(data, width, height, s,
-                                               bpp, 16384,
+                                               bpp, MAX_ORDERS_SIZE,
                                                i - 1, temp_s, e, 0x10);
     }
     else
     {
-        lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384,
+        lines_sending = xrdp_bitmap_compress(data, width, height, s,
+                                             bpp, MAX_ORDERS_SIZE,
                                              i - 1, temp_s, e);
     }
 
     if (lines_sending != height)
     {
-        g_writeln("error in xrdp_orders_send_bitmap2, lines_sending(%d) != \
-height(%d)", lines_sending, height);
-        return 1;
+        height = lines_sending;
     }
 
     bufsize = (int)(s->p - p);