diff options
Diffstat (limited to 'libxrdp/xrdp_bitmap32_compress.c')
| -rw-r--r-- | libxrdp/xrdp_bitmap32_compress.c | 122 |
1 files changed, 71 insertions, 51 deletions
diff --git a/libxrdp/xrdp_bitmap32_compress.c b/libxrdp/xrdp_bitmap32_compress.c index 083c4409..daec9f28 100644 --- a/libxrdp/xrdp_bitmap32_compress.c +++ b/libxrdp/xrdp_bitmap32_compress.c @@ -107,7 +107,7 @@ fsplit3(char *in_data, int start_line, int width, int e, } start_line--; cy++; - if (out_index > 64 * 64) + if (out_index + width + e > 64 * 64) { break; } @@ -195,7 +195,7 @@ fsplit4(char *in_data, int start_line, int width, int e, } start_line--; cy++; - if (out_index > 64 * 64) + if (out_index + width + e > 64 * 64) { break; } @@ -422,6 +422,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, char *sr_data; char *sg_data; char *sb_data; + char *hold_p; int a_bytes; int r_bytes; int g_bytes; @@ -449,6 +450,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, r_data = a_data + max_bytes; g_data = r_data + max_bytes; b_data = g_data + max_bytes; + hold_p = s->p; if (header & FLAGS_NOALPHA) { @@ -459,35 +461,44 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, fdelta(sr_data, r_data, cx, cy); fdelta(sg_data, g_data, cx, cy); fdelta(sb_data, b_data, cx, cy); - out_uint8(s, header); - r_bytes = fpack(r_data, cx, cy, s); - g_bytes = fpack(g_data, cx, cy, s); - b_bytes = fpack(b_data, cx, cy, s); - total_bytes = r_bytes + g_bytes + b_bytes; - if (1 + total_bytes > byte_limit) + while (cy > 0) { - /* failed */ - LLOGLN(0, ("xrdp_bitmap32_compress: too big, rgb " - "bytes %d %d %d total_bytes %d cx %d cy %d " - "byte_limit %d", r_bytes, g_bytes, b_bytes, - total_bytes, cx, cy, byte_limit)); - return 0; - } - max_bytes = cx * cy * 3; - if (total_bytes > max_bytes) - { - /* raw is better */ - LLOGLN(10, ("xrdp_bitmap32_compress: too big, rgb " - "bytes %d %d %d total_bytes %d cx %d cy %d " - "max_bytes %d", r_bytes, g_bytes, b_bytes, - total_bytes, cx, cy, max_bytes)); - init_stream(s, 0); - foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); + s->p = hold_p; + out_uint8(s, header); + r_bytes = fpack(r_data, cx, cy, s); + g_bytes = fpack(g_data, cx, cy, s); + b_bytes = fpack(b_data, cx, cy, s); + max_bytes = cx * cy * 3; + total_bytes = r_bytes + g_bytes + b_bytes; + if (total_bytes > max_bytes) + { + if (2 + max_bytes <= byte_limit) + { + s->p = hold_p; + foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); + break; + } + } + if (1 + total_bytes <= byte_limit) + { + break; + } + cy--; } } else { - foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); + while (cy > 0) + { + max_bytes = cx * cy * 3; + if (2 + max_bytes <= byte_limit) + { + s->p = hold_p; + foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); + break; + } + cy--; + } } } else @@ -500,36 +511,45 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, fdelta(sr_data, r_data, cx, cy); fdelta(sg_data, g_data, cx, cy); fdelta(sb_data, b_data, cx, cy); - out_uint8(s, header); - a_bytes = fpack(a_data, cx, cy, s); - r_bytes = fpack(r_data, cx, cy, s); - g_bytes = fpack(g_data, cx, cy, s); - b_bytes = fpack(b_data, cx, cy, s); - max_bytes = cx * cy * 4; - total_bytes = a_bytes + r_bytes + g_bytes + b_bytes; - if (1 + total_bytes > byte_limit) + while (cy > 0) { - /* failed */ - LLOGLN(0, ("xrdp_bitmap32_compress: too big, argb " - "bytes %d %d %d %d total_bytes %d cx %d cy %d " - "byte_limit %d", a_bytes, r_bytes, g_bytes, b_bytes, - total_bytes, cx, cy, byte_limit)); - return 0; - } - if (total_bytes > max_bytes) - { - /* raw is better */ - LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb " - "bytes %d %d %d %d total_bytes %d cx %d cy %d " - "max_bytes %d", a_bytes, r_bytes, g_bytes, b_bytes, - total_bytes, cx, cy, max_bytes)); - init_stream(s, 0); - foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); + s->p = hold_p; + out_uint8(s, header); + a_bytes = fpack(a_data, cx, cy, s); + r_bytes = fpack(r_data, cx, cy, s); + g_bytes = fpack(g_data, cx, cy, s); + b_bytes = fpack(b_data, cx, cy, s); + max_bytes = cx * cy * 4; + total_bytes = a_bytes + r_bytes + g_bytes + b_bytes; + if (total_bytes > max_bytes) + { + if (2 + max_bytes <= byte_limit) + { + s->p = hold_p; + foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); + break; + } + } + if (1 + total_bytes <= byte_limit) + { + break; + } + cy--; } } else { - foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); + while (cy > 0) + { + max_bytes = cx * cy * 4; + if (2 + max_bytes <= byte_limit) + { + s->p = hold_p; + foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); + break; + } + cy--; + } } } return cy; |