work on codec mode jpeg

5 files changed, 89 insertions, 31 deletions

diff --git a/xrdp/xrdp.h b/xrdp/xrdp.h
index 0812a74f..32129c01 100644
--- a/xrdp/xrdp.h
+++ b/xrdp/xrdp.h
@@ -404,7 +404,8 @@ server_composite(struct xrdp_mod* mod, int srcidx, int srcformat, int srcwidth,
 int DEFAULT_CC
 server_paint_rects(struct xrdp_mod* mod, int num_drects, short *drects,
                    int num_crects, short *crects,
-                   char *data, int width, int height, int flags);
+                   char *data, int width, int height,
+                   int flags, int frame_id);
 int DEFAULT_CC
 server_set_pointer(struct xrdp_mod* mod, int x, int y,
                    char* data, char* mask);
diff --git a/xrdp/xrdp_cache.c b/xrdp/xrdp_cache.c
index 9c8098f6..834271db 100644
--- a/xrdp/xrdp_cache.c
+++ b/xrdp/xrdp_cache.c
@@ -79,7 +79,7 @@ xrdp_cache_reset_crc(struct xrdp_cache *self)
     {
         for (jndex = 0; jndex < 64 * 1024; jndex++)
         {
-            /* it's ok it deinit a zero'ed out struct list16 */
+            /* it's ok to deinit a zero'ed out struct list16 */
             list16_deinit(&(self->crc16[index][jndex]));
             list16_init(&(self->crc16[index][jndex]));
         }
diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c
index 02b371ec..2ae5381f 100644
--- a/xrdp/xrdp_encoder.c
+++ b/xrdp/xrdp_encoder.c
@@ -45,6 +45,7 @@ int APP_CC
 init_xrdp_encoder(struct xrdp_mm *self)
 {
     char buf[1024];
+    int pid;
 
     LLOGLN(0, ("init_xrdp_encoder: initing encoder"));
 
@@ -58,10 +59,14 @@ init_xrdp_encoder(struct xrdp_mm *self)
     self->fifo_processed = fifo_create();
     self->mutex = tc_mutex_create();
 
+    pid = g_getpid();
     /* setup wait objects for signalling */
-    g_snprintf(buf, 1024, "xrdp_encoder_%8.8x", g_getpid());
+    g_snprintf(buf, 1024, "xrdp_%8.8x_encoder_event_to_proc", pid);
     self->xrdp_encoder_event_to_proc = g_create_wait_obj(buf);
+    g_snprintf(buf, 1024, "xrdp_%8.8x_encoder_event_processed", pid);
     self->xrdp_encoder_event_processed = g_create_wait_obj(buf);
+    g_snprintf(buf, 1024, "xrdp_%8.8x_encoder_term", pid);
+    self->xrdp_encoder_term = g_create_wait_obj(buf);
 
     /* create thread to process messages */
     tc_thread_create(proc_enc_msg, self);
@@ -72,11 +77,12 @@ init_xrdp_encoder(struct xrdp_mm *self)
 /**
  * Deinit xrdp encoder
  *****************************************************************************/
-
+/* called from main thread */
 void APP_CC
 deinit_xrdp_encoder(struct xrdp_mm *self)
 {
     XRDP_ENC_DATA *enc;
+    XRDP_ENC_DATA_DONE *enc_done;
     FIFO          *fifo;
 
     LLOGLN(0, ("deinit_xrdp_encoder: deiniting encoder"));
@@ -86,9 +92,18 @@ deinit_xrdp_encoder(struct xrdp_mm *self)
         return;
     }
 
+    if (self->in_codec_mode == 0)
+    {
+        return;
+    }
+    /* tell worker thread to shut down */
+    g_set_wait_obj(self->xrdp_encoder_term);
+    g_sleep(1000);
+
     /* destroy wait objects used for signalling */
     g_delete_wait_obj(self->xrdp_encoder_event_to_proc);
     g_delete_wait_obj(self->xrdp_encoder_event_processed);
+    g_delete_wait_obj(self->xrdp_encoder_term);
 
     /* cleanup fifo_to_proc */
     fifo = self->fifo_to_proc;
@@ -97,9 +112,10 @@ deinit_xrdp_encoder(struct xrdp_mm *self)
         while (!fifo_is_empty(fifo))
         {
             enc = fifo_remove_item(fifo);
-            if (!enc)
+            if (enc == 0)
+            {
                 continue;
-
+            }
             g_free(enc->drects);
             g_free(enc->crects);
             g_free(enc);
@@ -114,14 +130,13 @@ deinit_xrdp_encoder(struct xrdp_mm *self)
     {
         while (!fifo_is_empty(fifo))
         {
-            enc = fifo_remove_item(fifo);
-            if (!enc)
+            enc_done = fifo_remove_item(fifo);
+            if (enc == 0)
             {
                 continue;
             }
-            g_free(enc->drects);
-            g_free(enc->crects);
-            g_free(enc);
+            g_free(enc_done->comp_pad_data);
+            g_free(enc_done);
         }
         fifo_delete(fifo);
     }
@@ -139,6 +154,7 @@ process_enc(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
     int                  quality;
     int                  error;
     int                  out_data_bytes;
+    int                  count;
     char                *out_data;
     XRDP_ENC_DATA_DONE  *enc_done;
     FIFO                *fifo_processed;
@@ -150,29 +166,37 @@ process_enc(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
     fifo_processed = self->fifo_processed;
     mutex = self->mutex;
     event_processed = self->xrdp_encoder_event_processed;
-    for (index = 0; index < enc->num_crects; index++)
+    count = enc->num_crects;
+    for (index = 0; index < count; index++)
     {
         x = enc->crects[index * 4 + 0];
         y = enc->crects[index * 4 + 1];
         cx = enc->crects[index * 4 + 2];
         cy = enc->crects[index * 4 + 3];
+        if (cx < 1 || cy < 1)
+        {
+            LLOGLN(0, ("process_enc: error 1"));
+            continue;
+        }
         out_data_bytes = MAX((cx + 4) * cy * 4, 8192);
         if ((out_data_bytes < 1) || (out_data_bytes > 16 * 1024 * 1024))
         {
-            LLOGLN(0, ("process_enc: error"));
+            LLOGLN(0, ("process_enc: error 2"));
             return 1;
         }
-        out_data = (char *) g_malloc(out_data_bytes + 256, 0);
+        out_data = (char *) g_malloc(out_data_bytes + 256 + 2, 0);
         if (out_data == 0)
         {
-            LLOGLN(0, ("process_enc: error"));
+            LLOGLN(0, ("process_enc: error 3"));
             return 1;
         }
+        out_data[256] = 0; /* header bytes */
+        out_data[257] = 0;
         error = libxrdp_codec_jpeg_compress(self->wm->session, 0, enc->data,
                                             enc->width, enc->height,
                                             enc->width * 4, x, y, cx, cy,
                                             quality,
-                                            out_data + 256, &out_data_bytes);
+                                            out_data + 256 + 2, &out_data_bytes);
         if (error < 0)
         {
             LLOGLN(0, ("process_enc: jpeg error %d bytes %d",
@@ -183,7 +207,7 @@ process_enc(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
         LLOGLN(10, ("jpeg error %d bytes %d", error, out_data_bytes));
         enc_done = (XRDP_ENC_DATA_DONE *)
                    g_malloc(sizeof(XRDP_ENC_DATA_DONE), 1);
-        enc_done->comp_bytes = out_data_bytes;
+        enc_done->comp_bytes = out_data_bytes + 2;
         enc_done->pad_bytes = 256;
         enc_done->comp_pad_data = out_data;
         enc_done->enc = enc;
@@ -213,6 +237,7 @@ proc_enc_msg(void *arg)
     tbus            mutex;
     tbus            event_to_proc;
     tbus            term_obj;
+    tbus            lterm_obj;
     int             robjs_count;
     int             wobjs_count;
     int             cont;
@@ -235,6 +260,7 @@ proc_enc_msg(void *arg)
     event_to_proc = self->xrdp_encoder_event_to_proc;
 
     term_obj = g_get_term_event();
+    lterm_obj = self->xrdp_encoder_term;
 
     cont = 1;
     while (cont)
@@ -243,6 +269,7 @@ proc_enc_msg(void *arg)
         robjs_count = 0;
         wobjs_count = 0;
         robjs[robjs_count++] = term_obj;
+        robjs[robjs_count++] = lterm_obj;
         robjs[robjs_count++] = event_to_proc;
 
         if (g_obj_wait(robjs, robjs_count, wobjs, wobjs_count, timeout) != 0)
@@ -251,8 +278,15 @@ proc_enc_msg(void *arg)
             g_sleep(100);
         }
 
-        if (g_is_wait_obj_set(term_obj)) /* term */
+        if (g_is_wait_obj_set(term_obj)) /* global term */
+        {
+            LLOGLN(0, ("proc_enc_msg: global term"));
+            break;
+        }
+
+        if (g_is_wait_obj_set(lterm_obj)) /* xrdp_mm term */
         {
+            LLOGLN(0, ("proc_enc_msg: xrdp_mm term"));
             break;
         }
 
diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c
index e7e338a5..58032a1e 100644
--- a/xrdp/xrdp_mm.c
+++ b/xrdp/xrdp_mm.c
@@ -55,10 +55,13 @@ xrdp_mm_create(struct xrdp_wm *owner)
     self->login_values = list_create();
     self->login_values->auto_free = 1;
 
-    /* setup thread to handle codec mode messages */
-    init_xrdp_encoder(self);
+    self->in_codec_mode = 0; /* TODO: */
 
-    //self->in_codec_mode = 1;
+    if (self->in_codec_mode)
+    {
+    /* setup thread to handle codec mode messages */
+        init_xrdp_encoder(self);
+    }
 
     return self;
 }
@@ -90,6 +93,7 @@ xrdp_mm_module_cleanup(struct xrdp_mm *self)
 {
     log_message(LOG_LEVEL_DEBUG,"xrdp_mm_module_cleanup");
 
+    /* shutdown thread */
     deinit_xrdp_encoder(self);
 
     if (self->mod != 0)
@@ -1911,6 +1915,10 @@ xrdp_mm_check_wait_objs(struct xrdp_mm *self)
 {
     XRDP_ENC_DATA_DONE *enc_done;
     int rv;
+    int x;
+    int y;
+    int cx;
+    int cy;
 
     if (self == 0)
     {
@@ -1971,7 +1979,7 @@ xrdp_mm_check_wait_objs(struct xrdp_mm *self)
             while (enc_done != 0)
             {
                 /* do something with msg */
-                LLOGLN(0, ("xrdp_mm_check_wait_objs: message back bytes %d",
+                LLOGLN(10, ("xrdp_mm_check_wait_objs: message back bytes %d",
                        enc_done->comp_bytes));
                 if (0)
                 {
@@ -1985,18 +1993,23 @@ xrdp_mm_check_wait_objs(struct xrdp_mm *self)
                     g_file_write(ii, enc_done->comp_pad_data + enc_done->pad_bytes, enc_done->comp_bytes);
                     g_file_close(ii);
                 }
-                
+
+                x = enc_done->enc->crects[enc_done->index * 4 + 0];
+                y = enc_done->enc->crects[enc_done->index * 4 + 1];
+                cx = enc_done->enc->crects[enc_done->index * 4 + 2];
+                cy = enc_done->enc->crects[enc_done->index * 4 + 3];
                 libxrdp_fastpath_send_surface(self->wm->session,
                                               enc_done->comp_pad_data,
                                               enc_done->pad_bytes,
                                               enc_done->comp_bytes,
-                                              0, 0, 0, 0, 32, 99, 0, 0);
-                
-                
+                                              x, y, x + cx, y + cy,
+                                              32, 2, cx, cy);
+
                 /* free enc_done */
                 if (enc_done->last)
                 {
                     LLOGLN(10, ("xrdp_mm_check_wait_objs: last set"));
+                    self->mod->mod_frame_ack(self->mod, enc_done->enc->flags, enc_done->enc->frame_id);
                     g_free(enc_done->enc->drects);
                     g_free(enc_done->enc->crects);
                     g_free(enc_done->enc);
@@ -2220,7 +2233,7 @@ server_composite(struct xrdp_mod* mod, int srcidx, int srcformat,
 int DEFAULT_CC
 server_paint_rects(struct xrdp_mod* mod, int num_drects, short *drects,
                    int num_crects, short *crects, char *data, int width,
-                   int height, int flags)
+                   int height, int flags, int frame_id)
 {
     struct xrdp_wm* wm;
     struct xrdp_mm* mm;
@@ -2239,7 +2252,7 @@ server_paint_rects(struct xrdp_mod* mod, int num_drects, short *drects,
     if (mm->in_codec_mode)
     {
         /* copy formal params to XRDP_ENC_DATA */
-        enc_data = (XRDP_ENC_DATA *) g_malloc(sizeof(XRDP_ENC_DATA), 0);
+        enc_data = (XRDP_ENC_DATA *) g_malloc(sizeof(XRDP_ENC_DATA), 1);
         if (enc_data == 0)
         {
             return 1;
@@ -2272,6 +2285,11 @@ server_paint_rects(struct xrdp_mod* mod, int num_drects, short *drects,
         enc_data->width = width;
         enc_data->height = height;
         enc_data->flags = flags;
+        enc_data->frame_id = frame_id;
+        if (width == 0 || height == 0)
+        {
+            LLOGLN(10, ("server_paint_rects: error"));
+        }
 
         /* insert into fifo for encoder thread to process */
         tc_mutex_lock(mm->mutex);
@@ -2281,7 +2299,7 @@ server_paint_rects(struct xrdp_mod* mod, int num_drects, short *drects,
         /* signal xrdp_encoder thread */
         g_set_wait_obj(mm->xrdp_encoder_event_to_proc);
 
-        //return 0;
+        return 0;
     }
 
     //g_writeln("server_paint_rects:");
@@ -2301,6 +2319,7 @@ server_paint_rects(struct xrdp_mod* mod, int num_drects, short *drects,
         s += 4;
     }
     xrdp_bitmap_delete(b);
+    mm->mod->mod_frame_ack(mm->mod, flags, frame_id);
     return 0;
 }
 
diff --git a/xrdp/xrdp_types.h b/xrdp/xrdp_types.h
index c50daf79..048bff0e 100644
--- a/xrdp/xrdp_types.h
+++ b/xrdp/xrdp_types.h
@@ -47,7 +47,8 @@ struct xrdp_mod
   int (*mod_get_wait_objs)(struct xrdp_mod* v, tbus* read_objs, int* rcount,
                            tbus* write_objs, int* wcount, int* timeout);
   int (*mod_check_wait_objs)(struct xrdp_mod* v);
-  long mod_dumby[100 - 9]; /* align, 100 minus the number of mod
+  int (*mod_frame_ack)(struct xrdp_mod* v, int flags, int frame_id);
+  long mod_dumby[100 - 10]; /* align, 100 minus the number of mod
                               functions above */
   /* server functions */
   int (*server_begin_update)(struct xrdp_mod* v);
@@ -143,7 +144,8 @@ struct xrdp_mod
   int (*server_paint_rects)(struct xrdp_mod* v,
                             int num_drects, short *drects,
                             int num_crects, short *crects,
-                            char *data, int width, int height, int flags);
+                            char *data, int width, int height,
+                            int flags, int frame_id);
   long server_dumby[100 - 43]; /* align, 100 minus the number of server
                                   functions above */
   /* common */
@@ -292,6 +294,7 @@ struct xrdp_mm
   int   in_codec_mode;
   tbus  xrdp_encoder_event_to_proc;
   tbus  xrdp_encoder_event_processed;
+  tbus  xrdp_encoder_term;
   FIFO *fifo_to_proc;
   FIFO *fifo_processed;
   tbus  mutex;
@@ -621,6 +624,7 @@ struct xrdp_enc_data
     int              width;
     int              height;
     int              flags;
+    int              frame_id;
 };
 
 typedef struct xrdp_enc_data XRDP_ENC_DATA;