work on codec mode jpeg

4 files changed, 291 insertions, 121 deletions

diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c
index 93544196..40c921f1 100644
--- a/xrdp/xrdp_encoder.c
+++ b/xrdp/xrdp_encoder.c
@@ -23,56 +23,73 @@
 #include "thread_calls.h"
 #include "fifo.h"
 
+#define LLOG_LEVEL 1
+#define LLOGLN(_level, _args) \
+  do \
+  { \
+    if (_level < LLOG_LEVEL) \
+    { \
+        g_write("xrdp:xrdp_encoder [%10.10u]: ", g_time3()); \
+        g_writeln _args ; \
+    } \
+  } \
+  while (0)
+
 /**
  * Init encoder
- * 
+ *
  * @return 0 on success, -1 on failure
- *****************************************************************************/ 
- 
-int init_xrdp_encoder(struct xrdp_mm *mm)
+ *****************************************************************************/
+
+int APP_CC
+init_xrdp_encoder(struct xrdp_mm *self)
 {
     char buf[1024];
-    
-    g_writeln("xrdp_encoder.c: initing encoder");
-        
-    if (!mm)
+
+    LLOGLN(0, ("init_xrdp_encoder: initing encoder"));
+
+    if (self == 0)
+    {
         return -1;
-        
-    /* save mm so thread can access it */
-    self = mm;
-    
+    }
+
     /* setup required FIFOs */
     self->fifo_to_proc = fifo_create();
     self->fifo_processed = fifo_create();
     self->mutex = tc_mutex_create();
-    
+
     /* setup wait objects for signalling */
     g_snprintf(buf, 1024, "xrdp_encoder_%8.8x", g_getpid());
-    self->xrdp_encoder_event = g_create_wait_obj(buf);
-    
+    self->xrdp_encoder_event_to_proc = g_create_wait_obj(buf);
+    self->xrdp_encoder_event_processed = g_create_wait_obj(buf);
+
     /* create thread to process messages */
-    tc_thread_create(proc_enc_msg, 0);
-    
+    tc_thread_create(proc_enc_msg, self);
+
     return 0;
 }
 
 /**
  * Deinit xrdp encoder
  *****************************************************************************/
- 
-void deinit_xrdp_encoder()
+
+void APP_CC
+deinit_xrdp_encoder(struct xrdp_mm *self)
 {
     XRDP_ENC_DATA *enc;
     FIFO          *fifo;
-    
-    g_writeln("xrdp_encoder.c: deiniting encoder");
-    
-    if (!self)
+
+    LLOGLN(0, ("deinit_xrdp_encoder: deiniting encoder"));
+
+    if (self == 0)
+    {
         return;
+    }
 
     /* destroy wait objects used for signalling */
-    g_delete_wait_obj(self->xrdp_encoder_event);
-    
+    g_delete_wait_obj(self->xrdp_encoder_event_to_proc);
+    g_delete_wait_obj(self->xrdp_encoder_event_processed);
+
     /* cleanup fifo_to_proc */
     fifo = self->fifo_to_proc;
     if (fifo)
@@ -82,15 +99,15 @@ void deinit_xrdp_encoder()
             enc = fifo_remove_item(fifo);
             if (!enc)
                 continue;
-                
+
             g_free(enc->drects);
             g_free(enc->crects);
             g_free(enc);
         }
-        
+
         fifo_delete(fifo);
     }
-    
+
     /* cleanup fifo_processed */
     fifo = self->fifo_processed;
     if (fifo)
@@ -99,61 +116,157 @@ void deinit_xrdp_encoder()
         {
             enc = fifo_remove_item(fifo);
             if (!enc)
+            {
                 continue;
-                
+            }
             g_free(enc->drects);
             g_free(enc->crects);
             g_free(enc);
         }
-        
         fifo_delete(fifo);
-    }    
+    }
 }
 
-void *proc_enc_msg(void *arg)
+/*****************************************************************************/
+static int
+process_enc(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
 {
-    XRDP_ENC_DATA *enc;
-    FIFO          *fifo_to_proc; 
-    FIFO          *fifo_processed; 
-    tbus           mutex;
-    tbus           event;
-    
-    g_writeln("xrdp_encoder.c: process_enc_msg thread is running");
-    
-    fifo_to_proc = self->fifo_to_proc;
+    int                  index;
+    int                  x;
+    int                  y;
+    int                  cx;
+    int                  cy;
+    int                  quality;
+    int                  error;
+    int                  out_data_bytes;
+    char                *out_data;
+    XRDP_ENC_DATA_DONE  *enc_done;
+    FIFO                *fifo_processed;
+    tbus                 mutex;
+    tbus                 event_processed;
+
+    LLOGLN(10, ("process_enc:"));
+    quality = 75;
     fifo_processed = self->fifo_processed;
     mutex = self->mutex;
-    event = self->xrdp_encoder_event;
-    
-    while (1)
+    event_processed = self->xrdp_encoder_event_processed;
+    for (index = 0; index < enc->num_crects; index++)
     {
-        /* get next msg */
-        tc_mutex_lock(mutex);
-        enc = fifo_remove_item(fifo_to_proc);
-        tc_mutex_unlock(mutex);
-        
-        /* if no msg available, wait for signal */
-        if (!enc)
+        x = enc->crects[index * 4 + 0];
+        y = enc->crects[index * 4 + 1];
+        cx = enc->crects[index * 4 + 2];
+        cy = enc->crects[index * 4 + 3];
+        out_data_bytes = MAX((cx + 4) * cy * 4, 8192);
+        if ((out_data_bytes < 1) || (out_data_bytes > 16 * 1024 * 1024))
         {
-            g_writeln("###### JAY_TODO: waiting for msg....");
-            g_tcp_can_recv(event, 1000);
-            g_reset_wait_obj(event);
-            continue;
+            LLOGLN(0, ("process_enc: error"));
+            return 1;
         }
-        
-        /* process msg in enc obj */
-        g_writeln("###### JAY_TODO: got msg....");
-        /* JAY_TODO */
-        
+        out_data = (char *) g_malloc(out_data_bytes, 0);
+        if (out_data == 0)
+        {
+            LLOGLN(0, ("process_enc: error"));
+            return 1;
+        }
+        error = libxrdp_codec_jpeg_compress(self->wm->session, 0, enc->data,
+                                            enc->width, enc->height,
+                                            enc->width * 4, x, y, cx, cy,
+                                            quality,
+                                            out_data, &out_data_bytes);
+        LLOGLN(10, ("jpeg error %d bytes %d", error, out_data_bytes));
+        enc_done = g_malloc(sizeof(XRDP_ENC_DATA_DONE), 1);
+        enc_done->comp_bytes = out_data_bytes;
+        enc_done->comp_data = out_data;
+        enc_done->enc = enc;
+        enc_done->last = index == (enc->num_crects - 1);
+        enc_done->index = index;
         /* done with msg */
+        /* inform main thread done */
         tc_mutex_lock(mutex);
-        fifo_add_item(fifo_processed, enc);
+        fifo_add_item(fifo_processed, enc_done);
         tc_mutex_unlock(mutex);
-        
-        /* signal completion */
-        g_set_wait_obj(event);
-        
-    } /* end while (1) */
-    
+        /* signal completion for main thread */
+        g_set_wait_obj(event_processed);
+    }
+    return 0;
+}
+
+/**
+ * Init encoder
+ *
+ * @return 0 on success, -1 on failure
+ *****************************************************************************/
+THREAD_RV THREAD_CC
+proc_enc_msg(void *arg)
+{
+    XRDP_ENC_DATA  *enc;
+    FIFO           *fifo_to_proc;
+    tbus            mutex;
+    tbus            event_to_proc;
+    tbus            term_obj;
+    int             robjs_count;
+    int             wobjs_count;
+    int             cont;
+    int             timeout;
+    tbus            robjs[32];
+    tbus            wobjs[32];
+    struct xrdp_mm *self;
+
+    LLOGLN(0, ("proc_enc_msg: thread is running"));
+
+    self = (struct xrdp_mm *) arg;
+    if (self == 0)
+    {
+        LLOGLN(0, ("proc_enc_msg: self nil"));
+        return 0;
+    }
+
+    fifo_to_proc = self->fifo_to_proc;
+    mutex = self->mutex;
+    event_to_proc = self->xrdp_encoder_event_to_proc;
+
+    term_obj = g_get_term_event();
+
+    cont = 1;
+    while (cont)
+    {
+        timeout = -1;
+        robjs_count = 0;
+        wobjs_count = 0;
+        robjs[robjs_count++] = term_obj;
+        robjs[robjs_count++] = event_to_proc;
+
+        if (g_obj_wait(robjs, robjs_count, wobjs, wobjs_count, timeout) != 0)
+        {
+            /* error, should not get here */
+            g_sleep(100);
+        }
+
+        if (g_is_wait_obj_set(term_obj)) /* term */
+        {
+            break;
+        }
+
+        if (g_is_wait_obj_set(event_to_proc))
+        {
+            /* clear it right away */
+            g_reset_wait_obj(event_to_proc);
+            /* get first msg */
+            tc_mutex_lock(mutex);
+            enc = (XRDP_ENC_DATA *) fifo_remove_item(fifo_to_proc);
+            tc_mutex_unlock(mutex);
+            while (enc != 0)
+            {
+                /* do work */
+                process_enc(self, enc);
+                /* get next msg */
+                tc_mutex_lock(mutex);
+                enc = (XRDP_ENC_DATA *) fifo_remove_item(fifo_to_proc);
+                tc_mutex_unlock(mutex);
+            }
+        }
+
+    } /* end while (cont) */
+    LLOGLN(0, ("proc_enc_msg: thread exit"));
     return 0;
 }
diff --git a/xrdp/xrdp_encoder.h b/xrdp/xrdp_encoder.h
index c538a378..1d525f12 100644
--- a/xrdp/xrdp_encoder.h
+++ b/xrdp/xrdp_encoder.h
@@ -1,10 +1,16 @@
+
 #ifndef _XRDP_ENCODER_H
 #define _XRDP_ENCODER_H
 
-struct xrdp_mm *self;
+#include "arch.h"
+
+struct xrdp_mm;
 
-int   init_xrdp_encoder(struct xrdp_mm *mm);
-void  deinit_xrdp_encoder();
-void *proc_enc_msg(void *arg);
+int APP_CC
+init_xrdp_encoder(struct xrdp_mm *self);
+void APP_CC
+deinit_xrdp_encoder(struct xrdp_mm *self);
+THREAD_RV THREAD_CC
+proc_enc_msg(void *arg);
 
 #endif
diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c
index 61bb3c31..daa82c80 100644
--- a/xrdp/xrdp_mm.c
+++ b/xrdp/xrdp_mm.c
@@ -28,6 +28,20 @@
 #endif
 #endif
 
+#include "xrdp_encoder.h"
+
+#define LLOG_LEVEL 1
+#define LLOGLN(_level, _args) \
+  do \
+  { \
+    if (_level < LLOG_LEVEL) \
+    { \
+        g_write("xrdp:xrdp_mm [%10.10u]: ", g_time3()); \
+        g_writeln _args ; \
+    } \
+  } \
+  while (0)
+
 /*****************************************************************************/
 struct xrdp_mm *APP_CC
 xrdp_mm_create(struct xrdp_wm *owner)
@@ -40,10 +54,12 @@ xrdp_mm_create(struct xrdp_wm *owner)
     self->login_names->auto_free = 1;
     self->login_values = list_create();
     self->login_values->auto_free = 1;
-    
+
     /* setup thread to handle codec mode messages */
     init_xrdp_encoder(self);
-    
+
+    //self->in_codec_mode = 1;
+
     return self;
 }
 
@@ -1881,9 +1897,9 @@ xrdp_mm_get_wait_objs(struct xrdp_mm *self,
 
     if (self->in_codec_mode)
     {
-        read_objs[(*rcount)++] = self->xrdp_encoder_event;
+        read_objs[(*rcount)++] = self->xrdp_encoder_event_processed;
     }
-    
+
     return rv;
 }
 
@@ -1891,6 +1907,7 @@ xrdp_mm_get_wait_objs(struct xrdp_mm *self,
 int APP_CC
 xrdp_mm_check_wait_objs(struct xrdp_mm *self)
 {
+    XRDP_ENC_DATA_DONE *enc_done;
     int rv;
 
     if (self == 0)
@@ -1942,33 +1959,47 @@ xrdp_mm_check_wait_objs(struct xrdp_mm *self)
 
     if (self->in_codec_mode)
     {
-        XRDP_ENC_DATA *enc;
-        
-        if (!g_is_wait_obj_set(self->xrdp_encoder_event))
-            return rv;
-            
-        g_reset_wait_obj(self->xrdp_encoder_event);
-        tc_mutex_lock(self->mutex);
-        
-        while ((enc = fifo_remove_item(self->fifo_processed)) != 0)
+        if (g_is_wait_obj_set(self->xrdp_encoder_event_processed))
         {
-            tc_mutex_unlock(self->mutex);
-            
-            /* do something with msg */
-            
-            /* JAY_TODO */
-            
-            /* free enc */
-            g_free(enc->drects);
-            g_free(enc->crects);
-            g_free(enc);   
-                         
+            g_reset_wait_obj(self->xrdp_encoder_event_processed);
             tc_mutex_lock(self->mutex);
+            enc_done = (XRDP_ENC_DATA_DONE*)
+                       fifo_remove_item(self->fifo_processed);
+            tc_mutex_unlock(self->mutex);
+            while (enc_done != 0)
+            {
+                /* do something with msg */
+                LLOGLN(0, ("xrdp_mm_check_wait_objs: message back bytes %d",
+                       enc_done->comp_bytes));
+                if (0)
+                {
+                    tbus ii;
+                    static int jj;
+                    char text[256];
+
+                    g_snprintf(text, 255, "/tmp/jj0x%8.8x.jpg", jj);
+                    jj++;
+                    ii = g_file_open(text);
+                    g_file_write(ii, enc_done->comp_data, enc_done->comp_bytes);
+                    g_file_close(ii);
+                }
+                /* free enc_done */
+                if (enc_done->last)
+                {
+                    LLOGLN(10, ("xrdp_mm_check_wait_objs: last set"));
+                    g_free(enc_done->enc->drects);
+                    g_free(enc_done->enc->crects);
+                    g_free(enc_done->enc);
+                }
+                g_free(enc_done->comp_data);
+                g_free(enc_done);
+                tc_mutex_lock(self->mutex);
+                enc_done = (XRDP_ENC_DATA_DONE*)
+                           fifo_remove_item(self->fifo_processed);
+                tc_mutex_unlock(self->mutex);
+            }
         }
-        
-        tc_mutex_unlock(self->mutex);
     }
-    
     return rv;
 }
 
@@ -2178,7 +2209,7 @@ server_composite(struct xrdp_mod* mod, int srcidx, int srcformat,
 /*****************************************************************************/
 int DEFAULT_CC
 server_paint_rects(struct xrdp_mod* mod, int num_drects, short *drects,
-                   int num_crects, short *crects, char *data, int width, 
+                   int num_crects, short *crects, char *data, int width,
                    int height, int flags)
 {
     struct xrdp_wm* wm;
@@ -2187,33 +2218,40 @@ server_paint_rects(struct xrdp_mod* mod, int num_drects, short *drects,
     struct xrdp_bitmap *b;
     short *s;
     int index;
-    int is_empty;
+    XRDP_ENC_DATA *enc_data;
 
     wm = (struct xrdp_wm*)(mod->wm);
     mm = wm->mm;
-    
+
+    LLOGLN(10, ("server_paint_rects:"));
+    LLOGLN(10, ("server_paint_rects: %d", mm->in_codec_mode));
+
     if (mm->in_codec_mode)
     {
         /* copy formal params to XRDP_ENC_DATA */
-        XRDP_ENC_DATA *enc_data = (XRDP_ENC_DATA *) g_malloc(sizeof(XRDP_ENC_DATA), 0);
-        if (!enc_data)
+        enc_data = (XRDP_ENC_DATA *) g_malloc(sizeof(XRDP_ENC_DATA), 0);
+        if (enc_data == 0)
+        {
             return 1;
-            
-        enc_data->drects = g_malloc(sizeof(short) * num_drects * 4, 0);
-        if (!enc_data->drects)
+        }
+
+        enc_data->drects = (short *)
+                           g_malloc(sizeof(short) * num_drects * 4, 0);
+        if (enc_data->drects == 0)
         {
             g_free(enc_data);
             return 1;
         }
-        
-        enc_data->crects = g_malloc(sizeof(short) * num_crects * 4, 0);
-        if (!enc_data->crects)
+
+        enc_data->crects = (short *)
+                           g_malloc(sizeof(short) * num_crects * 4, 0);
+        if (enc_data->crects == 0)
         {
             g_free(enc_data);
             g_free(enc_data->drects);
             return 1;
         }
-        
+
         g_memcpy(enc_data->drects, drects, sizeof(short) * num_drects * 4);
         g_memcpy(enc_data->crects, crects, sizeof(short) * num_crects * 4);
 
@@ -2224,22 +2262,20 @@ server_paint_rects(struct xrdp_mod* mod, int num_drects, short *drects,
         enc_data->width = width;
         enc_data->height = height;
         enc_data->flags = flags;
-        
+
         /* insert into fifo for encoder thread to process */
         tc_mutex_lock(mm->mutex);
-        is_empty = fifo_is_empty(mm->fifo_to_proc);
         fifo_add_item(mm->fifo_to_proc, (void *) enc_data);
         tc_mutex_unlock(mm->mutex);
-    
+
         /* signal xrdp_encoder thread */
-        if (is_empty)
-            g_set_wait_obj(mm->xrdp_encoder_event);
-        
-        return 0;
+        g_set_wait_obj(mm->xrdp_encoder_event_to_proc);
+
+        //return 0;
     }
 
     //g_writeln("server_paint_rects:");
-    
+
     p = (struct xrdp_painter*)(mod->painter);
     if (p == 0)
     {
diff --git a/xrdp/xrdp_types.h b/xrdp/xrdp_types.h
index 00cf435f..3ac1b752 100644
--- a/xrdp/xrdp_types.h
+++ b/xrdp/xrdp_types.h
@@ -287,10 +287,11 @@ struct xrdp_mm
   int chan_trans_up; /* true once connected to chansrv */
   int delete_chan_trans; /* boolean set when done with channel connection */
   int usechansrv; /* true if chansrvport is set in xrdp.ini or using sesman */
-  
+
   /* for codec mode operations */
   int   in_codec_mode;
-  tbus  xrdp_encoder_event;
+  tbus  xrdp_encoder_event_to_proc;
+  tbus  xrdp_encoder_event_processed;
   FIFO *fifo_to_proc;
   FIFO *fifo_processed;
   tbus  mutex;
@@ -619,9 +620,23 @@ struct xrdp_enc_data
     char            *data;
     int              width;
     int              height;
-    int              flags;  
+    int              flags;
 };
 
 typedef struct xrdp_enc_data XRDP_ENC_DATA;
 
+/* used when scheduling tasks from xrdp_encoder.c */
+struct xrdp_enc_data_done
+{
+    int                   comp_bytes;
+    char                 *comp_data;
+    struct xrdp_enc_data *enc;
+    int                   last; /* true is this is last message for enc */
+    int                   index; /* depends on codec */
+};
+
+typedef struct xrdp_enc_data_done XRDP_ENC_DATA_DONE;
+
+
+
 #endif