codecs: move jpeg hack to bitmap cache v3

4 files changed, 165 insertions, 109 deletions

diff --git a/libxrdp/libxrdp.h b/libxrdp/libxrdp.h
index 82f7eed1..b066db95 100644
--- a/libxrdp/libxrdp.h
+++ b/libxrdp/libxrdp.h
@@ -411,7 +411,7 @@ int APP_CC
 xrdp_jpeg_compress(char* in_data, int width, int height,
                    struct stream* s, int bpp, int byte_limit,
                    int start_line, struct stream* temp_s,
-                   int e);
+                   int e, int quality);
 
 /* xrdp_channel.c */
 struct xrdp_channel* APP_CC
diff --git a/libxrdp/xrdp_jpeg_compress.c b/libxrdp/xrdp_jpeg_compress.c
index 48f12359..9b9a9e1d 100644
--- a/libxrdp/xrdp_jpeg_compress.c
+++ b/libxrdp/xrdp_jpeg_compress.c
@@ -148,9 +148,8 @@ jp_do_compress(char* data, int width, int height, int bpp, int quality,
 /*****************************************************************************/
 static int APP_CC
 jpeg_compress(char* in_data, int width, int height,
-              struct stream* s, int bpp, int byte_limit,
-              int start_line, struct stream* temp_s,
-              int e)
+              struct stream* s, struct stream* temp_s, int bpp,
+              int byte_limit, int e, int quality)
 {
   char* data;
   tui32* src32;
@@ -164,7 +163,7 @@ jpeg_compress(char* in_data, int width, int height,
   int i;
   int cdata_bytes;
 
-  data = g_malloc((width + e) * height * 3, 0);
+  data = temp_s->data;
   dst8 = data;
   if (bpp == 24)
   {
@@ -192,9 +191,8 @@ jpeg_compress(char* in_data, int width, int height,
     g_writeln("bpp wrong %d", bpp);
   }
   cdata_bytes = byte_limit;
-  jp_do_compress(data, width + e, height, 24, JP_QUALITY, s->p, &cdata_bytes);
+  jp_do_compress(data, width + e, height, 24, quality, s->p, &cdata_bytes);
   s->p += cdata_bytes;
-  g_free(data);
   return cdata_bytes;
 }
 
@@ -203,10 +201,10 @@ int APP_CC
 xrdp_jpeg_compress(char* in_data, int width, int height,
                    struct stream* s, int bpp, int byte_limit,
                    int start_line, struct stream* temp_s,
-                   int e)
+                   int e, int quality)
 {
-  jpeg_compress(in_data, width, height, s, bpp, byte_limit,
-                start_line, temp_s, e);
+  jpeg_compress(in_data, width, height, s, temp_s, bpp, byte_limit,
+                e, quality);
   return height;
 }
 
diff --git a/libxrdp/xrdp_orders.c b/libxrdp/xrdp_orders.c
index 35666492..caeef7ff 100644
--- a/libxrdp/xrdp_orders.c
+++ b/libxrdp/xrdp_orders.c
@@ -26,6 +26,16 @@
 #include <freerdp/codec/rfx.h>
 #endif
 
+#define LLOG_LEVEL 2
+#define LLOGLN(_log_level, _params) \
+{ \
+  if (_log_level < LLOG_LEVEL) \
+  { \
+    g_write("xrdp_orders.c [%10.10u]: ", g_time3()); \
+    g_writeln _params ; \
+  } \
+}
+
 /*****************************************************************************/
 struct xrdp_orders* APP_CC
 xrdp_orders_create(struct xrdp_session* session, struct xrdp_rdp* rdp_layer)
@@ -1833,30 +1843,6 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders* self,
 }
 
 /*****************************************************************************/
-static int
-xrdp_orders_send_as_jpeg(struct xrdp_orders* self,
-                         int width, int height, int bpp, int hints)
-{
-  if (hints & 1)
-  {
-    return 0;
-  }
-  if (bpp != 24)
-  {
-    return 0;
-  }
-  if (self->rdp_layer->client_info.jpeg == 0)
-  {
-    return 0;
-  }
-  if (width * height < 64)
-  {
-    return 0;
-  }
-  return 1;
-}
-
-/*****************************************************************************/
 /* returns error */
 /* max size width * height * Bpp + 14 */
 int APP_CC
@@ -1871,7 +1857,6 @@ xrdp_orders_send_bitmap2(struct xrdp_orders* self,
   int i = 0;
   int lines_sending = 0;
   int e = 0;
-  int is_jpeg;
   struct stream* s = NULL;
   struct stream* temp_s = NULL;
   char* p = NULL;
@@ -1897,18 +1882,8 @@ xrdp_orders_send_bitmap2(struct xrdp_orders* self,
   init_stream(temp_s, 16384);
   p = s->p;
   i = height;
-  is_jpeg = 0;
-  if (xrdp_orders_send_as_jpeg(self, width, height, bpp, hints))
-  {
-    lines_sending = xrdp_jpeg_compress(data, width, height, s, bpp, 16384,
-                                       i - 1, temp_s, e);
-    is_jpeg = 1;
-  }
-  else
-  {
-    lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384,
+  lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384,
                                          i - 1, temp_s, e);
-  }
   if (lines_sending != height)
   {
     free_stream(s);
@@ -1927,10 +1902,6 @@ height(%d)", lines_sending, height);
   out_uint16_le(self->out_s, len);
   i = (((Bpp + 2) << 3) & 0x38) | (cache_id & 7);
   i = i | (0x08 << 7); /* CBR2_NO_BITMAP_COMPRESSION_HDR */
-  if (is_jpeg)
-  {
-    i = i | (0x80 << 7); /* unsed flag, jpeg hack */
-  }
   out_uint16_le(self->out_s, i); /* flags */
   out_uint8(self->out_s, RDP_ORDER_BMPCACHE2); /* type */
   out_uint8(self->out_s, width + e);
@@ -1946,11 +1917,31 @@ height(%d)", lines_sending, height);
   return 0;
 }
 
+/*****************************************************************************/
+static int
+xrdp_orders_send_as_jpeg(struct xrdp_orders* self,
+                         int width, int height, int bpp, int hints)
+{
+  if (hints & 1)
+  {
+    return 0;
+  }
+  if (bpp != 24)
+  {
+    return 0;
+  }
+  if (width * height < 64)
+  {
+    return 0;
+  }
+  return 1;
+}
+
 #if defined(XRDP_FREERDP1)
 /*****************************************************************************/
 /*  secondary drawing order (bitmap v3) using remotefx compression */
 static int APP_CC
-xrdp_orders_send_in_rfx(struct xrdp_orders* self,
+xrdp_orders_send_as_rfx(struct xrdp_orders* self,
                         int width, int height, int bpp,
                         int hints)
 {
@@ -1958,56 +1949,21 @@ xrdp_orders_send_in_rfx(struct xrdp_orders* self,
   {
     return 0;
   }
-  if (self->rdp_layer->client_info.rfx_codec_id == 0)
-  {
-    return 0;
-  }
   return 1;
 }
 #endif
 
 /*****************************************************************************/
-/*  secondary drawing order (bitmap v3) using remotefx compression */
-int APP_CC
-xrdp_orders_send_bitmap3(struct xrdp_orders* self,
-                         int width, int height, int bpp, char* data,
-                         int cache_id, int cache_idx, int hints)
+static int APP_CC
+xrdp_orders_out_v3(struct xrdp_orders* self, int cache_id, int cache_idx,
+                   char* buf, int bufsize, int width, int height, int bpp,
+                   int codec_id)
 {
-#if defined(XRDP_FREERDP1)
-  int bufsize;
   int Bpp;
   int order_flags;
   int len;
   int i;
-  STREAM* fr_s; /* FreeRDP stream */
-  struct stream* xr_s; /* xrdp stream */
-  RFX_CONTEXT* context = (RFX_CONTEXT*)(self->rdp_layer->rfx_enc);
-  RFX_RECT rect;
 
-  if (width > 64)
-  {
-    g_writeln("error, width > 64");
-    return 1;
-  }
-  if (height > 64)
-  {
-    g_writeln("error, height > 64");
-    return 1;
-  }
-  if (!xrdp_orders_send_in_rfx(self, width, height, bpp, hints))
-  {
-    return 2;
-  }
-  make_stream(xr_s);
-  init_stream(xr_s, 16384);
-  fr_s = stream_new(0);
-  stream_attach(fr_s, xr_s->data, 16384);
-  rect.x = 0;
-  rect.y = 0;
-  rect.width = width;
-  rect.height = height;
-  rfx_compose_message(context, fr_s, &rect, 1, data, width, height, width * 4);
-  bufsize = stream_get_length(fr_s);
   Bpp = (bpp + 7) / 8;
   xrdp_orders_check(self, bufsize + 30);
   self->order_count++;
@@ -2027,15 +1983,105 @@ xrdp_orders_send_bitmap3(struct xrdp_orders* self,
   out_uint8(self->out_s, bpp);
   out_uint8(self->out_s, 0); /* reserved */
   out_uint8(self->out_s, 0); /* reserved */
-  out_uint8(self->out_s, self->rdp_layer->client_info.rfx_codec_id);
+  out_uint8(self->out_s, codec_id);
   out_uint16_le(self->out_s, width);
   out_uint16_le(self->out_s, height);
   out_uint32_le(self->out_s, bufsize);
-  out_uint8a(self->out_s, fr_s->data, bufsize);
-  stream_detach(fr_s);
-  stream_free(fr_s);
-  free_stream(xr_s);
+  out_uint8a(self->out_s, buf, bufsize);
+  return 0;
+}
+
+/*****************************************************************************/
+/*  secondary drawing order (bitmap v3) using remotefx compression */
+int APP_CC
+xrdp_orders_send_bitmap3(struct xrdp_orders* self,
+                         int width, int height, int bpp, char* data,
+                         int cache_id, int cache_idx, int hints)
+{
+  int e;
+  int bufsize;
+  int quality;
+  struct stream* xr_s; /* xrdp stream */
+  struct stream* temp_s; /* xrdp stream */
+  struct xrdp_client_info* ci;
+#if defined(XRDP_FREERDP1)
+  STREAM* fr_s; /* FreeRDP stream */
+  RFX_CONTEXT* context;
+  RFX_RECT rect;
+#endif
+
+  ci = &(self->rdp_layer->client_info);
+  if (ci->v3_codec_id == 0)
+  {
+    return 2;
+  }
+  if (ci->v3_codec_id == ci->rfx_codec_id)
+  {
+#if defined(XRDP_FREERDP1)
+    if (!xrdp_orders_send_as_rfx(self, width, height, bpp, hints))
+    {
+      return 2;
+    }
+    LLOGLN(10, ("xrdp_orders_send_bitmap3: rfx"));
+    context = (RFX_CONTEXT*)(self->rdp_layer->rfx_enc);
+    make_stream(xr_s);
+    init_stream(xr_s, 16384);
+    fr_s = stream_new(0);
+    stream_attach(fr_s, (tui8*)(xr_s->data), 16384);
+    rect.x = 0;
+    rect.y = 0;
+    rect.width = width;
+    rect.height = height;
+    rfx_compose_message(context, fr_s, &rect, 1, (tui8*)data, width,
+                        height, width * 4);
+    bufsize = stream_get_length(fr_s);
+    xrdp_orders_out_v3(self, cache_id, cache_idx, (char*)(fr_s->data), bufsize,
+                       width, height, bpp,ci->v3_codec_id);
+    stream_detach(fr_s);
+    stream_free(fr_s);
+    free_stream(xr_s);
+    return 0;
+#else
+    return 2;
 #endif
+  }
+  else if (ci->v3_codec_id == ci->jpeg_codec_id)
+  {
+#if defined(XRDP_JPEG)
+    if (!xrdp_orders_send_as_jpeg(self, width, height, bpp, hints))
+    {
+      LLOGLN(10, ("xrdp_orders_send_bitmap3: jpeg skipped"));
+      return 2;
+    }
+    LLOGLN(10, ("xrdp_orders_send_bitmap3: jpeg"));
+    e = width % 4;
+    if (e != 0)
+    {
+      e = 4 - e;
+    }
+    make_stream(xr_s);
+    init_stream(xr_s, 16384);
+    make_stream(temp_s);
+    init_stream(temp_s, 16384);
+    quality = ci->jpeg_prop[0];
+    xrdp_jpeg_compress(data, width, height, xr_s, bpp, 16384,
+                       height - 1, temp_s, e, quality);
+    s_mark_end(xr_s);
+    bufsize = (int)(xr_s->end - xr_s->data);
+    xrdp_orders_out_v3(self, cache_id, cache_idx, (char*)(xr_s->data), bufsize,
+                       width + e, height, bpp,ci->v3_codec_id);
+    free_stream(xr_s);
+    free_stream(temp_s);
+    return 0;
+#else
+    return 2;
+#endif
+  }
+  else
+  {
+    g_writeln("xrdp_orders_send_bitmap3: todo unknown codec");
+    return 1;
+  }
   return 0;
 }
 
diff --git a/libxrdp/xrdp_rdp.c b/libxrdp/xrdp_rdp.c
index 1c381274..a42d3b90 100644
--- a/libxrdp/xrdp_rdp.c
+++ b/libxrdp/xrdp_rdp.c
@@ -559,9 +559,13 @@ xrdp_rdp_send_demand_active(struct xrdp_rdp* self)
   struct stream* s;
   int caps_count;
   int caps_size;
+  int codec_caps_count;
+  int codec_caps_size;
   char* caps_count_ptr;
   char* caps_size_ptr;
   char* caps_ptr;
+  char* codec_caps_count_ptr;
+  char* codec_caps_size_ptr;
 
   make_stream(s);
   init_stream(s, 8192);
@@ -684,25 +688,37 @@ xrdp_rdp_send_demand_active(struct xrdp_rdp* self)
   /* Output bmpcodecs capability set */
   caps_count++;
   out_uint16_le(s, RDP_CAPSET_BMPCODECS);
-  out_uint16_le(s, 5 + 22 + 275 + 20); /* cap len */
-  out_uint8(s, 3); /* bitmapCodecCount */
+  codec_caps_size_ptr = s->p;
+  out_uint8s(s, 2); /* cap len set later */
+  codec_caps_count = 0;
+  codec_caps_count_ptr = s->p;
+  out_uint8s(s, 1); /* bitmapCodecCount set later */
   /* nscodec */
+  codec_caps_count++;
   out_uint8a(s, XR_CODEC_GUID_NSCODEC, 16);
-  out_uint8(s, 1); /* codec id */
+  out_uint8(s, 1); /* codec id, must be 1 */
   out_uint16_le(s, 3);
   out_uint8(s, 0x01); /* fAllowDynamicFidelity */
   out_uint8(s, 0x01); /* fAllowSubsampling */
   out_uint8(s, 0x03); /* colorLossLevel */
   /* remotefx */
+  codec_caps_count++;
   out_uint8a(s, XR_CODEC_GUID_REMOTEFX, 16);
-  out_uint8(s, 0); /* codec id */
+  out_uint8(s, 0); /* codec id, client sets */
   out_uint16_le(s, 256);
   out_uint8s(s, 256);
   /* jpeg */
+  codec_caps_count++;
   out_uint8a(s, XR_CODEC_GUID_JPEG, 16);
-  out_uint8(s, 2); /* codec id */
+  out_uint8(s, 0); /* codec id, client sets */
   out_uint16_le(s, 1); /* ext length */
   out_uint8(s, 75);
+  /* calculate and set size and count */
+  codec_caps_size = (int)(s->p - codec_caps_size_ptr);
+  codec_caps_size += 2; /* 2 bytes for RDP_CAPSET_BMPCODECS above */
+  codec_caps_size_ptr[0] = codec_caps_size;
+  codec_caps_size_ptr[1] = codec_caps_size >> 8;
+  codec_caps_count_ptr[0] = codec_caps_count;
 
   /* Output color cache capability set */
   caps_count++;
@@ -823,8 +839,7 @@ xrdp_process_capset_order(struct xrdp_rdp* self, struct stream* s,
   in_uint16_le(s, ex_flags); /* Ex flags */
   if (ex_flags & XR_ORDERFLAGS_EX_CACHE_BITMAP_REV3_SUPPORT)
   {
-    g_writeln("RDP_CAPSET_BMPCACHE3");
-    DEBUG(("RDP_CAPSET_BMPCACHE3"));
+    g_writeln("xrdp_process_capset_order: bitmap cache v3 supported");
     self->client_info.bitmap_cache_version |= 4;
   }
   in_uint8s(s, 4); /* Pad */
@@ -872,13 +887,6 @@ xrdp_process_capset_bmpcache2(struct xrdp_rdp* self, struct stream* s,
   self->client_info.bitmap_cache_version |= 2;
   Bpp = (self->client_info.bpp + 7) / 8;
   in_uint16_le(s, i); /* cache flags */
-#if defined(XRDP_JPEG)
-  if (i & 0x80)
-  {
-    g_writeln("xrdp_process_capset_bmpcache2: client supports jpeg");
-    self->client_info.jpeg = 1;
-  }
-#endif
   self->client_info.bitmap_cache_persist_enable = i;
   in_uint8s(s, 2); /* number of caches in set, 3 */
   in_uint32_le(s, i);
@@ -1028,6 +1036,7 @@ xrdp_process_capset_codecs(struct xrdp_rdp* self, struct stream* s, int len)
       i1 = MIN(64, codec_properties_length);
       g_memcpy(self->client_info.ns_prop, s->p, i1);
       self->client_info.ns_prop_len = i1;
+      self->client_info.v3_codec_id = codec_id;
     }
     else if (g_memcmp(codec_guid, XR_CODEC_GUID_REMOTEFX, 16) == 0)
     {
@@ -1037,6 +1046,7 @@ xrdp_process_capset_codecs(struct xrdp_rdp* self, struct stream* s, int len)
       i1 = MIN(64, codec_properties_length);
       g_memcpy(self->client_info.rfx_prop, s->p, i1);
       self->client_info.rfx_prop_len = i1;
+      self->client_info.v3_codec_id = codec_id;
     }
     else if (g_memcmp(codec_guid, XR_CODEC_GUID_JPEG, 16) == 0)
     {
@@ -1046,6 +1056,8 @@ xrdp_process_capset_codecs(struct xrdp_rdp* self, struct stream* s, int len)
       i1 = MIN(64, codec_properties_length);
       g_memcpy(self->client_info.jpeg_prop, s->p, i1);
       self->client_info.jpeg_prop_len = i1;
+      self->client_info.v3_codec_id = codec_id;
+      g_writeln("  jpeg quality %d", self->client_info.jpeg_prop[0]);
     }
     else
     {