jifmux: Make it more colorspace smart

This makes it able to recombine rgb images, making it possible
to add tags to rgb jpegs as well.

Uses a simple strategy to check what are the possible colorspaces
and avoid adding jfif to ones that aren't YUV/Gray.

gst/jpegformat/gstjifmux.c

@@ -78,6 +78,13 @@ GST_STATIC_PAD_TEMPLATE ("sink",
 GST_DEBUG_CATEGORY_STATIC (jif_mux_debug);
 #define GST_CAT_DEFAULT jif_mux_debug
 
+#define COLORSPACE_UNKNOWN         (0 << 0)
+#define COLORSPACE_GRAYSCALE       (1 << 0)
+#define COLORSPACE_YUV             (1 << 1)
+#define COLORSPACE_RGB             (1 << 2)
+#define COLORSPACE_CMYK            (1 << 3)
+#define COLORSPACE_YCCK            (1 << 4)
+
 typedef struct _GstJifMuxMarker
 {
   guint8 marker;
@@ -302,39 +309,6 @@ gst_jif_mux_parse_image (GstJifMux * self, GstBuffer * buf)
         if (!gst_byte_reader_get_data (&reader, size - 2, &data))
           goto error;
 
-        if (marker == APP14) {
-          gboolean valid = FALSE;
-
-          /* check if this contains RGB and reject it */
-          /*
-           * This marker should have:
-           * - 'Adobe\0'
-           * - 2 bytes DCTEncodeVersion
-           * - 2 bytes flags0
-           * - 2 bytes flags1
-           * - 1 byte  ColorTransform
-           *             - 0 means unknown (RGB or CMYK)
-           *             - 1 YCbCr
-           *             - 2 YCCK
-           */
-
-          if (size >= 14) {
-            if (strncmp ((gchar *) data, "Adobe\0", 6) == 0) {
-              valid = TRUE;
-
-              if (data[11] == 0) {
-                /* this is either RGB or CMYK, reject it */
-                goto not_yuv;
-              }
-
-            }
-          }
-          if (!valid) {
-            GST_WARNING_OBJECT (self, "Not checking suspicious APP14 "
-                "marker of size due to its size (%u) or name string", size);
-          }
-        }
-
         m = gst_jif_mux_new_marker (marker, size - 2, data, FALSE);
         self->priv->markers = g_list_prepend (self->priv->markers, m);
 
@@ -385,10 +359,6 @@ error:
       "Error parsing image header (need more that %u bytes available)",
       gst_byte_reader_get_remaining (&reader));
   return FALSE;
-not_yuv:
-  GST_WARNING_OBJECT (self,
-      "This image is in RGB/CMYK format and JFIF only allows YCbCr");
-  return FALSE;
 }
 
 static gboolean
@@ -402,6 +372,7 @@ gst_jif_mux_mangle_markers (GstJifMux * self)
   GList *app0_jfif = NULL, *app1_exif = NULL, *app1_xmp = NULL, *com = NULL;
   GstBuffer *xmp_data;
   gchar *str = NULL;
+  gint colorspace = COLORSPACE_UNKNOWN;
 
   /* update the APP markers
    * - put any JFIF APP0 first
@@ -420,6 +391,7 @@ gst_jif_mux_mangle_markers (GstJifMux * self)
       case APP0:
         if (m->size > 5 && !memcmp (m->data, "JFIF\0", 5)) {
           GST_DEBUG_OBJECT (self, "found APP0 JFIF");
+          colorspace |= COLORSPACE_GRAYSCALE | COLORSPACE_YUV;
           if (!app0_jfif)
             app0_jfif = node;
         }
@@ -436,6 +408,38 @@ gst_jif_mux_mangle_markers (GstJifMux * self)
           if (!app1_xmp)
             app1_xmp = node;
         }
+        break;
+      case APP14:
+        /* check if this contains RGB */
+        /*
+         * This marker should have:
+         * - 'Adobe\0'
+         * - 2 bytes DCTEncodeVersion
+         * - 2 bytes flags0
+         * - 2 bytes flags1
+         * - 1 byte  ColorTransform
+         *             - 0 means unknown (RGB or CMYK)
+         *             - 1 YCbCr
+         *             - 2 YCCK
+         */
+
+        if ((m->size >= 14)
+            && (strncmp ((gchar *) m->data, "Adobe\0", 6) == 0)) {
+          switch (m->data[11]) {
+            case 0:
+              colorspace |= COLORSPACE_RGB | COLORSPACE_CMYK;
+              break;
+            case 1:
+              colorspace |= COLORSPACE_YUV;
+              break;
+            case 2:
+              colorspace |= COLORSPACE_YCCK;
+              break;
+            default:
+              break;
+          }
+        }
+
         break;
       case COM:
         GST_INFO_OBJECT (self, "found COM, will be replaced");
@@ -471,7 +475,7 @@ gst_jif_mux_mangle_markers (GstJifMux * self)
 
   /* if we want combined or JFIF */
   /* check if we don't have JFIF APP0 */
-  if (!app0_jfif) {
+  if (!app0_jfif && (colorspace & (COLORSPACE_GRAYSCALE | COLORSPACE_YUV))) {
     /* build jfif header */
     static const struct
     {
@@ -668,7 +672,8 @@ gst_jif_mux_recombine_image (GstJifMux * self, GstBuffer ** new_buf,
 
     if (m->marker == SOS) {
       GST_DEBUG_OBJECT (self, "scan data, size = %u", self->priv->scan_size);
-      writer_status &= gst_byte_writer_put_data (writer, self->priv->scan_data,
+      writer_status &=
+          gst_byte_writer_put_data (writer, self->priv->scan_data,
           self->priv->scan_size);
     }
   }