From eb7565b60eff63017fa2ead542da6362c8d8f728 Mon Sep 17 00:00:00 2001
From: Nicolas Dufresne <nicolas.dufresne@collabora.com>
Date: Tue, 6 Feb 2018 16:16:15 -0500
Subject: [PATCH] video-format: Fix 10LE32 formats packing function

The source offset (soff) was not incremented for each component and then
each group of 3 components were inverted. This was causing a staircase
effect combined with some noise.

https://bugzilla.gnome.org/show_bug.cgi?id=789876
---
 gst-libs/gst/video/video-format.c | 60 +++++++++++++++----------------
 1 file changed, 28 insertions(+), 32 deletions(-)

diff --git a/gst-libs/gst/video/video-format.c b/gst-libs/gst/video/video-format.c
index 4360ead29c..9fc424bef0 100644
--- a/gst-libs/gst/video/video-format.c
+++ b/gst-libs/gst/video/video-format.c
@@ -4616,13 +4616,11 @@ pack_GRAY10_LE32 (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
     guint32 Y = 0;
 
     for (c = 0; c < num_comps; c++) {
-      Y <<= 10;
-      Y |= s[soff + 1] >> 6;
+      Y |= s[soff + 1] >> 6 << (10 * c);
+      soff += 4;
     }
 
     GST_WRITE_UINT32_LE (dy + i, Y);
-
-    soff += 4;
   }
 }
 
@@ -4745,31 +4743,31 @@ pack_NV12_10LE32 (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
     guint32 Y = 0;
 
     for (c = 0; c < num_comps; c++) {
-      Y <<= 10;
-      Y |= s[soff + 1] >> 6;
+      Y |= s[soff + 1] >> 6 << (10 * c);
 
-      if (!IS_CHROMA_LINE_420 (y, flags))
-        continue;
-
-      switch ((pix + c) % 6) {
-        case 0:
-          UV = s[soff + 2] >> 6;
-          UV |= s[soff + 3] >> 6 << 10;
-          break;
-        case 2:
-          UV |= s[soff + 2] >> 6 << 20;
-          GST_WRITE_UINT32_LE (duv + i, UV);
-          UV = s[soff + 3] >> 6;
-          break;
-        case 4:
-          UV |= s[soff + 2] >> 6 << 10;
-          UV |= s[soff + 3] >> 6 << 20;
-          GST_WRITE_UINT32_LE (duv + i, UV);
-          break;
-        default:
-          /* keep value */
-          break;
+      if (IS_CHROMA_LINE_420 (y, flags)) {
+        switch ((pix + c) % 6) {
+          case 0:
+            UV = s[soff + 2] >> 6;
+            UV |= s[soff + 3] >> 6 << 10;
+            break;
+          case 2:
+            UV |= s[soff + 2] >> 6 << 20;
+            GST_WRITE_UINT32_LE (duv + i, UV);
+            UV = s[soff + 3] >> 6;
+            break;
+          case 4:
+            UV |= s[soff + 2] >> 6 << 10;
+            UV |= s[soff + 3] >> 6 << 20;
+            GST_WRITE_UINT32_LE (duv + i, UV);
+            break;
+          default:
+            /* keep value */
+            break;
+        }
       }
+
+      soff += 4;
     }
 
     GST_WRITE_UINT32_LE (dy + i, Y);
@@ -4777,7 +4775,6 @@ pack_NV12_10LE32 (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
     if (IS_CHROMA_LINE_420 (y, flags) && num_comps < 3)
       GST_WRITE_UINT32_LE (duv + i, UV);
 
-    soff += 4;
   }
 }
 
@@ -4898,8 +4895,7 @@ pack_NV16_10LE32 (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
     guint32 Y = 0;
 
     for (c = 0; c < num_comps; c++) {
-      Y <<= 10;
-      Y |= s[soff + 1] >> 6;
+      Y |= s[soff + 1] >> 6 << (10 * c);
 
       switch ((pix + c) % 6) {
         case 0:
@@ -4920,14 +4916,14 @@ pack_NV16_10LE32 (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
           /* keep value */
           break;
       }
+
+      soff += 4;
     }
 
     GST_WRITE_UINT32_LE (dy + i, Y);
 
     if (num_comps < 3)
       GST_WRITE_UINT32_LE (duv + i, UV);
-
-    soff += 4;
   }
 }