diff --git a/sys/msdk/gstmsdkdec.c b/sys/msdk/gstmsdkdec.c
index cb81400188..4a52999744 100644
--- a/sys/msdk/gstmsdkdec.c
+++ b/sys/msdk/gstmsdkdec.c
@@ -476,6 +476,7 @@ gst_msdkdec_set_src_caps (GstMsdkDec * thiz, gboolean need_allocation)
   GstVideoInfo *vinfo;
   GstVideoAlignment align;
   GstCaps *allocation_caps = NULL;
+  GstVideoFormat format;
   guint width, height;
   const gchar *format_str;
 
@@ -487,9 +488,19 @@ gst_msdkdec_set_src_caps (GstMsdkDec * thiz, gboolean need_allocation)
   height =
       thiz->param.mfx.FrameInfo.CropH ? thiz->param.mfx.
       FrameInfo.CropH : GST_VIDEO_INFO_HEIGHT (&thiz->input_state->info);
+
+  format =
+      gst_msdk_get_video_format_from_mfx_fourcc (thiz->param.mfx.
+      FrameInfo.FourCC);
+
+  if (format == GST_VIDEO_FORMAT_UNKNOWN) {
+    GST_WARNING_OBJECT (thiz, "Failed to find a valid video format\n");
+    return FALSE;
+  }
+
   output_state =
       gst_video_decoder_set_output_state (GST_VIDEO_DECODER (thiz),
-      GST_VIDEO_FORMAT_NV12, width, height, thiz->input_state);
+      format, width, height, thiz->input_state);
   if (!output_state)
     return FALSE;
 
diff --git a/sys/msdk/msdk.c b/sys/msdk/msdk.c
index 718039639a..28ba1994f1 100644
--- a/sys/msdk/msdk.c
+++ b/sys/msdk/msdk.c
@@ -312,3 +312,16 @@ gst_msdk_get_surface_from_buffer (GstBuffer * buf)
 
   return NULL;
 }
+
+GstVideoFormat
+gst_msdk_get_video_format_from_mfx_fourcc (mfxU32 fourcc)
+{
+  const struct map *m = gst_msdk_video_format_to_mfx_map;
+
+  for (; m->mfx_fourcc != 0; m++) {
+    if (m->mfx_fourcc == fourcc)
+      return m->format;
+  }
+
+  return GST_VIDEO_FORMAT_UNKNOWN;
+}
diff --git a/sys/msdk/msdk.h b/sys/msdk/msdk.h
index d56bb3a814..a2775a65b1 100644
--- a/sys/msdk/msdk.h
+++ b/sys/msdk/msdk.h
@@ -75,6 +75,9 @@ gst_msdk_is_msdk_buffer (GstBuffer * buf);
 mfxFrameSurface1 *
 gst_msdk_get_surface_from_buffer (GstBuffer * buf);
 
+GstVideoFormat
+gst_msdk_get_video_format_from_mfx_fourcc (mfxU32 fourcc);
+
 G_END_DECLS
 
 #endif /* __MSDK_H__ */