diff --git a/sys/msdk/gstmsdkbufferpool.c b/sys/msdk/gstmsdkbufferpool.c
index 2181f1ea6e..c8c6157dc2 100644
--- a/sys/msdk/gstmsdkbufferpool.c
+++ b/sys/msdk/gstmsdkbufferpool.c
@@ -135,7 +135,7 @@ gst_msdk_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config)
           GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT)) {
     GstVideoAlignment alignment;
 
-    gst_msdk_set_video_alignment (&video_info, &alignment);
+    gst_msdk_set_video_alignment (&video_info, 0, 0, &alignment);
     gst_video_info_align (&video_info, &alignment);
     gst_buffer_pool_config_set_video_alignment (config, &alignment);
   }
diff --git a/sys/msdk/gstmsdkdec.c b/sys/msdk/gstmsdkdec.c
index 20694bfad8..0d591382c2 100644
--- a/sys/msdk/gstmsdkdec.c
+++ b/sys/msdk/gstmsdkdec.c
@@ -478,6 +478,7 @@ gst_msdkdec_set_src_caps (GstMsdkDec * thiz, gboolean need_allocation)
   GstCaps *allocation_caps = NULL;
   GstVideoFormat format;
   guint width, height;
+  guint alloc_w, alloc_h;
   const gchar *format_str;
 
   /* use display width and display height in output state which
@@ -504,10 +505,18 @@ gst_msdkdec_set_src_caps (GstMsdkDec * thiz, gboolean need_allocation)
   if (!output_state)
     return FALSE;
 
+  /* Find allocation width and height */
+  alloc_w =
+      GST_ROUND_UP_16 (thiz->param.mfx.FrameInfo.Width ? thiz->param.mfx.
+      FrameInfo.Width : width);
+  alloc_h =
+      GST_ROUND_UP_32 (thiz->param.mfx.FrameInfo.Height ? thiz->param.mfx.
+      FrameInfo.Height : height);
+
   /* Ensure output_state->caps and info has same width and height
    * Also mandate the 32 bit alignment */
   vinfo = &output_state->info;
-  gst_msdk_set_video_alignment (vinfo, &align);
+  gst_msdk_set_video_alignment (vinfo, alloc_w, alloc_h, &align);
   gst_video_info_align (vinfo, &align);
   output_state->caps = gst_video_info_to_caps (vinfo);
   if (srcpad_can_dmabuf (thiz))
diff --git a/sys/msdk/gstmsdkenc.c b/sys/msdk/gstmsdkenc.c
index 49de8557da..d02e11daa2 100644
--- a/sys/msdk/gstmsdkenc.c
+++ b/sys/msdk/gstmsdkenc.c
@@ -936,7 +936,7 @@ gst_msdkenc_create_buffer_pool (GstMsdkEnc * thiz, GstCaps * caps,
     return NULL;
   }
 
-  gst_msdk_set_video_alignment (&info, &align);
+  gst_msdk_set_video_alignment (&info, 0, 0, &align);
   gst_video_info_align (&info, &align);
 
   if (thiz->use_dmabuf)
diff --git a/sys/msdk/gstmsdkvpp.c b/sys/msdk/gstmsdkvpp.c
index e3417a01cc..5a3eac5e33 100644
--- a/sys/msdk/gstmsdkvpp.c
+++ b/sys/msdk/gstmsdkvpp.c
@@ -293,7 +293,7 @@ gst_msdkvpp_create_buffer_pool (GstMsdkVPP * thiz, GstPadDirection direction,
   if (!gst_video_info_from_caps (&info, caps))
     goto error_no_video_info;
 
-  gst_msdk_set_video_alignment (&info, &align);
+  gst_msdk_set_video_alignment (&info, 0, 0, &align);
   gst_video_info_align (&info, &align);
 
   if (use_dmabuf)
diff --git a/sys/msdk/msdk.c b/sys/msdk/msdk.c
index de186a637f..c977535e06 100644
--- a/sys/msdk/msdk.c
+++ b/sys/msdk/msdk.c
@@ -215,7 +215,7 @@ msdk_is_available (void)
 }
 
 void
-gst_msdk_set_video_alignment (GstVideoInfo * info,
+gst_msdk_set_video_alignment (GstVideoInfo * info, guint alloc_w, guint alloc_h,
     GstVideoAlignment * alignment)
 {
   guint i, width, height;
@@ -223,14 +223,21 @@ gst_msdk_set_video_alignment (GstVideoInfo * info,
   width = GST_VIDEO_INFO_WIDTH (info);
   height = GST_VIDEO_INFO_HEIGHT (info);
 
+  g_assert (alloc_w == 0 || alloc_w >= width);
+  g_assert (alloc_h == 0 || alloc_h >= height);
+
+  if (alloc_w == 0)
+    alloc_w = width;
+
+  if (alloc_h == 0)
+    alloc_h = height;
+
   gst_video_alignment_reset (alignment);
   for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++)
     alignment->stride_align[i] = 15;    /* 16-byte alignment */
 
-  if (width & 15)
-    alignment->padding_right = GST_MSDK_ALIGNMENT_PADDING (width, 16);
-  if (height & 31)
-    alignment->padding_bottom = GST_MSDK_ALIGNMENT_PADDING (height, 32);
+  alignment->padding_right = GST_ROUND_UP_16 (alloc_w) - width;
+  alignment->padding_bottom = GST_ROUND_UP_32 (alloc_h) - height;
 }
 
 static const struct map *
@@ -267,8 +274,20 @@ gst_msdk_set_mfx_frame_info_from_video_info (mfxFrameInfo * mfx_info,
 {
   g_return_if_fail (info && mfx_info);
 
-  mfx_info->Width = GST_ROUND_UP_16 (GST_VIDEO_INFO_WIDTH (info));
-  mfx_info->Height = GST_ROUND_UP_32 (GST_VIDEO_INFO_HEIGHT (info));
+  /* Use the first component in info to calculate mfx width / height */
+  mfx_info->Width =
+      GST_ROUND_UP_16 (GST_VIDEO_INFO_COMP_STRIDE (info,
+          0) / GST_VIDEO_INFO_COMP_PSTRIDE (info, 0));
+
+  if (GST_VIDEO_INFO_N_PLANES (info) > 1)
+    mfx_info->Height =
+        GST_ROUND_UP_32 (GST_VIDEO_INFO_COMP_OFFSET (info,
+            1) / GST_VIDEO_INFO_COMP_STRIDE (info, 0));
+  else
+    mfx_info->Height =
+        GST_ROUND_UP_32 (GST_VIDEO_INFO_SIZE (info) /
+        GST_VIDEO_INFO_COMP_STRIDE (info, 0));
+
   mfx_info->CropW = GST_VIDEO_INFO_WIDTH (info);
   mfx_info->CropH = GST_VIDEO_INFO_HEIGHT (info);
   mfx_info->FrameRateExtN = GST_VIDEO_INFO_FPS_N (info);
diff --git a/sys/msdk/msdk.h b/sys/msdk/msdk.h
index 12b5d5adfe..ba662c5e67 100644
--- a/sys/msdk/msdk.h
+++ b/sys/msdk/msdk.h
@@ -56,7 +56,7 @@ void msdk_frame_to_surface (GstVideoFrame * frame, mfxFrameSurface1 * surface);
 
 const gchar *msdk_status_to_string (mfxStatus status);
 
-void gst_msdk_set_video_alignment (GstVideoInfo * info,
+void gst_msdk_set_video_alignment (GstVideoInfo * info, guint alloc_w, guint alloc_h,
     GstVideoAlignment * alignment);
 
 /* Conversion from Gstreamer to libmfx */