diff --git a/subprojects/gst-plugins-bad/sys/va/gstvabase.c b/subprojects/gst-plugins-bad/sys/va/gstvabase.c
index 834430b701..8c356b4130 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvabase.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvabase.c
@@ -40,26 +40,20 @@ _try_import_dmabuf_unlocked (GstVaBufferImporter * importer, GstBuffer * inbuf)
 {
   GstVideoMeta *meta;
   GstVideoInfo in_info = *importer->in_info;
-  GstVideoInfoDmaDrm drm_info;
+  GstVideoInfoDmaDrm drm_info = *importer->in_drm_info;
   GstMemory *mems[GST_VIDEO_MAX_PLANES];
-  guint i, n_mem, n_planes, usage_hint;
+  guint i, n_planes, usage_hint;
   gsize offset[GST_VIDEO_MAX_PLANES];
   uintptr_t fd[GST_VIDEO_MAX_PLANES];
   gsize plane_size[GST_VIDEO_MAX_PLANES];
   GstVideoAlignment align = { 0, };
-  GstVideoFormat format;
 
   /* This will eliminate most non-dmabuf out there */
   if (!gst_is_dmabuf_memory (gst_buffer_peek_memory (inbuf, 0)))
     return FALSE;
 
-  n_mem = gst_buffer_n_memory (inbuf);
   n_planes = GST_VIDEO_INFO_N_PLANES (&in_info);
 
-  /* We cannot have multiple dmabuf per plane */
-  if (n_mem > n_planes)
-    return FALSE;
-
   meta = gst_buffer_get_video_meta (inbuf);
 
   /* Update video info importerd on video meta */
@@ -67,7 +61,9 @@ _try_import_dmabuf_unlocked (GstVaBufferImporter * importer, GstBuffer * inbuf)
     GST_VIDEO_INFO_WIDTH (&in_info) = meta->width;
     GST_VIDEO_INFO_HEIGHT (&in_info) = meta->height;
 
-    for (i = 0; i < meta->n_planes; i++) {
+    g_assert (n_planes == meta->n_planes);
+
+    for (i = 0; i < n_planes; i++) {
       GST_VIDEO_INFO_PLANE_OFFSET (&in_info, i) = meta->offset[i];
       GST_VIDEO_INFO_PLANE_STRIDE (&in_info, i) = meta->stride[i];
     }
@@ -104,12 +100,6 @@ _try_import_dmabuf_unlocked (GstVaBufferImporter * importer, GstBuffer * inbuf)
   usage_hint = va_get_surface_usage_hint (importer->display,
       importer->entrypoint, GST_PAD_SINK, TRUE);
 
-  /* FIXME(victor): don't assume the modifier   */
-  format = GST_VIDEO_INFO_FORMAT (&in_info);
-  drm_info.drm_fourcc = gst_va_drm_fourcc_from_video_format (format);
-  drm_info.drm_modifier = DRM_FORMAT_MOD_LINEAR;
-  drm_info.vinfo = in_info;
-
   /* Now create a VASurfaceID for the buffer */
   return gst_va_dmabuf_memories_setup (importer->display, &drm_info, mems, fd,
       offset, usage_hint);
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvabase.h b/subprojects/gst-plugins-bad/sys/va/gstvabase.h
index a4e97b847c..c385852e75 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvabase.h
+++ b/subprojects/gst-plugins-bad/sys/va/gstvabase.h
@@ -36,7 +36,10 @@ struct _GstVaBufferImporter
   GstVaDisplay *display;
   VAEntrypoint entrypoint;
 
-  GstVideoInfo *in_info;
+  union {
+    GstVideoInfo *in_info;
+    GstVideoInfoDmaDrm *in_drm_info;
+  };
   GstVideoInfo *sinkpad_info;
 
   gpointer pool_data;
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c
index 4e42a4a8cf..9d112a934f 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c
@@ -261,7 +261,7 @@ gst_va_base_enc_import_input_buffer (GstVaBaseEnc * base,
 #endif
     .display = base->display,
     .entrypoint = GST_VA_BASE_ENC_ENTRYPOINT (base),
-    .in_info = &base->in_info,
+    .in_drm_info = &base->in_drm_info,
     .sinkpad_info = &base->priv->sinkpad_info,
     .get_sinkpad_pool = _get_sinkpad_pool,
   };
@@ -679,8 +679,18 @@ gst_va_base_enc_set_format (GstVideoEncoder * venc, GstVideoCodecState * state)
 
   g_return_val_if_fail (state->caps != NULL, FALSE);
 
-  if (!gst_va_video_info_from_caps (&base->in_info, NULL, state->caps))
-    return FALSE;
+  if (!gst_video_is_dma_drm_caps (state->caps)) {
+    gst_video_info_dma_drm_init (&base->in_drm_info);
+    base->in_info = state->info;
+  } else {
+    GstVideoInfo info;
+
+    if (!gst_video_info_dma_drm_from_caps (&base->in_drm_info, state->caps))
+      return FALSE;
+    if (!gst_va_dma_drm_info_to_video_info (&base->in_drm_info, &info))
+      return FALSE;
+    base->in_info = info;
+  }
 
   if (base->input_state)
     gst_video_codec_state_unref (base->input_state);
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h
index ecf2454d66..b7d045448a 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h
+++ b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h
@@ -65,7 +65,10 @@ struct _GstVaBaseEnc
   GQueue output_list;
 
   GstVideoCodecState *input_state;
-  GstVideoInfo in_info;
+  union {
+    GstVideoInfo in_info;
+    GstVideoInfoDmaDrm in_drm_info;
+  };
 
   /*< private >*/
   GstVaBaseEncPrivate *priv;
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvabasetransform.c b/subprojects/gst-plugins-bad/sys/va/gstvabasetransform.c
index be57878ab7..98f9e8d4ac 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvabasetransform.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvabasetransform.c
@@ -178,8 +178,16 @@ gst_va_base_transform_set_caps (GstBaseTransform * trans, GstCaps * incaps,
   gboolean res;
 
   /* input caps */
-  if (!gst_va_video_info_from_caps (&in_info, NULL, incaps))
-    goto invalid_caps;
+  if (!gst_video_is_dma_drm_caps (incaps)) {
+    gst_video_info_dma_drm_init (&self->in_drm_info);
+    if (!gst_video_info_from_caps (&in_info, incaps))
+      goto invalid_caps;
+  } else {
+    if (!gst_video_info_dma_drm_from_caps (&self->in_drm_info, incaps))
+      goto invalid_caps;
+    if (!gst_va_dma_drm_info_to_video_info (&self->in_drm_info, &in_info))
+      goto invalid_caps;
+  }
 
   /* output caps */
   if (!gst_va_video_info_from_caps (&out_info, NULL, outcaps))
@@ -864,7 +872,7 @@ gst_va_base_transform_import_buffer (GstVaBaseTransform * self,
 #endif
     .display = self->display,
     .entrypoint = VAEntrypointVideoProc,
-    .in_info = &self->in_info,
+    .in_drm_info = &self->in_drm_info,
     .sinkpad_info = &self->priv->sinkpad_info,
     .get_sinkpad_pool = _get_sinkpad_pool,
   };
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvabasetransform.h b/subprojects/gst-plugins-bad/sys/va/gstvabasetransform.h
index 04575cabe3..5b8b6c4d5a 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvabasetransform.h
+++ b/subprojects/gst-plugins-bad/sys/va/gstvabasetransform.h
@@ -47,7 +47,10 @@ struct _GstVaBaseTransform
 
   GstCaps *in_caps;
   GstCaps *out_caps;
-  GstVideoInfo in_info;
+  union {
+    GstVideoInfo in_info;
+    GstVideoInfoDmaDrm in_drm_info;
+  };
   GstVideoInfo out_info;
 
   gboolean negotiated;