msdkvpp: Fix dma caps negotiation

Modify the fix_output_format in vpp to directly generate caps with
negotiated src caps, and we have the correct dma caps negotiation in
fix_output_format function. And thus, we can remove the redundant
negotiation of using function pad_accept_memory in vpp.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5845>

subprojects/gst-plugins-bad/sys/msdk/gstmsdkvpp.c

@@ -63,6 +63,7 @@
 #include <libdrm/drm_fourcc.h>
 #include "gstmsdkallocator_libva.h"
 #include <gst/va/gstvaallocator.h>
+#include <gst/va/gstvavideoformat.h>
 #else
 #include <gst/d3d11/gstd3d11.h>
 #endif
@@ -1483,40 +1484,6 @@ error_no_video_info:
   return FALSE;
 }
 
-static gboolean
-pad_accept_memory (GstMsdkVPP * thiz, const gchar * mem_type,
-    GstPadDirection direction, GstCaps * filter)
-{
-  gboolean ret = FALSE;
-  GstCaps *caps, *out_caps;
-  GstPad *pad;
-  GstBaseTransform *trans = GST_BASE_TRANSFORM (thiz);
-
-  if (direction == GST_PAD_SRC)
-    pad = GST_BASE_TRANSFORM_SRC_PAD (trans);
-  else
-    pad = GST_BASE_TRANSFORM_SINK_PAD (trans);
-
-  /* make a copy of filter caps since we need to alter the structure
-   * by adding dmabuf-capsfeatures */
-  caps = gst_caps_copy (filter);
-  gst_caps_set_features (caps, 0, gst_caps_features_from_string (mem_type));
-
-  out_caps = gst_pad_peer_query_caps (pad, caps);
-
-  if (!out_caps || gst_caps_is_empty (out_caps))
-    goto done;
-
-  if (gst_msdkcaps_has_feature (out_caps, mem_type))
-    ret = TRUE;
-done:
-  if (caps)
-    gst_caps_unref (caps);
-  if (out_caps)
-    gst_caps_unref (out_caps);
-  return ret;
-}
-
 static GstCaps *
 gst_msdkvpp_fixate_caps (GstBaseTransform * trans,
     GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
@@ -1541,27 +1508,6 @@ gst_msdkvpp_fixate_caps (GstBaseTransform * trans,
   GST_DEBUG_OBJECT (trans, "fixated to %" GST_PTR_FORMAT, result);
   gst_caps_unref (othercaps);
 
-  /* We let msdkvpp srcpad first query if downstream has va memory type caps,
-   * if not, will check the type of dma memory.
-   */
-#ifndef _WIN32
-  if (pad_accept_memory (thiz, GST_CAPS_FEATURE_MEMORY_VA,
-          direction == GST_PAD_SRC ? GST_PAD_SINK : GST_PAD_SRC, result)) {
-    gst_caps_set_features (result, 0,
-        gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_VA, NULL));
-  } else if (pad_accept_memory (thiz, GST_CAPS_FEATURE_MEMORY_DMABUF,
-          direction == GST_PAD_SRC ? GST_PAD_SINK : GST_PAD_SRC, result)) {
-    gst_caps_set_features (result, 0,
-        gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_DMABUF, NULL));
-  }
-#else
-  if (pad_accept_memory (thiz, GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY,
-          direction == GST_PAD_SRC ? GST_PAD_SINK : GST_PAD_SRC, result)) {
-    gst_caps_set_features (result, 0,
-        gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, NULL));
-  }
-#endif
-
   return result;
 }
 

subprojects/gst-plugins-bad/sys/msdk/gstmsdkvpputil.c

@@ -32,6 +32,8 @@
 #ifndef _WIN32
 #include <libdrm/drm_fourcc.h>
 #include <gst/va/gstvavideoformat.h>
+#else
+#include <gst/d3d11/gstd3d11.h>
 #endif
 
 #define SWAP_GINT(a, b) do {      \
@@ -72,7 +74,7 @@ fixate_output_format (GstMsdkVPP * thiz, GstVideoInfo * vinfo, GstCaps * caps)
   GstCapsFeatures *features;
   GstCaps *ret;
   const GValue *format;
-  gboolean is_dma = FALSE;
+  gboolean is_va = FALSE, is_dma = FALSE, is_d3d = FALSE;
   gboolean fixate = FALSE;
 #ifndef _WIN32
   guint64 modifier = DRM_FORMAT_MOD_INVALID;
@@ -86,13 +88,21 @@ fixate_output_format (GstMsdkVPP * thiz, GstVideoInfo * vinfo, GstCaps * caps)
   for (i = 0; i < size; i++) {
     s = gst_caps_get_structure (caps, i);
     features = gst_caps_get_features (caps, i);
+    is_va = is_dma = is_d3d = FALSE;
 
     if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_DMABUF)) {
       format = gst_structure_get_value (s, "drm-format");
       is_dma = TRUE;
     } else {
       format = gst_structure_get_value (s, "format");
-      is_dma = FALSE;
+#ifndef _WIN32
+      if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_VA))
+        is_va = TRUE;
+#else
+      if (gst_caps_features_contains (features,
+              GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY))
+        is_d3d = TRUE;
+#endif
     }
 
     if (format == NULL)
@@ -142,6 +152,12 @@ fixate_output_format (GstMsdkVPP * thiz, GstVideoInfo * vinfo, GstCaps * caps)
 #endif
       if (fmt == GST_VIDEO_FORMAT_UNKNOWN)
         continue;
+
+      /* size > 1 means downstream doesn't specify the caps directly, so we
+       * still need to compare fmt with the one in vpp sinkcaps */
+      if (size > 1 && fmt != GST_VIDEO_INFO_FORMAT (vinfo))
+        continue;
+
       fixate = TRUE;
       break;
     }
@@ -149,6 +165,9 @@ fixate_output_format (GstMsdkVPP * thiz, GstVideoInfo * vinfo, GstCaps * caps)
       break;
   }
 
+  if (!fixate)
+    fmt = GST_VIDEO_FORMAT_NV12;
+
   out = gst_structure_copy (gst_caps_get_structure (caps, fixated_idx));
   features = gst_caps_features_copy (gst_caps_get_features (caps, fixated_idx));
 
@@ -174,6 +193,18 @@ fixate_output_format (GstMsdkVPP * thiz, GstVideoInfo * vinfo, GstCaps * caps)
 
   ret = gst_caps_new_full (out, NULL);
   gst_caps_set_features_simple (ret, features);
+#ifndef _WIN32
+  if (is_va)
+    gst_caps_set_features (ret, 0,
+        gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_VA));
+#else
+  if (is_d3d)
+    gst_caps_set_features (ret, 0,
+        gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY));
+#endif
+  else if (is_dma)
+    gst_caps_set_features (ret, 0,
+        gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_DMABUF));
 
   return ret;
 }
@@ -657,6 +688,7 @@ _get_preferred_src_caps (GstMsdkVPP * thiz, GstVideoInfo * vinfo,
 {
   GstStructure *structure = NULL;
   GstCaps *outcaps, *fixate_caps;
+  GstCapsFeatures *features;
 
   /* Fixate the format */
   fixate_caps = fixate_output_format (thiz, vinfo, srccaps);
@@ -664,8 +696,11 @@ _get_preferred_src_caps (GstMsdkVPP * thiz, GstVideoInfo * vinfo,
     goto fixate_failed;
 
   structure = gst_caps_get_structure (fixate_caps, 0);
+  features = gst_caps_get_features (fixate_caps, 0);
   /* make a copy */
   structure = gst_structure_copy (structure);
+  features = gst_caps_features_copy (features);
+
   gst_caps_unref (fixate_caps);
 
   if (thiz->keep_aspect)
@@ -692,6 +727,7 @@ _get_preferred_src_caps (GstMsdkVPP * thiz, GstVideoInfo * vinfo,
 
   outcaps = gst_caps_new_empty ();
   gst_caps_append_structure (outcaps, structure);
+  gst_caps_set_features (outcaps, 0, features);
 
   return outcaps;