hipmemorycopy: Add support for GL interop

Enable memory copy between HIP and GL

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/8923>

subprojects/gst-plugins-bad/sys/hip/gsthipmemorycopy.cpp

@@ -25,8 +25,12 @@
 
 #ifdef HAVE_GST_CUDA
 #include <gst/cuda/gstcuda.h>
-#else
+#endif
-#define GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY "memory:CUDAMemory"
+
+#ifdef HAVE_GST_GL
+#include <gst/gl/gl.h>
+#include "gsthiploader-gl.h"
+#include "gsthip-interop-gl.h"
 #endif
 
 #include "gsthip.h"
@@ -42,53 +46,13 @@ GST_DEBUG_CATEGORY_STATIC (gst_hip_memory_copy_debug);
     "BGR, BGR10A2_LE, RGB10A2_LE, Y42B, I422_10LE, I422_12LE, YUY2, UYVY, RGBP, " \
     "BGRP, GBR, GBR_10LE, GBR_12LE, GBR_16LE, GBRA, VUYA }"
 
-#ifdef HAVE_GST_CUDA
-#define CAPS_STR \
-  GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
-    (GST_CAPS_FEATURE_MEMORY_HIP_MEMORY, GST_HIP_FORMATS) "; " \
-    GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
-    (GST_CAPS_FEATURE_MEMORY_HIP_MEMORY "," \
-        GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, \
-        GST_HIP_FORMATS) "; " \
-    GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
-    (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, GST_HIP_FORMATS) "; " \
-    GST_VIDEO_CAPS_MAKE_WITH_FEATURES  \
-    (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY "," \
-        GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, \
-        GST_HIP_FORMATS) "; " \
-    GST_VIDEO_CAPS_MAKE (GST_HIP_FORMATS) "; " \
-    GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
-    (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY "," \
-        GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, \
-        GST_HIP_FORMATS)
-#else
-#define CAPS_STR \
-  GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
-    (GST_CAPS_FEATURE_MEMORY_HIP_MEMORY, GST_HIP_FORMATS) "; " \
-    GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
-    (GST_CAPS_FEATURE_MEMORY_HIP_MEMORY "," \
-        GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, \
-        GST_HIP_FORMATS) "; " \
-    GST_VIDEO_CAPS_MAKE (GST_HIP_FORMATS) "; " \
-    GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
-    (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY "," \
-        GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, \
-        GST_HIP_FORMATS)
-#endif
-
-static GstStaticPadTemplate sink_template =
-GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
-    GST_STATIC_CAPS (CAPS_STR));
-
-static GstStaticPadTemplate src_template =
-GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
-    GST_STATIC_CAPS (CAPS_STR));
-
 enum class TransferType
 {
   SYSTEM,
   CUDA_TO_HIP,
+  GL_TO_HIP,
   HIP_TO_CUDA,
+  HIP_TO_GL,
 };
 
 enum class MemoryType
@@ -96,6 +60,7 @@ enum class MemoryType
   SYSTEM,
   HIP,
   CUDA,
+  GL,
 };
 
 enum class DeviceSearchType
@@ -138,6 +103,11 @@ struct _GstHipMemoryCopyPrivate
       gst_clear_object (&device);
 #ifdef HAVE_GST_CUDA
       gst_clear_object (&cuda_ctx);
+#endif
+#ifdef HAVE_GST_GL
+      gst_clear_object (&other_gl_ctx);
+      gst_clear_object (&gl_ctx);
+      gst_clear_object (&gl_dpy);
 #endif
     }
   }
@@ -155,6 +125,12 @@ struct _GstHipMemoryCopyPrivate
   GstCudaContext *cuda_ctx = nullptr;
 #endif
 
+#ifdef HAVE_GST_GL
+  GstGLDisplay *gl_dpy = nullptr;
+  GstGLContext *gl_ctx = nullptr;
+  GstGLContext *other_gl_ctx = nullptr;
+#endif
+
   DeviceSearchType search_type = DeviceSearchType::PROPERTY;
   TransferType transfer_type = TransferType::SYSTEM;
   MemoryType in_type = MemoryType::SYSTEM;
@@ -222,9 +198,6 @@ gst_hip_memory_copy_class_init (GstHipMemoryCopyClass * klass)
   element_class->set_context =
       GST_DEBUG_FUNCPTR (gst_hip_memory_copy_set_context);
 
-  gst_element_class_add_static_pad_template (element_class, &sink_template);
-  gst_element_class_add_static_pad_template (element_class, &src_template);
-
   trans_class->passthrough_on_same_caps = TRUE;
 
   trans_class->start = GST_DEBUG_FUNCPTR (gst_hip_memory_copy_start);
@@ -314,6 +287,11 @@ gst_hip_memory_copy_set_context (GstElement * element, GstContext * context)
 
   {
     std::lock_guard < std::recursive_mutex > lk (priv->lock);
+#ifdef HAVE_GST_GL
+    gst_gl_handle_set_context (element, context, &priv->gl_dpy,
+        &priv->other_gl_ctx);
+#endif
+
     switch (priv->search_type) {
       case DeviceSearchType::ANY:
         gst_hip_handle_set_context (element, context, GST_HIP_VENDOR_UNKNOWN,
@@ -446,6 +424,64 @@ gst_hip_memory_copy_ensure_device (GstHipMemoryCopy * self)
 }
 #endif
 
+#ifdef HAVE_GST_GL
+static gboolean
+gst_hip_memory_copy_ensure_gl_context (GstHipMemoryCopy * self)
+{
+  auto priv = self->priv;
+
+  if (!gst_gl_ensure_element_data (GST_ELEMENT (self),
+          &priv->gl_dpy, &priv->other_gl_ctx)) {
+    GST_DEBUG_OBJECT (self, "No available OpenGL display");
+    return FALSE;
+  }
+
+  auto gl_dpy = priv->gl_dpy;
+  if (!gst_gl_query_local_gl_context (GST_ELEMENT (self), GST_PAD_SRC,
+          &priv->gl_ctx) &&
+      !gst_gl_query_local_gl_context (GST_ELEMENT (self), GST_PAD_SINK,
+          &priv->gl_ctx)) {
+    GST_INFO_OBJECT (self, "failed to query local OpenGL context");
+
+    gst_clear_object (&priv->gl_ctx);
+    priv->gl_ctx = gst_gl_display_get_gl_context_for_thread (gl_dpy, nullptr);
+    if (!priv->gl_ctx
+        || !gst_gl_display_add_context (gl_dpy,
+            GST_GL_CONTEXT (priv->gl_ctx))) {
+      gst_clear_object (&priv->gl_ctx);
+      if (!gst_gl_display_create_context (gl_dpy,
+              priv->other_gl_ctx, &priv->gl_ctx, nullptr)) {
+        GST_WARNING_OBJECT (self, "failed to create OpenGL context");
+        return FALSE;
+      }
+
+      if (!gst_gl_display_add_context (gl_dpy, priv->gl_ctx)) {
+        GST_WARNING_OBJECT (self,
+            "failed to add the OpenGL context to the display");
+        return FALSE;
+      }
+    }
+  }
+
+  auto gl_ctx = priv->gl_ctx;
+  if (!gst_gl_context_check_gl_version (gl_ctx,
+          (GstGLAPI) (GST_GL_API_OPENGL | GST_GL_API_OPENGL3), 3, 0)) {
+    GST_WARNING_OBJECT (self, "OpenGL context could not support PBO");
+    return FALSE;
+  }
+
+  GST_DEBUG_OBJECT (self, "Found GL context");
+
+  return TRUE;
+}
+#else
+static gboolean
+gst_hip_memory_copy_ensure_gl_context (GstHipMemoryCopy * self)
+{
+  return TRUE;
+}
+#endif
+
 static gboolean
 gst_hip_memory_copy_set_caps (GstBaseTransform * trans, GstCaps * incaps,
     GstCaps * outcaps)
@@ -478,6 +514,12 @@ gst_hip_memory_copy_set_caps (GstBaseTransform * trans, GstCaps * incaps,
     priv->in_type = MemoryType::CUDA;
   }
 #endif
+#ifdef HAVE_GST_GL
+  else if (gst_caps_features_contains (features,
+          GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
+    priv->in_type = MemoryType::GL;
+  }
+#endif
 
   features = gst_caps_get_features (outcaps, 0);
   if (features && gst_caps_features_contains (features,
@@ -490,8 +532,27 @@ gst_hip_memory_copy_set_caps (GstBaseTransform * trans, GstCaps * incaps,
     priv->out_type = MemoryType::CUDA;
   }
 #endif
+#ifdef HAVE_GST_GL
+  else if (gst_caps_features_contains (features,
+          GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
+    priv->out_type = MemoryType::GL;
+  }
+#endif
 
   priv->transfer_type = TransferType::SYSTEM;
+#ifdef HAVE_GST_GL
+  if (priv->in_type == MemoryType::GL && priv->out_type == MemoryType::HIP &&
+      gst_hip_memory_copy_ensure_gl_context (self)) {
+    priv->transfer_type = TransferType::GL_TO_HIP;
+    return TRUE;
+  } else if (priv->out_type == MemoryType::GL &&
+      priv->in_type == MemoryType::HIP &&
+      gst_hip_memory_copy_ensure_gl_context (self)) {
+    priv->transfer_type = TransferType::HIP_TO_GL;
+    return TRUE;
+  }
+#endif
+
   return gst_hip_memory_copy_ensure_device (self);
 }
 
@@ -508,6 +569,13 @@ gst_hip_memory_copy_query (GstBaseTransform * trans,
     if (gst_hip_handle_context_query (elem, query, priv->device))
       return TRUE;
 
+#ifdef HAVE_GST_GL
+    if (gst_gl_handle_context_query (elem, query,
+            priv->gl_dpy, priv->gl_ctx, priv->other_gl_ctx)) {
+      return TRUE;
+    }
+#endif
+
 #ifdef HAVE_GST_CUDA
     if (gst_cuda_handle_context_query (elem, query, priv->cuda_ctx))
       return TRUE;
@@ -629,6 +697,18 @@ _set_caps_features (const GstCaps * caps, const gchar * feature_name)
   return tmp;
 }
 
+static void
+_remove_field (GstCaps * caps, const gchar * field)
+{
+  guint n = gst_caps_get_size (caps);
+  guint i = 0;
+
+  for (i = 0; i < n; i++) {
+    GstStructure *s = gst_caps_get_structure (caps, i);
+    gst_structure_remove_field (s, field);
+  }
+}
+
 static GstCaps *
 gst_hip_memory_copy_transform_caps (GstBaseTransform * trans,
     GstPadDirection direction, GstCaps * caps, GstCaps * filter)
@@ -647,29 +727,41 @@ gst_hip_memory_copy_transform_caps (GstBaseTransform * trans,
           _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_HIP_MEMORY);
       tmp = gst_caps_merge (caps_hip, gst_caps_ref (caps));
     } else {
-      auto caps_sys =
+      auto ret = gst_caps_ref (caps);
-          _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
+
 #ifdef HAVE_GST_CUDA
       auto caps_cuda =
           _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY);
-      tmp = gst_caps_merge (caps_cuda, caps_sys);
+      ret = gst_caps_merge (ret, caps_cuda);
-      tmp = gst_caps_merge (gst_caps_ref (caps), tmp);
-#else
-      tmp = gst_caps_merge (gst_caps_ref (caps), caps_sys);
 #endif
+#ifdef HAVE_GST_GL
+      auto caps_gl =
+          _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+      ret = gst_caps_merge (ret, caps_gl);
+#endif
+
+      auto caps_sys =
+          _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
+      tmp = gst_caps_merge (ret, caps_sys);
     }
   } else {
     if (priv->is_uploader) {
-      auto caps_sys =
+      auto ret = gst_caps_ref (caps);
-          _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
+
 #ifdef HAVE_GST_CUDA
       auto caps_cuda =
           _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY);
-      tmp = gst_caps_merge (caps_cuda, caps_sys);
+      ret = gst_caps_merge (ret, caps_cuda);
-      tmp = gst_caps_merge (tmp, gst_caps_ref (caps));
-#else
-      tmp = gst_caps_merge (caps_sys, gst_caps_ref (caps));
 #endif
+#ifdef HAVE_GST_GL
+      auto caps_gl =
+          _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+      ret = gst_caps_merge (ret, caps_gl);
+#endif
+
+      auto caps_sys =
+          _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
+      tmp = gst_caps_merge (ret, caps_sys);
     } else {
       auto caps_hip =
           _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_HIP_MEMORY);
@@ -677,6 +769,9 @@ gst_hip_memory_copy_transform_caps (GstBaseTransform * trans,
     }
   }
 
+  tmp = gst_caps_make_writable (tmp);
+  _remove_field (tmp, "texture-target");
+
   if (filter) {
     result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
     gst_caps_unref (tmp);
@@ -737,6 +832,15 @@ gst_hip_memory_copy_propose_allocation (GstBaseTransform * trans,
       is_system = false;
     }
 #endif
+#ifdef HAVE_GST_GL
+    else if (gst_caps_features_contains (features,
+            GST_CAPS_FEATURE_MEMORY_GL_MEMORY) &&
+        gst_hip_memory_copy_ensure_gl_context (self)) {
+      GST_DEBUG_OBJECT (self, "upstream support gl memory");
+      pool = gst_gl_buffer_pool_new (priv->gl_ctx);
+      is_system = false;
+    }
+#endif
 
     if (!pool)
       pool = gst_video_buffer_pool_new ();
@@ -769,8 +873,6 @@ gst_hip_memory_copy_propose_allocation (GstBaseTransform * trans,
   }
 
   gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, nullptr);
-  gst_query_add_allocation_meta (query,
-      GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, nullptr);
 
   return TRUE;
 }
@@ -841,6 +943,18 @@ gst_hip_memory_copy_decide_allocation (GstBaseTransform * trans,
       pool = gst_cuda_buffer_pool_new (priv->cuda_ctx);
   }
 #endif
+#ifdef HAVE_GST_GL
+  else if (gst_caps_features_contains (features,
+          GST_CAPS_FEATURE_MEMORY_GL_MEMORY) &&
+      gst_hip_memory_copy_ensure_gl_context (self)) {
+    GST_DEBUG_OBJECT (self, "downstream support gl memory");
+    if (pool && !GST_IS_GL_BUFFER_POOL (pool))
+      gst_clear_object (&pool);
+
+    if (!pool)
+      pool = gst_gl_buffer_pool_new (priv->gl_ctx);
+  }
+#endif
 
   if (!pool)
     pool = gst_video_buffer_pool_new ();
@@ -965,15 +1079,220 @@ gst_hip_memory_copy_device_copy (GstHipMemoryCopy * self, GstBuffer * inbuf,
 }
 #endif
 
+#ifdef HAVE_GST_GL
+struct GLCopyData
+{
+  GstHipMemoryCopy *self;
+  GstHipDevice *device;
+  GstBuffer *gl_buf;
+  GstBuffer *hip_buf;
+  gboolean gl_to_hip;
+  gboolean ret = FALSE;
+};
+
+static void
+gl_copy_thread_func (GstGLContext * gl_ctx, GLCopyData * data)
+{
+  auto self = data->self;
+  auto priv = self->priv;
+  auto vendor = gst_hip_device_get_vendor (data->device);
+  guint device_count = 0;
+  int device_list[1] = { 0, };
+  GstHipGraphicsResource *resources[4] = { };
+  GstVideoFrame hip_frame;
+  /* TODO: use stream */
+  hipStream_t stream = nullptr;
+
+  data->ret = FALSE;
+
+  auto hip_ret = HipGLGetDevices (vendor,
+      &device_count, device_list, 1, hipGLDeviceListAll);
+  if (!gst_hip_result (hip_ret, vendor) || device_count == 0) {
+    GST_WARNING_OBJECT (self, "GL context is not compatible with HIP device");
+    return;
+  }
+
+  if (!gst_hip_device_set_current (data->device)) {
+    GST_ERROR_OBJECT (self, "Couldn't set device");
+    return;
+  }
+
+  auto n_mem = gst_buffer_n_memory (data->gl_buf);
+  if (n_mem != GST_VIDEO_INFO_N_PLANES (&priv->info)) {
+    GST_ERROR_OBJECT (self, "Plane count mismatch");
+    return;
+  }
+
+  for (guint i = 0; i < n_mem; i++) {
+    auto mem = gst_buffer_peek_memory (data->gl_buf, i);
+    auto pbo_mem = (GstGLMemoryPBO *) mem;
+
+    hip_ret = gst_hip_get_graphics_resource_from_gl_memory (data->device,
+        mem, &resources[i]);
+    if (!gst_hip_result (hip_ret, vendor)) {
+      GST_WARNING_OBJECT (self, "Couldn't get graphics resource");
+      for (guint j = 0; j < i; j++)
+        gst_clear_hip_graphics_resource (&resources[j]);
+
+      return;
+    }
+
+    if (data->gl_to_hip) {
+      /* get the texture into the PBO */
+      gst_gl_memory_pbo_upload_transfer (pbo_mem);
+      gst_gl_memory_pbo_download_transfer (pbo_mem);
+    } else {
+      /* Need PBO -> texture */
+      GST_MINI_OBJECT_FLAG_SET (mem, GST_GL_BASE_MEMORY_TRANSFER_NEED_UPLOAD);
+
+      /* PBO -> sysmem */
+      GST_MINI_OBJECT_FLAG_SET (pbo_mem->pbo,
+          GST_GL_BASE_MEMORY_TRANSFER_NEED_DOWNLOAD);
+    }
+  }
+
+  GstMapFlags map_flags;
+  if (data->gl_to_hip)
+    map_flags = GST_MAP_WRITE_HIP;
+  else
+    map_flags = GST_MAP_READ_HIP;
+
+  if (!gst_video_frame_map (&hip_frame, &priv->info, data->hip_buf, map_flags)) {
+    GST_ERROR_OBJECT (self, "Couldn't map HIP frame");
+    for (guint i = 0; i < n_mem; i++)
+      gst_clear_hip_graphics_resource (&resources[i]);
+
+    return;
+  }
+
+  gboolean copy_ret = TRUE;
+  for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES (&hip_frame); i++) {
+    hip_ret = gst_hip_graphics_resource_map (resources[i], stream);
+    copy_ret = gst_hip_result (hip_ret, vendor);
+    if (!copy_ret) {
+      GST_ERROR_OBJECT (self, "Couldn't map resource %d", i);
+      break;
+    }
+
+    void *gl_dev_ptr;
+    size_t gl_size;
+    hip_ret = gst_hip_graphics_resource_get_mapped_pointer (resources[i],
+        &gl_dev_ptr, &gl_size);
+    copy_ret = gst_hip_result (hip_ret, vendor);
+    if (!copy_ret) {
+      GST_ERROR_OBJECT (self, "Couldn't get mapped pointer %d", i);
+      gst_hip_graphics_resource_unmap (resources[i], stream);
+      break;
+    }
+
+    hip_Memcpy2D param = { };
+
+    param.srcMemoryType = hipMemoryTypeDevice;
+    param.dstMemoryType = hipMemoryTypeDevice;
+    param.Height = GST_VIDEO_FRAME_COMP_HEIGHT (&hip_frame, i);
+
+    if (data->gl_to_hip) {
+      param.srcDevice = gl_dev_ptr;
+      param.srcPitch = GST_VIDEO_INFO_PLANE_STRIDE (&priv->info, i);
+
+      param.dstDevice = GST_VIDEO_FRAME_PLANE_DATA (&hip_frame, i);
+      param.dstPitch = GST_VIDEO_FRAME_PLANE_STRIDE (&hip_frame, i);
+
+    } else {
+      param.dstDevice = gl_dev_ptr;
+      param.dstPitch = GST_VIDEO_INFO_PLANE_STRIDE (&priv->info, i);
+
+      param.srcDevice = GST_VIDEO_FRAME_PLANE_DATA (&hip_frame, i);
+      param.srcPitch = GST_VIDEO_FRAME_PLANE_STRIDE (&hip_frame, i);
+    }
+
+    param.WidthInBytes = MIN (param.srcPitch, param.dstPitch);
+
+    hip_ret = HipMemcpyParam2DAsync (vendor, &param, stream);
+    copy_ret = gst_hip_result (hip_ret, vendor);
+    gst_hip_graphics_resource_unmap (resources[i], stream);
+
+    if (!copy_ret) {
+      GST_ERROR_OBJECT (self, "Couldn't copy plane %d", i);
+      break;
+    }
+  }
+
+  if (copy_ret)
+    HipStreamSynchronize (vendor, stream);
+
+  for (guint i = 0; i < n_mem; i++)
+    gst_clear_hip_graphics_resource (&resources[i]);
+
+  gst_video_frame_unmap (&hip_frame);
+
+  data->ret = copy_ret;
+}
+
+static gboolean
+gst_hip_memory_copy_gl_copy (GstHipMemoryCopy * self, GstBuffer * inbuf,
+    GstBuffer * outbuf)
+{
+  auto priv = self->priv;
+  GstGLContext *gl_ctx = nullptr;
+  GLCopyData data;
+
+  data.self = self;
+  data.ret = FALSE;
+
+  if (priv->transfer_type == TransferType::GL_TO_HIP) {
+    data.gl_buf = inbuf;
+    data.hip_buf = outbuf;
+    data.gl_to_hip = TRUE;
+  } else {
+    data.gl_buf = outbuf;
+    data.hip_buf = inbuf;
+    data.gl_to_hip = FALSE;
+  }
+
+  auto mem = gst_buffer_peek_memory (data.gl_buf, 0);
+  if (!gst_is_gl_memory_pbo (mem)) {
+    GST_WARNING_OBJECT (self, "Not a GL PBO buffer");
+    return FALSE;
+  }
+  gl_ctx = GST_GL_MEMORY_CAST (mem)->mem.context;
+
+  mem = gst_buffer_peek_memory (data.hip_buf, 0);
+  if (!gst_is_hip_memory (mem)) {
+    GST_WARNING_OBJECT (self, "Not a HIP buffer");
+    return FALSE;
+  }
+
+  data.device = GST_HIP_MEMORY_CAST (mem)->device;
+
+  gst_gl_context_thread_add (gl_ctx,
+      (GstGLContextThreadFunc) gl_copy_thread_func, &data);
+
+  return data.ret;
+}
+#endif
+
 static GstFlowReturn
 gst_hip_memory_copy_transform (GstBaseTransform * trans, GstBuffer * inbuf,
     GstBuffer * outbuf)
 {
   auto self = GST_HIP_MEMORY_COPY (trans);
-
-#ifdef HAVE_GST_CUDA
   auto priv = self->priv;
-  if (priv->transfer_type != TransferType::SYSTEM) {
+
+  if (priv->transfer_type == TransferType::GL_TO_HIP ||
+      priv->transfer_type == TransferType::HIP_TO_GL) {
+    if (gst_hip_memory_copy_gl_copy (self, inbuf, outbuf)) {
+      GST_TRACE_OBJECT (self, "Done GL interop copy");
+      return GST_FLOW_OK;
+    }
+
+    GST_WARNING_OBJECT (self,
+        "GL interop copy failed, fallback to system copy");
+    priv->transfer_type = TransferType::SYSTEM;
+  }
+#ifdef HAVE_GST_CUDA
+  if (priv->transfer_type == TransferType::HIP_TO_CUDA ||
+      priv->transfer_type == TransferType::CUDA_TO_HIP) {
     auto ret = gst_hip_memory_copy_device_copy (self, inbuf, outbuf);
     if (ret) {
       GST_TRACE_OBJECT (self, "Done using device copy");
@@ -1003,6 +1322,36 @@ gst_hip_upload_class_init (GstHipUploadClass * klass)
       "HIP Uploader", "Filter/Video",
       "Uploads system memory into HIP device memory",
       "Seungha Yang <seungha@centricular.com>");
+
+  auto sys_caps = gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_HIP_FORMATS));
+  auto hip_caps = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
+      (GST_CAPS_FEATURE_MEMORY_HIP_MEMORY, GST_HIP_FORMATS));
+
+  auto src_caps = gst_caps_merge (gst_caps_ref (hip_caps),
+      gst_caps_ref (sys_caps));
+
+  auto sink_caps = sys_caps;
+#ifdef HAVE_GST_GL
+  auto gl_caps = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
+      (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, GST_HIP_FORMATS));
+  sink_caps = gst_caps_merge (sink_caps, gl_caps);
+#endif
+
+#ifdef HAVE_GST_CUDA
+  auto cuda_caps = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
+      (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, GST_HIP_FORMATS));
+  sink_caps = gst_caps_merge (sink_caps, cuda_caps);
+#endif
+
+  sink_caps = gst_caps_merge (sink_caps, hip_caps);
+
+  GST_MINI_OBJECT_FLAG_SET (sink_caps, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
+  GST_MINI_OBJECT_FLAG_SET (src_caps, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
+
+  gst_element_class_add_pad_template (element_class,
+      gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, sink_caps));
+  gst_element_class_add_pad_template (element_class,
+      gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, src_caps));
 }
 
 static void
@@ -1028,6 +1377,36 @@ gst_hip_download_class_init (GstHipDownloadClass * klass)
       "HIP Downloader", "Filter/Video",
       "Downloads HIP device memory into system memory",
       "Seungha Yang <seungha@centricular.com>");
+
+  auto sys_caps = gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_HIP_FORMATS));
+  auto hip_caps = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
+      (GST_CAPS_FEATURE_MEMORY_HIP_MEMORY, GST_HIP_FORMATS));
+
+  auto sink_caps = gst_caps_merge (gst_caps_ref (hip_caps),
+      gst_caps_ref (sys_caps));
+
+  auto src_caps = sys_caps;
+#ifdef HAVE_GST_GL
+  auto gl_caps = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
+      (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, GST_HIP_FORMATS));
+  src_caps = gst_caps_merge (src_caps, gl_caps);
+#endif
+
+#ifdef HAVE_GST_CUDA
+  auto cuda_caps = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
+      (GST_CAPS_FEATURE_MEMORY_CUDA_MEMORY, GST_HIP_FORMATS));
+  src_caps = gst_caps_merge (src_caps, cuda_caps);
+#endif
+
+  src_caps = gst_caps_merge (src_caps, hip_caps);
+
+  GST_MINI_OBJECT_FLAG_SET (sink_caps, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
+  GST_MINI_OBJECT_FLAG_SET (src_caps, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
+
+  gst_element_class_add_pad_template (element_class,
+      gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, sink_caps));
+  gst_element_class_add_pad_template (element_class,
+      gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, src_caps));
 }
 
 static void