diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudabufferpool.c b/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudabufferpool.c
index fe3f6336fa..6f6b34156d 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudabufferpool.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudabufferpool.c
@@ -30,7 +30,6 @@ GST_DEBUG_CATEGORY_STATIC (gst_cuda_buffer_pool_debug);
 
 struct _GstCudaBufferPoolPrivate
 {
-  GstCudaAllocator *allocator;
   GstVideoInfo info;
 };
 
@@ -74,15 +73,7 @@ gst_cuda_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config)
     return FALSE;
   }
 
-  gst_clear_object (&priv->allocator);
-  priv->allocator = (GstCudaAllocator *)
-      gst_allocator_find (GST_CUDA_MEMORY_TYPE_NAME);
-  if (!priv->allocator) {
-    GST_WARNING_OBJECT (self, "CudaAllocator is unavailable");
-    return FALSE;
-  }
-
-  mem = gst_cuda_allocator_alloc (priv->allocator, self->context, &info);
+  mem = gst_cuda_allocator_alloc (NULL, self->context, &info);
   if (!mem) {
     GST_WARNING_OBJECT (self, "Failed to allocate memory");
     return FALSE;
@@ -111,7 +102,7 @@ gst_cuda_buffer_pool_alloc (GstBufferPool * pool, GstBuffer ** buffer,
   GstMemory *mem;
   GstCudaMemory *cmem;
 
-  mem = gst_cuda_allocator_alloc (priv->allocator, self->context, &priv->info);
+  mem = gst_cuda_allocator_alloc (NULL, self->context, &priv->info);
   if (!mem) {
     GST_WARNING_OBJECT (pool, "Cannot create CUDA memory");
     return GST_FLOW_ERROR;
@@ -161,9 +152,7 @@ static void
 gst_cuda_buffer_pool_dispose (GObject * object)
 {
   GstCudaBufferPool *self = GST_CUDA_BUFFER_POOL_CAST (object);
-  GstCudaBufferPoolPrivate *priv = self->priv;
 
-  gst_clear_object (&priv->allocator);
   gst_clear_object (&self->context);
 
   G_OBJECT_CLASS (parent_class)->dispose (object);
diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudacontext.c b/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudacontext.c
index 05bfb7b102..636e64d11a 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudacontext.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudacontext.c
@@ -24,6 +24,7 @@
 #include "gstcudaloader.h"
 #include "gstcudacontext.h"
 #include "gstcudautils.h"
+#include "gstcudamemory.h"
 
 #ifdef GST_CUDA_HAS_D3D
 #include <gst/d3d11/gstd3d11.h>
@@ -99,6 +100,8 @@ gst_cuda_context_class_init (GstCudaContextClass * klass)
           GST_PARAM_CONDITIONALLY_AVAILABLE | G_PARAM_READABLE |
           G_PARAM_STATIC_STRINGS));
 #endif
+
+  gst_cuda_memory_init_once ();
 }
 
 static void
diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudamemory.c b/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudamemory.c
index 50ff67bfa5..9cda172775 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudamemory.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/cuda/gstcudamemory.c
@@ -462,6 +462,7 @@ gst_cuda_memory_init_once (void)
     _gst_cuda_allocator =
         (GstAllocator *) g_object_new (GST_TYPE_CUDA_ALLOCATOR, NULL);
     gst_object_ref_sink (_gst_cuda_allocator);
+    gst_object_ref (_gst_cuda_allocator);
 
     gst_allocator_register (GST_CUDA_MEMORY_TYPE_NAME, _gst_cuda_allocator);
     g_once_init_leave (&_init, 1);
@@ -485,6 +486,11 @@ gst_is_cuda_memory (GstMemory * mem)
 
 /**
  * gst_cuda_allocator_alloc:
+ * @allocator: (transfer none) (allow-none): a #GstCudaAllocator
+ * @context: (transfer none): a #GstCudaContext
+ * @info: a #GstVideoInfo
+ *
+ * Returns: (transfer full) (nullable): a newly allocated #GstCudaMemory
  *
  * Since: 1.22
  */
@@ -494,10 +500,12 @@ gst_cuda_allocator_alloc (GstCudaAllocator * allocator,
 {
   guint alloc_height;
 
-  g_return_val_if_fail (GST_IS_CUDA_ALLOCATOR (allocator), NULL);
   g_return_val_if_fail (GST_IS_CUDA_CONTEXT (context), NULL);
   g_return_val_if_fail (info != NULL, NULL);
 
+  if (!allocator)
+    allocator = (GstCudaAllocator *) _gst_cuda_allocator;
+
   alloc_height = GST_VIDEO_INFO_HEIGHT (info);
 
   /* make sure valid height for subsampled formats */