diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.cpp b/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.cpp
index 3bdbbee3f0..8a5c5bf8d3 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.cpp
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.cpp
@@ -262,6 +262,8 @@ struct _GstD3D11MemoryPrivate
   guint num_render_target_views;
 
   ID3D11VideoDecoderOutputView *decoder_output_view;
+  ID3D11VideoDecoder *decoder_handle;
+
   ID3D11VideoProcessorInputView *processor_input_view;
   ID3D11VideoProcessorOutputView *processor_output_view;
 
@@ -992,7 +994,8 @@ gst_d3d11_memory_get_render_target_view (GstD3D11Memory * mem, guint index)
 
 static gboolean
 gst_d3d11_memory_ensure_decoder_output_view (GstD3D11Memory * mem,
-    ID3D11VideoDevice * video_device, GUID * decoder_profile)
+    ID3D11VideoDevice * video_device, ID3D11VideoDecoder * decoder,
+    const GUID * decoder_profile)
 {
   GstD3D11MemoryPrivate *dmem_priv = mem->priv;
   GstD3D11Allocator *allocator;
@@ -1014,13 +1017,15 @@ gst_d3d11_memory_ensure_decoder_output_view (GstD3D11Memory * mem,
   GST_D3D11_MEMORY_LOCK (mem);
   if (dmem_priv->decoder_output_view) {
     dmem_priv->decoder_output_view->GetDesc (&desc);
-    if (IsEqualGUID (desc.DecodeProfile, *decoder_profile)) {
+    if (IsEqualGUID (desc.DecodeProfile, *decoder_profile) &&
+        dmem_priv->decoder_handle == decoder) {
       goto succeeded;
     } else {
       /* Shouldn't happen, but try again anyway */
       GST_WARNING_OBJECT (allocator,
           "Existing view has different decoder profile");
       GST_D3D11_CLEAR_COM (dmem_priv->decoder_output_view);
+      GST_D3D11_CLEAR_COM (dmem_priv->decoder_handle);
     }
   }
 
@@ -1039,6 +1044,12 @@ gst_d3d11_memory_ensure_decoder_output_view (GstD3D11Memory * mem,
     goto done;
   }
 
+  /* XXX: decoder output view is bound to video device, not decoder handle
+   * from API point of view. But some driver seems to be unhappy
+   * when decoder handle is released while there are outstanding view objects */
+  dmem_priv->decoder_handle = decoder;
+  decoder->AddRef ();
+
 succeeded:
   ret = TRUE;
 
@@ -1051,6 +1062,9 @@ done:
 /**
  * gst_d3d11_memory_get_decoder_output_view:
  * @mem: a #GstD3D11Memory
+ * @video_device: (transfer none): a ID3D11VideoDevice handle
+ * @decoder: (transfer none): a ID3D11VideoDecoder handle
+ * @decoder_profile: a DXVA decoder profile GUID
  *
  * Returns: (transfer none) (nullable): a pointer to the
  * ID3D11VideoDecoderOutputView or %NULL if ID3D11VideoDecoderOutputView is
@@ -1060,14 +1074,16 @@ done:
  */
 ID3D11VideoDecoderOutputView *
 gst_d3d11_memory_get_decoder_output_view (GstD3D11Memory * mem,
-    ID3D11VideoDevice * video_device, GUID * decoder_profile)
+    ID3D11VideoDevice * video_device, ID3D11VideoDecoder * decoder,
+    const GUID * decoder_profile)
 {
   g_return_val_if_fail (gst_is_d3d11_memory (GST_MEMORY_CAST (mem)), NULL);
   g_return_val_if_fail (video_device != NULL, NULL);
+  g_return_val_if_fail (decoder != NULL, NULL);
   g_return_val_if_fail (decoder_profile != NULL, NULL);
 
   if (!gst_d3d11_memory_ensure_decoder_output_view (mem,
-          video_device, decoder_profile))
+          video_device, decoder, decoder_profile))
     return NULL;
 
   return mem->priv->decoder_output_view;
@@ -1409,6 +1425,8 @@ gst_d3d11_allocator_free (GstAllocator * allocator, GstMemory * mem)
   GST_D3D11_CLEAR_COM (dmem_priv->staging);
   GST_D3D11_CLEAR_COM (dmem_priv->buffer);
 
+  GST_D3D11_CLEAR_COM (dmem_priv->decoder_handle);
+
   gst_clear_object (&dmem->device);
   g_mutex_clear (&dmem_priv->lock);
   g_free (dmem->priv);
diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.h b/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.h
index d0f5f1fff8..3b4a3004dc 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.h
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11memory.h
@@ -210,7 +210,8 @@ ID3D11RenderTargetView *   gst_d3d11_memory_get_render_target_view        (GstD3
 GST_D3D11_API
 ID3D11VideoDecoderOutputView *    gst_d3d11_memory_get_decoder_output_view  (GstD3D11Memory * mem,
                                                                              ID3D11VideoDevice * video_device,
-                                                                             GUID * decoder_profile);
+                                                                             ID3D11VideoDecoder * decoder,
+                                                                             const GUID * decoder_profile);
 
 GST_D3D11_API
 ID3D11VideoProcessorInputView *   gst_d3d11_memory_get_processor_input_view  (GstD3D11Memory * mem,
diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11decoder.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11decoder.cpp
index 77d6469bba..40427ce47b 100644
--- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11decoder.cpp
+++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11decoder.cpp
@@ -409,7 +409,7 @@ gst_d3d11_decoder_ensure_output_view (GstD3D11Decoder * self,
 
   mem = (GstD3D11Memory *) gst_buffer_peek_memory (buffer, 0);
   if (!gst_d3d11_memory_get_decoder_output_view (mem, self->video_device,
-          &self->decoder_profile)) {
+          self->decoder_handle, &self->decoder_profile)) {
     GST_ERROR_OBJECT (self, "Decoder output view is unavailable");
     return FALSE;
   }
@@ -1367,7 +1367,7 @@ gst_d3d11_decoder_get_output_view_from_buffer (GstD3D11Decoder * decoder,
 
   dmem = (GstD3D11Memory *) mem;
   view = gst_d3d11_memory_get_decoder_output_view (dmem, decoder->video_device,
-      &decoder->decoder_profile);
+      decoder->decoder_handle, &decoder->decoder_profile);
 
   if (!view) {
     GST_ERROR_OBJECT (decoder, "Decoder output view is unavailable");