diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11decoder.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11decoder.cpp
index 3e0aae152d..53fe381806 100644
--- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11decoder.cpp
+++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11decoder.cpp
@@ -1709,6 +1709,10 @@ gst_d3d11_decoder_decide_allocation (GstD3D11Decoder * decoder,
     }
 
     GST_DEBUG_OBJECT (videodec, "Downstream min buffres: %d", min);
+
+    /* We will not use downstream pool for decoding, and therefore preallocation
+     * is unnecessary. So, Non-zero min buffer will be a waste of GPU memory */
+    min = 0;
   }
 
   gst_buffer_pool_set_config (pool, config);