diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c
index a6e72bc108..e49762b6d6 100644
--- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c
+++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c
@@ -1874,6 +1874,10 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
             GST_IS_VA_DMABUF_ALLOCATOR (allocator)))
       thiz->ds_has_known_allocator = FALSE;
   }
+#else
+  if (!GST_IS_D3D11_BUFFER_POOL (pool)) {
+    thiz->ds_has_known_allocator = FALSE;
+  }
 #endif
 
   /* If downstream supports video meta and video alignment, or downstream