diff --git a/sys/v4l2/gstv4l2videodec.c b/sys/v4l2/gstv4l2videodec.c
index 5699519fd2..e461a4612d 100644
--- a/sys/v4l2/gstv4l2videodec.c
+++ b/sys/v4l2/gstv4l2videodec.c
@@ -233,19 +233,20 @@ gst_v4l2_video_dec_set_format (GstVideoDecoder * decoder,
     gst_v4l2_video_dec_finish (decoder);
     gst_v4l2_object_stop (self->v4l2output);
 
-    /* The renegotiation flow don't blend with the base class flow. To
-     * properly stop the capture pool we need to reclaim our buffers, which
-     * will happend through the allocation query. The allocation query is
-     * triggered by gst_video_decoder_negotiate() which requires the output
-     * caps to be set, but we can't know this information as we rely on the
-     * decoder, which requires the capture queue to be stopped.
+    /* The renegotiation flow don't blend with the base class flow. To properly
+     * stop the capture pool, if the buffers can't be orphaned, we need to
+     * reclaim our buffers, which will happend through the allocation query.
+     * The allocation query is triggered by gst_video_decoder_negotiate() which
+     * requires the output caps to be set, but we can't know this information
+     * as we rely on the decoder, which requires the capture queue to be
+     * stopped.
      *
      * To workaround this issue, we simply run an allocation query with the
      * old negotiated caps in order to drain/reclaim our buffers. That breaks
      * the complexity and should not have much impact in performance since the
      * following allocation query will happen on a drained pipeline and won't
      * block. */
-    {
+    if (!gst_v4l2_buffer_pool_orphan (&self->v4l2capture->pool)) {
       GstCaps *caps = gst_pad_get_current_caps (decoder->srcpad);
       if (caps) {
         GstQuery *query = gst_query_new_allocation (caps, FALSE);