diff --git a/sys/ximage/ximagesink.c b/sys/ximage/ximagesink.c
index 35d97d2af3..7413b78253 100644
--- a/sys/ximage/ximagesink.c
+++ b/sys/ximage/ximagesink.c
@@ -1150,8 +1150,8 @@ gst_ximagesink_setcaps (GstBaseSink * bsink, GstCaps * caps)
 
   /* unref the old sink */
   if (oldpool) {
-    /* deactivate */
-    gst_buffer_pool_set_active (oldpool, FALSE);
+    /* we don't deactivate, some elements might still be using it, it will be
+     * deactivated when the last ref is gone */
     gst_object_unref (oldpool);
   }
   g_mutex_unlock (ximagesink->flow_lock);
diff --git a/sys/xvimage/xvimagesink.c b/sys/xvimage/xvimagesink.c
index 22811ce25e..2533d3ab7d 100644
--- a/sys/xvimage/xvimagesink.c
+++ b/sys/xvimage/xvimagesink.c
@@ -1660,8 +1660,8 @@ gst_xvimagesink_setcaps (GstBaseSink * bsink, GstCaps * caps)
 
   /* unref the old sink */
   if (oldpool) {
-    /* deactivate */
-    gst_buffer_pool_set_active (oldpool, FALSE);
+    /* we don't deactivate, some elements might still be using it, it will
+     * be deactivated when the last ref is gone */
     gst_object_unref (oldpool);
   }