diff --git a/gst/colorspace/gstcolorspace.c b/gst/colorspace/gstcolorspace.c
index 8a823fa0d9..508fb8a708 100644
--- a/gst/colorspace/gstcolorspace.c
+++ b/gst/colorspace/gstcolorspace.c
@@ -229,6 +229,10 @@ gst_csp_set_caps (GstBaseTransform * btrans, GstCaps * incaps,
 
   space = GST_CSP (btrans);
 
+  if (space->convert) {
+    colorspace_convert_free (space->convert);
+  }
+
   /* input caps */
 
   ret = gst_video_format_parse_caps (incaps, &in_format, &in_width, &in_height);