diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c
index 069cf78f40..bd9095f3eb 100644
--- a/gst-libs/gst/video/gstvideoaggregator.c
+++ b/gst-libs/gst/video/gstvideoaggregator.c
@@ -450,8 +450,13 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg)
       GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d",
           GST_VIDEO_INFO_FORMAT (&pad->info),
           GST_VIDEO_INFO_FORMAT (&best_info));
-      pad->priv->convert =
-          gst_video_converter_new (&pad->info, &best_info, NULL);
+      /* TODO: GstVideoConverter currently can't rescale! */
+      if (pad->info.width == best_info.width &&
+          pad->info.height == best_info.height &&
+          pad->info.par_n == best_info.par_n &&
+          pad->info.par_d == best_info.par_d)
+        pad->priv->convert =
+            gst_video_converter_new (&pad->info, &best_info, NULL);
       pad->need_conversion_update = TRUE;
       if (!pad->priv->convert) {
         g_free (colorimetry);