diff --git a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvert.c b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvert.c
index e87fb15790..0f63b23ea9 100644
--- a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvert.c
+++ b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvert.c
@@ -59,7 +59,6 @@ gst_video_convert_class_init (GstVideoConvertClass * klass)
       "Converts video from one colorspace to another",
       "Wim Taymans <wim.taymans@gmail.com>");
 
-  convertscale_class->any_memory = TRUE;
   convertscale_class->converts = TRUE;
   convertscale_class->scales = FALSE;
 }
diff --git a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.c b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.c
index dd46b55747..556f59b7b4 100644
--- a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.c
+++ b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.c
@@ -405,7 +405,6 @@ gst_video_convert_scale_class_init (GstVideoConvertScaleClass * klass)
   filter_class->transform_frame =
       GST_DEBUG_FUNCPTR (gst_video_convert_scale_transform_frame);
 
-  klass->any_memory = FALSE;
   klass->converts = TRUE;
   klass->scales = TRUE;
 
@@ -631,7 +630,6 @@ gst_video_convert_scale_transform_caps (GstBaseTransform * trans,
     GstPadDirection direction, GstCaps * caps, GstCaps * filter)
 {
   GstVideoConvertScale *self = GST_VIDEO_CONVERT_SCALE (trans);
-  gint i;
   GstCaps *ret;
 
   GST_DEBUG_OBJECT (trans,
@@ -648,30 +646,6 @@ gst_video_convert_scale_transform_caps (GstBaseTransform * trans,
     ret = intersection;
   }
 
-  if (GST_VIDEO_CONVERT_SCALE_GET_CLASS (trans)->any_memory)
-    return ret;
-
-  for (i = 0; i < gst_caps_get_size (ret); i++) {
-    gint j;
-    GstCapsFeatures *f = gst_caps_get_features (ret, i);
-
-    if (!f || gst_caps_features_is_any (f) ||
-        gst_caps_features_is_equal (f, GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY))
-      continue;
-
-    for (j = 0; j < gst_caps_features_get_size (f); j++) {
-      const gchar *feature = gst_caps_features_get_nth (f, j);
-
-      if (g_str_has_prefix (feature, "memory:")) {
-        GST_DEBUG_OBJECT (trans, "Can not work with memory `%s`", feature);
-        gst_caps_remove_structure (ret, i);
-        break;
-      }
-    }
-  }
-
-  GST_DEBUG_OBJECT (trans, "returning caps: %" GST_PTR_FORMAT, ret);
-
   return ret;
 }
 
diff --git a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.h b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.h
index 1b24add274..67581156aa 100644
--- a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.h
+++ b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoconvertscale.h
@@ -37,7 +37,6 @@ struct _GstVideoConvertScaleClass
 {
   GstVideoFilterClass parent;
 
-  gboolean any_memory;
   gboolean scales;
   gboolean converts;
 };
diff --git a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoscale.c b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoscale.c
index 5e6cfbdf7b..d866f0373b 100644
--- a/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoscale.c
+++ b/subprojects/gst-plugins-base/gst/videoconvertscale/gstvideoscale.c
@@ -124,7 +124,6 @@ gst_video_scale_class_init (GstVideoScaleClass * klass)
       "Video scaler", "Filter/Converter/Video/Scaler",
       "Resizes video", "Wim Taymans <wim.taymans@gmail.com>");
 
-  convertscale_class->any_memory = TRUE;
   convertscale_class->converts = FALSE;
   convertscale_class->scales = TRUE;
 }