diff --git a/ext/pango/gstbasetextoverlay.c b/ext/pango/gstbasetextoverlay.c
index e76e2d21fa..9c7aba7a92 100644
--- a/ext/pango/gstbasetextoverlay.c
+++ b/ext/pango/gstbasetextoverlay.c
@@ -669,14 +669,13 @@ gst_base_text_overlay_setcaps_txt (GstBaseTextOverlay * overlay, GstCaps * caps)
 static gboolean
 gst_base_text_overlay_negotiate (GstBaseTextOverlay * overlay, GstCaps * caps)
 {
-  GstQuery *query;
+  gboolean upstream_has_meta = FALSE;
+  gboolean caps_has_meta = FALSE;
+  gboolean alloc_has_meta = FALSE;
   gboolean attach = FALSE;
-  gboolean caps_has_meta = TRUE;
-  gboolean ret;
+  gboolean ret = TRUE;
   GstCapsFeatures *f;
-  GstCaps *original_caps;
-  gboolean original_has_meta = FALSE;
-  gboolean allocation_ret = TRUE;
+  GstCaps *overlay_caps;
 
   GST_DEBUG_OBJECT (overlay, "performing negotiation");
 
@@ -688,76 +687,77 @@ gst_base_text_overlay_negotiate (GstBaseTextOverlay * overlay, GstCaps * caps)
   if (!caps || gst_caps_is_empty (caps))
     goto no_format;
 
-  original_caps = caps;
+  /* Check if upstream caps have meta */
+  if ((f = gst_caps_get_features (caps, 0))) {
+    caps_has_meta = gst_caps_features_contains (f,
+        GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
+  }
 
-  /* Try to use the overlay meta if possible */
-  f = gst_caps_get_features (caps, 0);
+  if (upstream_has_meta) {
+    overlay_caps = gst_caps_ref (caps);
+  } else {
+    GstQuery *query;
 
-  /* if the caps doesn't have the overlay meta, we query if downstream
-   * accepts it before trying the version without the meta
-   * If upstream already is using the meta then we can only use it */
-  if (!f
-      || !gst_caps_features_contains (f,
-          GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) {
-    GstCaps *overlay_caps;
+    /* First check if the allocation meta has compositon */
+    query = gst_query_new_allocation (caps, FALSE);
 
-    /* In this case we added the meta, but we can work without it
-     * so preserve the original caps so we can use it as a fallback */
+    if (!gst_pad_peer_query (overlay->srcpad, query)) {
+      /* no problem, we use the query defaults */
+      GST_DEBUG_OBJECT (overlay, "ALLOCATION query failed");
+
+      /* In case we were flushing, mark reconfigure and fail this method,
+       * will make it retry */
+      if (overlay->video_flushing)
+        ret = FALSE;
+    }
+
+    alloc_has_meta = gst_query_find_allocation_meta (query,
+        GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL);
+
+    gst_query_unref (query);
+
+    /* Then check if downstream accept overlay composition in caps */
     overlay_caps = gst_caps_copy (caps);
 
     f = gst_caps_get_features (overlay_caps, 0);
     gst_caps_features_add (f,
         GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
 
-    ret = gst_pad_peer_query_accept_caps (overlay->srcpad, overlay_caps);
-    GST_DEBUG_OBJECT (overlay, "Downstream accepts the overlay meta: %d", ret);
-    if (ret) {
-      gst_caps_unref (caps);
-      caps = overlay_caps;
+    caps_has_meta = gst_pad_peer_query_accept_caps (overlay->srcpad,
+        overlay_caps);
+  }
 
+  /* For backward compatbility, we will prefer bliting if downstream
+   * allocation does not support the meta. In other case we will prefer
+   * attaching, and will fail the negotiation in the unlikely case we are
+   * force to blit, but format isn't supported. */
+
+  if (upstream_has_meta) {
+    attach = TRUE;
+  } else if (caps_has_meta) {
+    if (alloc_has_meta) {
+      attach = TRUE;
     } else {
-      /* fallback to the original */
-      gst_caps_unref (overlay_caps);
-      caps_has_meta = FALSE;
+      /* Don't attach unless we cannot handle the format */
+      attach = !gst_base_text_overlay_can_handle_caps (caps);
     }
   } else {
-    original_has_meta = TRUE;
+    ret = gst_base_text_overlay_can_handle_caps (caps);
   }
-  GST_DEBUG_OBJECT (overlay, "Using caps %" GST_PTR_FORMAT, caps);
-  ret = gst_pad_set_caps (overlay->srcpad, caps);
 
+  /* If we attach, then pick the overlay caps */
+  if (attach) {
+    gst_caps_unref (caps);
+    caps = overlay_caps;
+  } else {
+    gst_caps_unref (overlay_caps);
+  }
+
+  /* If negotiation worked, set the caps and remember to attach */
   if (ret) {
-    /* find supported meta */
-    query = gst_query_new_allocation (caps, FALSE);
-
-    if (!gst_pad_peer_query (overlay->srcpad, query)) {
-      /* no problem, we use the query defaults */
-      GST_DEBUG_OBJECT (overlay, "ALLOCATION query failed");
-      allocation_ret = FALSE;
-    }
-
-    if (caps_has_meta && gst_query_find_allocation_meta (query,
-            GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL))
-      attach = TRUE;
-
-    gst_query_unref (query);
-  }
-
-  overlay->attach_compo_to_buffer = attach;
-
-  if (!allocation_ret && overlay->video_flushing) {
-    ret = FALSE;
-  } else if (original_caps && !original_has_meta && !attach) {
-    if (caps_has_meta) {
-      /* Some elements (fakesink) claim to accept the meta on caps but won't
-         put it in the allocation query result, this leads below
-         check to fail. Prevent this by removing the meta from caps */
-      gst_caps_unref (caps);
-      caps = gst_caps_ref (original_caps);
-      ret = gst_pad_set_caps (overlay->srcpad, caps);
-      if (ret && !gst_base_text_overlay_can_handle_caps (caps))
-        ret = FALSE;
-    }
+    GST_DEBUG_OBJECT (overlay, "Using caps %" GST_PTR_FORMAT, caps);
+    overlay->attach_compo_to_buffer = attach;
+    ret = gst_pad_set_caps (overlay->srcpad, caps);
   }
 
   if (!ret) {