diff --git a/gst/rtp/gstrtpgstdepay.c b/gst/rtp/gstrtpgstdepay.c
index 6a03f7ecb2..489f6a8bc8 100644
--- a/gst/rtp/gstrtpgstdepay.c
+++ b/gst/rtp/gstrtpgstdepay.c
@@ -116,12 +116,19 @@ gst_rtp_gst_depay_finalize (GObject * object)
   G_OBJECT_CLASS (parent_class)->finalize (object);
 }
 
-static void
+static gboolean
 store_cache (GstRtpGSTDepay * rtpgstdepay, guint CV, GstCaps * caps)
 {
+  gboolean changed = FALSE;
+
+  if (caps && rtpgstdepay->CV_cache[CV])
+    changed = !gst_caps_is_strictly_equal (caps, rtpgstdepay->CV_cache[CV]);
+
   if (rtpgstdepay->CV_cache[CV])
     gst_caps_unref (rtpgstdepay->CV_cache[CV]);
   rtpgstdepay->CV_cache[CV] = caps;
+
+  return changed;
 }
 
 static void
@@ -464,7 +471,8 @@ gst_rtp_gst_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp)
       GST_DEBUG_OBJECT (rtpgstdepay,
           "inline caps %u, length %u, %" GST_PTR_FORMAT, CV, size, outcaps);
 
-      store_cache (rtpgstdepay, CV, outcaps);
+      if (store_cache (rtpgstdepay, CV, outcaps))
+        gst_pad_set_caps (depayload->srcpad, outcaps);
 
       /* skip caps */
       offset += size;