diff --git a/gst/rtp/gstrtph264depay.c b/gst/rtp/gstrtph264depay.c
index bef0b16094..83356cb3d6 100644
--- a/gst/rtp/gstrtph264depay.c
+++ b/gst/rtp/gstrtph264depay.c
@@ -440,8 +440,9 @@ gst_rtp_h264_set_src_caps (GstRtpH264Depay * rtph264depay)
 
   gst_caps_unref (srccaps);
 
-  /* Insert SPS and PPS into the stream on next opportunity */
-  if (rtph264depay->sps->len > 0 || rtph264depay->pps->len > 0) {
+  /* Insert SPS and PPS into the stream on next opportunity (if bytestream) */
+  if (rtph264depay->byte_stream
+      && (rtph264depay->sps->len > 0 || rtph264depay->pps->len > 0)) {
     gint i;
     GstBuffer *codec_data;
     GstMapInfo map;