diff --git a/ext/theora/gsttheoraparse.c b/ext/theora/gsttheoraparse.c
index b7e1c716ee..289b92802e 100644
--- a/ext/theora/gsttheoraparse.c
+++ b/ext/theora/gsttheoraparse.c
@@ -307,8 +307,6 @@ theora_parse_set_streamheader (GstTheoraParse * parse)
     if (buf == NULL)
       continue;
 
-    gst_buffer_set_caps (buf, GST_PAD_CAPS (parse->srcpad));
-
     packet.packet = GST_BUFFER_DATA (buf);
     packet.bytes = GST_BUFFER_SIZE (buf);
     packet.granulepos = GST_BUFFER_OFFSET_END (buf);
@@ -368,8 +366,12 @@ theora_parse_push_headers (GstTheoraParse * parse)
   for (i = 0; i < 3; i++) {
     GstBuffer *buf;
 
-    if ((buf = parse->streamheader[i]))
-      gst_pad_push (parse->srcpad, gst_buffer_ref (buf));
+    if ((buf = parse->streamheader[i])) {
+      buf = gst_buffer_make_metadata_writable (buf);
+      gst_buffer_set_caps (buf, GST_PAD_CAPS (parse->srcpad));
+      gst_pad_push (parse->srcpad, buf);
+      parse->streamheader[i] = NULL;
+    }
   }
 }