diff --git a/gst-libs/gst/rtp/gstrtpbasepayload.c b/gst-libs/gst/rtp/gstrtpbasepayload.c
index 21012123ba..d092fc75e6 100644
--- a/gst-libs/gst/rtp/gstrtpbasepayload.c
+++ b/gst-libs/gst/rtp/gstrtpbasepayload.c
@@ -1764,13 +1764,17 @@ set_headers (GstBuffer ** buffer, guint idx, gpointer user_data)
     g_ptr_array_foreach (data->payload->priv->header_exts,
         (GFunc) write_header_extension, &hdrext);
 
-    wordlen = hdrext.written_size / 4 + ((hdrext.written_size % 4) ? 1 : 0);
+    if (hdrext.written_size > 0) {
+      wordlen = hdrext.written_size / 4 + ((hdrext.written_size % 4) ? 1 : 0);
 
-    /* zero-fill the hdrext padding bytes */
-    memset (&hdrext.data[hdrext.written_size], 0,
-        wordlen * 4 - hdrext.written_size);
+      /* zero-fill the hdrext padding bytes */
+      memset (&hdrext.data[hdrext.written_size], 0,
+          wordlen * 4 - hdrext.written_size);
 
-    gst_rtp_buffer_set_extension_data (&rtp, bit_pattern, wordlen);
+      gst_rtp_buffer_set_extension_data (&rtp, bit_pattern, wordlen);
+    } else {
+      gst_rtp_buffer_remove_extension_data (&rtp);
+    }
   }
   GST_OBJECT_UNLOCK (data->payload);
   gst_rtp_buffer_unmap (&rtp);