diff --git a/gst/rtp/gstrtph264pay.c b/gst/rtp/gstrtph264pay.c
index da3faf43ab..8590d99352 100644
--- a/gst/rtp/gstrtph264pay.c
+++ b/gst/rtp/gstrtph264pay.c
@@ -835,8 +835,10 @@ gst_rtp_h264_pay_payload_nal (GstRTPBasePayload * basepayload,
      * checking when we need to send SPS/PPS but convert to running_time first. */
     rtph264pay->send_spspps = FALSE;
     ret = gst_rtp_h264_pay_send_sps_pps (basepayload, rtph264pay, dts, pts);
-    if (ret != GST_FLOW_OK)
+    if (ret != GST_FLOW_OK) {
+      gst_buffer_unref (paybuf);
       return ret;
+    }
   }
 
   packet_len = gst_rtp_buffer_calc_packet_len (size, 0, 0);