diff --git a/gst/rtp/gstrtpg723pay.c b/gst/rtp/gstrtpg723pay.c
index e791fc4006..8c03a313fc 100644
--- a/gst/rtp/gstrtpg723pay.c
+++ b/gst/rtp/gstrtpg723pay.c
@@ -146,27 +146,23 @@ gst_rtp_g723_pay_set_caps (GstRTPBasePayload * payload, GstCaps * caps)
 static GstFlowReturn
 gst_rtp_g723_pay_flush (GstRTPG723Pay * pay)
 {
-  GstBuffer *outbuf;
+  GstBuffer *outbuf, *payload_buf;
   GstFlowReturn ret;
-  guint8 *payload;
   guint avail;
   GstRTPBuffer rtp = { NULL };
 
   avail = gst_adapter_available (pay->adapter);
 
-  outbuf = gst_rtp_buffer_new_allocate (avail, 0, 0);
+  outbuf = gst_rtp_buffer_new_allocate (0, 0, 0);
 
   gst_rtp_buffer_map (outbuf, GST_MAP_WRITE, &rtp);
-  payload = gst_rtp_buffer_get_payload (&rtp);
 
   GST_BUFFER_PTS (outbuf) = pay->timestamp;
   GST_BUFFER_DURATION (outbuf) = pay->duration;
 
   /* copy G723 data as payload */
-  gst_adapter_copy (pay->adapter, payload, 0, avail);
+  payload_buf = gst_adapter_take_buffer_fast (pay->adapter, avail);
 
-  /* flush bytes from adapter */
-  gst_adapter_flush (pay->adapter, avail);
   pay->timestamp = GST_CLOCK_TIME_NONE;
   pay->duration = 0;
 
@@ -178,6 +174,8 @@ gst_rtp_g723_pay_flush (GstRTPG723Pay * pay)
   }
   gst_rtp_buffer_unmap (&rtp);
 
+  outbuf = gst_buffer_append (outbuf, payload_buf);
+
   ret = gst_rtp_base_payload_push (GST_RTP_BASE_PAYLOAD (pay), outbuf);
 
   return ret;