diff --git a/gst/rtp/gstrtph264depay.c b/gst/rtp/gstrtph264depay.c
index 586cf73427..756730cb80 100644
--- a/gst/rtp/gstrtph264depay.c
+++ b/gst/rtp/gstrtph264depay.c
@@ -955,10 +955,12 @@ gst_rtp_h264_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf)
         }
 
         outsize = gst_adapter_available (rtph264depay->adapter);
-        outbuf = gst_adapter_take_buffer (rtph264depay->adapter, outsize);
-
-        outbuf = gst_rtp_h264_depay_handle_nal (rtph264depay, outbuf, timestamp,
-            marker);
+        if (outsize > 0) {
+          outbuf = gst_adapter_take_buffer (rtph264depay->adapter, outsize);
+          outbuf =
+              gst_rtp_h264_depay_handle_nal (rtph264depay, outbuf, timestamp,
+              marker);
+        }
         break;
       }
       case 26: