diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
index eecf8bbd70..d20bb377f6 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
@@ -4640,6 +4640,7 @@ gst_vaapi_decoder_h264_parse (GstVaapiDecoder * base_decoder,
 
 exit:
   {
+    gst_adapter_flush (adapter, unit->size);
     gst_vaapi_parser_info_h264_unref (pi);
     return status;
   }
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h265.c b/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
index 156237bb90..8e6d2e5763 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
@@ -2984,6 +2984,7 @@ gst_vaapi_decoder_h265_parse (GstVaapiDecoder * base_decoder,
   return GST_VAAPI_DECODER_STATUS_SUCCESS;
 
 exit:
+  gst_adapter_flush (adapter, unit->size);
   gst_vaapi_parser_info_h265_unref (pi);
   return status;
 }
diff --git a/gst/vaapi/gstvaapidecode.c b/gst/vaapi/gstvaapidecode.c
index 3e470a8050..cdb907fc6d 100644
--- a/gst/vaapi/gstvaapidecode.c
+++ b/gst/vaapi/gstvaapidecode.c
@@ -1179,7 +1179,8 @@ gst_vaapidecode_parse_frame (GstVideoDecoder * vdec,
       break;
     default:
       GST_ERROR ("parse error %d", status);
-      ret = GST_FLOW_EOS;
+      /* just keep parsing, the decoder should have flushed the broken unit */
+      ret = GST_VAAPI_DECODE_FLOW_PARSE_DATA;
       decode->current_frame_size = 0;
       break;
   }