From 783ebbeecbfc578feb18b0bbed287685abb9662c Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Thu, 27 Apr 2023 13:54:58 +0800
Subject: [PATCH] v4l2videoenc: fix set format failure when needs reset encoder

In cases that encoder needs to reset format, there is race while draining.
v4l2videoenc finish() sends CMD_STOP command to driver, and desire to return
GST_FLOW_OK. But at this time, encoder CAPTURE may have dequeued the last
buffer and got eos. finish() return value changes to be GST_FLOW_EOS which
causes set format fail. So there is no need to check return value for finish()
when set format.

Also need to flush encoder after draining to make sure flush is finished.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4495>
---
 .../sys/v4l2/gstv4l2videoenc.c                | 60 +++++++++----------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videoenc.c b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videoenc.c
index 1adb72054d..a78bf540d5 100644
--- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videoenc.c
+++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videoenc.c
@@ -301,6 +301,34 @@ done:
   return ret;
 }
 
+static gboolean
+gst_v4l2_video_enc_flush (GstVideoEncoder * encoder)
+{
+  GstV4l2VideoEnc *self = GST_V4L2_VIDEO_ENC (encoder);
+
+  GST_DEBUG_OBJECT (self, "Flushing");
+
+  /* Ensure the processing thread has stopped for the reverse playback
+   * iscount case */
+  if (g_atomic_int_get (&self->processing)) {
+    GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder);
+
+    gst_v4l2_object_unlock_stop (self->v4l2output);
+    gst_v4l2_object_unlock_stop (self->v4l2capture);
+    gst_pad_stop_task (encoder->srcpad);
+
+    GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder);
+
+  }
+
+  self->output_flow = GST_FLOW_OK;
+
+  gst_v4l2_object_unlock_stop (self->v4l2output);
+  gst_v4l2_object_unlock_stop (self->v4l2capture);
+
+  return TRUE;
+}
+
 static gboolean
 gst_v4l2_video_enc_set_format (GstVideoEncoder * encoder,
     GstVideoCodecState * state)
@@ -319,8 +347,8 @@ gst_v4l2_video_enc_set_format (GstVideoEncoder * encoder,
       return TRUE;
     }
 
-    if (gst_v4l2_video_enc_finish (encoder) != GST_FLOW_OK)
-      return FALSE;
+    gst_v4l2_video_enc_finish (encoder);
+    gst_v4l2_video_enc_flush (encoder);
 
     gst_v4l2_object_stop (self->v4l2output);
     gst_v4l2_object_stop (self->v4l2capture);
@@ -352,34 +380,6 @@ gst_v4l2_video_enc_set_format (GstVideoEncoder * encoder,
   return ret;
 }
 
-static gboolean
-gst_v4l2_video_enc_flush (GstVideoEncoder * encoder)
-{
-  GstV4l2VideoEnc *self = GST_V4L2_VIDEO_ENC (encoder);
-
-  GST_DEBUG_OBJECT (self, "Flushing");
-
-  /* Ensure the processing thread has stopped for the reverse playback
-   * iscount case */
-  if (g_atomic_int_get (&self->processing)) {
-    GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder);
-
-    gst_v4l2_object_unlock_stop (self->v4l2output);
-    gst_v4l2_object_unlock_stop (self->v4l2capture);
-    gst_pad_stop_task (encoder->srcpad);
-
-    GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder);
-
-  }
-
-  self->output_flow = GST_FLOW_OK;
-
-  gst_v4l2_object_unlock_stop (self->v4l2output);
-  gst_v4l2_object_unlock_stop (self->v4l2capture);
-
-  return TRUE;
-}
-
 struct ProfileLevelCtx
 {
   GstV4l2VideoEnc *self;