From 88e0b6ba2dc76799502fc7fbbde6f3602757bbb0 Mon Sep 17 00:00:00 2001
From: Benjamin Gaignard <benjamin.gaignard@collabora.com>
Date: Mon, 21 Oct 2024 17:03:00 +0200
Subject: [PATCH] codecs: av1 decoder: Drain output buffers resolution change

We must drain the pending output picture so that subclass can renegotiate
the caps. Not doing so while still renegotiating would mean that the
subclass would have to do an allocation query before pushing the caps.
Pushing the caps now without this would also not work since these caps
won't match the pending buffers format.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/8064>
---
 .../gst-libs/gst/codecs/gstav1decoder.c       | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c
index c1ab7f9994..81b41b08eb 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c
@@ -39,6 +39,8 @@ struct _GstAV1DecoderPrivate
 {
   gint max_width;
   gint max_height;
+  gint frame_width;
+  gint frame_height;
   GstAV1Profile profile;
   GstAV1Parser *parser;
   GstAV1Dpb *dpb;
@@ -149,6 +151,8 @@ gst_av1_decoder_reset (GstAV1Decoder * self)
 
   priv->max_width = 0;
   priv->max_height = 0;
+  priv->frame_width = 0;
+  priv->frame_height = 0;
   gst_clear_av1_picture (&priv->current_picture);
   priv->current_frame = NULL;
   priv->profile = GST_AV1_PROFILE_UNDEFINED;
@@ -419,6 +423,8 @@ gst_av1_decoder_process_sequence (GstAV1Decoder * self, GstAV1OBU * obu)
   priv->profile = seq_header.seq_profile;
   priv->max_width = seq_header.max_frame_width_minus_1 + 1;
   priv->max_height = seq_header.max_frame_height_minus_1 + 1;
+  priv->frame_width = seq_header.frame_width_bits_minus_1 + 1;
+  priv->frame_height = seq_header.frame_height_bits_minus_1 + 1;
 
   return GST_FLOW_OK;
 }
@@ -456,6 +462,22 @@ gst_av1_decoder_decode_tile_group (GstAV1Decoder * self,
   return GST_FLOW_OK;
 }
 
+static gboolean
+gst_av1_decoder_is_format_change (GstAV1Decoder * self,
+    GstAV1FrameHeaderOBU * frame_header)
+{
+  GstAV1DecoderPrivate *priv = self->priv;
+
+  if (priv->frame_width != frame_header->upscaled_width
+      || priv->frame_height != frame_header->frame_height) {
+    GST_INFO_OBJECT (self, "frame resolution changed %dx%d",
+        frame_header->upscaled_width, frame_header->frame_height);
+    return TRUE;
+  }
+
+  return FALSE;
+}
+
 static GstFlowReturn
 gst_av1_decoder_decode_frame_header (GstAV1Decoder * self,
     GstAV1OBU * obu, GstAV1FrameHeaderOBU * frame_header)
@@ -509,6 +531,18 @@ gst_av1_decoder_decode_frame_header (GstAV1Decoder * self,
     g_assert (picture->spatial_id <= self->highest_spatial_layer);
     g_assert (self->highest_spatial_layer < GST_AV1_MAX_NUM_SPATIAL_LAYERS);
 
+    if (gst_av1_decoder_is_format_change (self, frame_header)) {
+      gst_av1_decoder_drain_output_queue (self, 0, &ret);
+      if (ret != GST_FLOW_OK) {
+        GST_WARNING_OBJECT (self, "Failed to drain pending frames, returned %s",
+            gst_flow_get_name (ret));
+        return ret;
+      }
+
+      priv->frame_width = frame_header->upscaled_width;
+      priv->frame_height = frame_header->frame_height;
+    }
+
     if (!frame_header->show_frame && !frame_header->showable_frame)
       GST_VIDEO_CODEC_FRAME_FLAG_SET (priv->current_frame,
           GST_VIDEO_CODEC_FRAME_FLAG_DECODE_ONLY);