From 3d56f133f86f045003793546b3b802b4d33fe910 Mon Sep 17 00:00:00 2001
From: He Junyan <junyan.he@intel.com>
Date: Thu, 12 May 2022 10:12:37 +0800
Subject: [PATCH] codecs: av1decoder: Add the highest_spatial_layer field.

As SPEC says, when multi spatial layer exists, we should only output
one frame with the highest spatial id from each TU. We now store the
highest spatial layer information in the base class in order to let
the sub class handle different layers easily.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2382>
---
 .../gst-libs/gst/codecs/gstav1decoder.c       | 25 ++++++++++++++++---
 .../gst-libs/gst/codecs/gstav1decoder.h       |  1 +
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c
index 4339e94279..7208821dfd 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c
@@ -140,6 +140,8 @@ gst_av1_decoder_reset (GstAV1Decoder * self)
 {
   GstAV1DecoderPrivate *priv = self->priv;
 
+  self->highest_spatial_layer = 0;
+
   priv->max_width = 0;
   priv->max_height = 0;
   gst_clear_av1_picture (&priv->current_picture);
@@ -379,6 +381,13 @@ gst_av1_decoder_process_sequence (GstAV1Decoder * self, GstAV1OBU * obu)
     priv->preferred_output_delay = 0;
   }
 
+  if (priv->parser->state.operating_point_idc) {
+    self->highest_spatial_layer =
+        _floor_log2 (priv->parser->state.operating_point_idc >> 8);
+    GST_INFO_OBJECT (self, "set highest spatial layer to %d",
+        self->highest_spatial_layer);
+  }
+
   ret = klass->new_sequence (self, &seq_header,
       GST_AV1_TOTAL_REFS_PER_FRAME + priv->preferred_output_delay);
   if (ret != GST_FLOW_OK) {
@@ -475,6 +484,9 @@ gst_av1_decoder_decode_frame_header (GstAV1Decoder * self,
     picture->temporal_id = obu->header.obu_temporal_id;
     picture->spatial_id = obu->header.obu_spatial_id;
 
+    g_assert (picture->spatial_id <= self->highest_spatial_layer);
+    g_assert (self->highest_spatial_layer < GST_AV1_MAX_NUM_SPATIAL_LAYERS);
+
     if (!frame_header->show_frame && !frame_header->showable_frame)
       GST_VIDEO_CODEC_FRAME_FLAG_SET (priv->current_frame,
           GST_VIDEO_CODEC_FRAME_FLAG_DECODE_ONLY);
@@ -683,6 +695,15 @@ gst_av1_decoder_handle_frame (GstVideoDecoder * decoder,
     goto out;
   }
 
+  if (priv->current_picture->temporal_id > self->highest_spatial_layer) {
+    ret = GST_FLOW_ERROR;
+    GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
+        ("current picture temporal_id %d should not be higher than "
+            "highest spatial layer %d", priv->current_picture->temporal_id,
+            self->highest_spatial_layer), (NULL), ret);
+    goto out;
+  }
+
   if (!priv->current_picture->frame_hdr.show_existing_frame) {
     if (klass->end_picture) {
       ret = klass->end_picture (self, priv->current_picture);
@@ -704,9 +725,7 @@ out:
       /* Only output one frame with the highest spatial id from each TU
        * when there are multiple spatial layers.
        */
-      if (priv->parser->state.operating_point_idc &&
-          obu.header.obu_spatial_id <
-          _floor_log2 (priv->parser->state.operating_point_idc >> 8)) {
+      if (obu.header.obu_spatial_id < self->highest_spatial_layer) {
         gst_av1_picture_unref (priv->current_picture);
         gst_video_decoder_release_frame (decoder, frame);
       } else {
diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.h b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.h
index faa760ecee..e249d54266 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.h
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.h
@@ -53,6 +53,7 @@ struct _GstAV1Decoder
 
   /*< protected >*/
   GstVideoCodecState * input_state;
+  guint highest_spatial_layer;
 
   /*< private >*/
   GstAV1DecoderPrivate *priv;