From 049655c82455a0c24bf95fba8f700dd4758cc656 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Wed, 16 Mar 2022 13:17:21 -0400 Subject: [PATCH] codecs: av1: Fix state when we show existing keyframe Showing existing keyframe have special meaning in AV1. All the references frame will be refreshed with the original keyframe information. The refresh process (7.20) is implemented by saving data from the frame_header into the state. To fix this special case, load all the relevant information into the frame_header. As there is nothing happening in between this and the loading of the key-frame into the state, this patch also remove the separate API function, using it internally instead. Fixes #1090 Part-of: --- .../gst-docs/symbols/symbol_index.json | 3 +- .../gst-libs/gst/codecparsers/gstav1parser.c | 141 ++++++++---------- .../gst-libs/gst/codecparsers/gstav1parser.h | 5 - .../gst-libs/gst/codecs/gstav1decoder.c | 8 - .../gst-libs/gst/vaapi/gstvaapidecoder_av1.c | 15 -- 5 files changed, 60 insertions(+), 112 deletions(-) diff --git a/subprojects/gst-docs/symbols/symbol_index.json b/subprojects/gst-docs/symbols/symbol_index.json index 02270e3ebb..7741aaec80 100644 --- a/subprojects/gst-docs/symbols/symbol_index.json +++ b/subprojects/gst-docs/symbols/symbol_index.json @@ -37478,7 +37478,6 @@ "gst_av1_parser_parse_temporal_delimiter_obu", "gst_av1_parser_parse_tile_group_obu", "gst_av1_parser_parse_tile_list_obu", - "gst_av1_parser_reference_frame_loading", "gst_av1_parser_reference_frame_update", "gst_av1_parser_reset", "gst_av1_parser_reset_annex_b", @@ -66199,4 +66198,4 @@ "zbar:message", "zebrastripe", "zebrastripe:threshold" -] \ No newline at end of file +] diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gstav1parser.c b/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gstav1parser.c index 5fa283c384..fb1050f369 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gstav1parser.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gstav1parser.c @@ -67,10 +67,6 @@ * should call gst_av1_parser_reference_frame_update() to update the parser's inside * state(such as reference information, global segmentation information, etc). * - * Note: If the frame is actived by show_existing_frame in #GST_AV1_OBU_FRAME_HEADER, - * the function of gst_av1_parser_reference_frame_loading() should be called before - * really showing that frame. - * * @since: 1.18.00 */ @@ -3496,6 +3492,64 @@ gst_av1_set_frame_refs (GstAV1Parser * parser, frame_header->ref_frame_idx[i] = ref; } +/* 7.21 */ +static void +gst_av1_parser_reference_frame_loading (GstAV1Parser * parser, + GstAV1FrameHeaderOBU * frame_header) +{ + GstAV1ReferenceFrameInfo *ref_info = &(parser->state.ref_info); + gint idx = frame_header->frame_to_show_map_idx; + GstAV1TileInfo *ref_tile_info = &ref_info->entry[idx].ref_tile_info; + const gint all_frames = (1 << GST_AV1_NUM_REF_FRAMES) - 1; + + /* copy the relevant frame information as these will be needed by + * all subclasses. */ + frame_header->frame_type = ref_info->entry[idx].ref_frame_type; + frame_header->upscaled_width = ref_info->entry[idx].ref_upscaled_width; + frame_header->frame_width = ref_info->entry[idx].ref_frame_width; + frame_header->frame_height = ref_info->entry[idx].ref_frame_height; + frame_header->render_width = ref_info->entry[idx].ref_render_width; + frame_header->render_height = ref_info->entry[idx].ref_render_height; + + if (parser->seq_header->film_grain_params_present) + frame_header->film_grain_params = + ref_info->entry[idx].ref_film_grain_params; + + /* the remaining is only relevant to ensure proper state update and only + * keyframe updates the state. */ + if (frame_header->frame_type != GST_AV1_KEY_FRAME) + return; + + frame_header->refresh_frame_flags = all_frames; + frame_header->current_frame_id = ref_info->entry[idx].ref_frame_id; + frame_header->order_hint = ref_info->entry[idx].ref_order_hint; + frame_header->segmentation_params = + ref_info->entry[idx].ref_segmentation_params; + frame_header->global_motion_params = + ref_info->entry[idx].ref_global_motion_params; + frame_header->loop_filter_params = ref_info->entry[idx].ref_lf_params; + frame_header->tile_info = *ref_tile_info; + + parser->state.current_frame_id = ref_info->entry[idx].ref_frame_id; + parser->state.upscaled_width = ref_info->entry[idx].ref_upscaled_width; + parser->state.frame_width = ref_info->entry[idx].ref_frame_width; + parser->state.frame_height = ref_info->entry[idx].ref_frame_height; + parser->state.render_width = ref_info->entry[idx].ref_render_width; + parser->state.render_height = ref_info->entry[idx].ref_render_height; + parser->state.mi_cols = ref_info->entry[idx].ref_mi_cols; + parser->state.mi_rows = ref_info->entry[idx].ref_mi_rows; + + memcpy (parser->state.mi_col_starts, ref_tile_info->mi_col_starts, + sizeof (guint32) * (GST_AV1_MAX_TILE_COLS + 1)); + memcpy (parser->state.mi_row_starts, ref_tile_info->mi_row_starts, + sizeof (guint32) * (GST_AV1_MAX_TILE_ROWS + 1)); + parser->state.tile_cols_log2 = ref_tile_info->tile_cols_log2; + parser->state.tile_cols = ref_tile_info->tile_cols; + parser->state.tile_rows_log2 = ref_tile_info->tile_rows_log2; + parser->state.tile_rows = ref_tile_info->tile_rows; + parser->state.tile_size_bytes = ref_tile_info->tile_size_bytes; +} + /* 5.9.2 */ static GstAV1ParserResult gst_av1_parse_uncompressed_frame_header (GstAV1Parser * parser, GstAV1OBU * obu, @@ -3581,16 +3635,7 @@ gst_av1_parse_uncompressed_frame_header (GstAV1Parser * parser, GstAV1OBU * obu, } } - frame_header->frame_type = - ref_info->entry[frame_header->frame_to_show_map_idx].ref_frame_type; - if (frame_header->frame_type == GST_AV1_KEY_FRAME) { - frame_header->refresh_frame_flags = all_frames; - } - - /* just use the frame_to_show's grain_params - * if (seq_header->film_grain_params_present) - * load_grain_params () */ - + gst_av1_parser_reference_frame_loading (parser, frame_header); goto success; } @@ -4174,74 +4219,6 @@ error: return retval; } -/* 7.21 */ -/** - * gst_av1_parser_reference_frame_loading: - * @parser: the #GstAV1Parser - * @frame_header: a #GstAV1FrameHeaderOBU to load - * - * Load the context of @frame_header to parser's state. This function is - * used when we want to show already parsed frames before. - * - * Returns: The #GstAV1ParserResult. - * - * Since: 1.18 - */ -GstAV1ParserResult -gst_av1_parser_reference_frame_loading (GstAV1Parser * parser, - GstAV1FrameHeaderOBU * frame_header) -{ - GstAV1ReferenceFrameInfo *ref_info; - GstAV1TileInfo *ref_tile_info; - - g_return_val_if_fail (parser != NULL, GST_AV1_PARSER_INVALID_OPERATION); - g_return_val_if_fail (frame_header != NULL, GST_AV1_PARSER_INVALID_OPERATION); - - if (!parser->seq_header) { - GST_WARNING ("Missing OBU Reference: seq_header"); - return GST_AV1_PARSER_MISSING_OBU_REFERENCE; - } - - ref_info = &(parser->state.ref_info); - - if (frame_header->frame_to_show_map_idx > GST_AV1_NUM_REF_FRAMES - 1) - return GST_AV1_PARSER_BITSTREAM_ERROR; - - g_assert (ref_info->entry[frame_header->frame_to_show_map_idx].ref_valid); - - parser->state.current_frame_id = - ref_info->entry[frame_header->frame_to_show_map_idx].ref_frame_id; - parser->state.upscaled_width = - ref_info->entry[frame_header->frame_to_show_map_idx].ref_upscaled_width; - parser->state.frame_width = - ref_info->entry[frame_header->frame_to_show_map_idx].ref_frame_width; - parser->state.frame_height = - ref_info->entry[frame_header->frame_to_show_map_idx].ref_frame_height; - parser->state.render_width = - ref_info->entry[frame_header->frame_to_show_map_idx].ref_render_width; - parser->state.render_height = - ref_info->entry[frame_header->frame_to_show_map_idx].ref_render_height; - parser->state.mi_cols = - ref_info->entry[frame_header->frame_to_show_map_idx].ref_mi_cols; - parser->state.mi_rows = - ref_info->entry[frame_header->frame_to_show_map_idx].ref_mi_rows; - - ref_tile_info = - &ref_info->entry[frame_header->frame_to_show_map_idx].ref_tile_info; - - memcpy (parser->state.mi_col_starts, ref_tile_info->mi_col_starts, - sizeof (guint32) * (GST_AV1_MAX_TILE_COLS + 1)); - memcpy (parser->state.mi_row_starts, ref_tile_info->mi_row_starts, - sizeof (guint32) * (GST_AV1_MAX_TILE_ROWS + 1)); - parser->state.tile_cols_log2 = ref_tile_info->tile_cols_log2; - parser->state.tile_cols = ref_tile_info->tile_cols; - parser->state.tile_rows_log2 = ref_tile_info->tile_rows_log2; - parser->state.tile_rows = ref_tile_info->tile_rows; - parser->state.tile_size_bytes = ref_tile_info->tile_size_bytes; - - return GST_AV1_PARSER_OK; -} - /** * gst_av1_parser_reference_frame_update: * @parser: the #GstAV1Parser diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gstav1parser.h b/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gstav1parser.h index f8df2efca9..a5f1c761f6 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gstav1parser.h +++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gstav1parser.h @@ -1820,11 +1820,6 @@ GstAV1ParserResult gst_av1_parser_parse_frame_obu (GstAV1Parser * parser, GstAV1OBU * obu, GstAV1FrameOBU * frame); -GST_CODEC_PARSERS_API -GstAV1ParserResult -gst_av1_parser_reference_frame_loading (GstAV1Parser * parser, - GstAV1FrameHeaderOBU * frame_header); - GST_CODEC_PARSERS_API GstAV1ParserResult gst_av1_parser_reference_frame_update (GstAV1Parser * parser, diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c index df727507bf..6fe9acc6a9 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c @@ -363,12 +363,6 @@ gst_av1_decoder_decode_frame_header (GstAV1Decoder * self, return GST_FLOW_ERROR; } - if (gst_av1_parser_reference_frame_loading (priv->parser, - &ref_picture->frame_hdr) != GST_AV1_PARSER_OK) { - GST_WARNING_OBJECT (self, "load the reference frame failed"); - return GST_FLOW_ERROR; - } - /* FIXME: duplicate picture might be optional feature like that of VP9 * decoder baseclass */ g_assert (klass->duplicate_picture); @@ -380,8 +374,6 @@ gst_av1_decoder_decode_frame_header (GstAV1Decoder * self, picture->system_frame_number = priv->current_frame->system_frame_number; picture->frame_hdr = *frame_header; - picture->frame_hdr.render_width = ref_picture->frame_hdr.render_width; - picture->frame_hdr.render_height = ref_picture->frame_hdr.render_height; priv->current_picture = picture; } else { picture = gst_av1_picture_new (); diff --git a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidecoder_av1.c b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidecoder_av1.c index eff62e635d..e8a0e310fb 100644 --- a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidecoder_av1.c +++ b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidecoder_av1.c @@ -753,12 +753,6 @@ av1_decode_frame_header (GstVaapiDecoderAV1 * decoder, return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN; } - if (gst_av1_parser_reference_frame_loading (priv->parser, - &to_show_picture->frame_header) != GST_AV1_PARSER_OK) { - GST_ERROR ("load frame to show ref frame failed"); - return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN; - } - picture = (GstVaapiPictureAV1 *) gst_vaapi_picture_new_clone (GST_VAAPI_PICTURE_CAST (to_show_picture)); if (!picture) @@ -769,15 +763,6 @@ av1_decode_frame_header (GstVaapiDecoderAV1 * decoder, GST_VAAPI_PICTURE_FLAG_UNSET (picture, GST_VAAPI_PICTURE_FLAG_SKIPPED); picture->frame_header = to_show_picture->frame_header; - /* only update references if the frame_to_show_map_idx is a KEY FRAME */ - if (picture->frame_header.frame_type == GST_AV1_KEY_FRAME) { - picture->frame_header = to_show_picture->frame_header; - g_assert (picture->frame_header.refresh_frame_flags == - ((1 << GST_AV1_NUM_REF_FRAMES) - 1)); - } else { - /* Just set to no update ref */ - picture->frame_header.refresh_frame_flags = 0; - } } else { /* Resolution changed */ if (priv->width != priv->seq_header->max_frame_width_minus_1 + 1 ||