From 50cad4a7faea2c1ccdfad037234e08a5b38ba9d6 Mon Sep 17 00:00:00 2001 From: Doug Nazar Date: Thu, 22 May 2025 09:10:49 -0400 Subject: [PATCH] libav: Only allocate extradata while decoding Part-of: --- subprojects/gst-libav/ext/libav/gstavauddec.c | 2 +- subprojects/gst-libav/ext/libav/gstavaudenc.c | 2 +- .../gst-libav/ext/libav/gstavcodecmap.c | 99 ++++++++++--------- .../gst-libav/ext/libav/gstavcodecmap.h | 3 +- subprojects/gst-libav/ext/libav/gstavviddec.c | 2 +- subprojects/gst-libav/ext/libav/gstavvidenc.c | 2 +- 6 files changed, 58 insertions(+), 52 deletions(-) diff --git a/subprojects/gst-libav/ext/libav/gstavauddec.c b/subprojects/gst-libav/ext/libav/gstavauddec.c index 6eb344258a..ad21cca372 100644 --- a/subprojects/gst-libav/ext/libav/gstavauddec.c +++ b/subprojects/gst-libav/ext/libav/gstavauddec.c @@ -297,7 +297,7 @@ gst_ffmpegauddec_set_format (GstAudioDecoder * decoder, GstCaps * caps) /* get size and so */ gst_ffmpeg_caps_with_codecid (oclass->in_plugin->id, - oclass->in_plugin->type, caps, ffmpegdec->context); + oclass->in_plugin->type, caps, ffmpegdec->context, FALSE); /* workaround encoder bugs */ ffmpegdec->context->workaround_bugs |= FF_BUG_AUTODETECT; diff --git a/subprojects/gst-libav/ext/libav/gstavaudenc.c b/subprojects/gst-libav/ext/libav/gstavaudenc.c index 0a6da7be0b..af22b07ecd 100644 --- a/subprojects/gst-libav/ext/libav/gstavaudenc.c +++ b/subprojects/gst-libav/ext/libav/gstavaudenc.c @@ -286,7 +286,7 @@ gst_ffmpegaudenc_set_format (GstAudioEncoder * encoder, GstAudioInfo * info) } GST_DEBUG_OBJECT (ffmpegaudenc, "chose caps %" GST_PTR_FORMAT, allowed_caps); gst_ffmpeg_caps_with_codecid (oclass->in_plugin->id, - oclass->in_plugin->type, allowed_caps, ffmpegaudenc->context); + oclass->in_plugin->type, allowed_caps, ffmpegaudenc->context, TRUE); /* open codec */ if (gst_ffmpeg_avcodec_open (ffmpegaudenc->context, oclass->in_plugin) < 0) { diff --git a/subprojects/gst-libav/ext/libav/gstavcodecmap.c b/subprojects/gst-libav/ext/libav/gstavcodecmap.c index 81b2827e97..7e67c57fd6 100644 --- a/subprojects/gst-libav/ext/libav/gstavcodecmap.c +++ b/subprojects/gst-libav/ext/libav/gstavcodecmap.c @@ -3858,7 +3858,8 @@ full_copy: void gst_ffmpeg_caps_with_codecid (enum AVCodecID codec_id, - enum AVMediaType codec_type, const GstCaps * caps, AVCodecContext * context) + enum AVMediaType codec_type, const GstCaps * caps, + AVCodecContext * context, gboolean encode) { GstStructure *str; const GValue *value; @@ -3872,59 +3873,62 @@ gst_ffmpeg_caps_with_codecid (enum AVCodecID codec_id, str = gst_caps_get_structure (caps, 0); - /* extradata parsing (esds [mpeg4], wma/wmv, msmpeg4v1/2/3, etc.) */ - if ((value = gst_structure_get_value (str, "codec_data"))) { - GstMapInfo map; - - buf = gst_value_get_buffer (value); - gst_buffer_map (buf, &map, GST_MAP_READ); - + /* only add extradata for decoding */ + if (!encode) { /* free the old one if it is there */ if (context->extradata) av_free (context->extradata); + /* extradata parsing (esds [mpeg4], wma/wmv, msmpeg4v1/2/3, etc.) */ + if ((value = gst_structure_get_value (str, "codec_data"))) { + GstMapInfo map; + + buf = gst_value_get_buffer (value); + gst_buffer_map (buf, &map, GST_MAP_READ); + #if 0 - if (codec_id == AV_CODEC_ID_H264) { - guint extrasize; + if (codec_id == AV_CODEC_ID_H264) { + guint extrasize; - GST_DEBUG ("copy, escaping codec_data %d", size); - /* ffmpeg h264 expects the codec_data to be escaped, there is no real - * reason for this but let's just escape it for now. Start by allocating - * enough space, x2 is more than enough. - * - * FIXME, we disabled escaping because some file already contain escaped - * codec_data and then we escape twice and fail. It's better to leave it - * as is, as that is what most players do. */ - context->extradata = - av_mallocz (GST_ROUND_UP_16 (size * 2 + - AV_INPUT_BUFFER_PADDING_SIZE)); - copy_config (context->extradata, data, size, &extrasize); - GST_DEBUG ("escaped size: %d", extrasize); - context->extradata_size = extrasize; - } else + GST_DEBUG ("copy, escaping codec_data %d", size); + /* ffmpeg h264 expects the codec_data to be escaped, there is no real + * reason for this but let's just escape it for now. Start by allocating + * enough space, x2 is more than enough. + * + * FIXME, we disabled escaping because some file already contain escaped + * codec_data and then we escape twice and fail. It's better to leave it + * as is, as that is what most players do. */ + context->extradata = + av_mallocz (GST_ROUND_UP_16 (size * 2 + + AV_INPUT_BUFFER_PADDING_SIZE)); + copy_config (context->extradata, data, size, &extrasize); + GST_DEBUG ("escaped size: %d", extrasize); + context->extradata_size = extrasize; + } else #endif - { - /* allocate with enough padding */ - GST_DEBUG ("copy codec_data"); - context->extradata = - av_mallocz (GST_ROUND_UP_16 (map.size + - AV_INPUT_BUFFER_PADDING_SIZE)); - memcpy (context->extradata, map.data, map.size); - context->extradata_size = map.size; + { + /* allocate with enough padding */ + GST_DEBUG ("copy codec_data"); + context->extradata = + av_mallocz (GST_ROUND_UP_16 (map.size + + AV_INPUT_BUFFER_PADDING_SIZE)); + memcpy (context->extradata, map.data, map.size); + context->extradata_size = map.size; + } + + /* Hack for VC1. Sometimes the first (length) byte is 0 for some files */ + if (codec_id == AV_CODEC_ID_VC1 && map.size > 0 && map.data[0] == 0) { + context->extradata[0] = (guint8) map.size; + } + + GST_DEBUG ("have codec data of size %" G_GSIZE_FORMAT, map.size); + + gst_buffer_unmap (buf, &map); + } else { + context->extradata = NULL; + context->extradata_size = 0; + GST_DEBUG ("no codec data"); } - - /* Hack for VC1. Sometimes the first (length) byte is 0 for some files */ - if (codec_id == AV_CODEC_ID_VC1 && map.size > 0 && map.data[0] == 0) { - context->extradata[0] = (guint8) map.size; - } - - GST_DEBUG ("have codec data of size %" G_GSIZE_FORMAT, map.size); - - gst_buffer_unmap (buf, &map); - } else { - context->extradata = NULL; - context->extradata_size = 0; - GST_DEBUG ("no codec data"); } switch (codec_id) { @@ -5107,7 +5111,8 @@ gst_ffmpeg_caps_to_codecid (const GstCaps * caps, AVCodecContext * context) context->codec_type = AVMEDIA_TYPE_UNKNOWN; } context->codec_id = id; - gst_ffmpeg_caps_with_codecid (id, context->codec_type, caps, context); + // Say we're encoding since we don't want to allocate the extradata region + gst_ffmpeg_caps_with_codecid (id, context->codec_type, caps, context, TRUE); } if (id != AV_CODEC_ID_NONE) { diff --git a/subprojects/gst-libav/ext/libav/gstavcodecmap.h b/subprojects/gst-libav/ext/libav/gstavcodecmap.h index c1cf495283..091f23f52e 100644 --- a/subprojects/gst-libav/ext/libav/gstavcodecmap.h +++ b/subprojects/gst-libav/ext/libav/gstavcodecmap.h @@ -81,7 +81,8 @@ void gst_ffmpeg_caps_with_codecid (enum AVCodecID codec_id, enum AVMediaType codec_type, const GstCaps *caps, - AVCodecContext *context); + AVCodecContext *context, + gboolean encode); /* * caps_with_codectype () transforms a GstCaps that belongs to diff --git a/subprojects/gst-libav/ext/libav/gstavviddec.c b/subprojects/gst-libav/ext/libav/gstavviddec.c index e476881113..a492ac837b 100644 --- a/subprojects/gst-libav/ext/libav/gstavviddec.c +++ b/subprojects/gst-libav/ext/libav/gstavviddec.c @@ -624,7 +624,7 @@ gst_ffmpegviddec_set_format (GstVideoDecoder * decoder, /* FIXME : Create a method that takes GstVideoCodecState instead */ /* get size and so */ gst_ffmpeg_caps_with_codecid (oclass->in_plugin->id, - oclass->in_plugin->type, state->caps, ffmpegdec->context); + oclass->in_plugin->type, state->caps, ffmpegdec->context, FALSE); GST_LOG_OBJECT (ffmpegdec, "size after %dx%d", ffmpegdec->context->width, ffmpegdec->context->height); diff --git a/subprojects/gst-libav/ext/libav/gstavvidenc.c b/subprojects/gst-libav/ext/libav/gstavvidenc.c index 2b2a04dcc9..2ae6a1b451 100644 --- a/subprojects/gst-libav/ext/libav/gstavvidenc.c +++ b/subprojects/gst-libav/ext/libav/gstavvidenc.c @@ -323,7 +323,7 @@ gst_ffmpegvidenc_set_format (GstVideoEncoder * encoder, } GST_DEBUG_OBJECT (ffmpegenc, "chose caps %" GST_PTR_FORMAT, allowed_caps); gst_ffmpeg_caps_with_codecid (oclass->in_plugin->id, - oclass->in_plugin->type, allowed_caps, ffmpegenc->context); + oclass->in_plugin->type, allowed_caps, ffmpegenc->context, TRUE); /* open codec */ if (gst_ffmpeg_avcodec_open (ffmpegenc->context, oclass->in_plugin) < 0) {