From 1cead20777cf7705b3926fe696df1e6d2f58c373 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sat, 4 Jul 2020 12:33:20 -0400 Subject: [PATCH] encodebin: Fix and refactor smart encoding It was not working properly and the implementation of the smartencoder element was weird. This introduce a number of changes (which are all in one single commit because they basically all work together and lead to basically reimplementing the element): * Make smartencoder a bin so that the reencoding chain of elements are inside of it instead of not having any parent. Those elements were not be visible when dumping the pipeline which was very confusing. * Make encodebin create the right encoder with a capsfilter (and parser) to properly enforce the format specified by the user, and so that the encoder properties specified in the encoding profile are respected. * Use `decodebin` to do the decoding instead of selecting a decoder ourself and not plug any parser etc... * Ensure that negotiated format in the sinkpad of smart encoder is fixed through time when the user requested a non dynamic output * Add a parser at the beginning of the smart encoder * Handle errors when reencoding Part-of: --- gst-libs/gst/pbutils/encoding-profile.c | 14 +- gst/encoding/gstencodebin.c | 193 +++- gst/encoding/gstsmartencoder.c | 1378 ++++++++++++----------- gst/encoding/gstsmartencoder.h | 39 +- 4 files changed, 949 insertions(+), 675 deletions(-) diff --git a/gst-libs/gst/pbutils/encoding-profile.c b/gst-libs/gst/pbutils/encoding-profile.c index d8a33ad436..8636402b18 100644 --- a/gst-libs/gst/pbutils/encoding-profile.c +++ b/gst-libs/gst/pbutils/encoding-profile.c @@ -665,11 +665,14 @@ gst_encoding_profile_get_single_segment (GstEncodingProfile * profile) /** * gst_encoding_profile_set_single_segment: * @profile: a #GstEncodingProfile - * @single_segment: #TRUE if the stream represented by @profile should use a single - * segment before the encoder #FALSE otherwise. + * @single_segment: #TRUE if the stream represented by @profile should use a + * single segment before the encoder, #FALSE otherwise. * - * If using a single segment, buffers will be retimestamped - * and segments will be eat so as to appear as one segment. + * If using a single segment, buffers will be retimestamped and segments will be + * eat so as to appear as one segment. + * + * > *NOTE*: Single segment is not property supported when using + * > #encodebin:avoid-reencoding * * Since: 1.18 */ @@ -913,6 +916,9 @@ gst_encoding_video_profile_get_pass (GstEncodingVideoProfile * prof) * gst_encoding_video_profile_get_variableframerate: * @prof: a #GstEncodingVideoProfile * + * > *NOTE*: Fixed framerate won't be enforced when #encodebin:avoid-reencoding + * > is set. + * * Returns: Whether non-constant video framerate is allowed for encoding. */ gboolean diff --git a/gst/encoding/gstencodebin.c b/gst/encoding/gstencodebin.c index c662b796d7..f0addaa16e 100644 --- a/gst/encoding/gstencodebin.c +++ b/gst/encoding/gstencodebin.c @@ -209,6 +209,8 @@ struct _StreamGroup GstElement *combiner; GstElement *parser; GstElement *smartencoder; + GstElement *smart_capsfilter; + gulong smart_capsfilter_sid; GstElement *outfilter; /* Output capsfilter (streamprofile.format) */ gulong outputfilter_caps_sid; GstElement *formatter; @@ -1128,10 +1130,17 @@ _profile_restriction_caps_cb (GstEncodingProfile * profile, static void _capsfilter_force_format (GstPad * pad, - GParamSpec * arg G_GNUC_UNUSED, gulong * signal_id) + GParamSpec * arg G_GNUC_UNUSED, StreamGroup * sgroup) { GstCaps *caps; GstStructure *structure; + GstElement *parent = + GST_ELEMENT_CAST (gst_object_get_parent (GST_OBJECT (pad))); + + if (!parent) { + GST_DEBUG_OBJECT (pad, "Doesn't have a parent anymore"); + return; + } g_object_get (pad, "caps", &caps, NULL); caps = gst_caps_copy (caps); @@ -1139,10 +1148,36 @@ _capsfilter_force_format (GstPad * pad, structure = gst_caps_get_structure (caps, 0); gst_structure_remove_field (structure, "streamheader"); GST_INFO_OBJECT (pad, "Forcing caps to %" GST_PTR_FORMAT, caps); - g_object_set (GST_OBJECT_PARENT (pad), "caps", caps, NULL); - g_signal_handler_disconnect (pad, *signal_id); - *signal_id = 0; + if (parent == sgroup->outfilter || parent == sgroup->smart_capsfilter) { + /* outfilter and the smart encoder internal capsfilter need to always be + * in sync so the caps match between the two */ + if (sgroup->smart_capsfilter) { + gst_structure_remove_field (structure, "codec_data"); + /* The smart encoder handles codec_data itself */ + g_object_set (sgroup->smart_capsfilter, "caps", caps, NULL); + + g_signal_handler_disconnect (sgroup->smart_capsfilter->sinkpads->data, + sgroup->smart_capsfilter_sid); + sgroup->smart_capsfilter_sid = 0; + } + + if (sgroup->outfilter) { + GstCaps *tmpcaps = gst_caps_copy (caps); + g_object_set (sgroup->outfilter, "caps", tmpcaps, NULL); + gst_caps_unref (tmpcaps); + g_signal_handler_disconnect (sgroup->outfilter->sinkpads->data, + sgroup->outputfilter_caps_sid); + sgroup->outputfilter_caps_sid = 0; + } + } else if (parent == sgroup->capsfilter) { + g_object_set (parent, "caps", caps, NULL); + g_signal_handler_disconnect (pad, sgroup->inputfilter_caps_sid); + } else { + g_assert_not_reached (); + } + gst_caps_unref (caps); + gst_object_unref (parent); } static void @@ -1155,8 +1190,7 @@ _set_group_caps_format (StreamGroup * sgroup, GstEncodingProfile * prof, if (!sgroup->outputfilter_caps_sid) { sgroup->outputfilter_caps_sid = g_signal_connect (sgroup->outfilter->sinkpads->data, - "notify::caps", G_CALLBACK (_capsfilter_force_format), - &sgroup->outputfilter_caps_sid); + "notify::caps", G_CALLBACK (_capsfilter_force_format), sgroup); } } } @@ -1204,6 +1238,112 @@ _set_up_fake_encoder_pad_probe (GstEncodeBin * ebin, StreamGroup * sgroup) gst_object_unref (pad); } +static GstElement * +setup_smart_encoder (GstEncodeBin * ebin, GstEncodingProfile * sprof, + StreamGroup * sgroup) +{ + GstElement *encoder = NULL, *parser = NULL; + GstElement *reencoder_bin = NULL; + GstElement *sinkelement, *convert = NULL; + GstElement *smartencoder = g_object_new (GST_TYPE_SMART_ENCODER, NULL); + GstPad *srcpad = gst_element_get_static_pad (smartencoder, "src"); + GstCaps *format = gst_encoding_profile_get_format (sprof); + GstCaps *tmpcaps = gst_pad_query_caps (srcpad, NULL); + const gboolean native_video = + ! !(ebin->flags & GST_ENCODEBIN_FLAG_NO_VIDEO_CONVERSION); + + /* Check if stream format is compatible */ + if (!gst_caps_can_intersect (tmpcaps, format)) { + GST_DEBUG_OBJECT (ebin, + "We don't have a smart encoder for the stream format: %" GST_PTR_FORMAT, + format); + goto err; + } + + sinkelement = encoder = _get_encoder (ebin, sprof); + if (!encoder) { + GST_INFO_OBJECT (ebin, "No encoder found... not using smart rendering"); + goto err; + } + + parser = _get_parser (ebin, sprof); + sgroup->smart_capsfilter = gst_element_factory_make ("capsfilter", NULL); + reencoder_bin = gst_bin_new (NULL); + g_object_set (sgroup->smart_capsfilter, "caps", format, NULL); + + gst_bin_add_many (GST_BIN (reencoder_bin), + gst_object_ref (encoder), + parser ? gst_object_ref (parser) : sgroup->smart_capsfilter, + parser ? gst_object_ref (sgroup->smart_capsfilter) : NULL, NULL); + if (!native_video) { + convert = gst_element_factory_make ("videoconvert", NULL); + if (!convert) { + GST_ERROR_OBJECT (ebin, "`videoconvert` element missing"); + goto err; + } + + gst_bin_add (GST_BIN (reencoder_bin), gst_object_ref (convert)); + if (!gst_element_link (convert, sinkelement)) { + GST_ERROR_OBJECT (ebin, "Can not link `videoconvert` to %" GST_PTR_FORMAT, + sinkelement); + goto err; + } + sinkelement = convert; + } + + if (!gst_element_link_many (encoder, + parser ? parser : sgroup->smart_capsfilter, + parser ? sgroup->smart_capsfilter : NULL, NULL)) { + GST_ERROR_OBJECT (ebin, "Can not link smart encoding elements"); + goto err; + } + + if (!gst_element_add_pad (reencoder_bin, + gst_ghost_pad_new ("sink", sinkelement->sinkpads->data))) { + GST_ERROR_OBJECT (ebin, "Can add smart encoding bin `srcpad`"); + goto err; + } + + if (!gst_element_add_pad (reencoder_bin, + gst_ghost_pad_new ("src", sgroup->smart_capsfilter->srcpads->data))) { + GST_ERROR_OBJECT (ebin, "Could not ghost smart encoder bin" + " srcpad, not being smart."); + goto err; + } + + if (!gst_encoding_profile_get_allow_dynamic_output (sprof)) { + /* Enforce no dynamic output in the smart encoder */ + if (!sgroup->smart_capsfilter_sid) { + sgroup->smart_capsfilter_sid = + g_signal_connect (sgroup->smart_capsfilter->sinkpads->data, + "notify::caps", G_CALLBACK (_capsfilter_force_format), sgroup); + } + } + + if (!gst_smart_encoder_set_encoder (GST_SMART_ENCODER (smartencoder), + format, reencoder_bin)) { + reencoder_bin = NULL; /* We do not own the ref anymore */ + GST_ERROR_OBJECT (ebin, "Could not set encoder to the smart encoder," + " disabling smartness"); + goto err; + } + +done: + gst_caps_unref (tmpcaps); + gst_caps_unref (format); + gst_object_unref (srcpad); + gst_clear_object (&encoder); + gst_clear_object (&parser); + gst_clear_object (&convert); + + return smartencoder; + +err: + gst_clear_object (&smartencoder); + gst_clear_object (&reencoder_bin); + goto done; +} + /* FIXME : Add handling of streams that don't require conversion elements */ /* * Create the elements, StreamGroup, add the sink pad, link it to the muxer @@ -1338,10 +1478,16 @@ _create_stream_group (GstEncodeBin * ebin, GstEncodingProfile * sprof, tosync = g_list_append (tosync, sgroup->splitter); if (gst_encoding_profile_get_single_segment (sprof)) { - sgroup->identity = gst_element_factory_make ("identity", NULL); - g_object_set (sgroup->identity, "single-segment", TRUE, NULL); - gst_bin_add (GST_BIN (ebin), sgroup->identity); - tosync = g_list_append (tosync, sgroup->identity); + + if (!ebin->avoid_reencoding) { + sgroup->identity = gst_element_factory_make ("identity", NULL); + g_object_set (sgroup->identity, "single-segment", TRUE, NULL); + gst_bin_add (GST_BIN (ebin), sgroup->identity); + tosync = g_list_append (tosync, sgroup->identity); + } else { + GST_INFO_OBJECT (ebin, "Single segment is not supported when avoiding" + " to reencode!"); + } } /* Input queue @@ -1386,26 +1532,16 @@ _create_stream_group (GstEncodeBin * ebin, GstEncodingProfile * sprof, goto no_combiner_sinkpad; if (ebin->avoid_reencoding) { - GstCaps *tmpcaps; - GST_DEBUG ("Asked to use Smart Encoder"); - sgroup->smartencoder = g_object_new (GST_TYPE_SMART_ENCODER, NULL); - - /* Check if stream format is compatible */ - srcpad = gst_element_get_static_pad (sgroup->smartencoder, "src"); - tmpcaps = gst_pad_query_caps (srcpad, NULL); - if (!gst_caps_can_intersect (tmpcaps, format)) { - GST_DEBUG ("We don't have a smart encoder for the stream format"); - gst_object_unref (sgroup->smartencoder); - sgroup->smartencoder = NULL; - } else { + sgroup->smartencoder = setup_smart_encoder (ebin, sprof, sgroup); + if (sgroup->smartencoder) { gst_bin_add ((GstBin *) ebin, sgroup->smartencoder); + srcpad = gst_element_get_static_pad (sgroup->smartencoder, "src"); fast_pad_link (srcpad, sinkpad); + gst_object_unref (srcpad); tosync = g_list_append (tosync, sgroup->smartencoder); sinkpad = gst_element_get_static_pad (sgroup->smartencoder, "sink"); } - gst_caps_unref (tmpcaps); - gst_object_unref (srcpad); } srcpad = @@ -1468,8 +1604,7 @@ _create_stream_group (GstEncodeBin * ebin, GstEncodingProfile * sprof, if (!sgroup->inputfilter_caps_sid) { sgroup->inputfilter_caps_sid = g_signal_connect (sgroup->capsfilter->sinkpads->data, - "notify::caps", G_CALLBACK (_capsfilter_force_format), - &sgroup->inputfilter_caps_sid); + "notify::caps", G_CALLBACK (_capsfilter_force_format), sgroup); } } @@ -2136,6 +2271,7 @@ stream_group_free (GstEncodeBin * ebin, StreamGroup * sgroup) } if (sgroup->smartencoder) gst_element_set_state (sgroup->smartencoder, GST_STATE_NULL); + gst_clear_object (&sgroup->smart_capsfilter); if (sgroup->capsfilter) { gst_element_set_state (sgroup->capsfilter, GST_STATE_NULL); @@ -2144,11 +2280,6 @@ stream_group_free (GstEncodeBin * ebin, StreamGroup * sgroup) else gst_element_unlink (sgroup->capsfilter, sgroup->fakesink); - if (sgroup->inputfilter_caps_sid) { - g_signal_handler_disconnect (sgroup->capsfilter->sinkpads->data, - sgroup->inputfilter_caps_sid); - sgroup->inputfilter_caps_sid = 0; - } gst_bin_remove ((GstBin *) ebin, sgroup->capsfilter); } diff --git a/gst/encoding/gstsmartencoder.c b/gst/encoding/gstsmartencoder.c index a950edfa14..5ded06733b 100644 --- a/gst/encoding/gstsmartencoder.c +++ b/gst/encoding/gstsmartencoder.c @@ -1,5 +1,6 @@ /* GStreamer Smart Video Encoder element * Copyright (C) <2010> Edward Hervey + * Copyright (C) <2020> Thibault Saunier * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -17,11 +18,6 @@ * Boston, MA 02110-1301, USA. */ -/* TODO: - * * Implement get_caps/set_caps (store/forward caps) - * * Adjust template caps to the formats we can support - **/ - #ifdef HAVE_CONFIG_H #include "config.h" #endif @@ -35,6 +31,7 @@ GST_DEBUG_CATEGORY_STATIC (smart_encoder_debug); /* FIXME : Update this with new caps */ /* WARNING : We can only allow formats with closed-GOP */ #define ALLOWED_CAPS "video/x-h263;video/x-intel-h263;"\ + "video/x-h264;"\ "video/mpeg,mpegversion=(int)1,systemstream=(boolean)false;"\ "video/mpeg,mpegversion=(int)2,systemstream=(boolean)false;" @@ -50,44 +47,747 @@ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_STATIC_CAPS (ALLOWED_CAPS) ); -static GQuark INTERNAL_ELEMENT; - -/* GstSmartEncoder signals and args */ -enum -{ - /* FILL ME */ - LAST_SIGNAL -}; - -enum -{ - PROP_0 - /* FILL ME */ -}; +G_DEFINE_TYPE (GstSmartEncoder, gst_smart_encoder, GST_TYPE_BIN); static void -_do_init (void) +smart_encoder_reset (GstSmartEncoder * self) { - INTERNAL_ELEMENT = g_quark_from_static_string ("internal-element"); -}; + gst_segment_init (&self->internal_segment, GST_FORMAT_UNDEFINED); + gst_segment_init (&self->input_segment, GST_FORMAT_UNDEFINED); + gst_segment_init (&self->output_segment, GST_FORMAT_UNDEFINED); -G_DEFINE_TYPE_EXTENDED (GstSmartEncoder, gst_smart_encoder, GST_TYPE_ELEMENT, 0, - _do_init ()); + if (self->decoder) { + /* Clean up/remove internal encoding elements */ + gst_element_set_state (self->encoder, GST_STATE_NULL); + gst_element_set_state (self->decoder, GST_STATE_NULL); + gst_clear_object (&self->internal_srcpad); + gst_element_remove_pad (GST_ELEMENT (self), self->internal_sinkpad); + gst_bin_remove (GST_BIN (self), gst_object_ref (self->encoder)); + gst_bin_remove (GST_BIN (self), self->decoder); -static void gst_smart_encoder_dispose (GObject * object); + self->decoder = NULL; + self->internal_sinkpad = NULL; + } + gst_clear_event (&self->segment_event); +} -static gboolean setup_recoder_pipeline (GstSmartEncoder * smart_encoder); +static void +translate_timestamp_from_internal_to_src (GstSmartEncoder * self, + GstClockTime * ts) +{ + GstClockTime running_time; + + if (gst_segment_to_running_time_full (&self->internal_segment, + GST_FORMAT_TIME, *ts, &running_time) > 0) + *ts = running_time + self->output_segment.start; + else /* Negative timestamp */ + *ts = self->output_segment.start - running_time; +} + +static GstFlowReturn +gst_smart_encoder_finish_buffer (GstSmartEncoder * self, GstBuffer * buf) +{ + translate_timestamp_from_internal_to_src (self, &GST_BUFFER_PTS (buf)); + translate_timestamp_from_internal_to_src (self, &GST_BUFFER_DTS (buf)); + GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (buf); + if (self->last_dts > GST_BUFFER_DTS (buf)) { + /* Hack to always produces dts increasing DTS-s that are close to what the + * encoder produced. */ + GST_BUFFER_DTS (buf) = self->last_dts + 1; + } + self->last_dts = GST_BUFFER_DTS (buf); + + return gst_pad_push (self->srcpad, buf); +} + +/***************************************** + * Internal encoder/decoder pipeline * + ******************************************/ +static gboolean +internal_event_func (GstPad * pad, GstObject * parent, GstEvent * event) +{ + GstSmartEncoder *self = GST_SMART_ENCODER (parent); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_EOS: + g_mutex_lock (&self->internal_flow_lock); + if (self->internal_flow == GST_FLOW_CUSTOM_SUCCESS) + self->internal_flow = GST_FLOW_OK; + g_cond_signal (&self->internal_flow_cond); + g_mutex_unlock (&self->internal_flow_lock); + break; + case GST_EVENT_SEGMENT: + gst_event_copy_segment (event, &self->internal_segment); + break; + case GST_EVENT_CAPS: + { + GstCaps *caps; + + gst_event_parse_caps (event, &caps); + caps = gst_caps_copy (caps); + if (self->last_caps) { + GstBuffer *codec_data; + GstCaps *new_caps; + GstStructure *last_struct = gst_caps_get_structure (self->last_caps, 0); + + gst_structure_get (last_struct, "codec_data", GST_TYPE_BUFFER, + &codec_data, NULL); + if (codec_data) + gst_structure_set (gst_caps_get_structure (caps, 0), "codec_data", + GST_TYPE_BUFFER, codec_data, NULL); + + new_caps = gst_caps_intersect (self->last_caps, caps); + if (!new_caps || gst_caps_is_empty (new_caps)) { + GST_ERROR_OBJECT (parent, "New caps from reencoder %" GST_PTR_FORMAT + " are not compatible with previous caps: %" GST_PTR_FORMAT, caps, + self->last_caps); + + g_mutex_lock (&self->internal_flow_lock); + self->internal_flow = GST_FLOW_NOT_NEGOTIATED; + g_cond_signal (&self->internal_flow_cond); + g_mutex_unlock (&self->internal_flow_lock); + + return FALSE; + } + + gst_caps_unref (caps); + caps = new_caps; + } + event = gst_event_new_caps (caps); + self->last_caps = caps; + + return gst_pad_push_event (self->srcpad, event); + } + default: + break; + } + + return gst_pad_event_default (pad, parent, event); +} + +static GstFlowReturn +internal_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) +{ + return gst_smart_encoder_finish_buffer (GST_SMART_ENCODER (parent), buf); +} + +static void +decodebin_src_pad_added_cb (GstElement * decodebin, GstPad * srcpad, + GstSmartEncoder * self) +{ + GstPadLinkReturn ret = gst_pad_link (srcpad, self->encoder->sinkpads->data); + + if (ret != GST_PAD_LINK_OK) { + GST_ERROR_OBJECT (self, "Could not link decoder with encoder! %s", + gst_pad_link_get_name (ret)); + g_mutex_lock (&self->internal_flow_lock); + self->internal_flow = GST_FLOW_NOT_LINKED; + g_mutex_unlock (&self->internal_flow_lock); + } +} + +static gboolean +setup_recoder_pipeline (GstSmartEncoder * self) +{ + GstPad *tmppad; + GstElement *capsfilter; + GstPadLinkReturn lret; + + /* Fast path */ + if (G_UNLIKELY (self->decoder)) + return TRUE; + + g_assert (self->encoder); + GST_DEBUG ("Creating internal decoder and encoder"); + + /* Create decoder/encoder */ + self->decoder = gst_element_factory_make ("decodebin", NULL); + if (G_UNLIKELY (self->decoder == NULL)) + goto no_decoder; + g_signal_connect (self->decoder, "pad-added", + G_CALLBACK (decodebin_src_pad_added_cb), self); + gst_element_set_locked_state (self->decoder, TRUE); + gst_bin_add (GST_BIN (self), self->decoder); + gst_bin_add (GST_BIN (self), gst_object_ref (self->encoder)); + + GST_DEBUG_OBJECT (self, "Creating internal pads"); + + /* Create internal pads */ + + /* Source pad which we'll use to feed data to decoders */ + self->internal_srcpad = gst_pad_new ("internal_src", GST_PAD_SRC); + self->internal_sinkpad = gst_pad_new ("internal_sink", GST_PAD_SINK); + gst_pad_set_iterate_internal_links_function (self->internal_sinkpad, NULL); + if (!gst_element_add_pad (GST_ELEMENT (self), self->internal_sinkpad)) { + GST_ERROR_OBJECT (self, "Could not add internal sinkpad %" GST_PTR_FORMAT, + self->internal_sinkpad); + return FALSE; + } + + gst_pad_set_chain_function (self->internal_sinkpad, + GST_DEBUG_FUNCPTR (internal_chain)); + gst_pad_set_event_function (self->internal_sinkpad, + GST_DEBUG_FUNCPTR (internal_event_func)); + gst_pad_set_active (self->internal_sinkpad, TRUE); + gst_pad_set_active (self->internal_srcpad, TRUE); + + GST_DEBUG_OBJECT (self, "Linking pads to elements"); + + /* Link everything */ + capsfilter = gst_element_factory_make ("capsfilter", NULL); + if (!gst_bin_add (GST_BIN (self), capsfilter)) { + GST_ERROR_OBJECT (self, "Could not add capsfilter!"); + return FALSE; + } + + gst_element_sync_state_with_parent (capsfilter); + if (!gst_element_link (self->encoder, capsfilter)) + goto encoder_capsfilter_link_fail; + tmppad = gst_element_get_static_pad (capsfilter, "src"); + if ((lret = + gst_pad_link_full (tmppad, self->internal_sinkpad, + GST_PAD_LINK_CHECK_NOTHING)) < GST_PAD_LINK_OK) + goto sinkpad_link_fail; + gst_object_unref (tmppad); + + tmppad = gst_element_get_static_pad (self->decoder, "sink"); + if (GST_PAD_LINK_FAILED (gst_pad_link_full (self->internal_srcpad, + tmppad, GST_PAD_LINK_CHECK_NOTHING))) + goto srcpad_link_fail; + gst_object_unref (tmppad); + + GST_DEBUG ("Done creating internal elements/pads"); + + return TRUE; + +no_decoder: + { + GST_WARNING ("Couldn't find a decodebin?!"); + return FALSE; + } + +srcpad_link_fail: + { + gst_object_unref (tmppad); + GST_WARNING ("Couldn't link internal srcpad to decoder"); + return FALSE; + } + +sinkpad_link_fail: + { + gst_object_unref (tmppad); + GST_WARNING ("Couldn't link encoder to internal sinkpad: %s", + gst_pad_link_get_name (lret)); + return FALSE; + } + +encoder_capsfilter_link_fail: + { + GST_WARNING ("Couldn't link encoder to capsfilter"); + return FALSE; + } +} + +static GstFlowReturn +gst_smart_encoder_reencode_gop (GstSmartEncoder * self) +{ + GstFlowReturn res = GST_FLOW_OK; + GstCaps *caps = NULL; + + GST_DEBUG_OBJECT (self, "Reencoding GOP!"); + if (self->decoder == NULL) { + if (!setup_recoder_pipeline (self)) { + GST_ERROR_OBJECT (self, "Could not setup reencoder pipeline"); + return GST_FLOW_ERROR; + } + } + + /* Activate elements */ + /* Set elements to PAUSED */ + gst_element_set_state (self->encoder, GST_STATE_PLAYING); + gst_element_set_state (self->decoder, GST_STATE_PLAYING); + + GST_INFO ("Pushing Flush start/stop to clean decoder/encoder"); + gst_pad_push_event (self->internal_srcpad, gst_event_new_flush_start ()); + gst_pad_push_event (self->internal_srcpad, gst_event_new_flush_stop (TRUE)); + + /* push segment_event */ + GST_INFO ("Pushing segment_event %" GST_PTR_FORMAT, self->segment_event); + gst_pad_push_event (self->internal_srcpad, + gst_event_ref (self->stream_start_event)); + caps = gst_pad_get_current_caps (self->sinkpad); + gst_pad_push_event (self->internal_srcpad, gst_event_new_caps (caps)); + gst_caps_unref (caps); + + gst_pad_push_event (self->internal_srcpad, + gst_event_ref (self->segment_event)); + + /* Push buffers through our pads */ + GST_DEBUG ("Pushing %d pending buffers", g_list_length (self->pending_gop)); + + g_mutex_lock (&self->internal_flow_lock); + self->internal_flow = GST_FLOW_CUSTOM_SUCCESS; + g_mutex_unlock (&self->internal_flow_lock); + while (self->pending_gop) { + GstBuffer *buf = (GstBuffer *) self->pending_gop->data; + + self->pending_gop = + g_list_remove_link (self->pending_gop, self->pending_gop); + res = gst_pad_push (self->internal_srcpad, buf); + if (res == GST_FLOW_EOS) { + GST_INFO_OBJECT (self, "Got eos... waiting for the event" + " waiting for encoding to be done"); + break; + } + + if (res != GST_FLOW_OK) { + GST_WARNING ("Error pushing pending buffers : %s", + gst_flow_get_name (res)); + goto done; + } + } + + GST_DEBUG_OBJECT (self, "-> Drain encoder."); + gst_pad_push_event (self->internal_srcpad, gst_event_new_eos ()); + + g_mutex_lock (&self->internal_flow_lock); + while (self->internal_flow == GST_FLOW_CUSTOM_SUCCESS) { + g_cond_wait (&self->internal_flow_cond, &self->internal_flow_lock); + } + g_mutex_unlock (&self->internal_flow_lock); + + res = self->internal_flow; + + GST_DEBUG_OBJECT (self, "Done reencoding GOP."); + gst_element_set_state (self->encoder, GST_STATE_NULL); + gst_element_set_state (self->decoder, GST_STATE_NULL); + GST_OBJECT_FLAG_UNSET (self->internal_sinkpad, GST_PAD_FLAG_EOS); + GST_OBJECT_FLAG_UNSET (self->internal_srcpad, GST_PAD_FLAG_EOS); + +done: + g_list_free_full (self->pending_gop, (GDestroyNotify) gst_buffer_unref); + self->pending_gop = NULL; + + return res; +} + +static GstFlowReturn +gst_smart_encoder_push_pending_gop (GstSmartEncoder * self) +{ + guint64 cstart, cstop; + GList *tmp; + GstFlowReturn res = GST_FLOW_OK; + + GST_DEBUG ("Pushing pending GOP (%" GST_TIME_FORMAT " -- %" GST_TIME_FORMAT + ")", GST_TIME_ARGS (self->gop_start), GST_TIME_ARGS (self->gop_stop)); + + if (self->output_segment.format == GST_FORMAT_UNDEFINED) { + gst_segment_init (&self->output_segment, GST_FORMAT_TIME); + + /* Ensure that we can represent negative DTS in our 'single' segment */ + self->output_segment.start = 60 * 60 * GST_SECOND * 1000; + if (!gst_pad_push_event (self->srcpad, + gst_event_new_segment (&self->output_segment))) { + GST_ERROR_OBJECT (self, "Could not push segment!"); + + GST_ELEMENT_FLOW_ERROR (self, GST_FLOW_ERROR); + + return GST_FLOW_ERROR; + } + } + + if (!self->pending_gop) { + /* This might happen on EOS */ + GST_INFO_OBJECT (self, "Empty gop!"); + goto done; + } + + if (!gst_segment_clip (&self->input_segment, GST_FORMAT_TIME, self->gop_start, + self->gop_stop, &cstart, &cstop)) { + /* The whole GOP is outside the segment, there's most likely + * a bug somewhere. */ + GST_DEBUG_OBJECT (self, + "GOP is entirely outside of the segment, upstream gave us too much data: (%" + GST_TIME_FORMAT " -- %" GST_TIME_FORMAT ")", + GST_TIME_ARGS (self->gop_start), GST_TIME_ARGS (self->gop_stop)); + for (tmp = self->pending_gop; tmp; tmp = tmp->next) + gst_buffer_unref ((GstBuffer *) tmp->data); + + goto done; + } + + if ((cstart != self->gop_start) + || (cstop != self->gop_stop)) { + GST_INFO_OBJECT (self, + "GOP needs to be re-encoded from %" GST_TIME_FORMAT " to %" + GST_TIME_FORMAT " - %" GST_SEGMENT_FORMAT, GST_TIME_ARGS (cstart), + GST_TIME_ARGS (cstop), &self->input_segment); + res = gst_smart_encoder_reencode_gop (self); + } else { + /* The whole GOP is within the segment, push all pending buffers downstream */ + GST_INFO_OBJECT (self, + "GOP doesn't need to be modified, pushing downstream: %" GST_TIME_FORMAT + " to %" GST_TIME_FORMAT, GST_TIME_ARGS (cstart), GST_TIME_ARGS (cstop)); + + self->internal_segment = self->input_segment; + for (tmp = self->pending_gop; tmp; tmp = tmp->next) { + GstBuffer *buf = (GstBuffer *) tmp->data; + + res = gst_smart_encoder_finish_buffer (self, buf); + if (G_UNLIKELY (res != GST_FLOW_OK)) + break; + } + } + +done: + g_list_free (self->pending_gop); + self->pending_gop = NULL; + self->gop_start = GST_CLOCK_TIME_NONE; + self->gop_stop = 0; + + return res; +} + +static GstFlowReturn +gst_smart_encoder_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) +{ + GstSmartEncoder *self; + GstFlowReturn res = GST_FLOW_OK; + gboolean discont, keyframe; + GstClockTime end_time; + + self = GST_SMART_ENCODER (parent->parent); + + discont = GST_BUFFER_IS_DISCONT (buf); + keyframe = !GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT); + end_time = GST_BUFFER_PTS (buf); + if (GST_CLOCK_TIME_IS_VALID (end_time)) + end_time += (GST_BUFFER_DURATION_IS_VALID (buf) ? buf->duration : 0); + + GST_DEBUG_OBJECT (pad, + "New buffer %s %s %" GST_TIME_FORMAT " - %" GST_TIME_FORMAT, + discont ? "discont" : "", keyframe ? "keyframe" : "", + GST_TIME_ARGS (GST_BUFFER_PTS (buf)), GST_TIME_ARGS (end_time)); + + if (keyframe) { + /* If there's a pending GOP, flush it out */ + if (self->pending_gop) { + /* Mark stop of previous gop */ + if (GST_BUFFER_PTS_IS_VALID (buf)) { + if (self->gop_stop > buf->pts) + GST_WARNING_OBJECT (self, "Next gop start < current gop" " end"); + self->gop_stop = buf->pts; + } + + /* flush pending */ + res = gst_smart_encoder_push_pending_gop (self); + if (G_UNLIKELY (res != GST_FLOW_OK)) + goto beach; + } + + /* Mark gop_start for new gop */ + self->gop_start = GST_BUFFER_TIMESTAMP (buf); + } + + /* Store buffer */ + self->pending_gop = g_list_append (self->pending_gop, buf); + + /* Update GOP stop position */ + if (GST_CLOCK_TIME_IS_VALID (end_time)) + self->gop_stop = MAX (self->gop_stop, end_time); + + GST_DEBUG_OBJECT (self, "Buffer stored , Current GOP : %" + GST_TIME_FORMAT " -- %" GST_TIME_FORMAT, + GST_TIME_ARGS (self->gop_start), GST_TIME_ARGS (self->gop_stop)); + +beach: + return res; +} + +static gboolean +smart_encoder_sink_event (GstPad * pad, GstObject * ghostpad, GstEvent * event) +{ + gboolean res = TRUE; + GstSmartEncoder *self = GST_SMART_ENCODER (ghostpad->parent); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_FLUSH_STOP: + smart_encoder_reset (self); + break; + case GST_EVENT_CAPS: + if (self->last_caps) { + gst_clear_event (&event); + } else { + gst_event_parse_caps (event, &self->last_caps); + self->last_caps = gst_caps_copy (self->last_caps); + } + break; + case GST_EVENT_STREAM_START: + gst_event_replace (&self->stream_start_event, gst_event_ref (event)); + break; + case GST_EVENT_SEGMENT: + { + GST_INFO_OBJECT (self, "Pushing pending GOP on new segment"); + gst_smart_encoder_push_pending_gop (self); + + gst_event_copy_segment (event, &self->input_segment); + + GST_DEBUG_OBJECT (self, "input_segment: %" GST_SEGMENT_FORMAT, + &self->input_segment); + if (self->input_segment.format != GST_FORMAT_TIME) { + GST_ERROR_OBJECT (self, "Can't handle streams %s format", + gst_format_get_name (self->input_segment.format)); + gst_event_unref (event); + + return FALSE; + } + self->segment_event = event; + event = NULL; + GST_INFO_OBJECT (self, "Eating segment"); + break; + } + case GST_EVENT_EOS: + if (self->input_segment.format == GST_FORMAT_TIME) + gst_smart_encoder_push_pending_gop (self); + break; + default: + break; + } + + if (event) + res = gst_pad_push_event (self->srcpad, event); + + return res; +} + +static GstCaps * +smart_encoder_sink_getcaps (GstSmartEncoder * self, GstPad * pad, + GstCaps * filter) +{ + GstCaps *peer, *tmpl, *res; + + tmpl = gst_static_pad_template_get_caps (&src_template); + + /* Try getting it from downstream */ + peer = gst_pad_peer_query_caps (self->srcpad, tmpl); + if (peer == NULL) { + res = tmpl; + } else { + res = peer; + gst_caps_unref (tmpl); + } + + if (filter) { + GstCaps *filtered_res = gst_caps_intersect (res, filter); + + gst_caps_unref (res); + if (!filtered_res || gst_caps_is_empty (filtered_res)) { + res = NULL; + } else { + res = filtered_res; + } + } + + return res; +} + +static gboolean +_pad_sink_acceptcaps (GstPad * pad, GstSmartEncoder * self, GstCaps * caps) +{ + gboolean ret; + GstCaps *modified_caps; + GstCaps *accepted_caps; + gint i, n; + GstStructure *s; + + GST_DEBUG_OBJECT (pad, "%" GST_PTR_FORMAT, caps); + + accepted_caps = gst_pad_get_current_caps (GST_PAD (self->srcpad)); + if (accepted_caps == NULL) + accepted_caps = gst_pad_get_pad_template_caps (GST_PAD (self->srcpad)); + accepted_caps = gst_caps_make_writable (accepted_caps); + + GST_LOG_OBJECT (pad, "src caps %" GST_PTR_FORMAT, accepted_caps); + + n = gst_caps_get_size (accepted_caps); + for (i = 0; i < n; i++) { + s = gst_caps_get_structure (accepted_caps, i); + gst_structure_remove_fields (s, "codec_data", NULL); + } + + modified_caps = gst_caps_copy (caps); + n = gst_caps_get_size (modified_caps); + for (i = 0; i < n; i++) { + s = gst_caps_get_structure (modified_caps, i); + gst_structure_remove_fields (s, "codec_data", NULL); + } + + ret = gst_caps_can_intersect (modified_caps, accepted_caps); + GST_DEBUG_OBJECT (pad, "%saccepted caps %" GST_PTR_FORMAT, + (ret ? "" : "Doesn't "), caps); + return ret; +} + +static gboolean +smart_encoder_sink_query (GstPad * pad, GstObject * ghostpad, GstQuery * query) +{ + gboolean res; + GstSmartEncoder *self = GST_SMART_ENCODER (ghostpad->parent); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CAPS: + { + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + caps = smart_encoder_sink_getcaps (self, pad, filter); + GST_DEBUG_OBJECT (self, "Got caps: %" GST_PTR_FORMAT, caps); + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + res = TRUE; + break; + } + case GST_QUERY_ACCEPT_CAPS: + { + GstCaps *caps; + + gst_query_parse_accept_caps (query, &caps); + res = _pad_sink_acceptcaps (GST_PAD (pad), self, caps); + gst_query_set_accept_caps_result (query, res); + res = TRUE; + break; + } + default: + res = gst_pad_query_default (pad, ghostpad, query); + break; + } + return res; +} + +static gboolean +gst_smart_encoder_add_parser (GstSmartEncoder * self, GstCaps * format) +{ + GstPad *chainpad, *internal_chainpad, *sinkpad; + GstElement *capsfilter = gst_element_factory_make ("capsfilter", NULL); + + gst_bin_add (GST_BIN (self), capsfilter); + g_object_set (capsfilter, "caps", format, NULL); + if (gst_structure_has_name (gst_caps_get_structure (format, 0), + "video/x-h264")) { + GstElement *parser = gst_element_factory_make ("h264parse", NULL); + if (!parser) { + GST_ERROR_OBJECT (self, "`h264parse` is missing, can't encode smartly"); + + goto failed; + } + + /* Add SPS/PPS before each gop to ensure that they can be decoded + * independently */ + g_object_set (parser, "config-interval", -1, NULL); + if (!gst_bin_add (GST_BIN (self), parser)) { + GST_ERROR_OBJECT (self, "Could not add parser."); + + goto failed; + } + + if (!gst_element_link (parser, capsfilter)) { + GST_ERROR_OBJECT (self, "Could not link capfilter and parser."); + + goto failed; + } + + sinkpad = gst_element_get_static_pad (parser, "sink"); + } else { + sinkpad = gst_element_get_static_pad (capsfilter, "sink"); + } + + g_assert (sinkpad); + + /* The chainpad is the pad that is linked to the srcpad of the chain + * of element that is linked to our public sinkpad, this is the pad where + * we chain the buffers either directly to our srcpad or through the + * reencoding sub chain. */ + chainpad = + GST_PAD (gst_ghost_pad_new ("chainpad", capsfilter->srcpads->data)); + gst_element_add_pad (GST_ELEMENT (self), chainpad); + internal_chainpad = + GST_PAD (gst_proxy_pad_get_internal (GST_PROXY_PAD (chainpad))); + gst_pad_set_chain_function (internal_chainpad, gst_smart_encoder_chain); + gst_pad_set_event_function (internal_chainpad, smart_encoder_sink_event); + gst_pad_set_query_function (internal_chainpad, smart_encoder_sink_query); + + gst_ghost_pad_set_target (GST_GHOST_PAD (self->sinkpad), sinkpad); + gst_object_unref (sinkpad); + + return TRUE; + +failed: + return FALSE; +} + +gboolean +gst_smart_encoder_set_encoder (GstSmartEncoder * self, GstCaps * format, + GstElement * encoder) +{ + self->encoder = g_object_ref_sink (encoder); + gst_element_set_locked_state (self->encoder, TRUE); + + return gst_smart_encoder_add_parser (self, format); +} + +/****************************************** + * GstElement vmethod implementations * + ******************************************/ -static GstFlowReturn gst_smart_encoder_chain (GstPad * pad, GstObject * parent, - GstBuffer * buf); -static gboolean smart_encoder_sink_event (GstPad * pad, GstObject * parent, - GstEvent * event); -static gboolean smart_encoder_sink_query (GstPad * pad, GstObject * parent, - GstQuery * query); -static GstCaps *smart_encoder_sink_getcaps (GstPad * pad, GstCaps * filter); static GstStateChangeReturn -gst_smart_encoder_change_state (GstElement * element, - GstStateChange transition); +gst_smart_encoder_change_state (GstElement * element, GstStateChange transition) +{ + GstSmartEncoder *self; + GstStateChangeReturn ret; + + g_return_val_if_fail (GST_IS_SMART_ENCODER (element), + GST_STATE_CHANGE_FAILURE); + + self = GST_SMART_ENCODER (element); + + ret = + GST_ELEMENT_CLASS (gst_smart_encoder_parent_class)->change_state (element, + transition); + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + smart_encoder_reset (self); + break; + default: + break; + } + + return ret; +} + +/****************************************** + * GObject vmethods * + ******************************************/ +static void +gst_smart_encoder_finalize (GObject * object) +{ + GstSmartEncoder *self = (GstSmartEncoder *) object; + g_mutex_clear (&self->internal_flow_lock); + g_cond_clear (&self->internal_flow_cond); + + G_OBJECT_CLASS (gst_smart_encoder_parent_class)->finalize (object); +} + +static void +gst_smart_encoder_dispose (GObject * object) +{ + GstSmartEncoder *self = (GstSmartEncoder *) object; + + gst_clear_object (&self->encoder); + + G_OBJECT_CLASS (gst_smart_encoder_parent_class)->dispose (object); +} + static void gst_smart_encoder_class_init (GstSmartEncoderClass * klass) @@ -108,7 +808,8 @@ gst_smart_encoder_class_init (GstSmartEncoderClass * klass) "Re-encodes portions of Video that lay on segment boundaries", "Edward Hervey "); - gobject_class->dispose = (GObjectFinalizeFunc) (gst_smart_encoder_dispose); + gobject_class->dispose = (GObjectFinalizeFunc) gst_smart_encoder_dispose; + gobject_class->finalize = (GObjectFinalizeFunc) gst_smart_encoder_finalize; element_class->change_state = gst_smart_encoder_change_state; GST_DEBUG_CATEGORY_INIT (smart_encoder_debug, "smartencoder", 0, @@ -116,602 +817,19 @@ gst_smart_encoder_class_init (GstSmartEncoderClass * klass) } static void -smart_encoder_reset (GstSmartEncoder * smart_encoder) +gst_smart_encoder_init (GstSmartEncoder * self) { - gst_segment_init (smart_encoder->segment, GST_FORMAT_UNDEFINED); + GstPadTemplate *template = gst_static_pad_template_get (&sink_template); - if (smart_encoder->encoder) { - /* Clean up/remove elements */ - gst_element_set_state (smart_encoder->encoder, GST_STATE_NULL); - gst_element_set_state (smart_encoder->decoder, GST_STATE_NULL); - gst_element_set_bus (smart_encoder->encoder, NULL); - gst_element_set_bus (smart_encoder->decoder, NULL); - gst_pad_set_active (smart_encoder->internal_srcpad, FALSE); - gst_pad_set_active (smart_encoder->internal_sinkpad, FALSE); - gst_object_unref (smart_encoder->encoder); - gst_object_unref (smart_encoder->decoder); - gst_object_unref (smart_encoder->internal_srcpad); - gst_object_unref (smart_encoder->internal_sinkpad); + self->sinkpad = gst_ghost_pad_new_no_target_from_template ("sink", template); + gst_element_add_pad (GST_ELEMENT (self), self->sinkpad); + gst_object_unref (template); - smart_encoder->encoder = NULL; - smart_encoder->decoder = NULL; - smart_encoder->internal_sinkpad = NULL; - smart_encoder->internal_srcpad = NULL; - } + self->srcpad = gst_pad_new_from_static_template (&src_template, "src"); + gst_pad_use_fixed_caps (self->srcpad); + gst_element_add_pad (GST_ELEMENT (self), self->srcpad); - if (smart_encoder->newsegment) { - gst_event_unref (smart_encoder->newsegment); - smart_encoder->newsegment = NULL; - } -} - - -static void -gst_smart_encoder_init (GstSmartEncoder * smart_encoder) -{ - smart_encoder->sinkpad = - gst_pad_new_from_static_template (&sink_template, "sink"); - gst_pad_set_chain_function (smart_encoder->sinkpad, gst_smart_encoder_chain); - gst_pad_set_event_function (smart_encoder->sinkpad, smart_encoder_sink_event); - gst_pad_set_query_function (smart_encoder->sinkpad, smart_encoder_sink_query); - gst_element_add_pad (GST_ELEMENT (smart_encoder), smart_encoder->sinkpad); - - smart_encoder->srcpad = - gst_pad_new_from_static_template (&src_template, "src"); - gst_pad_use_fixed_caps (smart_encoder->srcpad); - gst_element_add_pad (GST_ELEMENT (smart_encoder), smart_encoder->srcpad); - - smart_encoder->segment = gst_segment_new (); - - smart_encoder_reset (smart_encoder); -} - -void -gst_smart_encoder_dispose (GObject * object) -{ - GstSmartEncoder *smart_encoder = (GstSmartEncoder *) object; - - if (smart_encoder->segment) - gst_segment_free (smart_encoder->segment); - smart_encoder->segment = NULL; - if (smart_encoder->available_caps) - gst_caps_unref (smart_encoder->available_caps); - smart_encoder->available_caps = NULL; - G_OBJECT_CLASS (gst_smart_encoder_parent_class)->dispose (object); -} - -static GstFlowReturn -gst_smart_encoder_reencode_gop (GstSmartEncoder * smart_encoder) -{ - GstFlowReturn res = GST_FLOW_OK; - GList *tmp; - - if (smart_encoder->encoder == NULL) { - if (!setup_recoder_pipeline (smart_encoder)) - return GST_FLOW_ERROR; - } - - /* Activate elements */ - /* Set elements to PAUSED */ - gst_element_set_state (smart_encoder->encoder, GST_STATE_PAUSED); - gst_element_set_state (smart_encoder->decoder, GST_STATE_PAUSED); - - GST_INFO ("Pushing Flush start/stop to clean decoder/encoder"); - gst_pad_push_event (smart_encoder->internal_srcpad, - gst_event_new_flush_start ()); - gst_pad_push_event (smart_encoder->internal_srcpad, - gst_event_new_flush_stop (TRUE)); - - /* push newsegment */ - GST_INFO ("Pushing newsegment %" GST_PTR_FORMAT, smart_encoder->newsegment); - gst_pad_push_event (smart_encoder->internal_srcpad, - gst_event_ref (smart_encoder->newsegment)); - - /* Push buffers through our pads */ - GST_DEBUG ("Pushing pending buffers"); - - for (tmp = smart_encoder->pending_gop; tmp; tmp = tmp->next) { - GstBuffer *buf = (GstBuffer *) tmp->data; - - res = gst_pad_push (smart_encoder->internal_srcpad, buf); - if (G_UNLIKELY (res != GST_FLOW_OK)) - break; - } - - if (G_UNLIKELY (res != GST_FLOW_OK)) { - GST_WARNING ("Error pushing pending buffers : %s", gst_flow_get_name (res)); - /* Remove pending bfufers */ - for (tmp = smart_encoder->pending_gop; tmp; tmp = tmp->next) { - gst_buffer_unref ((GstBuffer *) tmp->data); - } - } else { - GST_INFO ("Pushing out EOS to flush out decoder/encoder"); - gst_pad_push_event (smart_encoder->internal_srcpad, gst_event_new_eos ()); - } - - /* Activate elements */ - /* Set elements to PAUSED */ - gst_element_set_state (smart_encoder->encoder, GST_STATE_NULL); - gst_element_set_state (smart_encoder->decoder, GST_STATE_NULL); - - g_list_free (smart_encoder->pending_gop); - smart_encoder->pending_gop = NULL; - - return res; -} - -static GstFlowReturn -gst_smart_encoder_push_pending_gop (GstSmartEncoder * smart_encoder) -{ - guint64 cstart, cstop; - GList *tmp; - GstFlowReturn res = GST_FLOW_OK; - - GST_DEBUG ("Pushing pending GOP (%" GST_TIME_FORMAT " -- %" GST_TIME_FORMAT - ")", GST_TIME_ARGS (smart_encoder->gop_start), - GST_TIME_ARGS (smart_encoder->gop_stop)); - - /* If GOP is entirely within segment, just push downstream */ - if (gst_segment_clip (smart_encoder->segment, GST_FORMAT_TIME, - smart_encoder->gop_start, smart_encoder->gop_stop, &cstart, &cstop)) { - if ((cstart != smart_encoder->gop_start) - || (cstop != smart_encoder->gop_stop)) { - GST_DEBUG ("GOP needs to be re-encoded from %" GST_TIME_FORMAT " to %" - GST_TIME_FORMAT, GST_TIME_ARGS (cstart), GST_TIME_ARGS (cstop)); - res = gst_smart_encoder_reencode_gop (smart_encoder); - } else { - /* The whole GOP is within the segment, push all pending buffers downstream */ - GST_DEBUG ("GOP doesn't need to be modified, pushing downstream"); - for (tmp = smart_encoder->pending_gop; tmp; tmp = tmp->next) { - GstBuffer *buf = (GstBuffer *) tmp->data; - res = gst_pad_push (smart_encoder->srcpad, buf); - if (G_UNLIKELY (res != GST_FLOW_OK)) - break; - } - } - } else { - /* The whole GOP is outside the segment, there's most likely - * a bug somewhere. */ - GST_WARNING - ("GOP is entirely outside of the segment, upstream gave us too much data"); - for (tmp = smart_encoder->pending_gop; tmp; tmp = tmp->next) { - gst_buffer_unref ((GstBuffer *) tmp->data); - } - } - - if (smart_encoder->pending_gop) { - g_list_free (smart_encoder->pending_gop); - smart_encoder->pending_gop = NULL; - } - smart_encoder->gop_start = GST_CLOCK_TIME_NONE; - smart_encoder->gop_stop = GST_CLOCK_TIME_NONE; - - return res; -} - -static GstFlowReturn -gst_smart_encoder_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) -{ - GstSmartEncoder *smart_encoder; - GstFlowReturn res = GST_FLOW_OK; - gboolean discont, keyframe; - - smart_encoder = GST_SMART_ENCODER (parent); - - discont = GST_BUFFER_IS_DISCONT (buf); - keyframe = !GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT); - - GST_DEBUG ("New buffer %s %s %" GST_TIME_FORMAT, - discont ? "discont" : "", - keyframe ? "keyframe" : "", GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf))); - - if (keyframe) { - GST_DEBUG ("Got a keyframe"); - - /* If there's a pending GOP, flush it out */ - if (smart_encoder->pending_gop) { - /* Mark gop_stop */ - smart_encoder->gop_stop = GST_BUFFER_TIMESTAMP (buf); - - /* flush pending */ - res = gst_smart_encoder_push_pending_gop (smart_encoder); - if (G_UNLIKELY (res != GST_FLOW_OK)) - goto beach; - } - - /* Mark gop_start for new gop */ - smart_encoder->gop_start = GST_BUFFER_TIMESTAMP (buf); - } - - /* Store buffer */ - smart_encoder->pending_gop = g_list_append (smart_encoder->pending_gop, buf); - /* Update GOP stop position */ - if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { - smart_encoder->gop_stop = GST_BUFFER_TIMESTAMP (buf); - if (GST_BUFFER_DURATION_IS_VALID (buf)) - smart_encoder->gop_stop += GST_BUFFER_DURATION (buf); - } - - GST_DEBUG ("Buffer stored , Current GOP : %" GST_TIME_FORMAT " -- %" - GST_TIME_FORMAT, GST_TIME_ARGS (smart_encoder->gop_start), - GST_TIME_ARGS (smart_encoder->gop_stop)); - -beach: - return res; -} - -static gboolean -smart_encoder_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) -{ - gboolean res = TRUE; - GstSmartEncoder *smart_encoder = GST_SMART_ENCODER (parent); - - switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_FLUSH_STOP: - smart_encoder_reset (smart_encoder); - break; - case GST_EVENT_SEGMENT: - { - gst_event_copy_segment (event, smart_encoder->segment); - - GST_DEBUG_OBJECT (smart_encoder, "segment: %" GST_SEGMENT_FORMAT, - smart_encoder->segment); - if (smart_encoder->segment->format != GST_FORMAT_TIME) { - GST_ERROR - ("smart_encoder can not handle streams not specified in GST_FORMAT_TIME"); - gst_event_unref (event); - return FALSE; - } - - /* And keep a copy for further usage */ - if (smart_encoder->newsegment) - gst_event_unref (smart_encoder->newsegment); - smart_encoder->newsegment = gst_event_ref (event); - } - break; - case GST_EVENT_EOS: - GST_DEBUG ("Eos, flushing remaining data"); - if (smart_encoder->segment->format == GST_FORMAT_TIME) - gst_smart_encoder_push_pending_gop (smart_encoder); - break; - default: - break; - } - - res = gst_pad_push_event (smart_encoder->srcpad, event); - - return res; -} - -static GstCaps * -smart_encoder_sink_getcaps (GstPad * pad, GstCaps * filter) -{ - GstCaps *peer, *tmpl, *res; - GstSmartEncoder *smart_encoder = GST_SMART_ENCODER (gst_pad_get_parent (pad)); - - /* Use computed caps */ - if (smart_encoder->available_caps) - tmpl = gst_caps_ref (smart_encoder->available_caps); - else - tmpl = gst_static_pad_template_get_caps (&src_template); - - /* Try getting it from downstream */ - peer = gst_pad_peer_query_caps (smart_encoder->srcpad, tmpl); - - if (peer == NULL) { - res = tmpl; - } else { - res = peer; - gst_caps_unref (tmpl); - } - - gst_object_unref (smart_encoder); - return res; -} - -static gboolean -smart_encoder_sink_query (GstPad * pad, GstObject * parent, GstQuery * query) -{ - gboolean res; - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_CAPS: - { - GstCaps *filter, *caps; - - gst_query_parse_caps (query, &filter); - caps = smart_encoder_sink_getcaps (pad, filter); - gst_query_set_caps_result (query, caps); - gst_caps_unref (caps); - res = TRUE; - break; - } - default: - res = gst_pad_query_default (pad, parent, query); - break; - } - return res; -} - -/***************************************** - * Internal encoder/decoder pipeline * - ******************************************/ - -static GstElementFactory * -get_decoder_factory (GstCaps * caps) -{ - GstElementFactory *fact = NULL; - GList *decoders, *tmp; - - tmp = - gst_element_factory_list_get_elements (GST_ELEMENT_FACTORY_TYPE_DECODER, - GST_RANK_MARGINAL); - decoders = gst_element_factory_list_filter (tmp, caps, GST_PAD_SINK, FALSE); - gst_plugin_feature_list_free (tmp); - - for (tmp = decoders; tmp; tmp = tmp->next) { - /* We just pick the first one */ - fact = (GstElementFactory *) tmp->data; - gst_object_ref (fact); - break; - } - - gst_plugin_feature_list_free (decoders); - - return fact; -} - -static GstElementFactory * -get_encoder_factory (GstCaps * caps) -{ - GstElementFactory *fact = NULL; - GList *encoders, *tmp; - - tmp = - gst_element_factory_list_get_elements (GST_ELEMENT_FACTORY_TYPE_ENCODER, - GST_RANK_MARGINAL); - encoders = gst_element_factory_list_filter (tmp, caps, GST_PAD_SRC, FALSE); - gst_plugin_feature_list_free (tmp); - - for (tmp = encoders; tmp; tmp = tmp->next) { - /* We just pick the first one */ - fact = (GstElementFactory *) tmp->data; - gst_object_ref (fact); - break; - } - - gst_plugin_feature_list_free (encoders); - - return fact; -} - -static GstElement * -get_decoder (GstCaps * caps) -{ - GstElementFactory *fact = get_decoder_factory (caps); - GstElement *res = NULL; - - if (fact) { - res = gst_element_factory_create (fact, "internal-decoder"); - gst_object_unref (fact); - } - return res; -} - -static GstElement * -get_encoder (GstCaps * caps) -{ - GstElementFactory *fact = get_encoder_factory (caps); - GstElement *res = NULL; - - if (fact) { - res = gst_element_factory_create (fact, "internal-encoder"); - gst_object_unref (fact); - } - return res; -} - -static GstFlowReturn -internal_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) -{ - GstSmartEncoder *smart_encoder = - g_object_get_qdata ((GObject *) pad, INTERNAL_ELEMENT); - - return gst_pad_push (smart_encoder->srcpad, buf); -} - -static gboolean -setup_recoder_pipeline (GstSmartEncoder * smart_encoder) -{ - GstPad *tmppad; - GstCaps *caps; - - /* Fast path */ - if (G_UNLIKELY (smart_encoder->encoder)) - return TRUE; - - GST_DEBUG ("Creating internal decoder and encoder"); - - /* Create decoder/encoder */ - caps = gst_pad_get_current_caps (smart_encoder->sinkpad); - smart_encoder->decoder = get_decoder (caps); - if (G_UNLIKELY (smart_encoder->decoder == NULL)) - goto no_decoder; - gst_caps_unref (caps); - gst_element_set_bus (smart_encoder->decoder, GST_ELEMENT_BUS (smart_encoder)); - - caps = gst_pad_get_current_caps (smart_encoder->sinkpad); - smart_encoder->encoder = get_encoder (caps); - if (G_UNLIKELY (smart_encoder->encoder == NULL)) - goto no_encoder; - gst_caps_unref (caps); - gst_element_set_bus (smart_encoder->encoder, GST_ELEMENT_BUS (smart_encoder)); - - GST_DEBUG ("Creating internal pads"); - - /* Create internal pads */ - - /* Source pad which we'll use to feed data to decoders */ - smart_encoder->internal_srcpad = gst_pad_new ("internal_src", GST_PAD_SRC); - g_object_set_qdata ((GObject *) smart_encoder->internal_srcpad, - INTERNAL_ELEMENT, smart_encoder); - gst_pad_set_active (smart_encoder->internal_srcpad, TRUE); - - /* Sink pad which will get the buffers from the encoder. - * Note: We don't need an event function since we'll be discarding all - * of them. */ - smart_encoder->internal_sinkpad = gst_pad_new ("internal_sink", GST_PAD_SINK); - g_object_set_qdata ((GObject *) smart_encoder->internal_sinkpad, - INTERNAL_ELEMENT, smart_encoder); - gst_pad_set_chain_function (smart_encoder->internal_sinkpad, internal_chain); - gst_pad_set_active (smart_encoder->internal_sinkpad, TRUE); - - GST_DEBUG ("Linking pads to elements"); - - /* Link everything */ - tmppad = gst_element_get_static_pad (smart_encoder->encoder, "src"); - if (GST_PAD_LINK_FAILED (gst_pad_link (tmppad, - smart_encoder->internal_sinkpad))) - goto sinkpad_link_fail; - gst_object_unref (tmppad); - - if (!gst_element_link (smart_encoder->decoder, smart_encoder->encoder)) - goto encoder_decoder_link_fail; - - tmppad = gst_element_get_static_pad (smart_encoder->decoder, "sink"); - if (GST_PAD_LINK_FAILED (gst_pad_link (smart_encoder->internal_srcpad, - tmppad))) - goto srcpad_link_fail; - gst_object_unref (tmppad); - - GST_DEBUG ("Done creating internal elements/pads"); - - return TRUE; - -no_decoder: - { - GST_WARNING ("Couldn't find a decoder for %" GST_PTR_FORMAT, caps); - gst_caps_unref (caps); - return FALSE; - } - -no_encoder: - { - GST_WARNING ("Couldn't find an encoder for %" GST_PTR_FORMAT, caps); - gst_caps_unref (caps); - return FALSE; - } - -srcpad_link_fail: - { - gst_object_unref (tmppad); - GST_WARNING ("Couldn't link internal srcpad to decoder"); - return FALSE; - } - -sinkpad_link_fail: - { - gst_object_unref (tmppad); - GST_WARNING ("Couldn't link encoder to internal sinkpad"); - return FALSE; - } - -encoder_decoder_link_fail: - { - GST_WARNING ("Couldn't link decoder to encoder"); - return FALSE; - } -} - -static GstStateChangeReturn -gst_smart_encoder_find_elements (GstSmartEncoder * smart_encoder) -{ - guint i, n; - GstCaps *tmpl, *st, *res; - GstElementFactory *dec, *enc; - GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; - - if (G_UNLIKELY (smart_encoder->available_caps)) - goto beach; - - /* Iterate over all pad template caps and see if we have both an - * encoder and a decoder for those media types */ - tmpl = gst_static_pad_template_get_caps (&src_template); - res = gst_caps_new_empty (); - n = gst_caps_get_size (tmpl); - - for (i = 0; i < n; i++) { - st = gst_caps_copy_nth (tmpl, i); - GST_DEBUG_OBJECT (smart_encoder, - "Checking for available decoder and encoder for %" GST_PTR_FORMAT, st); - if (!(dec = get_decoder_factory (st))) { - gst_caps_unref (st); - continue; - } - gst_object_unref (dec); - if (!(enc = get_encoder_factory (st))) { - gst_caps_unref (st); - continue; - } - gst_object_unref (enc); - GST_DEBUG_OBJECT (smart_encoder, "OK"); - gst_caps_append (res, st); - } - - gst_caps_unref (tmpl); - - if (gst_caps_is_empty (res)) { - gst_caps_unref (res); - ret = GST_STATE_CHANGE_FAILURE; - } else - smart_encoder->available_caps = res; - - GST_DEBUG_OBJECT (smart_encoder, "Done, available_caps:%" GST_PTR_FORMAT, - smart_encoder->available_caps); - -beach: - return ret; -} - -/****************************************** - * GstElement vmethod implementations * - ******************************************/ - -static GstStateChangeReturn -gst_smart_encoder_change_state (GstElement * element, GstStateChange transition) -{ - GstSmartEncoder *smart_encoder; - GstStateChangeReturn ret; - - g_return_val_if_fail (GST_IS_SMART_ENCODER (element), - GST_STATE_CHANGE_FAILURE); - - smart_encoder = GST_SMART_ENCODER (element); - - switch (transition) { - case GST_STATE_CHANGE_NULL_TO_READY: - /* Figure out which elements are available */ - if ((ret = - gst_smart_encoder_find_elements (smart_encoder)) == - GST_STATE_CHANGE_FAILURE) - goto beach; - break; - default: - break; - } - - ret = - GST_ELEMENT_CLASS (gst_smart_encoder_parent_class)->change_state (element, - transition); - - switch (transition) { - case GST_STATE_CHANGE_PAUSED_TO_READY: - smart_encoder_reset (smart_encoder); - break; - default: - break; - } - -beach: - return ret; + g_mutex_init (&self->internal_flow_lock); + g_cond_init (&self->internal_flow_cond); + smart_encoder_reset (self); } diff --git a/gst/encoding/gstsmartencoder.h b/gst/encoding/gstsmartencoder.h index ec5d0396db..71e79011f9 100644 --- a/gst/encoding/gstsmartencoder.h +++ b/gst/encoding/gstsmartencoder.h @@ -24,21 +24,35 @@ G_BEGIN_DECLS #define GST_TYPE_SMART_ENCODER (gst_smart_encoder_get_type()) -G_DECLARE_FINAL_TYPE (GstSmartEncoder, gst_smart_encoder, GST, SMART_ENCODER, - GstElement) +G_DECLARE_FINAL_TYPE (GstSmartEncoder, gst_smart_encoder, GST, SMART_ENCODER, GstBin) struct _GstSmartEncoder { - GstElement element; + GstBin parent; GstPad *sinkpad, *srcpad; - GstSegment *segment; - GstEvent *newsegment; + gboolean pushed_segment; + + /* Segment received upstream */ + GstSegment input_segment; + + /* The segment we pushed downstream */ + GstSegment output_segment; + + /* Internal segments to compute buffers running time before pushing + * them downstream. It is the encoder segment when reecoding gops, + * and the input segment when pushing them unmodified. */ + GstSegment internal_segment; + GstClockTime last_dts; + + GstCaps *last_caps; + GstEvent *segment_event; + GstEvent *stream_start_event; /* Pending GOP to be checked */ - GList *pending_gop; - guint64 gop_start; /* GOP start in running time */ - guint64 gop_stop; /* GOP end in running time */ + GList* pending_gop; + guint64 gop_start; /* GOP start PTS in the `input_segment` scale. */ + guint64 gop_stop; /* GOP end PTS in the `input_segment` scale. */ /* Internal recoding elements */ GstPad *internal_sinkpad; @@ -46,10 +60,15 @@ struct _GstSmartEncoder { GstElement *decoder; GstElement *encoder; - /* Available caps at runtime */ - GstCaps *available_caps; + GstFlowReturn internal_flow; + GMutex internal_flow_lock; + GCond internal_flow_cond; }; +gboolean gst_smart_encoder_set_encoder (GstSmartEncoder *self, + GstCaps *format, + GstElement *encoder); + G_END_DECLS #endif /* __SMART_ENCODER_H__ */