From 61cc3529025f81b8f8308993b6f977683bfdc0b0 Mon Sep 17 00:00:00 2001 From: Mart Raudsepp Date: Thu, 10 Nov 2011 15:13:34 +0200 Subject: [PATCH 01/25] mimic, opencv, vp8, acmmp3dec, linsys: Don't build static plugins Pass --tag=disable-static to libtool everywhere where it's been forgotten https://bugzilla.gnome.org/show_bug.cgi?id=663768 --- ext/mimic/Makefile.am | 1 + ext/opencv/Makefile.am | 1 + ext/vp8/Makefile.am | 1 + sys/acmmp3dec/Makefile.am | 2 +- sys/linsys/Makefile.am | 2 +- 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ext/mimic/Makefile.am b/ext/mimic/Makefile.am index 7cf08e468a..b16741449e 100644 --- a/ext/mimic/Makefile.am +++ b/ext/mimic/Makefile.am @@ -5,5 +5,6 @@ libgstmimic_la_SOURCES = gstmimic.c gstmimdec.c gstmimenc.c libgstmimic_la_CFLAGS = $(GST_BASE_CFLAGS) $(GST_CFLAGS) $(MIMIC_CFLAGS) libgstmimic_la_LIBADD = $(GST_BASE_LIBS) $(GST_LIBS) $(MIMIC_LIBS) libgstmimic_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) +libgstmimic_la_LIBTOOLFLAGS = --tag=disable-static noinst_HEADERS = gstmimdec.h gstmimenc.h diff --git a/ext/opencv/Makefile.am b/ext/opencv/Makefile.am index a32e16cfa2..d466bbb0b5 100644 --- a/ext/opencv/Makefile.am +++ b/ext/opencv/Makefile.am @@ -36,6 +36,7 @@ libgstopencv_la_LIBADD = $(GST_LIBS) $(GST_BASE_LIBS) $(OPENCV_LIBS) \ $(GSTPB_BASE_LIBS) -lgstvideo-$(GST_MAJORMINOR) libgstopencv_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) +libgstopencv_la_LIBTOOLFLAGS = --tag=disable-static # headers we need but don't want installed noinst_HEADERS = gstopencvvideofilter.h gstopencvutils.h \ diff --git a/ext/vp8/Makefile.am b/ext/vp8/Makefile.am index 474b3cec4e..d839666fda 100644 --- a/ext/vp8/Makefile.am +++ b/ext/vp8/Makefile.am @@ -18,6 +18,7 @@ libgstvp8_la_LIBADD = \ $(GST_PLUGINS_BASE_LIBS) -lgsttag-@GST_MAJORMINOR@ -lgstvideo-@GST_MAJORMINOR@ \ $(GST_BASE_LIBS) $(GST_LIBS) $(VPX_LIBS) libgstvp8_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) +libgstvp8_la_LIBTOOLFLAGS = --tag=disable-static noinst_HEADERS = \ gstvp8dec.h \ diff --git a/sys/acmmp3dec/Makefile.am b/sys/acmmp3dec/Makefile.am index 27c895c545..f927c7233c 100644 --- a/sys/acmmp3dec/Makefile.am +++ b/sys/acmmp3dec/Makefile.am @@ -11,4 +11,4 @@ libgstacmmp3dec_la_LIBADD = \ -lgsttag-$(GST_MAJORMINOR) \ -lmsacm32 libgstacmmp3dec_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) $(DIRECTSOUND_LDFLAGS) - +libgstacmmp3dec_la_LIBTOOLFLAGS = --tag=disable-static diff --git a/sys/linsys/Makefile.am b/sys/linsys/Makefile.am index 3362738746..ac7eed003f 100644 --- a/sys/linsys/Makefile.am +++ b/sys/linsys/Makefile.am @@ -23,4 +23,4 @@ libgstlinsys_la_CFLAGS = \ libgstlinsys_la_LDFLAGS = \ $(GST_PLUGIN_LDFLAGS) libgstlinsys_la_LIBADD = $(GST_BASE_LIBS) $(GST_LIBS) - +libgstlinsys_la_LIBTOOLFLAGS = --tag=disable-static From f509960feac8458f25c4ac45623b48572d73351f Mon Sep 17 00:00:00 2001 From: Sreerenj Balachandran Date: Thu, 3 Nov 2011 14:09:52 +0200 Subject: [PATCH 02/25] mpegvideoparse:Correct the skipping of vbv_delay in picture header and some typo fixes. https://bugzilla.gnome.org/show_bug.cgi?id=663309 --- gst-libs/gst/codecparsers/gstmpegvideoparser.c | 4 ++-- gst-libs/gst/codecparsers/gstmpegvideoparser.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/codecparsers/gstmpegvideoparser.c b/gst-libs/gst/codecparsers/gstmpegvideoparser.c index 2d762fbaaa..85d8b1d63b 100644 --- a/gst-libs/gst/codecparsers/gstmpegvideoparser.c +++ b/gst-libs/gst/codecparsers/gstmpegvideoparser.c @@ -681,8 +681,8 @@ gst_mpeg_video_parse_picture_header (GstMpegVideoPictureHdr * hdr, if (hdr->pic_type == 0 || hdr->pic_type > 4) goto failed; /* Corrupted picture packet */ - /* skype VBV delay */ - if (!gst_bit_reader_skip (&br, 8)) + /* skip VBV delay */ + if (!gst_bit_reader_skip (&br, 16)) goto failed; if (hdr->pic_type == GST_MPEG_VIDEO_PICTURE_TYPE_P diff --git a/gst-libs/gst/codecparsers/gstmpegvideoparser.h b/gst-libs/gst/codecparsers/gstmpegvideoparser.h index 444092ef41..7c23114eab 100644 --- a/gst-libs/gst/codecparsers/gstmpegvideoparser.h +++ b/gst-libs/gst/codecparsers/gstmpegvideoparser.h @@ -38,7 +38,7 @@ G_BEGIN_DECLS /** * GstMpegVideoPacketTypeCode: * @GST_MPEG_VIDEO_PACKET_PICTURE: Picture packet starting code - * @GST_MPEG_VIDEO_PACKET_SLICE_MIN: Picture packet starting code + * @GST_MPEG_VIDEO_PACKET_SLICE_MIN: Slice min packet starting code * @GST_MPEG_VIDEO_PACKET_SLICE_MAX: Slice max packet starting code * @GST_MPEG_VIDEO_PACKET_USER_DATA: User data packet starting code * @GST_MPEG_VIDEO_PACKET_SEQUENCE : Sequence packet starting code @@ -186,7 +186,7 @@ typedef struct _GstMpegVideoTypeOffsetSize GstMpegVideoTypeOffsetSize; * @width: Width of each frame * @height: Height of each frame * @par_w: Calculated Pixel Aspect Ratio width - * @par_h: Pixel Aspect Ratio height + * @par_h: Calculated Pixel Aspect Ratio height * @fps_n: Calculated Framrate nominator * @fps_d: Calculated Framerate denominator * @bitrate_value: Value of the bitrate as is in the stream (400bps unit) From c635e4bc3baff6f51abebd954293557079601847 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Thu, 10 Nov 2011 16:11:56 -0300 Subject: [PATCH 03/25] camerabin2: keep track of video recording state Adds a new variable to keep track of the state of the video recording in camerabin2. This allows start-capture to reject new video recording requests when one is already ongoing. This fixes one of check tests. --- gst/camerabin2/gstcamerabin2.c | 54 ++++++++++++++++++++++++++-------- gst/camerabin2/gstcamerabin2.h | 11 +++++++ 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/gst/camerabin2/gstcamerabin2.c b/gst/camerabin2/gstcamerabin2.c index afae1014a3..2e642d174a 100644 --- a/gst/camerabin2/gstcamerabin2.c +++ b/gst/camerabin2/gstcamerabin2.c @@ -368,11 +368,21 @@ gst_camera_bin_start_capture (GstCameraBin2 * camerabin) GST_DEBUG_OBJECT (camerabin, "Received start-capture"); /* check that we have a valid location */ - if (camerabin->mode == MODE_VIDEO && camerabin->location == NULL) { - GST_ELEMENT_ERROR (camerabin, RESOURCE, OPEN_WRITE, - (_("File location is set to NULL, please set it to a valid filename")), - (NULL)); - return; + if (camerabin->mode == MODE_VIDEO) { + if (camerabin->location == NULL) { + GST_ELEMENT_ERROR (camerabin, RESOURCE, OPEN_WRITE, + (_("File location is set to NULL, please set it to a valid filename")), (NULL)); + return; + } + + g_mutex_lock (camerabin->video_capture_mutex); + if (camerabin->video_state != GST_CAMERA_BIN_VIDEO_IDLE) { + GST_WARNING_OBJECT (camerabin, "Another video recording is ongoing" + " (state %d), cannot start a new one", camerabin->video_state); + g_mutex_unlock (camerabin->video_capture_mutex); + return; + } + camerabin->video_state = GST_CAMERA_BIN_VIDEO_STARTING; } GST_CAMERA_BIN2_PROCESSING_INC (camerabin); @@ -420,8 +430,13 @@ gst_camera_bin_start_capture (GstCameraBin2 * camerabin) } g_signal_emit_by_name (camerabin->src, "start-capture", NULL); - if (camerabin->mode == MODE_VIDEO && camerabin->audio_src) - gst_element_set_state (camerabin->audio_src, GST_STATE_PLAYING); + if (camerabin->mode == MODE_VIDEO) { + if (camerabin->audio_src) + gst_element_set_state (camerabin->audio_src, GST_STATE_PLAYING); + + camerabin->video_state = GST_CAMERA_BIN_VIDEO_RECORDING; + g_mutex_unlock (camerabin->video_capture_mutex); + } /* * We have to push tags after start capture because the video elements @@ -458,12 +473,19 @@ static void gst_camera_bin_stop_capture (GstCameraBin2 * camerabin) { GST_DEBUG_OBJECT (camerabin, "Received stop-capture"); - if (camerabin->src) - g_signal_emit_by_name (camerabin->src, "stop-capture", NULL); + if (camerabin->mode == MODE_VIDEO) { + g_mutex_lock (camerabin->video_capture_mutex); + if (camerabin->video_state == GST_CAMERA_BIN_VIDEO_RECORDING) { + if (camerabin->src) + g_signal_emit_by_name (camerabin->src, "stop-capture", NULL); - if (camerabin->mode == MODE_VIDEO && camerabin->audio_src) { - camerabin->audio_drop_eos = FALSE; - gst_element_send_event (camerabin->audio_src, gst_event_new_eos ()); + camerabin->video_state = GST_CAMERA_BIN_VIDEO_FINISHING; + if (camerabin->audio_src) { + camerabin->audio_drop_eos = FALSE; + gst_element_send_event (camerabin->audio_src, gst_event_new_eos ()); + } + } + g_mutex_unlock (camerabin->video_capture_mutex); } } @@ -535,6 +557,7 @@ gst_camera_bin_dispose (GObject * object) g_free (camerabin->location); g_mutex_free (camerabin->preview_list_mutex); g_mutex_free (camerabin->image_capture_mutex); + g_mutex_free (camerabin->video_capture_mutex); if (camerabin->src_capture_notify_id) g_signal_handler_disconnect (camerabin->src, @@ -892,6 +915,7 @@ gst_camera_bin_init (GstCameraBin2 * camera) camera->flags = DEFAULT_FLAGS; camera->preview_list_mutex = g_mutex_new (); camera->image_capture_mutex = g_mutex_new (); + camera->video_capture_mutex = g_mutex_new (); /* capsfilters are created here as we proxy their caps properties and * this way we avoid having to store the caps while on NULL state to @@ -1035,9 +1059,14 @@ gst_camera_bin_handle_message (GstBin * bin, GstMessage * message) case GST_MESSAGE_EOS:{ GstElement *src = GST_ELEMENT (GST_MESSAGE_SRC (message)); if (src == GST_CAMERA_BIN2_CAST (bin)->videosink) { + g_mutex_lock (camerabin->video_capture_mutex); GST_DEBUG_OBJECT (bin, "EOS from video branch"); + g_assert (camerabin->video_state == GST_CAMERA_BIN_VIDEO_FINISHING); + gst_video_capture_bin_post_video_done (GST_CAMERA_BIN2_CAST (bin)); dec_counter = TRUE; + camerabin->video_state = GST_CAMERA_BIN_VIDEO_IDLE; + g_mutex_unlock (camerabin->video_capture_mutex); } } break; @@ -1810,6 +1839,7 @@ gst_camera_bin_change_state (GstElement * element, GstStateChange trans) gst_tag_setter_reset_tags (GST_TAG_SETTER (camera)); GST_CAMERA_BIN2_RESET_PROCESSING_COUNTER (camera); + camera->video_state = GST_CAMERA_BIN_VIDEO_IDLE; g_mutex_lock (camera->image_capture_mutex); g_slist_foreach (camera->image_location_list, (GFunc) g_free, NULL); diff --git a/gst/camerabin2/gstcamerabin2.h b/gst/camerabin2/gstcamerabin2.h index ec740a57f4..0c786e20f7 100644 --- a/gst/camerabin2/gstcamerabin2.h +++ b/gst/camerabin2/gstcamerabin2.h @@ -45,6 +45,14 @@ typedef enum } GstCamFlags; +typedef enum _GstCameraBinVideoState +{ + GST_CAMERA_BIN_VIDEO_IDLE=0, + GST_CAMERA_BIN_VIDEO_STARTING=1, + GST_CAMERA_BIN_VIDEO_RECORDING=2, + GST_CAMERA_BIN_VIDEO_FINISHING=3 +} GstCameraBinVideoState; + typedef struct _GstCameraBin2 GstCameraBin2; typedef struct _GstCameraBin2Class GstCameraBin2Class; @@ -119,6 +127,9 @@ struct _GstCameraBin2 gboolean audio_drop_eos; + GMutex *video_capture_mutex; + GstCameraBinVideoState video_state; + /* properties */ gint mode; gchar *location; From 54351a012929582e2da29ac2c52f512679904bba Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Thu, 10 Nov 2011 16:53:40 -0300 Subject: [PATCH 04/25] camerabin2: Remove video elements' state clearing from start-capture Reduce start-capture workload by moving the elements' state reseting to the finishing steps of the capture. This reduces the time start-capture takes to actually start a capture and return to its caller, improving user experience. As the elements' state reset is now triggered from the message handling function, it needs to spawn a new thread, changing state from the pad's task would cause a deadlock. --- gst/camerabin2/gstcamerabin2.c | 47 +++++++++++++++++++++++++++++----- gst/camerabin2/gstcamerabin2.h | 1 + 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/gst/camerabin2/gstcamerabin2.c b/gst/camerabin2/gstcamerabin2.c index 2e642d174a..16137ae287 100644 --- a/gst/camerabin2/gstcamerabin2.c +++ b/gst/camerabin2/gstcamerabin2.c @@ -376,6 +376,9 @@ gst_camera_bin_start_capture (GstCameraBin2 * camerabin) } g_mutex_lock (camerabin->video_capture_mutex); + while (camerabin->video_state == GST_CAMERA_BIN_VIDEO_FINISHING) { + g_cond_wait (camerabin->video_state_cond, camerabin->video_capture_mutex); + } if (camerabin->video_state != GST_CAMERA_BIN_VIDEO_IDLE) { GST_WARNING_OBJECT (camerabin, "Another video recording is ongoing" " (state %d), cannot start a new one", camerabin->video_state); @@ -516,11 +519,8 @@ gst_camera_bin_src_notify_readyforcapture (GObject * obj, GParamSpec * pspec, gchar *location = NULL; if (camera->mode == MODE_VIDEO) { - /* a video recording is about to start, we reset the videobin to clear eos/flushing state - * also need to clean the queue ! capsfilter before it */ + /* a video recording is about to start, change the filesink location */ gst_element_set_state (camera->videosink, GST_STATE_NULL); - gst_element_set_state (camera->video_encodebin, GST_STATE_NULL); - gst_element_set_state (camera->videobin_capsfilter, GST_STATE_NULL); location = g_strdup_printf (camera->location, camera->capture_index); GST_DEBUG_OBJECT (camera, "Switching videobin location to %s", location); g_object_set (camera->videosink, "location", location, NULL); @@ -531,8 +531,6 @@ gst_camera_bin_src_notify_readyforcapture (GObject * obj, GParamSpec * pspec, * and could cause problems in a camerabin2 state change */ gst_element_set_state (camera->videosink, GST_STATE_NULL); } - gst_element_set_state (camera->video_encodebin, GST_STATE_PLAYING); - gst_element_set_state (camera->videobin_capsfilter, GST_STATE_PLAYING); } camera->capture_index++; @@ -558,6 +556,7 @@ gst_camera_bin_dispose (GObject * object) g_mutex_free (camerabin->preview_list_mutex); g_mutex_free (camerabin->image_capture_mutex); g_mutex_free (camerabin->video_capture_mutex); + g_cond_free (camerabin->video_state_cond); if (camerabin->src_capture_notify_id) g_signal_handler_disconnect (camerabin->src, @@ -916,6 +915,7 @@ gst_camera_bin_init (GstCameraBin2 * camera) camera->preview_list_mutex = g_mutex_new (); camera->image_capture_mutex = g_mutex_new (); camera->video_capture_mutex = g_mutex_new (); + camera->video_state_cond = g_cond_new (); /* capsfilters are created here as we proxy their caps properties and * this way we avoid having to store the caps while on NULL state to @@ -987,6 +987,29 @@ gst_camera_bin_skip_next_preview (GstCameraBin2 * camerabin) g_mutex_unlock (camerabin->preview_list_mutex); } +static gpointer +gst_camera_bin_video_reset_elements (gpointer u_data) +{ + GstCameraBin2 *camerabin = GST_CAMERA_BIN2_CAST (u_data); + + GST_DEBUG_OBJECT (camerabin, "Resetting video elements state"); + g_mutex_lock (camerabin->video_capture_mutex); + + /* reset element states to clear eos/flushing pads */ + gst_element_set_state (camerabin->video_encodebin, GST_STATE_READY); + gst_element_set_state (camerabin->videobin_capsfilter, GST_STATE_READY); + gst_element_set_state (camerabin->videobin_capsfilter, GST_STATE_PLAYING); + gst_element_set_state (camerabin->video_encodebin, GST_STATE_PLAYING); + + GST_DEBUG_OBJECT (camerabin, "Setting video state to idle"); + camerabin->video_state = GST_CAMERA_BIN_VIDEO_IDLE; + g_cond_signal (camerabin->video_state_cond); + g_mutex_unlock (camerabin->video_capture_mutex); + + gst_object_unref (camerabin); + return NULL; +} + static void gst_camera_bin_handle_message (GstBin * bin, GstMessage * message) { @@ -1059,13 +1082,23 @@ gst_camera_bin_handle_message (GstBin * bin, GstMessage * message) case GST_MESSAGE_EOS:{ GstElement *src = GST_ELEMENT (GST_MESSAGE_SRC (message)); if (src == GST_CAMERA_BIN2_CAST (bin)->videosink) { + g_mutex_lock (camerabin->video_capture_mutex); GST_DEBUG_OBJECT (bin, "EOS from video branch"); g_assert (camerabin->video_state == GST_CAMERA_BIN_VIDEO_FINISHING); gst_video_capture_bin_post_video_done (GST_CAMERA_BIN2_CAST (bin)); dec_counter = TRUE; - camerabin->video_state = GST_CAMERA_BIN_VIDEO_IDLE; + + if (!g_thread_create (gst_camera_bin_video_reset_elements, + gst_object_ref (camerabin), FALSE, NULL)) { + GST_WARNING_OBJECT (camerabin, "Failed to create thread to " + "reset video elements' state, video recordings may not work " + "anymore"); + gst_object_unref (camerabin); + camerabin->video_state = GST_CAMERA_BIN_VIDEO_IDLE; + } + g_mutex_unlock (camerabin->video_capture_mutex); } } diff --git a/gst/camerabin2/gstcamerabin2.h b/gst/camerabin2/gstcamerabin2.h index 0c786e20f7..4c509bb908 100644 --- a/gst/camerabin2/gstcamerabin2.h +++ b/gst/camerabin2/gstcamerabin2.h @@ -128,6 +128,7 @@ struct _GstCameraBin2 gboolean audio_drop_eos; GMutex *video_capture_mutex; + GCond *video_state_cond; GstCameraBinVideoState video_state; /* properties */ From aab3a73ccd44b7167801c2e231f06ec84f8af05c Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Fri, 11 Nov 2011 09:58:30 -0300 Subject: [PATCH 05/25] camerabin2: Add one debug line about camerabin2 being idle --- gst/camerabin2/gstcamerabin2.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gst/camerabin2/gstcamerabin2.c b/gst/camerabin2/gstcamerabin2.c index 16137ae287..8a02f447be 100644 --- a/gst/camerabin2/gstcamerabin2.c +++ b/gst/camerabin2/gstcamerabin2.c @@ -183,8 +183,10 @@ #define GST_CAMERA_BIN2_PROCESSING_DEC(c) \ { \ - if (g_atomic_int_dec_and_test (&c->processing_counter)) \ + if (g_atomic_int_dec_and_test (&c->processing_counter)) { \ g_object_notify (G_OBJECT (c), "idle"); \ + GST_DEBUG_OBJECT ((c), "Camerabin now idle"); \ + } \ GST_DEBUG_OBJECT ((c), "Processing counter decremented"); \ } From 1d5b324e7d218af826b81151404f74b9d6303834 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Mon, 14 Nov 2011 16:55:37 -0300 Subject: [PATCH 06/25] camerabin2: Also reset audio elements when video capture finishes Audio elements also need to be reset after each capture, do it together with the video elements' reset --- gst/camerabin2/gstcamerabin2.c | 37 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/gst/camerabin2/gstcamerabin2.c b/gst/camerabin2/gstcamerabin2.c index 8a02f447be..492c4eb411 100644 --- a/gst/camerabin2/gstcamerabin2.c +++ b/gst/camerabin2/gstcamerabin2.c @@ -399,12 +399,6 @@ gst_camera_bin_start_capture (GstCameraBin2 * camerabin) if (camerabin->audio_src) { GstClock *clock = gst_pipeline_get_clock (GST_PIPELINE_CAST (camerabin)); - /* need to reset eos status (pads could be flushing) */ - gst_element_set_state (camerabin->audio_capsfilter, GST_STATE_READY); - gst_element_set_state (camerabin->audio_volume, GST_STATE_READY); - - gst_element_sync_state_with_parent (camerabin->audio_capsfilter); - gst_element_sync_state_with_parent (camerabin->audio_volume); gst_element_set_state (camerabin->audio_src, GST_STATE_PAUSED); gst_element_set_base_time (camerabin->audio_src, @@ -536,16 +530,6 @@ gst_camera_bin_src_notify_readyforcapture (GObject * obj, GParamSpec * pspec, } camera->capture_index++; - } else { - if (camera->mode == MODE_VIDEO && camera->audio_src) { - /* FIXME We need to set audiosrc to null to make it resync the ringbuffer - * while bug https://bugzilla.gnome.org/show_bug.cgi?id=648359 isn't - * fixed. - * - * Also, we set to NULL here to stop capturing audio through to the next - * video mode start capture and to clear EOS. */ - gst_element_set_state (camera->audio_src, GST_STATE_NULL); - } } } @@ -1000,8 +984,25 @@ gst_camera_bin_video_reset_elements (gpointer u_data) /* reset element states to clear eos/flushing pads */ gst_element_set_state (camerabin->video_encodebin, GST_STATE_READY); gst_element_set_state (camerabin->videobin_capsfilter, GST_STATE_READY); - gst_element_set_state (camerabin->videobin_capsfilter, GST_STATE_PLAYING); - gst_element_set_state (camerabin->video_encodebin, GST_STATE_PLAYING); + gst_element_sync_state_with_parent (camerabin->videobin_capsfilter); + gst_element_sync_state_with_parent (camerabin->video_encodebin); + + if (camerabin->audio_src) { + gst_element_set_state (camerabin->audio_capsfilter, GST_STATE_READY); + gst_element_set_state (camerabin->audio_volume, GST_STATE_READY); + + /* FIXME We need to set audiosrc to null to make it resync the ringbuffer + * while bug https://bugzilla.gnome.org/show_bug.cgi?id=648359 isn't + * fixed. + * + * Also, we don't reinit the audiosrc to keep audio devices from being open + * and running until we really need them */ + gst_element_set_state (camerabin->audio_src, GST_STATE_NULL); + + gst_element_sync_state_with_parent (camerabin->audio_capsfilter); + gst_element_sync_state_with_parent (camerabin->audio_volume); + + } GST_DEBUG_OBJECT (camerabin, "Setting video state to idle"); camerabin->video_state = GST_CAMERA_BIN_VIDEO_IDLE; From 316091d452d1577faafd01384fb1541a6712aace Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Mon, 14 Nov 2011 18:34:18 -0300 Subject: [PATCH 07/25] camerabin2: Fix some racyness in tests Fix some racyness as the test was checking the idle property right after it got the preview message for video recordings. In some conditions, it might happen that camerabin2 still hasn't decremented the processing counter after posting the preview and/or the image/video-done message and the test will get idle=false and fail. The approach for checking for the idle property was a busy loop with a sleep. Far from elegant, but good enough for these tests. --- tests/check/elements/camerabin2.c | 40 ++++++++++++++++++------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/tests/check/elements/camerabin2.c b/tests/check/elements/camerabin2.c index b310b8654a..977ff26ea6 100644 --- a/tests/check/elements/camerabin2.c +++ b/tests/check/elements/camerabin2.c @@ -622,6 +622,22 @@ wait_for_element_message (GstElement * camera, const gchar * name, return msg; } +static void +wait_for_idle_state (void) +{ + gboolean idle = FALSE; + + /* not the ideal way, but should be enough for testing */ + while (idle == FALSE) { + g_object_get (camera, "idle", &idle, NULL); + if (idle) + break; + + g_usleep (GST_SECOND / 5); + } + fail_unless (idle); +} + GST_START_TEST (test_single_image_capture) { gboolean idle; @@ -652,8 +668,7 @@ GST_START_TEST (test_single_image_capture) /* check that we got a preview image */ check_preview_image (camera, image_filename, 0); - g_object_get (camera, "idle", &idle, NULL); - fail_unless (idle); + wait_for_idle_state (); gst_element_set_state (GST_ELEMENT (camera), GST_STATE_NULL); check_file_validity (image_filename, 0, NULL, 0, 0, NO_AUDIO); } @@ -705,8 +720,7 @@ GST_START_TEST (test_multiple_image_captures) check_preview_image (camera, image_filename, i); } - g_object_get (camera, "idle", &idle, NULL); - fail_unless (idle); + wait_for_idle_state (); gst_element_set_state (GST_ELEMENT (camera), GST_STATE_NULL); for (i = 0; i < 3; i++) { check_file_validity (image_filename, i, NULL, widths[i], heights[i], @@ -756,8 +770,7 @@ GST_START_TEST (test_single_video_recording) fail_unless (msg != NULL); gst_message_unref (msg); - g_object_get (camera, "idle", &idle, NULL); - fail_unless (idle); + wait_for_idle_state (); gst_element_set_state (GST_ELEMENT (camera), GST_STATE_NULL); check_file_validity (video_filename, 0, NULL, 0, 0, WITH_AUDIO); @@ -819,8 +832,7 @@ GST_START_TEST (test_multiple_video_recordings) check_preview_image (camera, video_filename, i); - g_object_get (camera, "idle", &idle, NULL); - fail_unless (idle); + wait_for_idle_state (); } gst_element_set_state (GST_ELEMENT (camera), GST_STATE_NULL); @@ -834,7 +846,6 @@ GST_END_TEST; GST_START_TEST (test_image_video_cycle) { - gboolean idle; gint i; if (!camera) @@ -854,8 +865,7 @@ GST_START_TEST (test_image_video_cycle) const gchar *img_filename; const gchar *vid_filename; - g_object_get (camera, "idle", &idle, NULL); - fail_unless (idle); + wait_for_idle_state (); /* take a picture */ img_filename = make_const_file_name (image_filename, i); @@ -885,10 +895,9 @@ GST_START_TEST (test_image_video_cycle) gst_message_unref (msg); check_preview_image (camera, vid_filename, i); - - /* wait for capture to finish */ - g_usleep (G_USEC_PER_SEC); } + + wait_for_idle_state (); gst_element_set_state (GST_ELEMENT (camera), GST_STATE_NULL); /* validate all the files */ @@ -1193,8 +1202,7 @@ GST_START_TEST (test_idle_property) check_preview_image (camera, video_filename, 0); - g_object_get (camera, "idle", &idle, NULL); - fail_unless (idle); + wait_for_idle_state (); gst_element_set_state (GST_ELEMENT (camera), GST_STATE_NULL); From 92f07e389813e8e6a1b30103d11bd02ad15df812 Mon Sep 17 00:00:00 2001 From: David King Date: Mon, 14 Nov 2011 11:30:13 +0100 Subject: [PATCH 08/25] camerabin: Document requirement for PLAYING state As described in GNOME bug 663998, the element must be in the PLAYING state before calling capture-start. https://bugzilla.gnome.org/show_bug.cgi?id=664048 --- gst/camerabin/gstcamerabin.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gst/camerabin/gstcamerabin.c b/gst/camerabin/gstcamerabin.c index 1a2b05f76a..be13b65d7c 100644 --- a/gst/camerabin/gstcamerabin.c +++ b/gst/camerabin/gstcamerabin.c @@ -84,6 +84,8 @@ * unreffed or replaced by a new user set element. Initially only elements * needed for view finder mode are created to speed up startup. Image bin and * video bin elements are created when setting the mode or starting capture. + * GstCameraBin must be in the PLAYING state before #GstCameraBin::capture-start + * is called. * * * From 5099ff23afb85539fa0f226598b358547c00fc27 Mon Sep 17 00:00:00 2001 From: Jonas Larsson Date: Wed, 16 Nov 2011 10:56:24 +0100 Subject: [PATCH 09/25] h264parse: outgoing byte stream prefix always has size 4 Fixes #664123. --- gst/videoparsers/gsth264parse.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c index d931111597..2127125f40 100644 --- a/gst/videoparsers/gsth264parse.c +++ b/gst/videoparsers/gsth264parse.c @@ -337,7 +337,7 @@ gst_h264_parse_wrap_nal (GstH264Parse * h264parse, guint format, guint8 * data, guint size) { GstBuffer *buf; - const guint nl = h264parse->nal_length_size; + guint nl = h264parse->nal_length_size; GST_DEBUG_OBJECT (h264parse, "nal length %d", size); @@ -345,7 +345,10 @@ gst_h264_parse_wrap_nal (GstH264Parse * h264parse, guint format, guint8 * data, if (format == GST_H264_PARSE_FORMAT_AVC) { GST_WRITE_UINT32_BE (GST_BUFFER_DATA (buf), size << (32 - 8 * nl)); } else { - g_assert (nl == 4); + /* HACK: nl should always be 4 here, otherwise this won't work. + * There are legit cases where nl in avc stream is 2, but byte-stream + * SC is still always 4 bytes. */ + nl = 4; GST_WRITE_UINT32_BE (GST_BUFFER_DATA (buf), 1); } From e500ec524ca3cc1a2437c5773eaa74902ac3db3e Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Wed, 16 Nov 2011 12:46:04 +0100 Subject: [PATCH 10/25] tsdemux: Add notes on synchronization and scheduling --- gst/mpegtsdemux/TODO | 238 ++++++++++++++++++++++---------------- gst/mpegtsdemux/tsdemux.c | 8 +- 2 files changed, 142 insertions(+), 104 deletions(-) diff --git a/gst/mpegtsdemux/TODO b/gst/mpegtsdemux/TODO index b8f636687a..66e1d5f1f1 100644 --- a/gst/mpegtsdemux/TODO +++ b/gst/mpegtsdemux/TODO @@ -1,117 +1,151 @@ -mpegtsparse rebasing +tsdemux/tsparse TODO -------------------- -Rationale : ------------ +* clock for live streams + In order for playback to happen at the same rate as on the producer, + we need to estimate the remote clock based on capture time and PCR + values. + For this estimation to be as accurate as possible, the capture time + needs to happen on the sources. + => Ensure live sources actually timestamp their buffers + Once we have accurate timestamps, we can use an algorithm to + calculate the PCR/local-clock skew. + => Use the EPTLA algorithm as used in -good/rtp/rtpmanager/ + gstrtpjitterbuffer - mpegtsparse code is more sane to handle and work with. +* Seeking + => Split out in a separate file/object. It is polluting tsdemux for + code readability/clarity. - We need a modular demuxer - - We need to avoid duplicating code regarding mpeg-ts in a gazillion - elements and allow easy creatiof new elements. - - -Battleplan : ------------- -* Figure out code from mpegtsparse which would be also needed for a -mpeg-ts demuxer (ex: packet/psi/pcr parsing). -* Extract common code into a base mpegtsbase class. -* Refactor mpegtsparse to subclass that base class. -* Create a minimalistic demuxer that creates pads (based on PSI info) -and outputs ES packets (let's say mpeg audio and video to start with) - -Potential subclasses : ----------------------- -* MpegTSParse : Program splitter. Given an incoming multi-program - mpeg-ts stream, it can provide request pads for each program. Each - of those pads will contain the ts packets specific to that program. - -* TSDemux : Program demuxer. Given an incoming single or multi-program - mpeg-ts stream, it will reconstruct the original Program Streams of - the selected program and output them on dynamically created pads. - -* HDVSplitter : Given an incoming HDV mpeg-ts stream, it will locate - the beginning of new scenes and output a mpeg-ts stream with the - PAT/PMT/AUX packets properly ordered and marked with DISCONT, so - that the following pipeline will automatically cut up a tape dump - into individual scenes: - filesrc ! hdvsplit ! multifilesink next-file=discont - -Code/Design common to a program-spliter and a demuxer : -------------------------------------------------------- -* Parsing TS packets -* Establishing PAT/PMT mapping -* Handling the notions of Programs/Streams -* Seeking ? - - One proposal... would be to have the base class automatically create - all the structures (and relationships) for the following objects: - - * Programs (from PAT/PMT, dunno if it could come from something - else) - * Program id - * Streams contained in that program (with links to them) - * Which stream contains the PCR - * Metadata ? - * Streams (ideally... in a table for fast access) - * We want to be able to have stream-type specific information - easily accessible also (like mpeg video specific data) - * Maybe some other info ??? - - The subclasses would then be able to make their own decision based - on those objects. - Maybe we could have some virtual methods that will be called when a - new program is detected, a new stream is added, etc... - - It is the subclass who decides what's to do with a given packet once - it's been parsed. - tsparse : forward it as-is to the pad corresponding to the program - tsdemux : forward it to the proper PS parser - hdvsplit : ? - - -Ideas to be taken into account for a proper demuxer : ------------------------------------------------------ -* Push-based (with inacurrate seeking) -* Pull-based (with fast *AND* accurate seeking) -* Modular system to add stream-type specific helper parsing - * Doesn't have to be fully fledged, just enough to help any kind of - seeking and scanning code. -* ... - -Problems to figure out : ------------------------- -* clock - Needed for proper dvb playback. mpegtsdemux currently does internal - clock estimation... to provide a clock with PCR estimations. - A proper way to solve that would be to timestamp the buffers at the - source element using the system clock, and then adjusting the PCR - against those values. (i.e. doing the opposite of what's done in - mpegtsdemux, but it will be more accurate since the timestamping is - done at the source). - - -Bugs that need fixing : ------------------------ * Perfomance : Creation/Destruction of buffers is slow * => This is due to g_type_instance_create using a dogslow rwlock which take up to 50% of gst_adapter_take_buffer() => Bugzilla #585375 (performance and contention problems) -Code structure: - - MpegTSBase - +--- MpegTSParse - +--- TSDemux - - -Known limitations and problems : --------------------------------- * mpegtspacketizer - * Assumes 188 bytes packets. It should support all modes. * offset/timestamp of incoming buffers need to be carried on to the sub-buffers in order for several demuxer features to work correctly. + * mpegtsparser * SERIOUS room for improvement performance-wise (see callgrind) + + + +Synchronization, Scheduling and Timestamping +-------------------------------------------- + + A mpeg-ts demuxer can be used in a variety of situations: + * lives streaming over DVB, UDP, RTP,.. + * play-as-you-download like HTTP Live Streaming or UPNP/DLNA + * random-access local playback, file, Bluray, ... + + Those use-cases can be categorized in 3 different categories: + * Push-based scheduling with live sources [0] + * Push-based scheduling with non-live sources + * Pull-based scheduling with fast random-access + + Due to the nature of timing within the mpeg-ts format, we need to +pay extra attention to the outgoing NEWSEGMENT event and buffer +timestamps in order to guarantee proper playback and synchronization +of the stream. + + + 1) Live push-based scheduling + + The NEWSEGMENT event will be in time format and is forwarded as is, + and the values are cached locally. + + Since the clock is running when the upstream buffers are captured, + the outgoing buffer timestamps need to correspond to the incoming + buffer timestamp values. + + => A delta, DTS_delta between incoming buffer timestamp and + DTS/PTS needs to be computed. + + => The outgoing buffers will be timestamped with their PTS values + (overflow corrected) offseted by that initial DTS_delta. + + A latency is introduced between the time the buffer containing the + first bit of a Access Unit is received in the demuxer and the moment + the demuxer pushed out the buffer corresponding to that Access Unit. + + => That latency needs to be reported. It corresponds to the + biggest Access Unit spacing, in this case 1/video-framerate. + + According to the ISO/IEC 13818-1:2007 specifications, D.0.1 Timing + mode, the "coded audio and video that represent sound and pictures + that are to be presented simultaneously may be separated in time + within the coded bit stream by ==>as much as one second<==" + + => The demuxer will therefore report an added latency of 1s to + handle this interleave. + + + 2) Non-live push-based scheduling + + If the upstream NEWSEGMENT is in time format, the NEWSEGMENT event + is forwarded as is, and the values are cached locally. + + If upstream does provide a NEWSEGMENT in another format, we need to + compute one by taking the default values: + start : 0 + stop : GST_CLOCK_TIME_NONE + time : 0 + + Since no prerolling is happening downstream and the incoming buffers + do not have capture timestamps, we need to ensure the first buffer + we push out corresponds to the base segment start runing time. + + => A delta between the first DTS to output and the segment start + position needs to be computed. + + => The outgoing buffers will be timestamped with their PTS values + (overflow corrected) offseted by that initial delta. + + Latency is reported just as with the live use-case. + + + 3) Random access pull-mode + + We do not get a NEWSEGMENT event from upstream, we therefore need to + compute the outgoing values. + + The base stream/running time corresponds to the DTS of the first + buffer we will output. The DTS_delta becomes that earliest DTS. + + => FILLME + + X) General notes + + It is assumed that PTS/DTS rollovers are detected and corrected such + as the outgoing timestamps never rollover. This can be easily + handled by correcting the DTS_delta when such rollovers are + detected. The maximum value of a GstClockTimeDiff is almost 3 + centuries, we therefore have enough margin to handle a decent number + of rollovers. + + The generic equation for calculating outgoing buffer timestamps + therefore becomes: + + D = DTS_delta, with rollover corrections + PTS = PTS of the buffer we are going to push out + TS = Timestamp of the outgoing buffer + + ==> TS = PTS + D + + If seeking is handled upstream for push-based cases, whether live or + not, no extra modification is required. + + If seeking is handled by the demuxer in the non-live push-based + cases (converting from TIME to BYTES), the demuxer will need to + set the segment start/time values to the requested seek position. + The DTS_delta will also have to be recomputed to take into account + the seek position. + + +[0] When talking about live sources, we mean this in the GStreamer +definition of live sources, which is to say sources where if we miss +the capture, we will miss the data to be captured. Sources which do +internal buffering (like TCP connections or file descriptors) are +*NOT* live sources. diff --git a/gst/mpegtsdemux/tsdemux.c b/gst/mpegtsdemux/tsdemux.c index 3facd64073..5685bd9eec 100644 --- a/gst/mpegtsdemux/tsdemux.c +++ b/gst/mpegtsdemux/tsdemux.c @@ -44,6 +44,12 @@ #include "payload_parsers.h" #include "pesparse.h" +/* + * tsdemux + * + * See TODO for explanations on improvements needed + */ + /* latency in mseconds */ #define TS_LATENCY 700 @@ -74,8 +80,6 @@ static GQuark QUARK_PTS; static GQuark QUARK_DTS; static GQuark QUARK_OFFSET; - - typedef enum { PENDING_PACKET_EMPTY = 0, /* No pending packet/buffer From da1eaa2d787b3b9c84be8a9bf2fd04ddea60758a Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 16 Nov 2011 16:56:43 +0000 Subject: [PATCH 11/25] opus: port to base audio encoder/decoder --- ext/opus/Makefile.am | 2 + ext/opus/gstopusdec.c | 819 +++++++++-------------------------- ext/opus/gstopusdec.h | 14 +- ext/opus/gstopusenc.c | 961 ++++++++++++++---------------------------- ext/opus/gstopusenc.h | 27 +- 5 files changed, 523 insertions(+), 1300 deletions(-) diff --git a/ext/opus/Makefile.am b/ext/opus/Makefile.am index aa50ba96ef..57e1692645 100644 --- a/ext/opus/Makefile.am +++ b/ext/opus/Makefile.am @@ -2,10 +2,12 @@ plugin_LTLIBRARIES = libgstopus.la libgstopus_la_SOURCES = gstopus.c gstopusdec.c gstopusenc.c libgstopus_la_CFLAGS = \ + -DGST_USE_UNSTABLE_API \ $(GST_PLUGINS_BASE_CFLAGS) \ $(GST_CFLAGS) \ $(OPUS_CFLAGS) libgstopus_la_LIBADD = \ + -lgstaudio-$(GST_MAJORMINOR) \ $(GST_PLUGINS_BASE_LIBS) -lgsttag-$(GST_MAJORMINOR) \ $(GST_BASE_LIBS) \ $(GST_LIBS) \ diff --git a/ext/opus/gstopusdec.c b/ext/opus/gstopusdec.c index 35f501a8ab..1abe3201cb 100644 --- a/ext/opus/gstopusdec.c +++ b/ext/opus/gstopusdec.c @@ -68,31 +68,17 @@ GST_STATIC_PAD_TEMPLATE ("sink", GST_STATIC_CAPS ("audio/x-opus") ); -GST_BOILERPLATE (GstOpusDec, gst_opus_dec, GstElement, GST_TYPE_ELEMENT); +GST_BOILERPLATE (GstOpusDec, gst_opus_dec, GstAudioDecoder, + GST_TYPE_AUDIO_DECODER); -static gboolean opus_dec_sink_event (GstPad * pad, GstEvent * event); -static GstFlowReturn opus_dec_chain (GstPad * pad, GstBuffer * buf); -static gboolean opus_dec_sink_setcaps (GstPad * pad, GstCaps * caps); -static GstStateChangeReturn opus_dec_change_state (GstElement * element, - GstStateChange transition); - -static gboolean opus_dec_src_event (GstPad * pad, GstEvent * event); -static gboolean opus_dec_src_query (GstPad * pad, GstQuery * query); -static gboolean opus_dec_sink_query (GstPad * pad, GstQuery * query); -static const GstQueryType *opus_get_src_query_types (GstPad * pad); -static const GstQueryType *opus_get_sink_query_types (GstPad * pad); -static gboolean opus_dec_convert (GstPad * pad, - GstFormat src_format, gint64 src_value, - GstFormat * dest_format, gint64 * dest_value); - -static GstFlowReturn opus_dec_chain_parse_data (GstOpusDec * dec, - GstBuffer * buf, GstClockTime timestamp, GstClockTime duration); -static GstFlowReturn opus_dec_chain_parse_header (GstOpusDec * dec, +static GstFlowReturn gst_opus_dec_parse_header (GstOpusDec * dec, GstBuffer * buf); -#if 0 -static GstFlowReturn opus_dec_chain_parse_comments (GstOpusDec * dec, - GstBuffer * buf); -#endif +static gboolean gst_opus_dec_start (GstAudioDecoder * dec); +static gboolean gst_opus_dec_stop (GstAudioDecoder * dec); +static GstFlowReturn gst_opus_dec_handle_frame (GstAudioDecoder * dec, + GstBuffer * buffer); +static gboolean gst_opus_dec_set_format (GstAudioDecoder * bdec, + GstCaps * caps); static void gst_opus_dec_base_init (gpointer g_class) @@ -112,11 +98,16 @@ gst_opus_dec_base_init (gpointer g_class) static void gst_opus_dec_class_init (GstOpusDecClass * klass) { + GstAudioDecoderClass *adclass; GstElementClass *gstelement_class; + adclass = (GstAudioDecoderClass *) klass; gstelement_class = (GstElementClass *) klass; - gstelement_class->change_state = GST_DEBUG_FUNCPTR (opus_dec_change_state); + adclass->start = GST_DEBUG_FUNCPTR (gst_opus_dec_start); + adclass->stop = GST_DEBUG_FUNCPTR (gst_opus_dec_stop); + adclass->handle_frame = GST_DEBUG_FUNCPTR (gst_opus_dec_handle_frame); + adclass->set_format = GST_DEBUG_FUNCPTR (gst_opus_dec_set_format); GST_DEBUG_CATEGORY_INIT (opusdec_debug, "opusdec", 0, "opus decoding element"); @@ -125,8 +116,6 @@ gst_opus_dec_class_init (GstOpusDecClass * klass) static void gst_opus_dec_reset (GstOpusDec * dec) { - gst_segment_init (&dec->segment, GST_FORMAT_UNDEFINED); - dec->granulepos = -1; dec->packetno = 0; dec->frame_size = 0; dec->frame_samples = 960; @@ -135,50 +124,14 @@ gst_opus_dec_reset (GstOpusDec * dec) opus_decoder_destroy (dec->state); dec->state = NULL; } -#if 0 - if (dec->mode) { - opus_mode_destroy (dec->mode); - dec->mode = NULL; - } -#endif gst_buffer_replace (&dec->streamheader, NULL); gst_buffer_replace (&dec->vorbiscomment, NULL); - g_list_foreach (dec->extra_headers, (GFunc) gst_mini_object_unref, NULL); - g_list_free (dec->extra_headers); - dec->extra_headers = NULL; - -#if 0 - memset (&dec->header, 0, sizeof (dec->header)); -#endif } static void gst_opus_dec_init (GstOpusDec * dec, GstOpusDecClass * g_class) { - dec->sinkpad = - gst_pad_new_from_static_template (&opus_dec_sink_factory, "sink"); - gst_pad_set_chain_function (dec->sinkpad, GST_DEBUG_FUNCPTR (opus_dec_chain)); - gst_pad_set_event_function (dec->sinkpad, - GST_DEBUG_FUNCPTR (opus_dec_sink_event)); - gst_pad_set_query_type_function (dec->sinkpad, - GST_DEBUG_FUNCPTR (opus_get_sink_query_types)); - gst_pad_set_query_function (dec->sinkpad, - GST_DEBUG_FUNCPTR (opus_dec_sink_query)); - gst_pad_set_setcaps_function (dec->sinkpad, - GST_DEBUG_FUNCPTR (opus_dec_sink_setcaps)); - gst_element_add_pad (GST_ELEMENT (dec), dec->sinkpad); - - dec->srcpad = gst_pad_new_from_static_template (&opus_dec_src_factory, "src"); - gst_pad_use_fixed_caps (dec->srcpad); - gst_pad_set_event_function (dec->srcpad, - GST_DEBUG_FUNCPTR (opus_dec_src_event)); - gst_pad_set_query_type_function (dec->srcpad, - GST_DEBUG_FUNCPTR (opus_get_src_query_types)); - gst_pad_set_query_function (dec->srcpad, - GST_DEBUG_FUNCPTR (opus_dec_src_query)); - gst_element_add_pad (GST_ELEMENT (dec), dec->srcpad); - dec->sample_rate = 48000; dec->n_channels = 2; @@ -186,532 +139,39 @@ gst_opus_dec_init (GstOpusDec * dec, GstOpusDecClass * g_class) } static gboolean -opus_dec_sink_setcaps (GstPad * pad, GstCaps * caps) +gst_opus_dec_start (GstAudioDecoder * dec) { - GstOpusDec *dec = GST_OPUS_DEC (gst_pad_get_parent (pad)); - gboolean ret = TRUE; - GstStructure *s; - const GValue *streamheader; + GstOpusDec *odec = GST_OPUS_DEC (dec); - GST_DEBUG_OBJECT (pad, "Setting sink caps to %" GST_PTR_FORMAT, caps); + gst_opus_dec_reset (odec); - s = gst_caps_get_structure (caps, 0); - if ((streamheader = gst_structure_get_value (s, "streamheader")) && - G_VALUE_HOLDS (streamheader, GST_TYPE_ARRAY) && - gst_value_array_get_size (streamheader) >= 2) { - const GValue *header; - GstBuffer *buf; - GstFlowReturn res = GST_FLOW_OK; + /* we know about concealment */ + gst_audio_decoder_set_plc_aware (dec, TRUE); - header = gst_value_array_get_value (streamheader, 0); - if (header && G_VALUE_HOLDS (header, GST_TYPE_BUFFER)) { - buf = gst_value_get_buffer (header); - res = opus_dec_chain_parse_header (dec, buf); - if (res != GST_FLOW_OK) - goto done; - gst_buffer_replace (&dec->streamheader, buf); - } -#if 0 - vorbiscomment = gst_value_array_get_value (streamheader, 1); - if (vorbiscomment && G_VALUE_HOLDS (vorbiscomment, GST_TYPE_BUFFER)) { - buf = gst_value_get_buffer (vorbiscomment); - res = opus_dec_chain_parse_comments (dec, buf); - if (res != GST_FLOW_OK) - goto done; - gst_buffer_replace (&dec->vorbiscomment, buf); - } -#endif - - g_list_foreach (dec->extra_headers, (GFunc) gst_mini_object_unref, NULL); - g_list_free (dec->extra_headers); - dec->extra_headers = NULL; - - if (gst_value_array_get_size (streamheader) > 2) { - gint i, n; - - n = gst_value_array_get_size (streamheader); - for (i = 2; i < n; i++) { - header = gst_value_array_get_value (streamheader, i); - buf = gst_value_get_buffer (header); - dec->extra_headers = - g_list_prepend (dec->extra_headers, gst_buffer_ref (buf)); - } - } - } - - if (!gst_structure_get_int (s, "frame-size", &dec->frame_size)) { - GST_WARNING_OBJECT (dec, "Frame size not included in caps"); - } - if (!gst_structure_get_int (s, "channels", &dec->n_channels)) { - GST_WARNING_OBJECT (dec, "Number of channels not included in caps"); - } - if (!gst_structure_get_int (s, "rate", &dec->sample_rate)) { - GST_WARNING_OBJECT (dec, "Sample rate not included in caps"); - } - switch (dec->frame_size) { - case 2: - dec->frame_samples = dec->sample_rate / 400; - break; - case 5: - dec->frame_samples = dec->sample_rate / 200; - break; - case 10: - dec->frame_samples = dec->sample_rate / 100; - break; - case 20: - dec->frame_samples = dec->sample_rate / 50; - break; - case 40: - dec->frame_samples = dec->sample_rate / 25; - break; - case 60: - dec->frame_samples = 3 * dec->sample_rate / 50; - break; - default: - GST_WARNING_OBJECT (dec, "Unsupported frame size: %d", dec->frame_size); - break; - } - - dec->frame_duration = gst_util_uint64_scale_int (dec->frame_samples, - GST_SECOND, dec->sample_rate); - - GST_INFO_OBJECT (dec, - "Got frame size %d, %d channels, %d Hz, giving %d samples per frame, frame duration %" - GST_TIME_FORMAT, dec->frame_size, dec->n_channels, dec->sample_rate, - dec->frame_samples, GST_TIME_ARGS (dec->frame_duration)); - -done: - gst_object_unref (dec); - return ret; + return TRUE; } static gboolean -opus_dec_convert (GstPad * pad, - GstFormat src_format, gint64 src_value, - GstFormat * dest_format, gint64 * dest_value) +gst_opus_dec_stop (GstAudioDecoder * dec) { - gboolean res = TRUE; - GstOpusDec *dec; - guint64 scale = 1; + GstOpusDec *odec = GST_OPUS_DEC (dec); - dec = GST_OPUS_DEC (gst_pad_get_parent (pad)); + gst_opus_dec_reset (odec); - if (dec->packetno < 1) { - res = FALSE; - goto cleanup; - } - - if (src_format == *dest_format) { - *dest_value = src_value; - res = TRUE; - goto cleanup; - } - - if (pad == dec->sinkpad && - (src_format == GST_FORMAT_BYTES || *dest_format == GST_FORMAT_BYTES)) { - res = FALSE; - goto cleanup; - } - - switch (src_format) { - case GST_FORMAT_TIME: - switch (*dest_format) { - case GST_FORMAT_BYTES: - scale = sizeof (gint16) * dec->n_channels; - case GST_FORMAT_DEFAULT: - *dest_value = - gst_util_uint64_scale_int (scale * src_value, - dec->sample_rate, GST_SECOND); - break; - default: - res = FALSE; - break; - } - break; - case GST_FORMAT_DEFAULT: - switch (*dest_format) { - case GST_FORMAT_BYTES: - *dest_value = src_value * sizeof (gint16) * dec->n_channels; - break; - case GST_FORMAT_TIME: - *dest_value = - gst_util_uint64_scale_int (src_value, GST_SECOND, - dec->sample_rate); - break; - default: - res = FALSE; - break; - } - break; - case GST_FORMAT_BYTES: - switch (*dest_format) { - case GST_FORMAT_DEFAULT: - *dest_value = src_value / (sizeof (gint16) * dec->n_channels); - break; - case GST_FORMAT_TIME: - *dest_value = gst_util_uint64_scale_int (src_value, GST_SECOND, - dec->sample_rate * sizeof (gint16) * dec->n_channels); - break; - default: - res = FALSE; - break; - } - break; - default: - res = FALSE; - break; - } - -cleanup: - gst_object_unref (dec); - return res; -} - -static const GstQueryType * -opus_get_sink_query_types (GstPad * pad) -{ - static const GstQueryType opus_dec_sink_query_types[] = { - GST_QUERY_CONVERT, - 0 - }; - - return opus_dec_sink_query_types; -} - -static gboolean -opus_dec_sink_query (GstPad * pad, GstQuery * query) -{ - GstOpusDec *dec; - gboolean res; - - dec = GST_OPUS_DEC (gst_pad_get_parent (pad)); - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_CONVERT: - { - GstFormat src_fmt, dest_fmt; - gint64 src_val, dest_val; - - gst_query_parse_convert (query, &src_fmt, &src_val, &dest_fmt, &dest_val); - res = opus_dec_convert (pad, src_fmt, src_val, &dest_fmt, &dest_val); - if (res) { - gst_query_set_convert (query, src_fmt, src_val, dest_fmt, dest_val); - } - break; - } - default: - res = gst_pad_query_default (pad, query); - break; - } - - gst_object_unref (dec); - return res; -} - -static const GstQueryType * -opus_get_src_query_types (GstPad * pad) -{ - static const GstQueryType opus_dec_src_query_types[] = { - GST_QUERY_POSITION, - GST_QUERY_DURATION, - 0 - }; - - return opus_dec_src_query_types; -} - -static gboolean -opus_dec_src_query (GstPad * pad, GstQuery * query) -{ - GstOpusDec *dec; - gboolean res = FALSE; - - dec = GST_OPUS_DEC (gst_pad_get_parent (pad)); - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_POSITION:{ - GstSegment segment; - GstFormat format; - gint64 cur; - - gst_query_parse_position (query, &format, NULL); - - GST_PAD_STREAM_LOCK (dec->sinkpad); - segment = dec->segment; - GST_PAD_STREAM_UNLOCK (dec->sinkpad); - - if (segment.format != GST_FORMAT_TIME) { - GST_DEBUG_OBJECT (dec, "segment not initialised yet"); - break; - } - - if ((res = opus_dec_convert (dec->srcpad, GST_FORMAT_TIME, - segment.last_stop, &format, &cur))) { - gst_query_set_position (query, format, cur); - } - break; - } - case GST_QUERY_DURATION:{ - GstFormat format = GST_FORMAT_TIME; - gint64 dur; - - /* get duration from demuxer */ - if (!gst_pad_query_peer_duration (dec->sinkpad, &format, &dur)) - break; - - gst_query_parse_duration (query, &format, NULL); - - /* and convert it into the requested format */ - if ((res = opus_dec_convert (dec->srcpad, GST_FORMAT_TIME, - dur, &format, &dur))) { - gst_query_set_duration (query, format, dur); - } - break; - } - default: - res = gst_pad_query_default (pad, query); - break; - } - - gst_object_unref (dec); - return res; -} - -static gboolean -opus_dec_src_event (GstPad * pad, GstEvent * event) -{ - gboolean res = FALSE; - GstOpusDec *dec = GST_OPUS_DEC (gst_pad_get_parent (pad)); - - GST_LOG_OBJECT (dec, "handling %s event", GST_EVENT_TYPE_NAME (event)); - - switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_SEEK:{ - GstFormat format, tformat; - gdouble rate; - GstEvent *real_seek; - GstSeekFlags flags; - GstSeekType cur_type, stop_type; - gint64 cur, stop; - gint64 tcur, tstop; - - gst_event_parse_seek (event, &rate, &format, &flags, &cur_type, &cur, - &stop_type, &stop); - - /* we have to ask our peer to seek to time here as we know - * nothing about how to generate a granulepos from the src - * formats or anything. - * - * First bring the requested format to time - */ - tformat = GST_FORMAT_TIME; - if (!(res = opus_dec_convert (pad, format, cur, &tformat, &tcur))) - break; - if (!(res = opus_dec_convert (pad, format, stop, &tformat, &tstop))) - break; - - /* then seek with time on the peer */ - real_seek = gst_event_new_seek (rate, GST_FORMAT_TIME, - flags, cur_type, tcur, stop_type, tstop); - - GST_LOG_OBJECT (dec, "seek to %" GST_TIME_FORMAT, GST_TIME_ARGS (tcur)); - - res = gst_pad_push_event (dec->sinkpad, real_seek); - gst_event_unref (event); - break; - } - default: - res = gst_pad_event_default (pad, event); - break; - } - - gst_object_unref (dec); - return res; -} - -static gboolean -opus_dec_sink_event (GstPad * pad, GstEvent * event) -{ - GstOpusDec *dec; - gboolean ret = FALSE; - - dec = GST_OPUS_DEC (gst_pad_get_parent (pad)); - - GST_LOG_OBJECT (dec, "handling %s event", GST_EVENT_TYPE_NAME (event)); - - switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_NEWSEGMENT:{ - GstFormat format; - gdouble rate, arate; - gint64 start, stop, time; - gboolean update; - - gst_event_parse_new_segment_full (event, &update, &rate, &arate, &format, - &start, &stop, &time); - - if (format != GST_FORMAT_TIME) - goto newseg_wrong_format; - - if (rate <= 0.0) - goto newseg_wrong_rate; - - if (update) { - /* time progressed without data, see if we can fill the gap with - * some concealment data */ - if (dec->segment.last_stop < start) { - GstClockTime duration; - - duration = start - dec->segment.last_stop; - opus_dec_chain_parse_data (dec, NULL, dec->segment.last_stop, - duration); - } - } - - /* now configure the values */ - gst_segment_set_newsegment_full (&dec->segment, update, - rate, arate, GST_FORMAT_TIME, start, stop, time); - - dec->granulepos = -1; - - GST_DEBUG_OBJECT (dec, "segment now: cur = %" GST_TIME_FORMAT " [%" - GST_TIME_FORMAT " - %" GST_TIME_FORMAT "]", - GST_TIME_ARGS (dec->segment.last_stop), - GST_TIME_ARGS (dec->segment.start), - GST_TIME_ARGS (dec->segment.stop)); - - ret = gst_pad_push_event (dec->srcpad, event); - break; - } - default: - ret = gst_pad_event_default (pad, event); - break; - } - - gst_object_unref (dec); - return ret; - - /* ERRORS */ -newseg_wrong_format: - { - GST_DEBUG_OBJECT (dec, "received non TIME newsegment"); - gst_object_unref (dec); - return FALSE; - } -newseg_wrong_rate: - { - GST_DEBUG_OBJECT (dec, "negative rates not supported yet"); - gst_object_unref (dec); - return FALSE; - } + return TRUE; } static GstFlowReturn -opus_dec_chain_parse_header (GstOpusDec * dec, GstBuffer * buf) +gst_opus_dec_parse_header (GstOpusDec * dec, GstBuffer * buf) { - GstCaps *caps; - int err; - -#if 0 - dec->samples_per_frame = opus_packet_get_samples_per_frame ( - (const unsigned char *) GST_BUFFER_DATA (buf), GST_BUFFER_SIZE (buf)); -#endif - -#if 0 - if (memcmp (dec->header.codec_id, "OPUS ", 8) != 0) - goto invalid_header; -#endif - - dec->state = opus_decoder_create (dec->sample_rate, dec->n_channels, &err); - if (!dec->state || err != OPUS_OK) - goto init_failed; - - dec->frame_duration = gst_util_uint64_scale_int (dec->frame_size, - GST_SECOND, dec->sample_rate); - - /* set caps */ - caps = gst_caps_new_simple ("audio/x-raw-int", - "rate", G_TYPE_INT, dec->sample_rate, - "channels", G_TYPE_INT, dec->n_channels, - "signed", G_TYPE_BOOLEAN, TRUE, - "endianness", G_TYPE_INT, G_BYTE_ORDER, - "width", G_TYPE_INT, 16, "depth", G_TYPE_INT, 16, NULL); - - GST_DEBUG_OBJECT (dec, "rate=%d channels=%d frame-size=%d", - dec->sample_rate, dec->n_channels, dec->frame_size); - - if (!gst_pad_set_caps (dec->srcpad, caps)) - goto nego_failed; - - gst_caps_unref (caps); return GST_FLOW_OK; - - /* ERRORS */ -#if 0 -invalid_header: - { - GST_ELEMENT_ERROR (GST_ELEMENT (dec), STREAM, DECODE, - (NULL), ("Invalid header")); - return GST_FLOW_ERROR; - } -mode_init_failed: - { - GST_ELEMENT_ERROR (GST_ELEMENT (dec), STREAM, DECODE, - (NULL), ("Mode initialization failed: %d", error)); - return GST_FLOW_ERROR; - } -#endif -init_failed: - { - GST_ELEMENT_ERROR (GST_ELEMENT (dec), STREAM, DECODE, - (NULL), ("couldn't initialize decoder")); - return GST_FLOW_ERROR; - } -nego_failed: - { - GST_ELEMENT_ERROR (GST_ELEMENT (dec), STREAM, DECODE, - (NULL), ("couldn't negotiate format")); - gst_caps_unref (caps); - return GST_FLOW_NOT_NEGOTIATED; - } } -#if 0 static GstFlowReturn -opus_dec_chain_parse_comments (GstOpusDec * dec, GstBuffer * buf) +gst_opus_dec_parse_comments (GstOpusDec * dec, GstBuffer * buf) { - GstTagList *list; - gchar *encoder = NULL; - - list = gst_tag_list_from_vorbiscomment_buffer (buf, NULL, 0, &encoder); - - if (!list) { - GST_WARNING_OBJECT (dec, "couldn't decode comments"); - list = gst_tag_list_new (); - } - - if (encoder) { - gst_tag_list_add (list, GST_TAG_MERGE_REPLACE, - GST_TAG_ENCODER, encoder, NULL); - } - - gst_tag_list_add (list, GST_TAG_MERGE_REPLACE, - GST_TAG_AUDIO_CODEC, "Opus", NULL); - - if (dec->header.bytes_per_packet > 0) { - gst_tag_list_add (list, GST_TAG_MERGE_REPLACE, - GST_TAG_BITRATE, (guint) dec->header.bytes_per_packet * 8, NULL); - } - - GST_INFO_OBJECT (dec, "tags: %" GST_PTR_FORMAT, list); - - gst_element_found_tags_for_pad (GST_ELEMENT (dec), dec->srcpad, list); - - g_free (encoder); - g_free (ver); - return GST_FLOW_OK; } -#endif static GstFlowReturn opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, @@ -724,11 +184,6 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, gint16 *out_data; int n, err; - if (timestamp != -1) { - dec->segment.last_stop = timestamp; - dec->granulepos = -1; - } - if (dec->state == NULL) { GstCaps *caps; @@ -747,7 +202,7 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, GST_DEBUG_OBJECT (dec, "rate=%d channels=%d frame-size=%d", dec->sample_rate, dec->n_channels, dec->frame_size); - if (!gst_pad_set_caps (dec->srcpad, caps)) + if (!gst_pad_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), caps)) GST_ERROR ("nego failure"); gst_caps_unref (caps); @@ -767,14 +222,15 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, size = 0; } + GST_DEBUG ("frames %d", opus_packet_get_nb_frames (data, size)); GST_DEBUG ("bandwidth %d", opus_packet_get_bandwidth (data)); GST_DEBUG ("samples_per_frame %d", opus_packet_get_samples_per_frame (data, 48000)); GST_DEBUG ("channels %d", opus_packet_get_nb_channels (data)); - res = gst_pad_alloc_buffer_and_set_caps (dec->srcpad, + res = gst_pad_alloc_buffer_and_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), GST_BUFFER_OFFSET_NONE, dec->frame_samples * dec->n_channels * 2, - GST_PAD_CAPS (dec->srcpad), &outbuf); + GST_PAD_CAPS (GST_AUDIO_DECODER_SRC_PAD (dec)), &outbuf); if (res != GST_FLOW_OK) { GST_DEBUG_OBJECT (dec, "buf alloc flow: %s", gst_flow_get_name (res)); @@ -783,7 +239,7 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, out_data = (gint16 *) GST_BUFFER_DATA (outbuf); - GST_LOG_OBJECT (dec, "decoding frame"); + GST_LOG_OBJECT (dec, "decoding %d sample frame", dec->frame_samples); n = opus_decode (dec->state, data, size, out_data, dec->frame_samples, 0); if (n < 0) { @@ -792,8 +248,7 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, } if (!GST_CLOCK_TIME_IS_VALID (timestamp)) { - timestamp = gst_util_uint64_scale_int (dec->granulepos - dec->frame_size, - GST_SECOND, dec->sample_rate); + GST_WARNING_OBJECT (dec, "No timestamp in -> no timestamp out"); } GST_DEBUG_OBJECT (dec, "timestamp=%" GST_TIME_FORMAT, @@ -801,18 +256,12 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, GST_BUFFER_TIMESTAMP (outbuf) = GST_BUFFER_TIMESTAMP (buf); GST_BUFFER_DURATION (outbuf) = GST_BUFFER_DURATION (buf); - if (dec->discont) { - GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DISCONT); - dec->discont = 0; - } - - dec->segment.last_stop += dec->frame_duration; GST_LOG_OBJECT (dec, "pushing buffer with ts=%" GST_TIME_FORMAT ", dur=%" GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)), GST_TIME_ARGS (dec->frame_duration)); - res = gst_pad_push (dec->srcpad, outbuf); + res = gst_audio_decoder_finish_frame (GST_AUDIO_DECODER (dec), outbuf, 1); if (res != GST_FLOW_OK) GST_DEBUG_OBJECT (dec, "flow: %s", gst_flow_get_name (res)); @@ -824,63 +273,173 @@ creation_failed: return GST_FLOW_ERROR; } +static gint +gst_opus_dec_get_frame_samples (GstOpusDec * dec) +{ + gint frame_samples = 0; + switch (dec->frame_size) { + case 2: + frame_samples = dec->sample_rate / 400; + break; + case 5: + frame_samples = dec->sample_rate / 200; + break; + case 10: + frame_samples = dec->sample_rate / 100; + break; + case 20: + frame_samples = dec->sample_rate / 50; + break; + case 40: + frame_samples = dec->sample_rate / 25; + break; + case 60: + frame_samples = 3 * dec->sample_rate / 50; + break; + default: + GST_WARNING_OBJECT (dec, "Unsupported frame size: %d", dec->frame_size); + frame_samples = 0; + break; + } + return frame_samples; +} + +static gboolean +gst_opus_dec_set_format (GstAudioDecoder * bdec, GstCaps * caps) +{ + GstOpusDec *dec = GST_OPUS_DEC (bdec); + gboolean ret = TRUE; + GstStructure *s; + const GValue *streamheader; + + GST_DEBUG_OBJECT (dec, "set_format: %" GST_PTR_FORMAT, caps); + + s = gst_caps_get_structure (caps, 0); + if ((streamheader = gst_structure_get_value (s, "streamheader")) && + G_VALUE_HOLDS (streamheader, GST_TYPE_ARRAY) && + gst_value_array_get_size (streamheader) >= 2) { + const GValue *header, *vorbiscomment; + GstBuffer *buf; + GstFlowReturn res = GST_FLOW_OK; + + header = gst_value_array_get_value (streamheader, 0); + if (header && G_VALUE_HOLDS (header, GST_TYPE_BUFFER)) { + buf = gst_value_get_buffer (header); + res = gst_opus_dec_parse_header (dec, buf); + if (res != GST_FLOW_OK) + goto done; + gst_buffer_replace (&dec->streamheader, buf); + } + + vorbiscomment = gst_value_array_get_value (streamheader, 1); + if (vorbiscomment && G_VALUE_HOLDS (vorbiscomment, GST_TYPE_BUFFER)) { + buf = gst_value_get_buffer (vorbiscomment); + res = gst_opus_dec_parse_comments (dec, buf); + if (res != GST_FLOW_OK) + goto done; + gst_buffer_replace (&dec->vorbiscomment, buf); + } + } + if (!gst_structure_get_int (s, "frame-size", &dec->frame_size)) { + GST_WARNING_OBJECT (dec, "Frame size not included in caps"); + } + if (!gst_structure_get_int (s, "channels", &dec->n_channels)) { + GST_WARNING_OBJECT (dec, "Number of channels not included in caps"); + } + if (!gst_structure_get_int (s, "rate", &dec->sample_rate)) { + GST_WARNING_OBJECT (dec, "Sample rate not included in caps"); + } + + dec->frame_samples = gst_opus_dec_get_frame_samples (dec); + dec->frame_duration = gst_util_uint64_scale_int (dec->frame_samples, + GST_SECOND, dec->sample_rate); + GST_INFO_OBJECT (dec, + "Got frame size %d, %d channels, %d Hz, giving %d samples per frame, frame duration %" + GST_TIME_FORMAT, dec->frame_size, dec->n_channels, dec->sample_rate, + dec->frame_samples, GST_TIME_ARGS (dec->frame_duration)); + + caps = gst_caps_new_simple ("audio/x-raw-int", + "rate", G_TYPE_INT, dec->sample_rate, + "channels", G_TYPE_INT, dec->n_channels, + "signed", G_TYPE_BOOLEAN, TRUE, + "endianness", G_TYPE_INT, G_BYTE_ORDER, + "width", G_TYPE_INT, 16, "depth", G_TYPE_INT, 16, NULL); + gst_pad_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), caps); + gst_caps_unref (caps); + +done: + return ret; +} + +static gboolean +memcmp_buffers (GstBuffer * buf1, GstBuffer * buf2) +{ + gsize size1, size2; + + size1 = GST_BUFFER_SIZE (buf1); + size2 = GST_BUFFER_SIZE (buf2); + + if (size1 != size2) + return FALSE; + + return !memcmp (GST_BUFFER_DATA (buf1), GST_BUFFER_DATA (buf2), size1); +} + static GstFlowReturn -opus_dec_chain (GstPad * pad, GstBuffer * buf) +gst_opus_dec_handle_frame (GstAudioDecoder * adec, GstBuffer * buf) { GstFlowReturn res; GstOpusDec *dec; - dec = GST_OPUS_DEC (gst_pad_get_parent (pad)); - GST_LOG_OBJECT (pad, + /* no fancy draining */ + if (G_UNLIKELY (!buf)) + return GST_FLOW_OK; + + dec = GST_OPUS_DEC (adec); + GST_LOG_OBJECT (dec, "Got buffer ts %" GST_TIME_FORMAT ", duration %" GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf)), GST_TIME_ARGS (GST_BUFFER_DURATION (buf))); - if (GST_BUFFER_IS_DISCONT (buf)) { - dec->discont = TRUE; + /* If we have the streamheader and vorbiscomment from the caps already + * ignore them here */ + if (dec->streamheader && dec->vorbiscomment) { + if (memcmp_buffers (dec->streamheader, buf)) { + GST_DEBUG_OBJECT (dec, "found streamheader"); + gst_audio_decoder_finish_frame (adec, NULL, 1); + res = GST_FLOW_OK; + } else if (memcmp_buffers (dec->vorbiscomment, buf)) { + GST_DEBUG_OBJECT (dec, "found vorbiscomments"); + gst_audio_decoder_finish_frame (adec, NULL, 1); + res = GST_FLOW_OK; + } else { + res = opus_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), + GST_BUFFER_DURATION (buf)); + } + } else { + /* Otherwise fall back to packet counting and assume that the + * first two packets are the headers. */ + switch (dec->packetno) { + case 0: + GST_DEBUG_OBJECT (dec, "counted streamheader"); + res = gst_opus_dec_parse_header (dec, buf); + gst_audio_decoder_finish_frame (adec, NULL, 1); + break; + case 1: + GST_DEBUG_OBJECT (dec, "counted vorbiscomments"); + res = gst_opus_dec_parse_comments (dec, buf); + gst_audio_decoder_finish_frame (adec, NULL, 1); + break; + default: + { + res = opus_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), + GST_BUFFER_DURATION (buf)); + break; + } + } } - res = opus_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), - GST_BUFFER_DURATION (buf)); - -//done: dec->packetno++; - gst_buffer_unref (buf); - gst_object_unref (dec); - return res; } - -static GstStateChangeReturn -opus_dec_change_state (GstElement * element, GstStateChange transition) -{ - GstStateChangeReturn ret; - GstOpusDec *dec = GST_OPUS_DEC (element); - - switch (transition) { - case GST_STATE_CHANGE_NULL_TO_READY: - case GST_STATE_CHANGE_READY_TO_PAUSED: - case GST_STATE_CHANGE_PAUSED_TO_PLAYING: - default: - break; - } - - ret = parent_class->change_state (element, transition); - if (ret != GST_STATE_CHANGE_SUCCESS) - return ret; - - switch (transition) { - case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - gst_opus_dec_reset (dec); - break; - case GST_STATE_CHANGE_READY_TO_NULL: - break; - default: - break; - } - - return ret; -} diff --git a/ext/opus/gstopusdec.h b/ext/opus/gstopusdec.h index 886a907532..8389580e3b 100644 --- a/ext/opus/gstopusdec.h +++ b/ext/opus/gstopusdec.h @@ -22,6 +22,7 @@ #define __GST_OPUS_DEC_H__ #include +#include #include G_BEGIN_DECLS @@ -41,11 +42,7 @@ typedef struct _GstOpusDec GstOpusDec; typedef struct _GstOpusDecClass GstOpusDecClass; struct _GstOpusDec { - GstElement element; - - /* pads */ - GstPad *sinkpad; - GstPad *srcpad; + GstAudioDecoder element; OpusDecoder *state; int frame_samples; @@ -54,20 +51,15 @@ struct _GstOpusDec { GstClockTime frame_duration; guint64 packetno; - GstSegment segment; /* STREAM LOCK */ - gint64 granulepos; /* -1 = needs to be set from current time */ - gboolean discont; - GstBuffer *streamheader; GstBuffer *vorbiscomment; - GList *extra_headers; int sample_rate; int n_channels; }; struct _GstOpusDecClass { - GstElementClass parent_class; + GstAudioDecoderClass parent_class; }; GType gst_opus_dec_get_type (void); diff --git a/ext/opus/gstopusenc.c b/ext/opus/gstopusenc.c index 8d40cdf812..4be63cb882 100644 --- a/ext/opus/gstopusenc.c +++ b/ext/opus/gstopusenc.c @@ -47,6 +47,7 @@ #include #include +#include #include #include "gstopusenc.h" @@ -125,18 +126,26 @@ enum static void gst_opus_enc_finalize (GObject * object); -static gboolean gst_opus_enc_sinkevent (GstPad * pad, GstEvent * event); -static GstFlowReturn gst_opus_enc_chain (GstPad * pad, GstBuffer * buf); +static gboolean gst_opus_enc_sink_event (GstAudioEncoder * benc, + GstEvent * event); static gboolean gst_opus_enc_setup (GstOpusEnc * enc); static void gst_opus_enc_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_opus_enc_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); -static GstStateChangeReturn gst_opus_enc_change_state (GstElement * element, - GstStateChange transition); -static GstFlowReturn gst_opus_enc_encode (GstOpusEnc * enc, gboolean flush); +static gboolean gst_opus_enc_start (GstAudioEncoder * benc); +static gboolean gst_opus_enc_stop (GstAudioEncoder * benc); +static gboolean gst_opus_enc_set_format (GstAudioEncoder * benc, + GstAudioInfo * info); +static GstFlowReturn gst_opus_enc_handle_frame (GstAudioEncoder * benc, + GstBuffer * buf); +static GstFlowReturn gst_opus_enc_pre_push (GstAudioEncoder * benc, + GstBuffer ** buffer); +static gint64 gst_opus_enc_get_latency (GstOpusEnc * enc); + +static GstFlowReturn gst_opus_enc_encode (GstOpusEnc * enc, GstBuffer * buffer); static void gst_opus_enc_setup_interfaces (GType opusenc_type) @@ -156,8 +165,8 @@ gst_opus_enc_setup_interfaces (GType opusenc_type) GST_DEBUG_CATEGORY_INIT (opusenc_debug, "opusenc", 0, "Opus encoder"); } -GST_BOILERPLATE_FULL (GstOpusEnc, gst_opus_enc, GstElement, GST_TYPE_ELEMENT, - gst_opus_enc_setup_interfaces); +GST_BOILERPLATE_FULL (GstOpusEnc, gst_opus_enc, GstAudioEncoder, + GST_TYPE_AUDIO_ENCODER, gst_opus_enc_setup_interfaces); static void gst_opus_enc_base_init (gpointer g_class) @@ -179,13 +188,22 @@ gst_opus_enc_class_init (GstOpusEncClass * klass) { GObjectClass *gobject_class; GstElementClass *gstelement_class; + GstAudioEncoderClass *base_class; gobject_class = (GObjectClass *) klass; gstelement_class = (GstElementClass *) klass; + base_class = (GstAudioEncoderClass *) klass; gobject_class->set_property = gst_opus_enc_set_property; gobject_class->get_property = gst_opus_enc_get_property; + base_class->start = GST_DEBUG_FUNCPTR (gst_opus_enc_start); + base_class->stop = GST_DEBUG_FUNCPTR (gst_opus_enc_stop); + base_class->set_format = GST_DEBUG_FUNCPTR (gst_opus_enc_set_format); + base_class->handle_frame = GST_DEBUG_FUNCPTR (gst_opus_enc_handle_frame); + base_class->pre_push = GST_DEBUG_FUNCPTR (gst_opus_enc_pre_push); + base_class->event = GST_DEBUG_FUNCPTR (gst_opus_enc_sink_event); + g_object_class_install_property (gobject_class, PROP_AUDIO, g_param_spec_boolean ("audio", "Audio or voice", "Audio or voice", DEFAULT_AUDIO, @@ -229,9 +247,6 @@ gst_opus_enc_class_init (GstOpusEncClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_opus_enc_finalize); - - gstelement_class->change_state = - GST_DEBUG_FUNCPTR (gst_opus_enc_change_state); } static void @@ -241,375 +256,15 @@ gst_opus_enc_finalize (GObject * object) enc = GST_OPUS_ENC (object); - g_object_unref (enc->adapter); - G_OBJECT_CLASS (parent_class)->finalize (object); } -static gboolean -gst_opus_enc_sink_setcaps (GstPad * pad, GstCaps * caps) -{ - GstOpusEnc *enc; - GstStructure *structure; - GstCaps *otherpadcaps; - - enc = GST_OPUS_ENC (GST_PAD_PARENT (pad)); - enc->setup = FALSE; - enc->frame_size = DEFAULT_FRAMESIZE; - otherpadcaps = gst_pad_get_allowed_caps (pad); - - structure = gst_caps_get_structure (caps, 0); - gst_structure_get_int (structure, "channels", &enc->n_channels); - gst_structure_get_int (structure, "rate", &enc->sample_rate); - - if (otherpadcaps) { - if (!gst_caps_is_empty (otherpadcaps)) { - GstStructure *ps = gst_caps_get_structure (otherpadcaps, 0); - gst_structure_get_int (ps, "frame-size", &enc->frame_size); - } - gst_caps_unref (otherpadcaps); - } - - GST_DEBUG_OBJECT (pad, "channels=%d rate=%d frame-size=%d", - enc->n_channels, enc->sample_rate, enc->frame_size); - switch (enc->frame_size) { - case 2: - enc->frame_samples = enc->sample_rate / 400; - break; - case 5: - enc->frame_samples = enc->sample_rate / 200; - break; - case 10: - enc->frame_samples = enc->sample_rate / 100; - break; - case 20: - enc->frame_samples = enc->sample_rate / 50; - break; - case 40: - enc->frame_samples = enc->sample_rate / 25; - break; - case 60: - enc->frame_samples = 3 * enc->sample_rate / 50; - break; - default: - GST_WARNING_OBJECT (enc, "Unsupported frame size: %d", enc->frame_size); - return FALSE; - break; - } - GST_DEBUG_OBJECT (pad, "frame_samples %d", enc->frame_samples); - - gst_opus_enc_setup (enc); - - return TRUE; -} - - -static GstCaps * -gst_opus_enc_sink_getcaps (GstPad * pad) -{ - GstCaps *caps = gst_caps_copy (gst_pad_get_pad_template_caps (pad)); - GstCaps *peercaps = NULL; - GstOpusEnc *enc = GST_OPUS_ENC (gst_pad_get_parent_element (pad)); - - peercaps = gst_pad_peer_get_caps (enc->srcpad); - - if (peercaps) { - if (!gst_caps_is_empty (peercaps) && !gst_caps_is_any (peercaps)) { - GstStructure *ps = gst_caps_get_structure (peercaps, 0); - GstStructure *s = gst_caps_get_structure (caps, 0); - gint rate, channels; - - if (gst_structure_get_int (ps, "rate", &rate)) { - gst_structure_fixate_field_nearest_int (s, "rate", rate); - } - - if (gst_structure_get_int (ps, "channels", &channels)) { - gst_structure_fixate_field_nearest_int (s, "channels", channels); - } - } - gst_caps_unref (peercaps); - } - - gst_object_unref (enc); - - return caps; -} - - -static gboolean -gst_opus_enc_convert_src (GstPad * pad, GstFormat src_format, gint64 src_value, - GstFormat * dest_format, gint64 * dest_value) -{ - gboolean res = TRUE; - GstOpusEnc *enc; - gint64 avg; - - enc = GST_OPUS_ENC (GST_PAD_PARENT (pad)); - - if (enc->samples_in == 0 || enc->bytes_out == 0 || enc->sample_rate == 0) - return FALSE; - - avg = (enc->bytes_out * enc->sample_rate) / (enc->samples_in); - - switch (src_format) { - case GST_FORMAT_BYTES: - switch (*dest_format) { - case GST_FORMAT_TIME: - *dest_value = src_value * GST_SECOND / avg; - break; - default: - res = FALSE; - } - break; - case GST_FORMAT_TIME: - switch (*dest_format) { - case GST_FORMAT_BYTES: - *dest_value = src_value * avg / GST_SECOND; - break; - default: - res = FALSE; - } - break; - default: - res = FALSE; - } - return res; -} - -static gboolean -gst_opus_enc_convert_sink (GstPad * pad, GstFormat src_format, - gint64 src_value, GstFormat * dest_format, gint64 * dest_value) -{ - gboolean res = TRUE; - guint scale = 1; - gint bytes_per_sample; - GstOpusEnc *enc; - - enc = GST_OPUS_ENC (GST_PAD_PARENT (pad)); - - bytes_per_sample = enc->n_channels * 2; - - switch (src_format) { - case GST_FORMAT_BYTES: - switch (*dest_format) { - case GST_FORMAT_DEFAULT: - if (bytes_per_sample == 0) - return FALSE; - *dest_value = src_value / bytes_per_sample; - break; - case GST_FORMAT_TIME: - { - gint byterate = bytes_per_sample * enc->sample_rate; - - if (byterate == 0) - return FALSE; - *dest_value = src_value * GST_SECOND / byterate; - break; - } - default: - res = FALSE; - } - break; - case GST_FORMAT_DEFAULT: - switch (*dest_format) { - case GST_FORMAT_BYTES: - *dest_value = src_value * bytes_per_sample; - break; - case GST_FORMAT_TIME: - if (enc->sample_rate == 0) - return FALSE; - *dest_value = src_value * GST_SECOND / enc->sample_rate; - break; - default: - res = FALSE; - } - break; - case GST_FORMAT_TIME: - switch (*dest_format) { - case GST_FORMAT_BYTES: - scale = bytes_per_sample; - /* fallthrough */ - case GST_FORMAT_DEFAULT: - *dest_value = src_value * scale * enc->sample_rate / GST_SECOND; - break; - default: - res = FALSE; - } - break; - default: - res = FALSE; - } - return res; -} - -static gint64 -gst_opus_enc_get_latency (GstOpusEnc * enc) -{ - return gst_util_uint64_scale (enc->frame_samples, GST_SECOND, - enc->sample_rate); -} - -static const GstQueryType * -gst_opus_enc_get_query_types (GstPad * pad) -{ - static const GstQueryType gst_opus_enc_src_query_types[] = { - GST_QUERY_POSITION, - GST_QUERY_DURATION, - GST_QUERY_CONVERT, - GST_QUERY_LATENCY, - 0 - }; - - return gst_opus_enc_src_query_types; -} - -static gboolean -gst_opus_enc_src_query (GstPad * pad, GstQuery * query) -{ - gboolean res = TRUE; - GstOpusEnc *enc; - - enc = GST_OPUS_ENC (gst_pad_get_parent (pad)); - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_POSITION: - { - GstFormat fmt, req_fmt; - gint64 pos, val; - - gst_query_parse_position (query, &req_fmt, NULL); - if ((res = gst_pad_query_peer_position (enc->sinkpad, &req_fmt, &val))) { - gst_query_set_position (query, req_fmt, val); - break; - } - - fmt = GST_FORMAT_TIME; - if (!(res = gst_pad_query_peer_position (enc->sinkpad, &fmt, &pos))) - break; - - if ((res = - gst_pad_query_peer_convert (enc->sinkpad, fmt, pos, &req_fmt, - &val))) - gst_query_set_position (query, req_fmt, val); - - break; - } - case GST_QUERY_DURATION: - { - GstFormat fmt, req_fmt; - gint64 dur, val; - - gst_query_parse_duration (query, &req_fmt, NULL); - if ((res = gst_pad_query_peer_duration (enc->sinkpad, &req_fmt, &val))) { - gst_query_set_duration (query, req_fmt, val); - break; - } - - fmt = GST_FORMAT_TIME; - if (!(res = gst_pad_query_peer_duration (enc->sinkpad, &fmt, &dur))) - break; - - if ((res = - gst_pad_query_peer_convert (enc->sinkpad, fmt, dur, &req_fmt, - &val))) { - gst_query_set_duration (query, req_fmt, val); - } - break; - } - case GST_QUERY_CONVERT: - { - GstFormat src_fmt, dest_fmt; - gint64 src_val, dest_val; - - gst_query_parse_convert (query, &src_fmt, &src_val, &dest_fmt, &dest_val); - if (!(res = gst_opus_enc_convert_src (pad, src_fmt, src_val, &dest_fmt, - &dest_val))) - goto error; - gst_query_set_convert (query, src_fmt, src_val, dest_fmt, dest_val); - break; - } - case GST_QUERY_LATENCY: - { - gboolean live; - GstClockTime min_latency, max_latency; - gint64 latency; - - if ((res = gst_pad_peer_query (enc->sinkpad, query))) { - gst_query_parse_latency (query, &live, &min_latency, &max_latency); - - latency = gst_opus_enc_get_latency (enc); - - /* add our latency */ - min_latency += latency; - if (max_latency != -1) - max_latency += latency; - - gst_query_set_latency (query, live, min_latency, max_latency); - } - break; - } - default: - res = gst_pad_peer_query (pad, query); - break; - } - -error: - - gst_object_unref (enc); - - return res; -} - -static gboolean -gst_opus_enc_sink_query (GstPad * pad, GstQuery * query) -{ - gboolean res = TRUE; - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_CONVERT: - { - GstFormat src_fmt, dest_fmt; - gint64 src_val, dest_val; - - gst_query_parse_convert (query, &src_fmt, &src_val, &dest_fmt, &dest_val); - if (!(res = - gst_opus_enc_convert_sink (pad, src_fmt, src_val, &dest_fmt, - &dest_val))) - goto error; - gst_query_set_convert (query, src_fmt, src_val, dest_fmt, dest_val); - break; - } - default: - res = gst_pad_query_default (pad, query); - break; - } - -error: - return res; -} - static void gst_opus_enc_init (GstOpusEnc * enc, GstOpusEncClass * klass) { - enc->sinkpad = gst_pad_new_from_static_template (&sink_factory, "sink"); - gst_element_add_pad (GST_ELEMENT (enc), enc->sinkpad); - gst_pad_set_event_function (enc->sinkpad, - GST_DEBUG_FUNCPTR (gst_opus_enc_sinkevent)); - gst_pad_set_chain_function (enc->sinkpad, - GST_DEBUG_FUNCPTR (gst_opus_enc_chain)); - gst_pad_set_setcaps_function (enc->sinkpad, - GST_DEBUG_FUNCPTR (gst_opus_enc_sink_setcaps)); - gst_pad_set_getcaps_function (enc->sinkpad, - GST_DEBUG_FUNCPTR (gst_opus_enc_sink_getcaps)); - gst_pad_set_query_function (enc->sinkpad, - GST_DEBUG_FUNCPTR (gst_opus_enc_sink_query)); + GstAudioEncoder *benc = GST_AUDIO_ENCODER (enc); - enc->srcpad = gst_pad_new_from_static_template (&src_factory, "src"); - gst_pad_set_query_function (enc->srcpad, - GST_DEBUG_FUNCPTR (gst_opus_enc_src_query)); - gst_pad_set_query_type_function (enc->srcpad, - GST_DEBUG_FUNCPTR (gst_opus_enc_get_query_types)); - gst_element_add_pad (GST_ELEMENT (enc), enc->srcpad); + GST_DEBUG_OBJECT (enc, "init"); enc->n_channels = -1; enc->sample_rate = -1; @@ -625,13 +280,140 @@ gst_opus_enc_init (GstOpusEnc * enc, GstOpusEncClass * klass) enc->dtx = DEFAULT_DTX; enc->packet_loss_percentage = DEFAULT_PACKET_LOSS_PERCENT; - enc->setup = FALSE; - enc->header_sent = FALSE; - - enc->adapter = gst_adapter_new (); + /* arrange granulepos marking (and required perfect ts) */ + gst_audio_encoder_set_mark_granule (benc, TRUE); + gst_audio_encoder_set_perfect_timestamp (benc, TRUE); +} + +static gboolean +gst_opus_enc_start (GstAudioEncoder * benc) +{ + GstOpusEnc *enc = GST_OPUS_ENC (benc); + + GST_DEBUG_OBJECT (enc, "start"); + enc->tags = gst_tag_list_new (); + enc->header_sent = FALSE; + return TRUE; +} + +static gboolean +gst_opus_enc_stop (GstAudioEncoder * benc) +{ + GstOpusEnc *enc = GST_OPUS_ENC (benc); + + GST_DEBUG_OBJECT (enc, "stop"); + enc->header_sent = FALSE; + if (enc->state) { + opus_encoder_destroy (enc->state); + enc->state = NULL; + } + gst_tag_list_free (enc->tags); + enc->tags = NULL; + g_slist_foreach (enc->headers, (GFunc) gst_buffer_unref, NULL); + enc->headers = NULL; + + return TRUE; +} + +static gint64 +gst_opus_enc_get_latency (GstOpusEnc * enc) +{ + gint64 latency = gst_util_uint64_scale (enc->frame_samples, GST_SECOND, + enc->sample_rate); + GST_DEBUG_OBJECT (enc, "Latency: %" GST_TIME_FORMAT, GST_TIME_ARGS (latency)); + return latency; +} + +static gint +gst_opus_enc_get_frame_samples (GstOpusEnc * enc) +{ + gint frame_samples = 0; + switch (enc->frame_size) { + case 2: + frame_samples = enc->sample_rate / 400; + break; + case 5: + frame_samples = enc->sample_rate / 200; + break; + case 10: + frame_samples = enc->sample_rate / 100; + break; + case 20: + frame_samples = enc->sample_rate / 50; + break; + case 40: + frame_samples = enc->sample_rate / 25; + break; + case 60: + frame_samples = 3 * enc->sample_rate / 50; + break; + default: + GST_WARNING_OBJECT (enc, "Unsupported frame size: %d", enc->frame_size); + frame_samples = 0; + break; + } + return frame_samples; +} + +static gboolean +gst_opus_enc_set_format (GstAudioEncoder * benc, GstAudioInfo * info) +{ + GstOpusEnc *enc; + + enc = GST_OPUS_ENC (benc); + + enc->n_channels = GST_AUDIO_INFO_CHANNELS (info); + enc->sample_rate = GST_AUDIO_INFO_RATE (info); + GST_DEBUG_OBJECT (benc, "Setup with %d channels, %d Hz", enc->n_channels, + enc->sample_rate); + + /* handle reconfigure */ + if (enc->state) { + opus_encoder_destroy (enc->state); + enc->state = NULL; + } + if (!gst_opus_enc_setup (enc)) + return FALSE; + + enc->frame_samples = gst_opus_enc_get_frame_samples (enc); + + /* feedback to base class */ + gst_audio_encoder_set_latency (benc, + gst_opus_enc_get_latency (enc), gst_opus_enc_get_latency (enc)); + gst_audio_encoder_set_frame_samples_min (benc, + enc->frame_samples * enc->n_channels * 2); + gst_audio_encoder_set_frame_samples_max (benc, + enc->frame_samples * enc->n_channels * 2); + gst_audio_encoder_set_frame_max (benc, 0); + + return TRUE; +} + +static GstBuffer * +gst_opus_enc_create_id_buffer (GstOpusEnc * enc) +{ + GstBuffer *buffer; + GstByteWriter bw; + + gst_byte_writer_init (&bw); + + /* See http://wiki.xiph.org/OggOpus */ + gst_byte_writer_put_string_utf8 (&bw, "OpusHead"); + gst_byte_writer_put_uint8 (&bw, 0); /* version number */ + gst_byte_writer_put_uint8 (&bw, enc->n_channels); + gst_byte_writer_put_uint16_le (&bw, 0); /* pre-skip *//* TODO: endianness ? */ + gst_byte_writer_put_uint32_le (&bw, enc->sample_rate); + gst_byte_writer_put_uint16_le (&bw, 0); /* output gain *//* TODO: endianness ? */ + gst_byte_writer_put_uint8 (&bw, 0); /* channel mapping *//* TODO: what is this ? */ + + buffer = gst_byte_writer_reset_and_get_buffer (&bw); + + GST_BUFFER_OFFSET (buffer) = 0; + GST_BUFFER_OFFSET_END (buffer) = 0; + + return buffer; } -#if 0 static GstBuffer * gst_opus_enc_create_metadata_buffer (GstOpusEnc * enc) { @@ -649,10 +431,11 @@ gst_opus_enc_create_metadata_buffer (GstOpusEnc * enc) empty_tags = gst_tag_list_new (); tags = empty_tags; } - comments = gst_tag_list_to_vorbiscomment_buffer (tags, NULL, - 0, "Encoded with GStreamer Opusenc"); + comments = + gst_tag_list_to_vorbiscomment_buffer (tags, (const guint8 *) "OpusTags", + 8, "Encoded with GStreamer Opusenc"); - GST_BUFFER_OFFSET (comments) = enc->bytes_out; + GST_BUFFER_OFFSET (comments) = 0; GST_BUFFER_OFFSET_END (comments) = 0; if (empty_tags) @@ -660,13 +443,14 @@ gst_opus_enc_create_metadata_buffer (GstOpusEnc * enc) return comments; } -#endif static gboolean gst_opus_enc_setup (GstOpusEnc * enc) { int error = OPUS_OK; + GST_DEBUG_OBJECT (enc, "setup"); + enc->setup = FALSE; enc->state = opus_encoder_create (enc->sample_rate, enc->n_channels, @@ -692,88 +476,20 @@ gst_opus_enc_setup (GstOpusEnc * enc) return TRUE; -#if 0 -mode_initialization_failed: - GST_ERROR_OBJECT (enc, "Mode initialization failed: %d", error); - return FALSE; -#endif - encoder_creation_failed: GST_ERROR_OBJECT (enc, "Encoder creation failed"); return FALSE; } - -/* push out the buffer and do internal bookkeeping */ -static GstFlowReturn -gst_opus_enc_push_buffer (GstOpusEnc * enc, GstBuffer * buffer) -{ - guint size; - - size = GST_BUFFER_SIZE (buffer); - - enc->bytes_out += size; - - GST_DEBUG_OBJECT (enc, "pushing output buffer of size %u", size); - - return gst_pad_push (enc->srcpad, buffer); -} - -#if 0 -static GstCaps * -gst_opus_enc_set_header_on_caps (GstCaps * caps, GstBuffer * buf1, - GstBuffer * buf2) -{ - GstStructure *structure = NULL; - GstBuffer *buf; - GValue array = { 0 }; - GValue value = { 0 }; - - caps = gst_caps_make_writable (caps); - structure = gst_caps_get_structure (caps, 0); - - g_assert (gst_buffer_is_metadata_writable (buf1)); - g_assert (gst_buffer_is_metadata_writable (buf2)); - - /* mark buffers */ - GST_BUFFER_FLAG_SET (buf1, GST_BUFFER_FLAG_IN_CAPS); - GST_BUFFER_FLAG_SET (buf2, GST_BUFFER_FLAG_IN_CAPS); - - /* put buffers in a fixed list */ - g_value_init (&array, GST_TYPE_ARRAY); - g_value_init (&value, GST_TYPE_BUFFER); - buf = gst_buffer_copy (buf1); - gst_value_set_buffer (&value, buf); - gst_buffer_unref (buf); - gst_value_array_append_value (&array, &value); - g_value_unset (&value); - g_value_init (&value, GST_TYPE_BUFFER); - buf = gst_buffer_copy (buf2); - gst_value_set_buffer (&value, buf); - gst_buffer_unref (buf); - gst_value_array_append_value (&array, &value); - gst_structure_set_value (structure, "streamheader", &array); - g_value_unset (&value); - g_value_unset (&array); - - return caps; -} -#endif - - static gboolean -gst_opus_enc_sinkevent (GstPad * pad, GstEvent * event) +gst_opus_enc_sink_event (GstAudioEncoder * benc, GstEvent * event) { - gboolean res = TRUE; GstOpusEnc *enc; - enc = GST_OPUS_ENC (gst_pad_get_parent (pad)); + enc = GST_OPUS_ENC (benc); + GST_DEBUG_OBJECT (enc, "sink event: %s", GST_EVENT_TYPE_NAME (event)); switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_EOS: - gst_opus_enc_encode (enc, TRUE); - res = gst_pad_event_default (pad, event); - break; case GST_EVENT_TAG: { GstTagList *list; @@ -782,62 +498,94 @@ gst_opus_enc_sinkevent (GstPad * pad, GstEvent * event) gst_event_parse_tag (event, &list); gst_tag_setter_merge_tags (setter, list, mode); - res = gst_pad_event_default (pad, event); break; } default: - res = gst_pad_event_default (pad, event); break; } - gst_object_unref (enc); - - return res; + return FALSE; } static GstFlowReturn -gst_opus_enc_encode (GstOpusEnc * enc, gboolean flush) +gst_opus_enc_pre_push (GstAudioEncoder * benc, GstBuffer ** buffer) { - GstFlowReturn ret = GST_FLOW_OK; - gint bytes = enc->frame_samples * 2 * enc->n_channels; - gint bytes_per_packet; + GstOpusEnc *enc; - bytes_per_packet = + enc = GST_OPUS_ENC (benc); + + /* FIXME 0.11 ? get rid of this special ogg stuff and have it + * put and use 'codec data' in caps like anything else, + * with all the usual out-of-band advantage etc */ + if (G_UNLIKELY (enc->headers)) { + GSList *header = enc->headers; + + /* try to push all of these, if we lose one, might as well lose all */ + while (header) { + if (ret == GST_FLOW_OK) + ret = gst_pad_push (GST_AUDIO_ENCODER_SRC_PAD (enc), header->data); + else + gst_pad_push (GST_AUDIO_ENCODER_SRC_PAD (enc), header->data); + header = g_slist_next (header); + } + + g_slist_free (enc->headers); + enc->headers = NULL; + } + + return ret; +} + +static GstFlowReturn +gst_opus_enc_encode (GstOpusEnc * enc, GstBuffer * buf) +{ + guint8 *bdata, *data, *mdata = NULL; + gsize bsize, size; + gsize bytes = enc->frame_samples * enc->n_channels * 2; + gsize bytes_per_packet = (enc->bitrate * enc->frame_samples / enc->sample_rate + 4) / 8; + gint ret = GST_FLOW_OK; - if (flush && gst_adapter_available (enc->adapter) % bytes != 0) { - guint diff = bytes - gst_adapter_available (enc->adapter) % bytes; - GstBuffer *buf = gst_buffer_new_and_alloc (diff); + if (G_LIKELY (buf)) { + bdata = GST_BUFFER_DATA (buf); + bsize = GST_BUFFER_SIZE (buf); + if (G_UNLIKELY (bsize % bytes)) { + GST_DEBUG_OBJECT (enc, "draining; adding silence samples"); - memset (GST_BUFFER_DATA (buf), 0, diff); - gst_adapter_push (enc->adapter, buf); + size = ((bsize / bytes) + 1) * bytes; + mdata = g_malloc0 (size); + memcpy (mdata, bdata, bsize); + bdata = NULL; + data = mdata; + } else { + data = bdata; + size = bsize; + } + } else { + GST_DEBUG_OBJECT (enc, "nothing to drain"); + goto done; } - while (gst_adapter_available (enc->adapter) >= bytes) { - gint16 *data; + while (size) { gint outsize; GstBuffer *outbuf; - ret = gst_pad_alloc_buffer_and_set_caps (enc->srcpad, - GST_BUFFER_OFFSET_NONE, bytes_per_packet, GST_PAD_CAPS (enc->srcpad), - &outbuf); + ret = gst_pad_alloc_buffer_and_set_caps (GST_AUDIO_ENCODER_SRC_PAD (enc), + GST_BUFFER_OFFSET_NONE, bytes_per_packet, + GST_PAD_CAPS (GST_AUDIO_ENCODER_SRC_PAD (enc)), &outbuf); if (GST_FLOW_OK != ret) goto done; - data = (gint16 *) gst_adapter_take (enc->adapter, bytes); - enc->samples_in += enc->frame_samples; + GST_DEBUG_OBJECT (enc, "encoding %d samples (%d bytes) to %d bytes", + enc->frame_samples, bytes, bytes_per_packet); - GST_DEBUG_OBJECT (enc, "encoding %d samples (%d bytes)", - enc->frame_samples, bytes); - - outsize = opus_encode (enc->state, data, enc->frame_samples, + outsize = + opus_encode (enc->state, (const gint16 *) data, enc->frame_samples, GST_BUFFER_DATA (outbuf), bytes_per_packet); - g_free (data); - if (outsize < 0) { GST_ERROR_OBJECT (enc, "Encoding failed: %d", outsize); ret = GST_FLOW_ERROR; @@ -850,149 +598,132 @@ gst_opus_enc_encode (GstOpusEnc * enc, gboolean flush) goto done; } - GST_BUFFER_TIMESTAMP (outbuf) = enc->start_ts + - gst_util_uint64_scale_int (enc->frameno_out * enc->frame_samples, - GST_SECOND, enc->sample_rate); - GST_BUFFER_DURATION (outbuf) = - gst_util_uint64_scale_int (enc->frame_samples, GST_SECOND, - enc->sample_rate); - GST_BUFFER_OFFSET (outbuf) = - gst_util_uint64_scale_int (GST_BUFFER_OFFSET_END (outbuf), GST_SECOND, - enc->sample_rate); - - enc->frameno++; - enc->frameno_out++; - - ret = gst_opus_enc_push_buffer (enc, outbuf); + ret = + gst_audio_encoder_finish_frame (GST_AUDIO_ENCODER (enc), outbuf, + enc->frame_samples); if ((GST_FLOW_OK != ret) && (GST_FLOW_NOT_LINKED != ret)) goto done; + + data += bytes; + size -= bytes; } done: + if (mdata) + g_free (mdata); + return ret; } +/* + * (really really) FIXME: move into core (dixit tpm) + */ +/** + * _gst_caps_set_buffer_array: + * @caps: a #GstCaps + * @field: field in caps to set + * @buf: header buffers + * + * Adds given buffers to an array of buffers set as the given @field + * on the given @caps. List of buffer arguments must be NULL-terminated. + * + * Returns: input caps with a streamheader field added, or NULL if some error + */ +static GstCaps * +_gst_caps_set_buffer_array (GstCaps * caps, const gchar * field, + GstBuffer * buf, ...) +{ + GstStructure *structure = NULL; + va_list va; + GValue array = { 0 }; + GValue value = { 0 }; + + g_return_val_if_fail (caps != NULL, NULL); + g_return_val_if_fail (gst_caps_is_fixed (caps), NULL); + g_return_val_if_fail (field != NULL, NULL); + + caps = gst_caps_make_writable (caps); + structure = gst_caps_get_structure (caps, 0); + + g_value_init (&array, GST_TYPE_ARRAY); + + va_start (va, buf); + /* put buffers in a fixed list */ + while (buf) { + g_assert (gst_buffer_is_writable (buf)); + + /* mark buffer */ + GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_IN_CAPS); + + g_value_init (&value, GST_TYPE_BUFFER); + buf = gst_buffer_copy (buf); + GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_IN_CAPS); + gst_value_set_buffer (&value, buf); + gst_buffer_unref (buf); + gst_value_array_append_value (&array, &value); + g_value_unset (&value); + + buf = va_arg (va, GstBuffer *); + } + + gst_structure_set_value (structure, field, &array); + g_value_unset (&array); + + return caps; +} + static GstFlowReturn -gst_opus_enc_chain (GstPad * pad, GstBuffer * buf) +gst_opus_enc_handle_frame (GstAudioEncoder * benc, GstBuffer * buf) { GstOpusEnc *enc; GstFlowReturn ret = GST_FLOW_OK; - enc = GST_OPUS_ENC (GST_PAD_PARENT (pad)); - - if (!enc->setup) - goto not_setup; + enc = GST_OPUS_ENC (benc); + GST_DEBUG_OBJECT (enc, "handle_frame"); if (!enc->header_sent) { + /* Opus streams in Ogg begin with two headers; the initial header (with + most of the codec setup parameters) which is mandated by the Ogg + bitstream spec. The second header holds any comment fields. */ + GstBuffer *buf1, *buf2; GstCaps *caps; - caps = gst_pad_get_caps (enc->srcpad); - gst_caps_set_simple (caps, - "rate", G_TYPE_INT, enc->sample_rate, - "channels", G_TYPE_INT, enc->n_channels, - "frame-size", G_TYPE_INT, enc->frame_size, NULL); + /* create header buffers */ + buf1 = gst_opus_enc_create_id_buffer (enc); + buf2 = gst_opus_enc_create_metadata_buffer (enc); + + /* mark and put on caps */ + caps = + gst_caps_new_simple ("audio/x-opus", "rate", G_TYPE_INT, + enc->sample_rate, "channels", G_TYPE_INT, enc->n_channels, "frame-size", + G_TYPE_INT, enc->frame_size, NULL); + caps = _gst_caps_set_buffer_array (caps, "streamheader", buf1, buf2, NULL); /* negotiate with these caps */ GST_DEBUG_OBJECT (enc, "here are the caps: %" GST_PTR_FORMAT, caps); - GST_LOG_OBJECT (enc, "rate=%d channels=%d frame-size=%d", - enc->sample_rate, enc->n_channels, enc->frame_size); - gst_pad_set_caps (enc->srcpad, caps); + gst_pad_set_caps (GST_AUDIO_ENCODER_SRC_PAD (enc), caps); + + /* push out buffers */ + /* store buffers for later pre_push sending */ + g_slist_foreach (enc->headers, (GFunc) gst_buffer_unref, NULL); + enc->headers = NULL; + GST_DEBUG_OBJECT (enc, "storing header buffers"); + enc->headers = g_slist_prepend (enc->headers, buf2); + enc->headers = g_slist_prepend (enc->headers, buf1); enc->header_sent = TRUE; } - GST_DEBUG_OBJECT (enc, "received buffer of %u bytes", GST_BUFFER_SIZE (buf)); + GST_DEBUG_OBJECT (enc, "received buffer %p of %u bytes", buf, + buf ? GST_BUFFER_SIZE (buf) : 0); - /* Save the timestamp of the first buffer. This will be later - * used as offset for all following buffers */ - if (enc->start_ts == GST_CLOCK_TIME_NONE) { - if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { - enc->start_ts = GST_BUFFER_TIMESTAMP (buf); - } else { - enc->start_ts = 0; - } - } - - - /* Check if we have a continous stream, if not drop some samples or the buffer or - * insert some silence samples */ - if (enc->next_ts != GST_CLOCK_TIME_NONE && - GST_BUFFER_TIMESTAMP (buf) < enc->next_ts) { - guint64 diff = enc->next_ts - GST_BUFFER_TIMESTAMP (buf); - guint64 diff_bytes; - - GST_WARNING_OBJECT (enc, "Buffer is older than previous " - "timestamp + duration (%" GST_TIME_FORMAT "< %" GST_TIME_FORMAT - "), cannot handle. Clipping buffer.", - GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf)), - GST_TIME_ARGS (enc->next_ts)); - - diff_bytes = - GST_CLOCK_TIME_TO_FRAMES (diff, enc->sample_rate) * enc->n_channels * 2; - if (diff_bytes >= GST_BUFFER_SIZE (buf)) { - gst_buffer_unref (buf); - return GST_FLOW_OK; - } - buf = gst_buffer_make_metadata_writable (buf); - GST_BUFFER_DATA (buf) += diff_bytes; - GST_BUFFER_SIZE (buf) -= diff_bytes; - - GST_BUFFER_TIMESTAMP (buf) += diff; - if (GST_BUFFER_DURATION_IS_VALID (buf)) - GST_BUFFER_DURATION (buf) -= diff; - } - - if (enc->next_ts != GST_CLOCK_TIME_NONE - && GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { - guint64 max_diff = - gst_util_uint64_scale (enc->frame_size, GST_SECOND, enc->sample_rate); - - if (GST_BUFFER_TIMESTAMP (buf) != enc->next_ts && - GST_BUFFER_TIMESTAMP (buf) - enc->next_ts > max_diff) { - GST_WARNING_OBJECT (enc, - "Discontinuity detected: %" G_GUINT64_FORMAT " > %" G_GUINT64_FORMAT, - GST_BUFFER_TIMESTAMP (buf) - enc->next_ts, max_diff); - - gst_opus_enc_encode (enc, TRUE); - - enc->frameno_out = 0; - enc->start_ts = GST_BUFFER_TIMESTAMP (buf); - } - } - - if (GST_BUFFER_TIMESTAMP_IS_VALID (buf) - && GST_BUFFER_DURATION_IS_VALID (buf)) - enc->next_ts = GST_BUFFER_TIMESTAMP (buf) + GST_BUFFER_DURATION (buf); - else - enc->next_ts = GST_CLOCK_TIME_NONE; - - /* push buffer to adapter */ - gst_adapter_push (enc->adapter, buf); - buf = NULL; - - ret = gst_opus_enc_encode (enc, FALSE); - -done: - - if (buf) - gst_buffer_unref (buf); + ret = gst_opus_enc_encode (enc, buf); return ret; - - /* ERRORS */ -not_setup: - { - GST_ELEMENT_ERROR (enc, CORE, NEGOTIATION, (NULL), - ("encoder not initialized (input is not audio?)")); - ret = GST_FLOW_NOT_NEGOTIATED; - goto done; - } - } - static void gst_opus_enc_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) @@ -1082,49 +813,3 @@ gst_opus_enc_set_property (GObject * object, guint prop_id, break; } } - -static GstStateChangeReturn -gst_opus_enc_change_state (GstElement * element, GstStateChange transition) -{ - GstOpusEnc *enc = GST_OPUS_ENC (element); - GstStateChangeReturn res; - - switch (transition) { - case GST_STATE_CHANGE_NULL_TO_READY: - break; - case GST_STATE_CHANGE_READY_TO_PAUSED: - enc->frameno = 0; - enc->samples_in = 0; - enc->frameno_out = 0; - enc->start_ts = GST_CLOCK_TIME_NONE; - enc->next_ts = GST_CLOCK_TIME_NONE; - break; - case GST_STATE_CHANGE_PAUSED_TO_PLAYING: - /* fall through */ - default: - break; - } - - res = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); - if (res == GST_STATE_CHANGE_FAILURE) - return res; - - switch (transition) { - case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - enc->setup = FALSE; - enc->header_sent = FALSE; - if (enc->state) { - opus_encoder_destroy (enc->state); - enc->state = NULL; - } - break; - case GST_STATE_CHANGE_READY_TO_NULL: - gst_tag_setter_reset_tags (GST_TAG_SETTER (enc)); - default: - break; - } - - return res; -} diff --git a/ext/opus/gstopusenc.h b/ext/opus/gstopusenc.h index 5cb54598af..fe7b94e68d 100644 --- a/ext/opus/gstopusenc.h +++ b/ext/opus/gstopusenc.h @@ -24,7 +24,7 @@ #include -#include +#include #include @@ -48,16 +48,9 @@ typedef struct _GstOpusEnc GstOpusEnc; typedef struct _GstOpusEncClass GstOpusEncClass; struct _GstOpusEnc { - GstElement element; + GstAudioEncoder element; - /* pads */ - GstPad *sinkpad; - GstPad *srcpad; - - //OpusHeader header; - //OpusMode *mode; OpusEncoder *state; - GstAdapter *adapter; /* properties */ gboolean audio_or_voip; @@ -71,28 +64,20 @@ struct _GstOpusEnc { gboolean dtx; gint packet_loss_percentage; - int frame_samples; - + gint frame_samples; gint n_channels; gint sample_rate; gboolean setup; gboolean header_sent; - gboolean eos; - guint64 samples_in; - guint64 bytes_out; + GSList *headers; - guint64 frameno; - guint64 frameno_out; - - GstClockTime start_ts; - GstClockTime next_ts; - guint64 granulepos_offset; + GstTagList *tags; }; struct _GstOpusEncClass { - GstElementClass parent_class; + GstAudioEncoderClass parent_class; /* signals */ void (*frame_encoded) (GstElement *element); From d10cbd0268807fce2f56fee0633b794830a44793 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 16 Nov 2011 17:05:17 +0000 Subject: [PATCH 12/25] opusdec: rewrite logic Parameters such as frame size, etc, are variable. Pretty much everything can change within a stream, so be prepared about it, and do not cache parameters in the decoder. --- ext/opus/gstopusdec.c | 123 ++++++++++++++++-------------------------- ext/opus/gstopusdec.h | 4 +- 2 files changed, 47 insertions(+), 80 deletions(-) diff --git a/ext/opus/gstopusdec.c b/ext/opus/gstopusdec.c index 1abe3201cb..f14e4e0ded 100644 --- a/ext/opus/gstopusdec.c +++ b/ext/opus/gstopusdec.c @@ -48,8 +48,6 @@ GST_DEBUG_CATEGORY_STATIC (opusdec_debug); #define GST_CAT_DEFAULT opusdec_debug -#define DEC_MAX_FRAME_SIZE 2000 - static GstStaticPadTemplate opus_dec_src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, @@ -117,14 +115,13 @@ static void gst_opus_dec_reset (GstOpusDec * dec) { dec->packetno = 0; - dec->frame_size = 0; - dec->frame_samples = 960; - dec->frame_duration = 0; if (dec->state) { opus_decoder_destroy (dec->state); dec->state = NULL; } + dec->next_ts = 0; + gst_buffer_replace (&dec->streamheader, NULL); gst_buffer_replace (&dec->vorbiscomment, NULL); } @@ -183,6 +180,8 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, GstBuffer *outbuf; gint16 *out_data; int n, err; + int samples; + unsigned int packet_size; if (dec->state == NULL) { GstCaps *caps; @@ -199,8 +198,8 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, "endianness", G_TYPE_INT, G_BYTE_ORDER, "width", G_TYPE_INT, 16, "depth", G_TYPE_INT, 16, NULL); - GST_DEBUG_OBJECT (dec, "rate=%d channels=%d frame-size=%d", - dec->sample_rate, dec->n_channels, dec->frame_size); + GST_DEBUG_OBJECT (dec, "rate=%d channels=%d", + dec->sample_rate, dec->n_channels); if (!gst_pad_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), caps)) GST_ERROR ("nego failure"); @@ -222,14 +221,15 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, size = 0; } - GST_DEBUG ("frames %d", opus_packet_get_nb_frames (data, size)); + samples = + opus_packet_get_samples_per_frame (data, + dec->sample_rate) * opus_packet_get_nb_frames (data, size); + packet_size = samples * dec->n_channels * 2; GST_DEBUG ("bandwidth %d", opus_packet_get_bandwidth (data)); - GST_DEBUG ("samples_per_frame %d", opus_packet_get_samples_per_frame (data, - 48000)); - GST_DEBUG ("channels %d", opus_packet_get_nb_channels (data)); + GST_DEBUG ("samples %d", samples); res = gst_pad_alloc_buffer_and_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), - GST_BUFFER_OFFSET_NONE, dec->frame_samples * dec->n_channels * 2, + GST_BUFFER_OFFSET_NONE, packet_size, GST_PAD_CAPS (GST_AUDIO_DECODER_SRC_PAD (dec)), &outbuf); if (res != GST_FLOW_OK) { @@ -239,27 +239,28 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, out_data = (gint16 *) GST_BUFFER_DATA (outbuf); - GST_LOG_OBJECT (dec, "decoding %d sample frame", dec->frame_samples); + GST_LOG_OBJECT (dec, "decoding %d samples, in size %u", samples, size); - n = opus_decode (dec->state, data, size, out_data, dec->frame_samples, 0); + n = opus_decode (dec->state, data, size, out_data, samples, 0); if (n < 0) { GST_ELEMENT_ERROR (dec, STREAM, DECODE, ("Decoding error: %d", n), (NULL)); return GST_FLOW_ERROR; } + GST_DEBUG_OBJECT (dec, "decoded %d samples", n); - if (!GST_CLOCK_TIME_IS_VALID (timestamp)) { - GST_WARNING_OBJECT (dec, "No timestamp in -> no timestamp out"); + if (GST_CLOCK_TIME_IS_VALID (timestamp)) { + GST_BUFFER_TIMESTAMP (outbuf) = timestamp; + } else { + GST_BUFFER_TIMESTAMP (outbuf) = dec->next_ts; } - GST_DEBUG_OBJECT (dec, "timestamp=%" GST_TIME_FORMAT, - GST_TIME_ARGS (timestamp)); - - GST_BUFFER_TIMESTAMP (outbuf) = GST_BUFFER_TIMESTAMP (buf); - GST_BUFFER_DURATION (outbuf) = GST_BUFFER_DURATION (buf); + GST_BUFFER_DURATION (outbuf) = + gst_util_uint64_scale (n, GST_SECOND, dec->sample_rate); + dec->next_ts = GST_BUFFER_TIMESTAMP (outbuf) + GST_BUFFER_DURATION (outbuf); GST_LOG_OBJECT (dec, "pushing buffer with ts=%" GST_TIME_FORMAT ", dur=%" GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)), - GST_TIME_ARGS (dec->frame_duration)); + GST_TIME_ARGS (GST_BUFFER_DURATION (outbuf))); res = gst_audio_decoder_finish_frame (GST_AUDIO_DECODER (dec), outbuf, 1); @@ -273,37 +274,6 @@ creation_failed: return GST_FLOW_ERROR; } -static gint -gst_opus_dec_get_frame_samples (GstOpusDec * dec) -{ - gint frame_samples = 0; - switch (dec->frame_size) { - case 2: - frame_samples = dec->sample_rate / 400; - break; - case 5: - frame_samples = dec->sample_rate / 200; - break; - case 10: - frame_samples = dec->sample_rate / 100; - break; - case 20: - frame_samples = dec->sample_rate / 50; - break; - case 40: - frame_samples = dec->sample_rate / 25; - break; - case 60: - frame_samples = 3 * dec->sample_rate / 50; - break; - default: - GST_WARNING_OBJECT (dec, "Unsupported frame size: %d", dec->frame_size); - frame_samples = 0; - break; - } - return frame_samples; -} - static gboolean gst_opus_dec_set_format (GstAudioDecoder * bdec, GstCaps * caps) { @@ -340,24 +310,6 @@ gst_opus_dec_set_format (GstAudioDecoder * bdec, GstCaps * caps) gst_buffer_replace (&dec->vorbiscomment, buf); } } - if (!gst_structure_get_int (s, "frame-size", &dec->frame_size)) { - GST_WARNING_OBJECT (dec, "Frame size not included in caps"); - } - if (!gst_structure_get_int (s, "channels", &dec->n_channels)) { - GST_WARNING_OBJECT (dec, "Number of channels not included in caps"); - } - if (!gst_structure_get_int (s, "rate", &dec->sample_rate)) { - GST_WARNING_OBJECT (dec, "Sample rate not included in caps"); - } - - dec->frame_samples = gst_opus_dec_get_frame_samples (dec); - dec->frame_duration = gst_util_uint64_scale_int (dec->frame_samples, - GST_SECOND, dec->sample_rate); - GST_INFO_OBJECT (dec, - "Got frame size %d, %d channels, %d Hz, giving %d samples per frame, frame duration %" - GST_TIME_FORMAT, dec->frame_size, dec->n_channels, dec->sample_rate, - dec->frame_samples, GST_TIME_ARGS (dec->frame_duration)); - caps = gst_caps_new_simple ("audio/x-raw-int", "rate", G_TYPE_INT, dec->sample_rate, "channels", G_TYPE_INT, dec->n_channels, @@ -385,6 +337,13 @@ memcmp_buffers (GstBuffer * buf1, GstBuffer * buf2) return !memcmp (GST_BUFFER_DATA (buf1), GST_BUFFER_DATA (buf2), size1); } +static gboolean +gst_opus_dec_is_header (GstBuffer * buf, const char *magic, guint magic_size) +{ + return (GST_BUFFER_SIZE (buf) >= magic_size + && !memcmp (magic, GST_BUFFER_DATA (buf), magic_size)); +} + static GstFlowReturn gst_opus_dec_handle_frame (GstAudioDecoder * adec, GstBuffer * buf) { @@ -421,14 +380,24 @@ gst_opus_dec_handle_frame (GstAudioDecoder * adec, GstBuffer * buf) * first two packets are the headers. */ switch (dec->packetno) { case 0: - GST_DEBUG_OBJECT (dec, "counted streamheader"); - res = gst_opus_dec_parse_header (dec, buf); - gst_audio_decoder_finish_frame (adec, NULL, 1); + if (gst_opus_dec_is_header (buf, "OpusHead", 8)) { + GST_DEBUG_OBJECT (dec, "found streamheader"); + res = gst_opus_dec_parse_header (dec, buf); + gst_audio_decoder_finish_frame (adec, NULL, 1); + } else { + res = opus_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), + GST_BUFFER_DURATION (buf)); + } break; case 1: - GST_DEBUG_OBJECT (dec, "counted vorbiscomments"); - res = gst_opus_dec_parse_comments (dec, buf); - gst_audio_decoder_finish_frame (adec, NULL, 1); + if (gst_opus_dec_is_header (buf, "OpusTags", 8)) { + GST_DEBUG_OBJECT (dec, "counted vorbiscomments"); + res = gst_opus_dec_parse_comments (dec, buf); + gst_audio_decoder_finish_frame (adec, NULL, 1); + } else { + res = opus_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), + GST_BUFFER_DURATION (buf)); + } break; default: { diff --git a/ext/opus/gstopusdec.h b/ext/opus/gstopusdec.h index 8389580e3b..38dd2799ad 100644 --- a/ext/opus/gstopusdec.h +++ b/ext/opus/gstopusdec.h @@ -45,11 +45,9 @@ struct _GstOpusDec { GstAudioDecoder element; OpusDecoder *state; - int frame_samples; - gint frame_size; - GstClockTime frame_duration; guint64 packetno; + GstClockTime next_ts; GstBuffer *streamheader; GstBuffer *vorbiscomment; From a02e18917fed445d73e8062657b64ba8c0dc7e90 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 16 Nov 2011 17:24:20 +0000 Subject: [PATCH 13/25] opusdec: allow negotiation of rate/channels with downstream Since an opus stream may be decoded to any (sensible) rate, and either stereo or mono, we try to accomodate downstream. --- ext/opus/gstopusdec.c | 74 +++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/ext/opus/gstopusdec.c b/ext/opus/gstopusdec.c index f14e4e0ded..d4cdabdded 100644 --- a/ext/opus/gstopusdec.c +++ b/ext/opus/gstopusdec.c @@ -129,8 +129,8 @@ gst_opus_dec_reset (GstOpusDec * dec) static void gst_opus_dec_init (GstOpusDec * dec, GstOpusDecClass * g_class) { - dec->sample_rate = 48000; - dec->n_channels = 2; + dec->sample_rate = 0; + dec->n_channels = 0; gst_opus_dec_reset (dec); } @@ -170,6 +170,48 @@ gst_opus_dec_parse_comments (GstOpusDec * dec, GstBuffer * buf) return GST_FLOW_OK; } +static void +gst_opus_dec_setup_from_peer_caps (GstOpusDec * dec) +{ + GstPad *srcpad, *peer; + GstStructure *s; + GstCaps *caps; + const GstCaps *template_caps; + const GstCaps *peer_caps; + + srcpad = GST_AUDIO_DECODER_SRC_PAD (dec); + peer = gst_pad_get_peer (srcpad); + + if (peer) { + template_caps = gst_pad_get_pad_template_caps (srcpad); + peer_caps = gst_pad_get_caps (peer); + GST_DEBUG_OBJECT (dec, "Peer caps: %" GST_PTR_FORMAT, peer_caps); + caps = gst_caps_intersect (template_caps, peer_caps); + gst_pad_fixate_caps (peer, caps); + GST_DEBUG_OBJECT (dec, "Fixated caps: %" GST_PTR_FORMAT, caps); + + s = gst_caps_get_structure (caps, 0); + if (!gst_structure_get_int (s, "channels", &dec->n_channels)) { + dec->n_channels = 2; + GST_WARNING_OBJECT (dec, "Failed to get channels, using default %d", + dec->n_channels); + } else { + GST_DEBUG_OBJECT (dec, "Got channels %d", dec->n_channels); + } + if (!gst_structure_get_int (s, "rate", &dec->sample_rate)) { + dec->sample_rate = 48000; + GST_WARNING_OBJECT (dec, "Failed to get rate, using default %d", + dec->sample_rate); + } else { + GST_DEBUG_OBJECT (dec, "Got sample rate %d", dec->sample_rate); + } + + gst_pad_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), caps); + } else { + GST_WARNING_OBJECT (dec, "Failed to get src pad peer"); + } +} + static GstFlowReturn opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, GstClockTime timestamp, GstClockTime duration) @@ -184,27 +226,13 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, unsigned int packet_size; if (dec->state == NULL) { - GstCaps *caps; + gst_opus_dec_setup_from_peer_caps (dec); + GST_DEBUG_OBJECT (dec, "Creating decoder with %d channels, %d Hz", + dec->n_channels, dec->sample_rate); dec->state = opus_decoder_create (dec->sample_rate, dec->n_channels, &err); if (!dec->state || err != OPUS_OK) goto creation_failed; - - /* set caps */ - caps = gst_caps_new_simple ("audio/x-raw-int", - "rate", G_TYPE_INT, dec->sample_rate, - "channels", G_TYPE_INT, dec->n_channels, - "signed", G_TYPE_BOOLEAN, TRUE, - "endianness", G_TYPE_INT, G_BYTE_ORDER, - "width", G_TYPE_INT, 16, "depth", G_TYPE_INT, 16, NULL); - - GST_DEBUG_OBJECT (dec, "rate=%d channels=%d", - dec->sample_rate, dec->n_channels); - - if (!gst_pad_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), caps)) - GST_ERROR ("nego failure"); - - gst_caps_unref (caps); } if (buf) { @@ -310,14 +338,6 @@ gst_opus_dec_set_format (GstAudioDecoder * bdec, GstCaps * caps) gst_buffer_replace (&dec->vorbiscomment, buf); } } - caps = gst_caps_new_simple ("audio/x-raw-int", - "rate", G_TYPE_INT, dec->sample_rate, - "channels", G_TYPE_INT, dec->n_channels, - "signed", G_TYPE_BOOLEAN, TRUE, - "endianness", G_TYPE_INT, G_BYTE_ORDER, - "width", G_TYPE_INT, 16, "depth", G_TYPE_INT, 16, NULL); - gst_pad_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), caps); - gst_caps_unref (caps); done: return ret; From 5efa9ebec80e316242efed7c5cb496c9b9090030 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Tue, 15 Nov 2011 19:53:33 +0000 Subject: [PATCH 14/25] opusparse: add opusparse element A very simple element that parses Opus streams from the ad hoc framing used by the Opus test vectors. --- ext/opus/Makefile.am | 4 +- ext/opus/gstopus.c | 5 ++ ext/opus/gstopusparse.c | 158 ++++++++++++++++++++++++++++++++++++++++ ext/opus/gstopusparse.h | 55 ++++++++++++++ 4 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 ext/opus/gstopusparse.c create mode 100644 ext/opus/gstopusparse.h diff --git a/ext/opus/Makefile.am b/ext/opus/Makefile.am index 57e1692645..6fe723ecce 100644 --- a/ext/opus/Makefile.am +++ b/ext/opus/Makefile.am @@ -1,6 +1,6 @@ plugin_LTLIBRARIES = libgstopus.la -libgstopus_la_SOURCES = gstopus.c gstopusdec.c gstopusenc.c +libgstopus_la_SOURCES = gstopus.c gstopusdec.c gstopusenc.c gstopusparse.c libgstopus_la_CFLAGS = \ -DGST_USE_UNSTABLE_API \ $(GST_PLUGINS_BASE_CFLAGS) \ @@ -15,4 +15,4 @@ libgstopus_la_LIBADD = \ libgstopus_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) $(LIBM) libgstopus_la_LIBTOOLFLAGS = --tag=disable-static -noinst_HEADERS = gstopusenc.h gstopusdec.h +noinst_HEADERS = gstopusenc.h gstopusdec.h gstopusparse.h diff --git a/ext/opus/gstopus.c b/ext/opus/gstopus.c index 65e9dcdc58..c5f68a131c 100644 --- a/ext/opus/gstopus.c +++ b/ext/opus/gstopus.c @@ -23,6 +23,7 @@ #include "gstopusdec.h" #include "gstopusenc.h" +#include "gstopusparse.h" #include @@ -38,6 +39,10 @@ plugin_init (GstPlugin * plugin) GST_TYPE_OPUS_DEC)) return FALSE; + if (!gst_element_register (plugin, "opusparse", GST_RANK_NONE, + GST_TYPE_OPUS_PARSE)) + return FALSE; + gst_tag_register_musicbrainz_tags (); return TRUE; diff --git a/ext/opus/gstopusparse.c b/ext/opus/gstopusparse.c new file mode 100644 index 0000000000..c94687589b --- /dev/null +++ b/ext/opus/gstopusparse.c @@ -0,0 +1,158 @@ +/* GStreamer + * Copyright (C) 2004 Wim Taymans + * Copyright (C) 2006 Tim-Philipp Müller + * Copyright (C) 2008 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +/** + * SECTION:element-opusparse + * @see_also: opusenc, opusdec + * + * This element parses OPUS packets. + * + * + * Example pipelines + * |[ + * gst-launch -v filesrc location=opusdata ! opusparse ! opusdec ! audioconvert ! audioresample ! alsasink + * ]| Decode and plays an unmuxed Opus file. + * + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include "gstopusparse.h" + +GST_DEBUG_CATEGORY_STATIC (opusparse_debug); +#define GST_CAT_DEFAULT opusparse_debug + +#define MAX_PAYLOAD_BYTES 1500 + +static GstStaticPadTemplate opus_parse_src_factory = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("audio/x-opus, framed = (boolean) true") + ); + +static GstStaticPadTemplate opus_parse_sink_factory = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("audio/x-opus") + ); + +G_DEFINE_TYPE (GstOpusParse, gst_opus_parse, GST_TYPE_BASE_PARSE); + +static gboolean gst_opus_parse_start (GstBaseParse * parse); +static gboolean gst_opus_parse_check_valid_frame (GstBaseParse * base, + GstBaseParseFrame * frame, guint * frame_size, gint * skip); + +static void +gst_opus_parse_class_init (GstOpusParseClass * klass) +{ + GstBaseParseClass *bpclass; + GstElementClass *element_class; + + bpclass = (GstBaseParseClass *) klass; + element_class = (GstElementClass *) klass; + + bpclass->start = GST_DEBUG_FUNCPTR (gst_opus_parse_start); + bpclass->check_valid_frame = + GST_DEBUG_FUNCPTR (gst_opus_parse_check_valid_frame); + + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&opus_parse_src_factory)); + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&opus_parse_sink_factory)); + gst_element_class_set_details_simple (element_class, "Opus audio parser", + "Codec/Parser/Audio", + "parses opus audio streams", + "Vincent Penquerc'h "); + + GST_DEBUG_CATEGORY_INIT (opusparse_debug, "opusparse", 0, + "opus parsing element"); +} + +static void +gst_opus_parse_init (GstOpusParse * parse) +{ +} + +static gboolean +gst_opus_parse_start (GstBaseParse * base) +{ + GstOpusParse *parse = GST_OPUS_PARSE (base); + GstCaps *caps; + + caps = gst_caps_from_string ("audio/x-opus"); + gst_pad_set_caps (GST_BASE_PARSE_SRC_PAD (GST_BASE_PARSE (parse)), caps); + gst_caps_unref (caps); + + return TRUE; +} + +static gboolean +gst_opus_parse_check_valid_frame (GstBaseParse * base, + GstBaseParseFrame * frame, guint * frame_size, gint * skip) +{ + GstOpusParse *parse; + guint8 *data; + gsize size; + guint32 packet_size; + int ret = FALSE; + int channels, bandwidth; + + parse = GST_OPUS_PARSE (base); + + data = GST_BUFFER_DATA (frame->buffer); + size = GST_BUFFER_SIZE (frame->buffer); + GST_DEBUG_OBJECT (parse, "Checking for frame, %u bytes in buffer", size); + if (size < 4) { + GST_DEBUG_OBJECT (parse, "Too small"); + goto beach; + } + packet_size = GST_READ_UINT32_BE (data); + GST_DEBUG_OBJECT (parse, "Packet size: %u bytes", packet_size); + if (packet_size > MAX_PAYLOAD_BYTES) { + GST_DEBUG_OBJECT (parse, "Too large"); + goto beach; + } + if (packet_size > size - 4) { + GST_DEBUG_OBJECT (parse, "Truncated"); + goto beach; + } + + channels = opus_packet_get_nb_channels (data + 8); + bandwidth = opus_packet_get_bandwidth (data + 8); + if (channels < 0 || bandwidth < 0) { + GST_DEBUG_OBJECT (parse, "It looked like a packet, but it is not"); + goto beach; + } + + GST_DEBUG_OBJECT (parse, "Got Opus packet, %d bytes"); + + *skip = 8; + *frame_size = packet_size; + ret = TRUE; + +beach: + return ret; +} diff --git a/ext/opus/gstopusparse.h b/ext/opus/gstopusparse.h new file mode 100644 index 0000000000..5f9f884d39 --- /dev/null +++ b/ext/opus/gstopusparse.h @@ -0,0 +1,55 @@ +/* GStreamer + * Copyright (C) <1999> Erik Walthinsen + * Copyright (C) <2008> Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifndef __GST_OPUS_PARSE_H__ +#define __GST_OPUS_PARSE_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_OPUS_PARSE \ + (gst_opus_parse_get_type()) +#define GST_OPUS_PARSE(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_OPUS_PARSE,GstOpusParse)) +#define GST_OPUS_PARSE_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_OPUS_PARSE,GstOpusParseClass)) +#define GST_IS_OPUS_PARSE(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_OPUS_PARSE)) +#define GST_IS_OPUS_PARSE_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_OPUS_PARSE)) + +typedef struct _GstOpusParse GstOpusParse; +typedef struct _GstOpusParseClass GstOpusParseClass; + +struct _GstOpusParse { + GstBaseParse element; +}; + +struct _GstOpusParseClass { + GstBaseParseClass parent_class; +}; + +GType gst_opus_parse_get_type (void); + +G_END_DECLS + +#endif /* __GST_OPUS_PARSE_H__ */ From c90f45df0059fa02b32e952abde4cd4de7e38861 Mon Sep 17 00:00:00 2001 From: Mark Nauwelaerts Date: Wed, 16 Nov 2011 19:11:24 +0100 Subject: [PATCH 15/25] celtenc: port to audioencoder --- ext/celt/Makefile.am | 5 +- ext/celt/gstceltenc.c | 1006 ++++++++++++----------------------------- ext/celt/gstceltenc.h | 27 +- 3 files changed, 303 insertions(+), 735 deletions(-) diff --git a/ext/celt/Makefile.am b/ext/celt/Makefile.am index 38119d8d1d..01153b1350 100644 --- a/ext/celt/Makefile.am +++ b/ext/celt/Makefile.am @@ -1,12 +1,13 @@ plugin_LTLIBRARIES = libgstcelt.la libgstcelt_la_SOURCES = gstcelt.c gstceltdec.c gstceltenc.c -libgstcelt_la_CFLAGS = \ +libgstcelt_la_CFLAGS = -DGST_USE_UNSTABLE_API \ $(GST_PLUGINS_BASE_CFLAGS) \ $(GST_CFLAGS) \ $(CELT_CFLAGS) libgstcelt_la_LIBADD = \ - $(GST_PLUGINS_BASE_LIBS) -lgsttag-$(GST_MAJORMINOR) \ + $(GST_PLUGINS_BASE_LIBS) \ + -lgstaudio-$(GST_MAJORMINOR) -lgsttag-$(GST_MAJORMINOR) \ $(GST_BASE_LIBS) \ $(GST_LIBS) \ $(CELT_LIBS) diff --git a/ext/celt/gstceltenc.c b/ext/celt/gstceltenc.c index c2011b31cc..37366a2a5d 100644 --- a/ext/celt/gstceltenc.c +++ b/ext/celt/gstceltenc.c @@ -19,7 +19,7 @@ */ /* - * Based on the speexenc element + * Based on the celtenc element */ /** @@ -115,41 +115,35 @@ enum PROP_START_BAND }; -static void gst_celt_enc_finalize (GObject * object); - -static gboolean gst_celt_enc_sinkevent (GstPad * pad, GstEvent * event); -static GstFlowReturn gst_celt_enc_chain (GstPad * pad, GstBuffer * buf); -static gboolean gst_celt_enc_setup (GstCeltEnc * enc); - static void gst_celt_enc_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_celt_enc_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); -static GstStateChangeReturn gst_celt_enc_change_state (GstElement * element, - GstStateChange transition); -static GstFlowReturn gst_celt_enc_encode (GstCeltEnc * enc, gboolean flush); +static gboolean gst_celt_enc_start (GstAudioEncoder * enc); +static gboolean gst_celt_enc_stop (GstAudioEncoder * enc); +static gboolean gst_celt_enc_set_format (GstAudioEncoder * enc, + GstAudioInfo * info); +static GstFlowReturn gst_celt_enc_handle_frame (GstAudioEncoder * enc, + GstBuffer * in_buf); +static gboolean gst_celt_enc_sink_event (GstAudioEncoder * enc, + GstEvent * event); +static GstFlowReturn gst_celt_enc_pre_push (GstAudioEncoder * benc, + GstBuffer ** buffer); static void gst_celt_enc_setup_interfaces (GType celtenc_type) { static const GInterfaceInfo tag_setter_info = { NULL, NULL, NULL }; - const GInterfaceInfo preset_interface_info = { - NULL, /* interface_init */ - NULL, /* interface_finalize */ - NULL /* interface_data */ - }; g_type_add_interface_static (celtenc_type, GST_TYPE_TAG_SETTER, &tag_setter_info); - g_type_add_interface_static (celtenc_type, GST_TYPE_PRESET, - &preset_interface_info); GST_DEBUG_CATEGORY_INIT (celtenc_debug, "celtenc", 0, "Celt encoder"); } -GST_BOILERPLATE_FULL (GstCeltEnc, gst_celt_enc, GstElement, GST_TYPE_ELEMENT, - gst_celt_enc_setup_interfaces); +GST_BOILERPLATE_FULL (GstCeltEnc, gst_celt_enc, GstAudioEncoder, + GST_TYPE_AUDIO_ENCODER, gst_celt_enc_setup_interfaces); static void gst_celt_enc_base_init (gpointer g_class) @@ -170,14 +164,21 @@ static void gst_celt_enc_class_init (GstCeltEncClass * klass) { GObjectClass *gobject_class; - GstElementClass *gstelement_class; + GstAudioEncoderClass *gstbase_class; gobject_class = (GObjectClass *) klass; - gstelement_class = (GstElementClass *) klass; + gstbase_class = (GstAudioEncoderClass *) klass; gobject_class->set_property = gst_celt_enc_set_property; gobject_class->get_property = gst_celt_enc_get_property; + gstbase_class->start = GST_DEBUG_FUNCPTR (gst_celt_enc_start); + gstbase_class->stop = GST_DEBUG_FUNCPTR (gst_celt_enc_stop); + gstbase_class->set_format = GST_DEBUG_FUNCPTR (gst_celt_enc_set_format); + gstbase_class->handle_frame = GST_DEBUG_FUNCPTR (gst_celt_enc_handle_frame); + gstbase_class->event = GST_DEBUG_FUNCPTR (gst_celt_enc_sink_event); + gstbase_class->pre_push = GST_DEBUG_FUNCPTR (gst_celt_enc_pre_push); + g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_BITRATE, g_param_spec_int ("bitrate", "Encoding Bit-rate", "Specify an encoding bit-rate (in bps).", @@ -210,411 +211,55 @@ gst_celt_enc_class_init (GstCeltEncClass * klass) "Controls the start band that should be used", 0, G_MAXINT, DEFAULT_START_BAND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_celt_enc_finalize); - - gstelement_class->change_state = - GST_DEBUG_FUNCPTR (gst_celt_enc_change_state); -} - -static void -gst_celt_enc_finalize (GObject * object) -{ - GstCeltEnc *enc; - - enc = GST_CELT_ENC (object); - - g_object_unref (enc->adapter); - - G_OBJECT_CLASS (parent_class)->finalize (object); -} - -static gboolean -gst_celt_enc_sink_setcaps (GstPad * pad, GstCaps * caps) -{ - GstCeltEnc *enc; - GstStructure *structure; - GstCaps *otherpadcaps; - - enc = GST_CELT_ENC (GST_PAD_PARENT (pad)); - enc->setup = FALSE; - enc->frame_size = DEFAULT_FRAMESIZE; - otherpadcaps = gst_pad_get_allowed_caps (pad); - - structure = gst_caps_get_structure (caps, 0); - gst_structure_get_int (structure, "channels", &enc->channels); - gst_structure_get_int (structure, "rate", &enc->rate); - - if (otherpadcaps) { - if (!gst_caps_is_empty (otherpadcaps)) { - GstStructure *ps = gst_caps_get_structure (otherpadcaps, 0); - gst_structure_get_int (ps, "frame-size", &enc->frame_size); - } - gst_caps_unref (otherpadcaps); - } - - if (enc->requested_frame_size > 0) - enc->frame_size = enc->requested_frame_size; - - GST_DEBUG_OBJECT (pad, "channels=%d rate=%d frame-size=%d", - enc->channels, enc->rate, enc->frame_size); - - gst_celt_enc_setup (enc); - - return enc->setup; -} - - -static GstCaps * -gst_celt_enc_sink_getcaps (GstPad * pad) -{ - GstCaps *caps = gst_caps_copy (gst_pad_get_pad_template_caps (pad)); - GstCaps *peercaps = NULL; - GstCeltEnc *enc = GST_CELT_ENC (gst_pad_get_parent_element (pad)); - - peercaps = gst_pad_peer_get_caps (enc->srcpad); - - if (peercaps) { - if (!gst_caps_is_empty (peercaps) && !gst_caps_is_any (peercaps)) { - GstStructure *ps = gst_caps_get_structure (peercaps, 0); - GstStructure *s = gst_caps_get_structure (caps, 0); - gint rate, channels; - - if (gst_structure_get_int (ps, "rate", &rate)) { - gst_structure_fixate_field_nearest_int (s, "rate", rate); - } - - if (gst_structure_get_int (ps, "channels", &channels)) { - gst_structure_fixate_field_nearest_int (s, "channels", channels); - } - } - gst_caps_unref (peercaps); - } - - gst_object_unref (enc); - - return caps; -} - - -static gboolean -gst_celt_enc_convert_src (GstPad * pad, GstFormat src_format, gint64 src_value, - GstFormat * dest_format, gint64 * dest_value) -{ - gboolean res = TRUE; - GstCeltEnc *enc; - gint64 avg; - - enc = GST_CELT_ENC (GST_PAD_PARENT (pad)); - - if (enc->samples_in == 0 || enc->bytes_out == 0 || enc->rate == 0) - return FALSE; - - avg = (enc->bytes_out * enc->rate) / (enc->samples_in); - - switch (src_format) { - case GST_FORMAT_BYTES: - switch (*dest_format) { - case GST_FORMAT_TIME: - *dest_value = src_value * GST_SECOND / avg; - break; - default: - res = FALSE; - } - break; - case GST_FORMAT_TIME: - switch (*dest_format) { - case GST_FORMAT_BYTES: - *dest_value = src_value * avg / GST_SECOND; - break; - default: - res = FALSE; - } - break; - default: - res = FALSE; - } - return res; -} - -static gboolean -gst_celt_enc_convert_sink (GstPad * pad, GstFormat src_format, - gint64 src_value, GstFormat * dest_format, gint64 * dest_value) -{ - gboolean res = TRUE; - guint scale = 1; - gint bytes_per_sample; - GstCeltEnc *enc; - - enc = GST_CELT_ENC (GST_PAD_PARENT (pad)); - - bytes_per_sample = enc->channels * 2; - - switch (src_format) { - case GST_FORMAT_BYTES: - switch (*dest_format) { - case GST_FORMAT_DEFAULT: - if (bytes_per_sample == 0) - return FALSE; - *dest_value = src_value / bytes_per_sample; - break; - case GST_FORMAT_TIME: - { - gint byterate = bytes_per_sample * enc->rate; - - if (byterate == 0) - return FALSE; - *dest_value = src_value * GST_SECOND / byterate; - break; - } - default: - res = FALSE; - } - break; - case GST_FORMAT_DEFAULT: - switch (*dest_format) { - case GST_FORMAT_BYTES: - *dest_value = src_value * bytes_per_sample; - break; - case GST_FORMAT_TIME: - if (enc->rate == 0) - return FALSE; - *dest_value = src_value * GST_SECOND / enc->rate; - break; - default: - res = FALSE; - } - break; - case GST_FORMAT_TIME: - switch (*dest_format) { - case GST_FORMAT_BYTES: - scale = bytes_per_sample; - /* fallthrough */ - case GST_FORMAT_DEFAULT: - *dest_value = src_value * scale * enc->rate / GST_SECOND; - break; - default: - res = FALSE; - } - break; - default: - res = FALSE; - } - return res; -} - -static gint64 -gst_celt_enc_get_latency (GstCeltEnc * enc) -{ - return gst_util_uint64_scale (enc->frame_size, GST_SECOND, enc->rate); -} - -static const GstQueryType * -gst_celt_enc_get_query_types (GstPad * pad) -{ - static const GstQueryType gst_celt_enc_src_query_types[] = { - GST_QUERY_POSITION, - GST_QUERY_DURATION, - GST_QUERY_CONVERT, - GST_QUERY_LATENCY, - 0 - }; - - return gst_celt_enc_src_query_types; -} - -static gboolean -gst_celt_enc_src_query (GstPad * pad, GstQuery * query) -{ - gboolean res = TRUE; - GstCeltEnc *enc; - - enc = GST_CELT_ENC (gst_pad_get_parent (pad)); - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_POSITION: - { - GstFormat fmt, req_fmt; - gint64 pos, val; - - gst_query_parse_position (query, &req_fmt, NULL); - if ((res = gst_pad_query_peer_position (enc->sinkpad, &req_fmt, &val))) { - gst_query_set_position (query, req_fmt, val); - break; - } - - fmt = GST_FORMAT_TIME; - if (!(res = gst_pad_query_peer_position (enc->sinkpad, &fmt, &pos))) - break; - - if ((res = - gst_pad_query_peer_convert (enc->sinkpad, fmt, pos, &req_fmt, - &val))) - gst_query_set_position (query, req_fmt, val); - - break; - } - case GST_QUERY_DURATION: - { - GstFormat fmt, req_fmt; - gint64 dur, val; - - gst_query_parse_duration (query, &req_fmt, NULL); - if ((res = gst_pad_query_peer_duration (enc->sinkpad, &req_fmt, &val))) { - gst_query_set_duration (query, req_fmt, val); - break; - } - - fmt = GST_FORMAT_TIME; - if (!(res = gst_pad_query_peer_duration (enc->sinkpad, &fmt, &dur))) - break; - - if ((res = - gst_pad_query_peer_convert (enc->sinkpad, fmt, dur, &req_fmt, - &val))) { - gst_query_set_duration (query, req_fmt, val); - } - break; - } - case GST_QUERY_CONVERT: - { - GstFormat src_fmt, dest_fmt; - gint64 src_val, dest_val; - - gst_query_parse_convert (query, &src_fmt, &src_val, &dest_fmt, &dest_val); - if (!(res = gst_celt_enc_convert_src (pad, src_fmt, src_val, &dest_fmt, - &dest_val))) - goto error; - gst_query_set_convert (query, src_fmt, src_val, dest_fmt, dest_val); - break; - } - case GST_QUERY_LATENCY: - { - gboolean live; - GstClockTime min_latency, max_latency; - gint64 latency; - - if ((res = gst_pad_peer_query (enc->sinkpad, query))) { - gst_query_parse_latency (query, &live, &min_latency, &max_latency); - - latency = gst_celt_enc_get_latency (enc); - - /* add our latency */ - min_latency += latency; - if (max_latency != -1) - max_latency += latency; - - gst_query_set_latency (query, live, min_latency, max_latency); - } - break; - } - default: - res = gst_pad_peer_query (pad, query); - break; - } - -error: - - gst_object_unref (enc); - - return res; -} - -static gboolean -gst_celt_enc_sink_query (GstPad * pad, GstQuery * query) -{ - gboolean res = TRUE; - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_CONVERT: - { - GstFormat src_fmt, dest_fmt; - gint64 src_val, dest_val; - - gst_query_parse_convert (query, &src_fmt, &src_val, &dest_fmt, &dest_val); - if (!(res = - gst_celt_enc_convert_sink (pad, src_fmt, src_val, &dest_fmt, - &dest_val))) - goto error; - gst_query_set_convert (query, src_fmt, src_val, dest_fmt, dest_val); - break; - } - default: - res = gst_pad_query_default (pad, query); - break; - } - -error: - return res; } static void gst_celt_enc_init (GstCeltEnc * enc, GstCeltEncClass * klass) { - enc->sinkpad = gst_pad_new_from_static_template (&sink_factory, "sink"); - gst_element_add_pad (GST_ELEMENT (enc), enc->sinkpad); - gst_pad_set_event_function (enc->sinkpad, - GST_DEBUG_FUNCPTR (gst_celt_enc_sinkevent)); - gst_pad_set_chain_function (enc->sinkpad, - GST_DEBUG_FUNCPTR (gst_celt_enc_chain)); - gst_pad_set_setcaps_function (enc->sinkpad, - GST_DEBUG_FUNCPTR (gst_celt_enc_sink_setcaps)); - gst_pad_set_getcaps_function (enc->sinkpad, - GST_DEBUG_FUNCPTR (gst_celt_enc_sink_getcaps)); - gst_pad_set_query_function (enc->sinkpad, - GST_DEBUG_FUNCPTR (gst_celt_enc_sink_query)); - - enc->srcpad = gst_pad_new_from_static_template (&src_factory, "src"); - gst_pad_set_query_function (enc->srcpad, - GST_DEBUG_FUNCPTR (gst_celt_enc_src_query)); - gst_pad_set_query_type_function (enc->srcpad, - GST_DEBUG_FUNCPTR (gst_celt_enc_get_query_types)); - gst_element_add_pad (GST_ELEMENT (enc), enc->srcpad); - - enc->channels = -1; - enc->rate = -1; - enc->bitrate = DEFAULT_BITRATE; enc->frame_size = DEFAULT_FRAMESIZE; - enc->requested_frame_size = -1; enc->cbr = DEFAULT_CBR; enc->complexity = DEFAULT_COMPLEXITY; enc->max_bitrate = DEFAULT_MAX_BITRATE; enc->prediction = DEFAULT_PREDICTION; - - enc->setup = FALSE; - enc->header_sent = FALSE; - - enc->adapter = gst_adapter_new (); } -static GstBuffer * -gst_celt_enc_create_metadata_buffer (GstCeltEnc * enc) +static gboolean +gst_celt_enc_start (GstAudioEncoder * benc) { - const GstTagList *tags; - GstTagList *empty_tags = NULL; - GstBuffer *comments = NULL; + GstCeltEnc *enc = GST_CELT_ENC (benc); - tags = gst_tag_setter_get_tag_list (GST_TAG_SETTER (enc)); + GST_DEBUG_OBJECT (enc, "start"); + enc->channels = -1; + enc->rate = -1; + enc->header_sent = FALSE; - GST_DEBUG_OBJECT (enc, "tags = %" GST_PTR_FORMAT, tags); + return TRUE; +} - if (tags == NULL) { - /* FIXME: better fix chain of callers to not write metadata at all, - * if there is none */ - empty_tags = gst_tag_list_new (); - tags = empty_tags; +static gboolean +gst_celt_enc_stop (GstAudioEncoder * benc) +{ + GstCeltEnc *enc = GST_CELT_ENC (benc); + + GST_DEBUG_OBJECT (enc, "stop"); + enc->header_sent = FALSE; + if (enc->state) { + celt_encoder_destroy (enc->state); + enc->state = NULL; } - comments = gst_tag_list_to_vorbiscomment_buffer (tags, NULL, - 0, "Encoded with GStreamer Celtenc"); + if (enc->mode) { + celt_mode_destroy (enc->mode); + enc->mode = NULL; + } + memset (&enc->header, 0, sizeof (enc->header)); - GST_BUFFER_OFFSET (comments) = enc->bytes_out; - GST_BUFFER_OFFSET_END (comments) = 0; + g_slist_foreach (enc->headers, (GFunc) gst_buffer_unref, NULL); + enc->headers = NULL; - if (empty_tags) - gst_tag_list_free (empty_tags); + gst_tag_setter_reset_tags (GST_TAG_SETTER (enc)); - return comments; + return TRUE; } static gboolean @@ -622,8 +267,6 @@ gst_celt_enc_setup (GstCeltEnc * enc) { gint error = CELT_OK; - enc->setup = FALSE; - #ifdef HAVE_CELT_0_7 enc->mode = celt_mode_create (enc->rate, enc->frame_size, &error); #else @@ -679,8 +322,6 @@ gst_celt_enc_setup (GstCeltEnc * enc) GST_LOG_OBJECT (enc, "we have frame size %d", enc->frame_size); - enc->setup = TRUE; - return TRUE; mode_initialization_failed: @@ -696,93 +337,70 @@ encoder_creation_failed: return FALSE; } -/* prepare a buffer for transmission */ -static GstBuffer * -gst_celt_enc_buffer_from_data (GstCeltEnc * enc, guchar * data, - guint data_len, gint64 granulepos) +static gint64 +gst_celt_enc_get_latency (GstCeltEnc * enc) { - GstBuffer *outbuf; - - outbuf = gst_buffer_new (); - GST_BUFFER_DATA (outbuf) = data; - GST_BUFFER_MALLOCDATA (outbuf) = data; - GST_BUFFER_SIZE (outbuf) = data_len; - GST_BUFFER_OFFSET (outbuf) = enc->bytes_out; - GST_BUFFER_OFFSET_END (outbuf) = granulepos; - - GST_LOG_OBJECT (enc, "encoded buffer of %u bytes", GST_BUFFER_SIZE (outbuf)); - return outbuf; + return gst_util_uint64_scale (enc->frame_size, GST_SECOND, enc->rate); } - -/* push out the buffer and do internal bookkeeping */ -static GstFlowReturn -gst_celt_enc_push_buffer (GstCeltEnc * enc, GstBuffer * buffer) -{ - guint size; - - size = GST_BUFFER_SIZE (buffer); - - enc->bytes_out += size; - - GST_DEBUG_OBJECT (enc, "pushing output buffer of size %u", size); - - return gst_pad_push (enc->srcpad, buffer); -} - -static GstCaps * -gst_celt_enc_set_header_on_caps (GstCaps * caps, GstBuffer * buf1, - GstBuffer * buf2) -{ - GstStructure *structure = NULL; - GstBuffer *buf; - GValue array = { 0 }; - GValue value = { 0 }; - - caps = gst_caps_make_writable (caps); - structure = gst_caps_get_structure (caps, 0); - - g_assert (gst_buffer_is_metadata_writable (buf1)); - g_assert (gst_buffer_is_metadata_writable (buf2)); - - /* mark buffers */ - GST_BUFFER_FLAG_SET (buf1, GST_BUFFER_FLAG_IN_CAPS); - GST_BUFFER_FLAG_SET (buf2, GST_BUFFER_FLAG_IN_CAPS); - - /* put buffers in a fixed list */ - g_value_init (&array, GST_TYPE_ARRAY); - g_value_init (&value, GST_TYPE_BUFFER); - buf = gst_buffer_copy (buf1); - gst_value_set_buffer (&value, buf); - gst_buffer_unref (buf); - gst_value_array_append_value (&array, &value); - g_value_unset (&value); - g_value_init (&value, GST_TYPE_BUFFER); - buf = gst_buffer_copy (buf2); - gst_value_set_buffer (&value, buf); - gst_buffer_unref (buf); - gst_value_array_append_value (&array, &value); - gst_structure_set_value (structure, "streamheader", &array); - g_value_unset (&value); - g_value_unset (&array); - - return caps; -} - - static gboolean -gst_celt_enc_sinkevent (GstPad * pad, GstEvent * event) +gst_celt_enc_set_format (GstAudioEncoder * benc, GstAudioInfo * info) +{ + GstCeltEnc *enc; + GstCaps *otherpadcaps; + + enc = GST_CELT_ENC (benc); + + enc->channels = GST_AUDIO_INFO_CHANNELS (info); + enc->rate = GST_AUDIO_INFO_RATE (info); + + /* handle reconfigure */ + if (enc->state) { + celt_encoder_destroy (enc->state); + enc->state = NULL; + } + if (enc->mode) { + celt_mode_destroy (enc->mode); + enc->mode = NULL; + } + memset (&enc->header, 0, sizeof (enc->header)); + + otherpadcaps = gst_pad_get_allowed_caps (GST_AUDIO_ENCODER_SRC_PAD (enc)); + if (otherpadcaps) { + if (!gst_caps_is_empty (otherpadcaps)) { + GstStructure *ps = gst_caps_get_structure (otherpadcaps, 0); + gst_structure_get_int (ps, "frame-size", &enc->frame_size); + } + gst_caps_unref (otherpadcaps); + } + + if (enc->requested_frame_size > 0) + enc->frame_size = enc->requested_frame_size; + + GST_DEBUG_OBJECT (enc, "channels=%d rate=%d frame-size=%d", + enc->channels, enc->rate, enc->frame_size); + + if (!gst_celt_enc_setup (enc)) + return FALSE; + + /* feedback to base class */ + gst_audio_encoder_set_latency (benc, + gst_celt_enc_get_latency (enc), gst_celt_enc_get_latency (enc)); + gst_audio_encoder_set_frame_samples_min (benc, enc->frame_size); + gst_audio_encoder_set_frame_samples_max (benc, enc->frame_size); + gst_audio_encoder_set_frame_max (benc, 1); + + return TRUE; +} + +static gboolean +gst_celt_enc_sink_event (GstAudioEncoder * benc, GstEvent * event) { - gboolean res = TRUE; GstCeltEnc *enc; - enc = GST_CELT_ENC (gst_pad_get_parent (pad)); + enc = GST_CELT_ENC (benc); switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_EOS: - gst_celt_enc_encode (enc, TRUE); - res = gst_pad_event_default (pad, event); - break; case GST_EVENT_TAG: { GstTagList *list; @@ -791,114 +409,177 @@ gst_celt_enc_sinkevent (GstPad * pad, GstEvent * event) gst_event_parse_tag (event, &list); gst_tag_setter_merge_tags (setter, list, mode); - res = gst_pad_event_default (pad, event); break; } default: - res = gst_pad_event_default (pad, event); break; } - gst_object_unref (enc); + /* we only peeked, let base class handle it */ + return FALSE; +} - return res; +static GstBuffer * +gst_celt_enc_create_metadata_buffer (GstCeltEnc * enc) +{ + const GstTagList *tags; + GstTagList *empty_tags = NULL; + GstBuffer *comments = NULL; + + tags = gst_tag_setter_get_tag_list (GST_TAG_SETTER (enc)); + + GST_DEBUG_OBJECT (enc, "tags = %" GST_PTR_FORMAT, tags); + + if (tags == NULL) { + /* FIXME: better fix chain of callers to not write metadata at all, + * if there is none */ + empty_tags = gst_tag_list_new (); + tags = empty_tags; + } + comments = gst_tag_list_to_vorbiscomment_buffer (tags, NULL, + 0, "Encoded with GStreamer Celtenc"); + + GST_BUFFER_OFFSET (comments) = 0; + GST_BUFFER_OFFSET_END (comments) = 0; + + if (empty_tags) + gst_tag_list_free (empty_tags); + + return comments; } static GstFlowReturn -gst_celt_enc_encode (GstCeltEnc * enc, gboolean flush) +gst_celt_enc_encode (GstCeltEnc * enc, GstBuffer * buf) { - GstFlowReturn ret = GST_FLOW_OK; gint frame_size = enc->frame_size; gint bytes = frame_size * 2 * enc->channels; gint bytes_per_packet; + gint16 *data, *data0 = NULL; + gint outsize, size; + GstBuffer *outbuf; - if (enc->cbr) { - bytes_per_packet = (enc->bitrate * enc->frame_size / enc->rate + 4) / 8; + if (G_LIKELY (buf)) { + data = (gint16 *) GST_BUFFER_DATA (buf); + size = GST_BUFFER_SIZE (buf); + + if (G_UNLIKELY (size % bytes)) { + GST_DEBUG_OBJECT (enc, "draining; adding silence samples"); + size = ((size / bytes) + 1) * bytes; + data0 = data = g_malloc0 (size); + memcpy (data, GST_BUFFER_DATA (buf), GST_BUFFER_SIZE (buf)); + } } else { - bytes_per_packet = (enc->max_bitrate * enc->frame_size / enc->rate + 4) / 8; + GST_DEBUG_OBJECT (enc, "nothing to drain"); + goto done; } - if (flush && gst_adapter_available (enc->adapter) % bytes != 0) { - guint diff = bytes - gst_adapter_available (enc->adapter) % bytes; - GstBuffer *buf = gst_buffer_new_and_alloc (diff); - - memset (GST_BUFFER_DATA (buf), 0, diff); - gst_adapter_push (enc->adapter, buf); + frame_size = size / (2 * enc->channels); + if (enc->cbr) { + bytes_per_packet = (enc->bitrate * frame_size / enc->rate + 4) / 8; + } else { + bytes_per_packet = (enc->max_bitrate * frame_size / enc->rate + 4) / 8; } + ret = gst_pad_alloc_buffer_and_set_caps (GST_AUDIO_ENCODER_SRC_PAD (enc), + GST_BUFFER_OFFSET_NONE, bytes_per_packet, + GST_PAD_CAPS (GST_AUDIO_ENCODER_SRC_PAD (enc)), &outbuf); - while (gst_adapter_available (enc->adapter) >= bytes) { - gint16 *data; - gint outsize; - GstBuffer *outbuf; + if (GST_FLOW_OK != ret) + goto done; - ret = gst_pad_alloc_buffer_and_set_caps (enc->srcpad, - GST_BUFFER_OFFSET_NONE, bytes_per_packet, GST_PAD_CAPS (enc->srcpad), - &outbuf); - - if (GST_FLOW_OK != ret) - goto done; - - data = (gint16 *) gst_adapter_take (enc->adapter, bytes); - enc->samples_in += frame_size; - - GST_DEBUG_OBJECT (enc, "encoding %d samples (%d bytes)", frame_size, bytes); + GST_DEBUG_OBJECT (enc, "encoding %d samples (%d bytes)", frame_size, bytes); #ifdef HAVE_CELT_0_8 - outsize = - celt_encode (enc->state, data, frame_size, - GST_BUFFER_DATA (outbuf), bytes_per_packet); + outsize = + celt_encode (enc->state, data, frame_size, + GST_BUFFER_DATA (outbuf), bytes_per_packet); #else - outsize = - celt_encode (enc->state, data, NULL, - GST_BUFFER_DATA (outbuf), bytes_per_packet); + outsize = + celt_encode (enc->state, data, NULL, + GST_BUFFER_DATA (outbuf), bytes_per_packet); #endif - g_free (data); - - if (outsize < 0) { - GST_ERROR_OBJECT (enc, "Encoding failed: %d", outsize); - ret = GST_FLOW_ERROR; - goto done; - } - - GST_BUFFER_TIMESTAMP (outbuf) = enc->start_ts + - gst_util_uint64_scale_int (enc->frameno_out * frame_size, GST_SECOND, - enc->rate); - GST_BUFFER_DURATION (outbuf) = - gst_util_uint64_scale_int (frame_size, GST_SECOND, enc->rate); - /* set gp time and granulepos; see gst-plugins-base/ext/ogg/README */ - GST_BUFFER_OFFSET_END (outbuf) = enc->granulepos_offset + - ((enc->frameno + 1) * frame_size); - GST_BUFFER_OFFSET (outbuf) = - gst_util_uint64_scale_int (GST_BUFFER_OFFSET_END (outbuf), GST_SECOND, - enc->rate); - - enc->frameno++; - enc->frameno_out++; - - ret = gst_celt_enc_push_buffer (enc, outbuf); - - if ((GST_FLOW_OK != ret) && (GST_FLOW_NOT_LINKED != ret)) - goto done; + if (outsize < 0) { + GST_ERROR_OBJECT (enc, "Encoding failed: %d", outsize); + ret = GST_FLOW_ERROR; + goto done; } -done: + GST_DEBUG_OBJECT (enc, "encoding %d bytes", bytes); + ret = gst_audio_encoder_finish_frame (GST_AUDIO_ENCODER (enc), + outbuf, frame_size); + +done: + g_free (data0); return ret; } +/* + * (really really) FIXME: move into core (dixit tpm) + */ +/** + * _gst_caps_set_buffer_array: + * @caps: a #GstCaps + * @field: field in caps to set + * @buf: header buffers + * + * Adds given buffers to an array of buffers set as the given @field + * on the given @caps. List of buffer arguments must be NULL-terminated. + * + * Returns: input caps with a streamheader field added, or NULL if some error + */ +static GstCaps * +_gst_caps_set_buffer_array (GstCaps * caps, const gchar * field, + GstBuffer * buf, ...) +{ + GstStructure *structure = NULL; + va_list va; + GValue array = { 0 }; + GValue value = { 0 }; + + g_return_val_if_fail (caps != NULL, NULL); + g_return_val_if_fail (gst_caps_is_fixed (caps), NULL); + g_return_val_if_fail (field != NULL, NULL); + + caps = gst_caps_make_writable (caps); + structure = gst_caps_get_structure (caps, 0); + + g_value_init (&array, GST_TYPE_ARRAY); + + va_start (va, buf); + /* put buffers in a fixed list */ + while (buf) { + g_assert (gst_buffer_is_metadata_writable (buf)); + + /* mark buffer */ + GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_IN_CAPS); + + g_value_init (&value, GST_TYPE_BUFFER); + buf = gst_buffer_copy (buf); + GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_IN_CAPS); + gst_value_set_buffer (&value, buf); + gst_buffer_unref (buf); + gst_value_array_append_value (&array, &value); + g_value_unset (&value); + + buf = va_arg (va, GstBuffer *); + } + + gst_structure_set_value (structure, field, &array); + g_value_unset (&array); + + return caps; +} + static GstFlowReturn -gst_celt_enc_chain (GstPad * pad, GstBuffer * buf) +gst_celt_enc_handle_frame (GstAudioEncoder * benc, GstBuffer * buf) { GstCeltEnc *enc; GstFlowReturn ret = GST_FLOW_OK; - enc = GST_CELT_ENC (GST_PAD_PARENT (pad)); - - if (!enc->setup) - goto not_setup; + enc = GST_CELT_ENC (benc); if (!enc->header_sent) { /* Celt streams begin with two headers; the initial header (with @@ -919,138 +600,53 @@ gst_celt_enc_chain (GstPad * pad, GstBuffer * buf) g_free (data); goto no_header; } - buf1 = gst_celt_enc_buffer_from_data (enc, data, header_size, 0); + buf1 = gst_buffer_new (); + GST_BUFFER_DATA (buf1) = GST_BUFFER_MALLOCDATA (buf1) = data; + GST_BUFFER_SIZE (buf1) = header_size; + GST_BUFFER_OFFSET_END (buf1) = 0; + GST_BUFFER_OFFSET (buf1) = 0; /* create comment buffer */ buf2 = gst_celt_enc_create_metadata_buffer (enc); /* mark and put on caps */ - caps = gst_pad_get_caps (enc->srcpad); - caps = gst_celt_enc_set_header_on_caps (caps, buf1, buf2); - + caps = gst_pad_get_caps (GST_AUDIO_ENCODER_SRC_PAD (enc)); gst_caps_set_simple (caps, "rate", G_TYPE_INT, enc->rate, "channels", G_TYPE_INT, enc->channels, "frame-size", G_TYPE_INT, enc->frame_size, NULL); + caps = _gst_caps_set_buffer_array (caps, "streamheader", buf1, buf2, NULL); /* negotiate with these caps */ GST_DEBUG_OBJECT (enc, "here are the caps: %" GST_PTR_FORMAT, caps); GST_LOG_OBJECT (enc, "rate=%d channels=%d frame-size=%d", enc->rate, enc->channels, enc->frame_size); - gst_pad_set_caps (enc->srcpad, caps); + gst_pad_set_caps (GST_AUDIO_ENCODER_SRC_PAD (enc), caps); gst_buffer_set_caps (buf1, caps); gst_buffer_set_caps (buf2, caps); gst_caps_unref (caps); /* push out buffers */ - ret = gst_celt_enc_push_buffer (enc, buf1); - - if (ret != GST_FLOW_OK) { - gst_buffer_unref (buf2); - goto done; - } - - ret = gst_celt_enc_push_buffer (enc, buf2); - - if (ret != GST_FLOW_OK) - goto done; + /* store buffers for later pre_push sending */ + g_slist_foreach (enc->headers, (GFunc) gst_buffer_unref, NULL); + enc->headers = NULL; + GST_DEBUG_OBJECT (enc, "storing header buffers"); + enc->headers = g_slist_prepend (enc->headers, buf2); + enc->headers = g_slist_prepend (enc->headers, buf1); enc->header_sent = TRUE; } - GST_DEBUG_OBJECT (enc, "received buffer of %u bytes", GST_BUFFER_SIZE (buf)); + GST_DEBUG_OBJECT (enc, "received buffer %p of %u bytes", buf, + buf ? GST_BUFFER_SIZE (buf) : 0); - /* Save the timestamp of the first buffer. This will be later - * used as offset for all following buffers */ - if (enc->start_ts == GST_CLOCK_TIME_NONE) { - if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { - enc->start_ts = GST_BUFFER_TIMESTAMP (buf); - enc->granulepos_offset = gst_util_uint64_scale - (GST_BUFFER_TIMESTAMP (buf), enc->rate, GST_SECOND); - } else { - enc->start_ts = 0; - enc->granulepos_offset = 0; - } - } - - - /* Check if we have a continous stream, if not drop some samples or the buffer or - * insert some silence samples */ - if (enc->next_ts != GST_CLOCK_TIME_NONE && - GST_BUFFER_TIMESTAMP_IS_VALID (buf) && - GST_BUFFER_TIMESTAMP (buf) < enc->next_ts) { - guint64 diff = enc->next_ts - GST_BUFFER_TIMESTAMP (buf); - guint64 diff_bytes; - - GST_WARNING_OBJECT (enc, "Buffer is older than previous " - "timestamp + duration (%" GST_TIME_FORMAT "< %" GST_TIME_FORMAT - "), cannot handle. Clipping buffer.", - GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf)), - GST_TIME_ARGS (enc->next_ts)); - - diff_bytes = GST_CLOCK_TIME_TO_FRAMES (diff, enc->rate) * enc->channels * 2; - if (diff_bytes >= GST_BUFFER_SIZE (buf)) { - gst_buffer_unref (buf); - return GST_FLOW_OK; - } - buf = gst_buffer_make_metadata_writable (buf); - GST_BUFFER_DATA (buf) += diff_bytes; - GST_BUFFER_SIZE (buf) -= diff_bytes; - - GST_BUFFER_TIMESTAMP (buf) += diff; - if (GST_BUFFER_DURATION_IS_VALID (buf)) - GST_BUFFER_DURATION (buf) -= diff; - } - - if (enc->next_ts != GST_CLOCK_TIME_NONE - && GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { - guint64 max_diff = - gst_util_uint64_scale (enc->frame_size, GST_SECOND, enc->rate); - - if (GST_BUFFER_TIMESTAMP (buf) != enc->next_ts && - GST_BUFFER_TIMESTAMP (buf) - enc->next_ts > max_diff) { - GST_WARNING_OBJECT (enc, - "Discontinuity detected: %" G_GUINT64_FORMAT " > %" G_GUINT64_FORMAT, - GST_BUFFER_TIMESTAMP (buf) - enc->next_ts, max_diff); - - gst_celt_enc_encode (enc, TRUE); - - enc->frameno_out = 0; - enc->start_ts = GST_BUFFER_TIMESTAMP (buf); - enc->granulepos_offset = gst_util_uint64_scale - (GST_BUFFER_TIMESTAMP (buf), enc->rate, GST_SECOND); - } - } - - if (GST_BUFFER_TIMESTAMP_IS_VALID (buf) - && GST_BUFFER_DURATION_IS_VALID (buf)) - enc->next_ts = GST_BUFFER_TIMESTAMP (buf) + GST_BUFFER_DURATION (buf); - else - enc->next_ts = GST_CLOCK_TIME_NONE; - - /* push buffer to adapter */ - gst_adapter_push (enc->adapter, buf); - buf = NULL; - - ret = gst_celt_enc_encode (enc, FALSE); + ret = gst_celt_enc_encode (enc, buf); done: - - if (buf) - gst_buffer_unref (buf); - return ret; /* ERRORS */ -not_setup: - { - GST_ELEMENT_ERROR (enc, CORE, NEGOTIATION, (NULL), - ("encoder not initialized (input is not audio?)")); - ret = GST_FLOW_NOT_NEGOTIATED; - goto done; - } - no_header: { GST_ELEMENT_ERROR (enc, STREAM, ENCODE, (NULL), @@ -1060,6 +656,48 @@ no_header: } } +/* push out the buffer */ +static GstFlowReturn +gst_celt_enc_push_buffer (GstCeltEnc * enc, GstBuffer * buffer) +{ + guint size; + + size = GST_BUFFER_SIZE (buffer); + GST_DEBUG_OBJECT (enc, "pushing output buffer of size %u", size); + + gst_buffer_set_caps (buffer, GST_PAD_CAPS (GST_AUDIO_ENCODER_SRC_PAD (enc))); + return gst_pad_push (GST_AUDIO_ENCODER_SRC_PAD (enc), buffer); +} + +static GstFlowReturn +gst_celt_enc_pre_push (GstAudioEncoder * benc, GstBuffer ** buffer) +{ + GstCeltEnc *enc; + GstFlowReturn ret = GST_FLOW_OK; + + enc = GST_CELT_ENC (benc); + + /* FIXME 0.11 ? get rid of this special ogg stuff and have it + * put and use 'codec data' in caps like anything else, + * with all the usual out-of-band advantage etc */ + if (G_UNLIKELY (enc->headers)) { + GSList *header = enc->headers; + + /* try to push all of these, if we lose one, might as well lose all */ + while (header) { + if (ret == GST_FLOW_OK) + ret = gst_celt_enc_push_buffer (enc, header->data); + else + gst_celt_enc_push_buffer (enc, header->data); + header = g_slist_next (header); + } + + g_slist_free (enc->headers); + enc->headers = NULL; + } + + return ret; +} static void gst_celt_enc_get_property (GObject * object, guint prop_id, GValue * value, @@ -1133,55 +771,3 @@ gst_celt_enc_set_property (GObject * object, guint prop_id, break; } } - -static GstStateChangeReturn -gst_celt_enc_change_state (GstElement * element, GstStateChange transition) -{ - GstCeltEnc *enc = GST_CELT_ENC (element); - GstStateChangeReturn res; - - switch (transition) { - case GST_STATE_CHANGE_NULL_TO_READY: - break; - case GST_STATE_CHANGE_READY_TO_PAUSED: - enc->frameno = 0; - enc->samples_in = 0; - enc->frameno_out = 0; - enc->start_ts = GST_CLOCK_TIME_NONE; - enc->next_ts = GST_CLOCK_TIME_NONE; - enc->granulepos_offset = 0; - break; - case GST_STATE_CHANGE_PAUSED_TO_PLAYING: - /* fall through */ - default: - break; - } - - res = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); - if (res == GST_STATE_CHANGE_FAILURE) - return res; - - switch (transition) { - case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - enc->setup = FALSE; - enc->header_sent = FALSE; - if (enc->state) { - celt_encoder_destroy (enc->state); - enc->state = NULL; - } - if (enc->mode) { - celt_mode_destroy (enc->mode); - enc->mode = NULL; - } - memset (&enc->header, 0, sizeof (enc->header)); - break; - case GST_STATE_CHANGE_READY_TO_NULL: - gst_tag_setter_reset_tags (GST_TAG_SETTER (enc)); - default: - break; - } - - return res; -} diff --git a/ext/celt/gstceltenc.h b/ext/celt/gstceltenc.h index 86fbc36f37..d0174ffa7c 100644 --- a/ext/celt/gstceltenc.h +++ b/ext/celt/gstceltenc.h @@ -24,7 +24,7 @@ #include -#include +#include #include #include @@ -49,16 +49,11 @@ typedef struct _GstCeltEnc GstCeltEnc; typedef struct _GstCeltEncClass GstCeltEncClass; struct _GstCeltEnc { - GstElement element; - - /* pads */ - GstPad *sinkpad; - GstPad *srcpad; + GstAudioEncoder element; CELTHeader header; CELTMode *mode; CELTEncoder *state; - GstAdapter *adapter; gint bitrate; gint frame_size; @@ -72,26 +67,12 @@ struct _GstCeltEnc { gint channels; gint rate; - gboolean setup; gboolean header_sent; - gboolean eos; - - guint64 samples_in; - guint64 bytes_out; - - guint64 frameno; - guint64 frameno_out; - - GstClockTime start_ts; - GstClockTime next_ts; - guint64 granulepos_offset; + GSList *headers; }; struct _GstCeltEncClass { - GstElementClass parent_class; - - /* signals */ - void (*frame_encoded) (GstElement *element); + GstAudioEncoderClass parent_class; }; GType gst_celt_enc_get_type (void); From 2842e14263755e828873100728944686838d0034 Mon Sep 17 00:00:00 2001 From: Mark Nauwelaerts Date: Wed, 16 Nov 2011 19:18:50 +0100 Subject: [PATCH 16/25] celtdec: port to audiodecoder --- ext/celt/gstceltdec.c | 661 +++++++++--------------------------------- ext/celt/gstceltdec.h | 12 +- 2 files changed, 134 insertions(+), 539 deletions(-) diff --git a/ext/celt/gstceltdec.c b/ext/celt/gstceltdec.c index 6d3d9433c6..0813a26499 100644 --- a/ext/celt/gstceltdec.c +++ b/ext/celt/gstceltdec.c @@ -68,29 +68,15 @@ GST_STATIC_PAD_TEMPLATE ("sink", GST_STATIC_CAPS ("audio/x-celt") ); -GST_BOILERPLATE (GstCeltDec, gst_celt_dec, GstElement, GST_TYPE_ELEMENT); +GST_BOILERPLATE (GstCeltDec, gst_celt_dec, GstAudioDecoder, + GST_TYPE_AUDIO_DECODER); -static gboolean celt_dec_sink_event (GstPad * pad, GstEvent * event); -static GstFlowReturn celt_dec_chain (GstPad * pad, GstBuffer * buf); -static gboolean celt_dec_sink_setcaps (GstPad * pad, GstCaps * caps); -static GstStateChangeReturn celt_dec_change_state (GstElement * element, - GstStateChange transition); - -static gboolean celt_dec_src_event (GstPad * pad, GstEvent * event); -static gboolean celt_dec_src_query (GstPad * pad, GstQuery * query); -static gboolean celt_dec_sink_query (GstPad * pad, GstQuery * query); -static const GstQueryType *celt_get_src_query_types (GstPad * pad); -static const GstQueryType *celt_get_sink_query_types (GstPad * pad); -static gboolean celt_dec_convert (GstPad * pad, - GstFormat src_format, gint64 src_value, - GstFormat * dest_format, gint64 * dest_value); - -static GstFlowReturn celt_dec_chain_parse_data (GstCeltDec * dec, - GstBuffer * buf, GstClockTime timestamp, GstClockTime duration); -static GstFlowReturn celt_dec_chain_parse_header (GstCeltDec * dec, - GstBuffer * buf); -static GstFlowReturn celt_dec_chain_parse_comments (GstCeltDec * dec, - GstBuffer * buf); +static gboolean gst_celt_dec_start (GstAudioDecoder * dec); +static gboolean gst_celt_dec_stop (GstAudioDecoder * dec); +static gboolean gst_celt_dec_set_format (GstAudioDecoder * bdec, + GstCaps * caps); +static GstFlowReturn gst_celt_dec_handle_frame (GstAudioDecoder * dec, + GstBuffer * buffer); static void gst_celt_dec_base_init (gpointer g_class) @@ -110,11 +96,14 @@ gst_celt_dec_base_init (gpointer g_class) static void gst_celt_dec_class_init (GstCeltDecClass * klass) { - GstElementClass *gstelement_class; + GstAudioDecoderClass *gstbase_class; - gstelement_class = (GstElementClass *) klass; + gstbase_class = (GstAudioDecoderClass *) klass; - gstelement_class->change_state = GST_DEBUG_FUNCPTR (celt_dec_change_state); + gstbase_class->start = GST_DEBUG_FUNCPTR (gst_celt_dec_start); + gstbase_class->stop = GST_DEBUG_FUNCPTR (gst_celt_dec_stop); + gstbase_class->set_format = GST_DEBUG_FUNCPTR (gst_celt_dec_set_format); + gstbase_class->handle_frame = GST_DEBUG_FUNCPTR (gst_celt_dec_handle_frame); GST_DEBUG_CATEGORY_INIT (celtdec_debug, "celtdec", 0, "celt decoding element"); @@ -123,11 +112,8 @@ gst_celt_dec_class_init (GstCeltDecClass * klass) static void gst_celt_dec_reset (GstCeltDec * dec) { - gst_segment_init (&dec->segment, GST_FORMAT_UNDEFINED); - dec->granulepos = -1; dec->packetno = 0; dec->frame_size = 0; - dec->frame_duration = 0; if (dec->state) { celt_decoder_destroy (dec->state); dec->state = NULL; @@ -150,411 +136,36 @@ gst_celt_dec_reset (GstCeltDec * dec) static void gst_celt_dec_init (GstCeltDec * dec, GstCeltDecClass * g_class) { - dec->sinkpad = - gst_pad_new_from_static_template (&celt_dec_sink_factory, "sink"); - gst_pad_set_chain_function (dec->sinkpad, GST_DEBUG_FUNCPTR (celt_dec_chain)); - gst_pad_set_event_function (dec->sinkpad, - GST_DEBUG_FUNCPTR (celt_dec_sink_event)); - gst_pad_set_query_type_function (dec->sinkpad, - GST_DEBUG_FUNCPTR (celt_get_sink_query_types)); - gst_pad_set_query_function (dec->sinkpad, - GST_DEBUG_FUNCPTR (celt_dec_sink_query)); - gst_pad_set_setcaps_function (dec->sinkpad, - GST_DEBUG_FUNCPTR (celt_dec_sink_setcaps)); - gst_element_add_pad (GST_ELEMENT (dec), dec->sinkpad); - - dec->srcpad = gst_pad_new_from_static_template (&celt_dec_src_factory, "src"); - gst_pad_use_fixed_caps (dec->srcpad); - gst_pad_set_event_function (dec->srcpad, - GST_DEBUG_FUNCPTR (celt_dec_src_event)); - gst_pad_set_query_type_function (dec->srcpad, - GST_DEBUG_FUNCPTR (celt_get_src_query_types)); - gst_pad_set_query_function (dec->srcpad, - GST_DEBUG_FUNCPTR (celt_dec_src_query)); - gst_element_add_pad (GST_ELEMENT (dec), dec->srcpad); - gst_celt_dec_reset (dec); } static gboolean -celt_dec_sink_setcaps (GstPad * pad, GstCaps * caps) +gst_celt_dec_start (GstAudioDecoder * dec) { - GstCeltDec *dec = GST_CELT_DEC (gst_pad_get_parent (pad)); - gboolean ret = TRUE; - GstStructure *s; - const GValue *streamheader; + GstCeltDec *cd = GST_CELT_DEC (dec); - s = gst_caps_get_structure (caps, 0); - if ((streamheader = gst_structure_get_value (s, "streamheader")) && - G_VALUE_HOLDS (streamheader, GST_TYPE_ARRAY) && - gst_value_array_get_size (streamheader) >= 2) { - const GValue *header, *vorbiscomment; - GstBuffer *buf; - GstFlowReturn res = GST_FLOW_OK; + GST_DEBUG_OBJECT (dec, "start"); + gst_celt_dec_reset (cd); - header = gst_value_array_get_value (streamheader, 0); - if (header && G_VALUE_HOLDS (header, GST_TYPE_BUFFER)) { - buf = gst_value_get_buffer (header); - res = celt_dec_chain_parse_header (dec, buf); - if (res != GST_FLOW_OK) - goto done; - gst_buffer_replace (&dec->streamheader, buf); - } + /* we know about concealment */ + gst_audio_decoder_set_plc_aware (dec, TRUE); - vorbiscomment = gst_value_array_get_value (streamheader, 1); - if (vorbiscomment && G_VALUE_HOLDS (vorbiscomment, GST_TYPE_BUFFER)) { - buf = gst_value_get_buffer (vorbiscomment); - res = celt_dec_chain_parse_comments (dec, buf); - if (res != GST_FLOW_OK) - goto done; - gst_buffer_replace (&dec->vorbiscomment, buf); - } - - g_list_foreach (dec->extra_headers, (GFunc) gst_mini_object_unref, NULL); - g_list_free (dec->extra_headers); - dec->extra_headers = NULL; - - if (gst_value_array_get_size (streamheader) > 2) { - gint i, n; - - n = gst_value_array_get_size (streamheader); - for (i = 2; i < n; i++) { - header = gst_value_array_get_value (streamheader, i); - buf = gst_value_get_buffer (header); - dec->extra_headers = - g_list_prepend (dec->extra_headers, gst_buffer_ref (buf)); - } - } - } - -done: - gst_object_unref (dec); - return ret; + return TRUE; } static gboolean -celt_dec_convert (GstPad * pad, - GstFormat src_format, gint64 src_value, - GstFormat * dest_format, gint64 * dest_value) +gst_celt_dec_stop (GstAudioDecoder * dec) { - gboolean res = TRUE; - GstCeltDec *dec; - guint64 scale = 1; + GstCeltDec *cd = GST_CELT_DEC (dec); - dec = GST_CELT_DEC (gst_pad_get_parent (pad)); + GST_DEBUG_OBJECT (dec, "stop"); + gst_celt_dec_reset (cd); - if (dec->packetno < 1) { - res = FALSE; - goto cleanup; - } - - if (src_format == *dest_format) { - *dest_value = src_value; - res = TRUE; - goto cleanup; - } - - if (pad == dec->sinkpad && - (src_format == GST_FORMAT_BYTES || *dest_format == GST_FORMAT_BYTES)) { - res = FALSE; - goto cleanup; - } - - switch (src_format) { - case GST_FORMAT_TIME: - switch (*dest_format) { - case GST_FORMAT_BYTES: - scale = sizeof (gint16) * dec->header.nb_channels; - case GST_FORMAT_DEFAULT: - *dest_value = - gst_util_uint64_scale_int (scale * src_value, - dec->header.sample_rate, GST_SECOND); - break; - default: - res = FALSE; - break; - } - break; - case GST_FORMAT_DEFAULT: - switch (*dest_format) { - case GST_FORMAT_BYTES: - *dest_value = src_value * sizeof (gint16) * dec->header.nb_channels; - break; - case GST_FORMAT_TIME: - *dest_value = - gst_util_uint64_scale_int (src_value, GST_SECOND, - dec->header.sample_rate); - break; - default: - res = FALSE; - break; - } - break; - case GST_FORMAT_BYTES: - switch (*dest_format) { - case GST_FORMAT_DEFAULT: - *dest_value = src_value / (sizeof (gint16) * dec->header.nb_channels); - break; - case GST_FORMAT_TIME: - *dest_value = gst_util_uint64_scale_int (src_value, GST_SECOND, - dec->header.sample_rate * sizeof (gint16) * - dec->header.nb_channels); - break; - default: - res = FALSE; - break; - } - break; - default: - res = FALSE; - break; - } - -cleanup: - gst_object_unref (dec); - return res; -} - -static const GstQueryType * -celt_get_sink_query_types (GstPad * pad) -{ - static const GstQueryType celt_dec_sink_query_types[] = { - GST_QUERY_CONVERT, - 0 - }; - - return celt_dec_sink_query_types; -} - -static gboolean -celt_dec_sink_query (GstPad * pad, GstQuery * query) -{ - GstCeltDec *dec; - gboolean res; - - dec = GST_CELT_DEC (gst_pad_get_parent (pad)); - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_CONVERT: - { - GstFormat src_fmt, dest_fmt; - gint64 src_val, dest_val; - - gst_query_parse_convert (query, &src_fmt, &src_val, &dest_fmt, &dest_val); - res = celt_dec_convert (pad, src_fmt, src_val, &dest_fmt, &dest_val); - if (res) { - gst_query_set_convert (query, src_fmt, src_val, dest_fmt, dest_val); - } - break; - } - default: - res = gst_pad_query_default (pad, query); - break; - } - - gst_object_unref (dec); - return res; -} - -static const GstQueryType * -celt_get_src_query_types (GstPad * pad) -{ - static const GstQueryType celt_dec_src_query_types[] = { - GST_QUERY_POSITION, - GST_QUERY_DURATION, - 0 - }; - - return celt_dec_src_query_types; -} - -static gboolean -celt_dec_src_query (GstPad * pad, GstQuery * query) -{ - GstCeltDec *dec; - gboolean res = FALSE; - - dec = GST_CELT_DEC (gst_pad_get_parent (pad)); - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_POSITION:{ - GstSegment segment; - GstFormat format; - gint64 cur; - - gst_query_parse_position (query, &format, NULL); - - GST_PAD_STREAM_LOCK (dec->sinkpad); - segment = dec->segment; - GST_PAD_STREAM_UNLOCK (dec->sinkpad); - - if (segment.format != GST_FORMAT_TIME) { - GST_DEBUG_OBJECT (dec, "segment not initialised yet"); - break; - } - - if ((res = celt_dec_convert (dec->srcpad, GST_FORMAT_TIME, - segment.last_stop, &format, &cur))) { - gst_query_set_position (query, format, cur); - } - break; - } - case GST_QUERY_DURATION:{ - GstFormat format = GST_FORMAT_TIME; - gint64 dur; - - /* get duration from demuxer */ - if (!gst_pad_query_peer_duration (dec->sinkpad, &format, &dur)) - break; - - gst_query_parse_duration (query, &format, NULL); - - /* and convert it into the requested format */ - if ((res = celt_dec_convert (dec->srcpad, GST_FORMAT_TIME, - dur, &format, &dur))) { - gst_query_set_duration (query, format, dur); - } - break; - } - default: - res = gst_pad_query_default (pad, query); - break; - } - - gst_object_unref (dec); - return res; -} - -static gboolean -celt_dec_src_event (GstPad * pad, GstEvent * event) -{ - gboolean res = FALSE; - GstCeltDec *dec = GST_CELT_DEC (gst_pad_get_parent (pad)); - - GST_LOG_OBJECT (dec, "handling %s event", GST_EVENT_TYPE_NAME (event)); - - switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_SEEK:{ - GstFormat format, tformat; - gdouble rate; - GstEvent *real_seek; - GstSeekFlags flags; - GstSeekType cur_type, stop_type; - gint64 cur, stop; - gint64 tcur, tstop; - - gst_event_parse_seek (event, &rate, &format, &flags, &cur_type, &cur, - &stop_type, &stop); - - /* we have to ask our peer to seek to time here as we know - * nothing about how to generate a granulepos from the src - * formats or anything. - * - * First bring the requested format to time - */ - tformat = GST_FORMAT_TIME; - if (!(res = celt_dec_convert (pad, format, cur, &tformat, &tcur))) - break; - if (!(res = celt_dec_convert (pad, format, stop, &tformat, &tstop))) - break; - - /* then seek with time on the peer */ - real_seek = gst_event_new_seek (rate, GST_FORMAT_TIME, - flags, cur_type, tcur, stop_type, tstop); - - GST_LOG_OBJECT (dec, "seek to %" GST_TIME_FORMAT, GST_TIME_ARGS (tcur)); - - res = gst_pad_push_event (dec->sinkpad, real_seek); - gst_event_unref (event); - break; - } - default: - res = gst_pad_event_default (pad, event); - break; - } - - gst_object_unref (dec); - return res; -} - -static gboolean -celt_dec_sink_event (GstPad * pad, GstEvent * event) -{ - GstCeltDec *dec; - gboolean ret = FALSE; - - dec = GST_CELT_DEC (gst_pad_get_parent (pad)); - - GST_LOG_OBJECT (dec, "handling %s event", GST_EVENT_TYPE_NAME (event)); - - switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_NEWSEGMENT:{ - GstFormat format; - gdouble rate, arate; - gint64 start, stop, time; - gboolean update; - - gst_event_parse_new_segment_full (event, &update, &rate, &arate, &format, - &start, &stop, &time); - - if (format != GST_FORMAT_TIME) - goto newseg_wrong_format; - - if (rate <= 0.0) - goto newseg_wrong_rate; - - if (update) { - /* time progressed without data, see if we can fill the gap with - * some concealment data */ - if (dec->segment.last_stop < start) { - GstClockTime duration; - - duration = start - dec->segment.last_stop; - celt_dec_chain_parse_data (dec, NULL, dec->segment.last_stop, - duration); - } - } - - /* now configure the values */ - gst_segment_set_newsegment_full (&dec->segment, update, - rate, arate, GST_FORMAT_TIME, start, stop, time); - - dec->granulepos = -1; - - GST_DEBUG_OBJECT (dec, "segment now: cur = %" GST_TIME_FORMAT " [%" - GST_TIME_FORMAT " - %" GST_TIME_FORMAT "]", - GST_TIME_ARGS (dec->segment.last_stop), - GST_TIME_ARGS (dec->segment.start), - GST_TIME_ARGS (dec->segment.stop)); - - ret = gst_pad_push_event (dec->srcpad, event); - break; - } - default: - ret = gst_pad_event_default (pad, event); - break; - } - - gst_object_unref (dec); - return ret; - - /* ERRORS */ -newseg_wrong_format: - { - GST_DEBUG_OBJECT (dec, "received non TIME newsegment"); - gst_object_unref (dec); - return FALSE; - } -newseg_wrong_rate: - { - GST_DEBUG_OBJECT (dec, "negative rates not supported yet"); - gst_object_unref (dec); - return FALSE; - } + return TRUE; } static GstFlowReturn -celt_dec_chain_parse_header (GstCeltDec * dec, GstBuffer * buf) +gst_celt_dec_parse_header (GstCeltDec * dec, GstBuffer * buf) { GstCaps *caps; gint error = CELT_OK; @@ -601,9 +212,6 @@ celt_dec_chain_parse_header (GstCeltDec * dec, GstBuffer * buf) celt_mode_info (dec->mode, CELT_GET_FRAME_SIZE, &dec->frame_size); #endif - dec->frame_duration = gst_util_uint64_scale_int (dec->frame_size, - GST_SECOND, dec->header.sample_rate); - /* set caps */ caps = gst_caps_new_simple ("audio/x-raw-int", "rate", G_TYPE_INT, dec->header.sample_rate, @@ -615,7 +223,7 @@ celt_dec_chain_parse_header (GstCeltDec * dec, GstBuffer * buf) GST_DEBUG_OBJECT (dec, "rate=%d channels=%d frame-size=%d", dec->header.sample_rate, dec->header.nb_channels, dec->frame_size); - if (!gst_pad_set_caps (dec->srcpad, caps)) + if (!gst_pad_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), caps)) goto nego_failed; gst_caps_unref (caps); @@ -655,7 +263,7 @@ nego_failed: } static GstFlowReturn -celt_dec_chain_parse_comments (GstCeltDec * dec, GstBuffer * buf) +gst_celt_dec_parse_comments (GstCeltDec * dec, GstBuffer * buf) { GstTagList *list; gchar *ver, *encoder = NULL; @@ -690,7 +298,8 @@ celt_dec_chain_parse_comments (GstCeltDec * dec, GstBuffer * buf) GST_INFO_OBJECT (dec, "tags: %" GST_PTR_FORMAT, list); - gst_element_found_tags_for_pad (GST_ELEMENT (dec), dec->srcpad, list); + gst_element_found_tags_for_pad (GST_ELEMENT (dec), + GST_AUDIO_DECODER_SRC_PAD (dec), list); g_free (encoder); g_free (ver); @@ -699,8 +308,7 @@ celt_dec_chain_parse_comments (GstCeltDec * dec, GstBuffer * buf) } static GstFlowReturn -celt_dec_chain_parse_data (GstCeltDec * dec, GstBuffer * buf, - GstClockTime timestamp, GstClockTime duration) +gst_celt_dec_parse_data (GstCeltDec * dec, GstBuffer * buf) { GstFlowReturn res = GST_FLOW_OK; gint size; @@ -710,33 +318,23 @@ celt_dec_chain_parse_data (GstCeltDec * dec, GstBuffer * buf, gint error = CELT_OK; int skip = 0; - if (timestamp != -1) { - dec->segment.last_stop = timestamp; - dec->granulepos = -1; - } + if (!dec->frame_size) + goto not_negotiated; - if (buf) { + if (G_LIKELY (GST_BUFFER_SIZE (buf))) { data = GST_BUFFER_DATA (buf); size = GST_BUFFER_SIZE (buf); - - GST_DEBUG_OBJECT (dec, "received buffer of size %u", size); - if (!GST_BUFFER_TIMESTAMP_IS_VALID (buf) - && GST_BUFFER_OFFSET_END_IS_VALID (buf)) { - dec->granulepos = GST_BUFFER_OFFSET_END (buf); - GST_DEBUG_OBJECT (dec, - "Taking granulepos from upstream: %" G_GUINT64_FORMAT, - dec->granulepos); - } - - /* copy timestamp */ } else { + /* FIXME ? actually consider how much concealment is needed */ /* concealment data, pass NULL as the bits parameters */ GST_DEBUG_OBJECT (dec, "creating concealment data"); data = NULL; size = 0; } - if (dec->discont) { + /* FIXME really needed ?; this might lead to skipping samples below + * which kind of messes with subsequent timestamping */ + if (G_UNLIKELY (GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT))) { #ifdef CELT_GET_LOOKAHEAD_REQUEST /* what will be 0.11.5, I guess, but no versioning yet in git */ celt_decoder_ctl (dec->state, CELT_GET_LOOKAHEAD_REQUEST, &skip); @@ -745,9 +343,9 @@ celt_dec_chain_parse_data (GstCeltDec * dec, GstBuffer * buf, #endif } - res = gst_pad_alloc_buffer_and_set_caps (dec->srcpad, + res = gst_pad_alloc_buffer_and_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), GST_BUFFER_OFFSET_NONE, dec->frame_size * dec->header.nb_channels * 2, - GST_PAD_CAPS (dec->srcpad), &outbuf); + GST_PAD_CAPS (GST_AUDIO_DECODER_SRC_PAD (dec)), &outbuf); if (res != GST_FLOW_OK) { GST_DEBUG_OBJECT (dec, "buf alloc flow: %s", gst_flow_get_name (res)); @@ -780,59 +378,88 @@ celt_dec_chain_parse_data (GstCeltDec * dec, GstBuffer * buf, skip * dec->header.nb_channels * 2; } - if (dec->granulepos == -1) { - if (dec->segment.format != GST_FORMAT_TIME) { - GST_WARNING_OBJECT (dec, "segment not initialized or not TIME format"); - dec->granulepos = dec->frame_size; - } else { - dec->granulepos = gst_util_uint64_scale_int (dec->segment.last_stop, - dec->header.sample_rate, GST_SECOND) + dec->frame_size; - } - GST_DEBUG_OBJECT (dec, "granulepos=%" G_GINT64_FORMAT, dec->granulepos); - } - - if (!GST_CLOCK_TIME_IS_VALID (timestamp)) - timestamp = gst_util_uint64_scale_int (dec->granulepos - dec->frame_size, - GST_SECOND, dec->header.sample_rate); - - GST_DEBUG_OBJECT (dec, "timestamp=%" GST_TIME_FORMAT, - GST_TIME_ARGS (timestamp)); - - GST_BUFFER_OFFSET (outbuf) = dec->granulepos - dec->frame_size; - GST_BUFFER_OFFSET_END (outbuf) = dec->granulepos; - GST_BUFFER_TIMESTAMP (outbuf) = timestamp; - GST_BUFFER_DURATION (outbuf) = dec->frame_duration; - if (dec->discont) { - GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DISCONT); - dec->discont = 0; - } - - dec->granulepos += dec->frame_size; - dec->segment.last_stop += dec->frame_duration; - - GST_LOG_OBJECT (dec, "pushing buffer with ts=%" GST_TIME_FORMAT ", dur=%" - GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)), - GST_TIME_ARGS (dec->frame_duration)); - - res = gst_pad_push (dec->srcpad, outbuf); + res = gst_audio_decoder_finish_frame (GST_AUDIO_DECODER (dec), outbuf, 1); if (res != GST_FLOW_OK) GST_DEBUG_OBJECT (dec, "flow: %s", gst_flow_get_name (res)); return res; + + /* ERRORS */ +not_negotiated: + { + GST_ELEMENT_ERROR (dec, CORE, NEGOTIATION, (NULL), + ("decoder not initialized")); + return GST_FLOW_NOT_NEGOTIATED; + } +} + +static gboolean +gst_celt_dec_set_format (GstAudioDecoder * bdec, GstCaps * caps) +{ + GstCeltDec *dec = GST_CELT_DEC (bdec); + gboolean ret = TRUE; + GstStructure *s; + const GValue *streamheader; + + s = gst_caps_get_structure (caps, 0); + if ((streamheader = gst_structure_get_value (s, "streamheader")) && + G_VALUE_HOLDS (streamheader, GST_TYPE_ARRAY) && + gst_value_array_get_size (streamheader) >= 2) { + const GValue *header, *vorbiscomment; + GstBuffer *buf; + GstFlowReturn res = GST_FLOW_OK; + + header = gst_value_array_get_value (streamheader, 0); + if (header && G_VALUE_HOLDS (header, GST_TYPE_BUFFER)) { + buf = gst_value_get_buffer (header); + res = gst_celt_dec_parse_header (dec, buf); + if (res != GST_FLOW_OK) + goto done; + gst_buffer_replace (&dec->streamheader, buf); + } + + vorbiscomment = gst_value_array_get_value (streamheader, 1); + if (vorbiscomment && G_VALUE_HOLDS (vorbiscomment, GST_TYPE_BUFFER)) { + buf = gst_value_get_buffer (vorbiscomment); + res = gst_celt_dec_parse_comments (dec, buf); + if (res != GST_FLOW_OK) + goto done; + gst_buffer_replace (&dec->vorbiscomment, buf); + } + + g_list_foreach (dec->extra_headers, (GFunc) gst_mini_object_unref, NULL); + g_list_free (dec->extra_headers); + dec->extra_headers = NULL; + + if (gst_value_array_get_size (streamheader) > 2) { + gint i, n; + + n = gst_value_array_get_size (streamheader); + for (i = 2; i < n; i++) { + header = gst_value_array_get_value (streamheader, i); + buf = gst_value_get_buffer (header); + dec->extra_headers = + g_list_prepend (dec->extra_headers, gst_buffer_ref (buf)); + } + } + } + +done: + return ret; } static GstFlowReturn -celt_dec_chain (GstPad * pad, GstBuffer * buf) +gst_celt_dec_handle_frame (GstAudioDecoder * bdec, GstBuffer * buf) { GstFlowReturn res; GstCeltDec *dec; - dec = GST_CELT_DEC (gst_pad_get_parent (pad)); + dec = GST_CELT_DEC (bdec); - if (GST_BUFFER_IS_DISCONT (buf)) { - dec->discont = TRUE; - } + /* no fancy draining */ + if (G_UNLIKELY (!buf)) + return GST_FLOW_OK; /* If we have the streamheader and vorbiscomment from the caps already * ignore them here */ @@ -840,10 +467,14 @@ celt_dec_chain (GstPad * pad, GstBuffer * buf) if (GST_BUFFER_SIZE (dec->streamheader) == GST_BUFFER_SIZE (buf) && memcmp (GST_BUFFER_DATA (dec->streamheader), GST_BUFFER_DATA (buf), GST_BUFFER_SIZE (buf)) == 0) { + GST_DEBUG_OBJECT (dec, "found streamheader"); + gst_audio_decoder_finish_frame (bdec, NULL, 1); res = GST_FLOW_OK; } else if (GST_BUFFER_SIZE (dec->vorbiscomment) == GST_BUFFER_SIZE (buf) && memcmp (GST_BUFFER_DATA (dec->vorbiscomment), GST_BUFFER_DATA (buf), GST_BUFFER_SIZE (buf)) == 0) { + GST_DEBUG_OBJECT (dec, "found vorbiscomments"); + gst_audio_decoder_finish_frame (bdec, NULL, 1); res = GST_FLOW_OK; } else { GList *l; @@ -853,66 +484,36 @@ celt_dec_chain (GstPad * pad, GstBuffer * buf) if (GST_BUFFER_SIZE (header) == GST_BUFFER_SIZE (buf) && memcmp (GST_BUFFER_DATA (header), GST_BUFFER_DATA (buf), GST_BUFFER_SIZE (buf)) == 0) { + GST_DEBUG_OBJECT (dec, "found extra header buffer"); + gst_audio_decoder_finish_frame (bdec, NULL, 1); res = GST_FLOW_OK; goto done; } } - res = - celt_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), - GST_BUFFER_DURATION (buf)); + res = gst_celt_dec_parse_data (dec, buf); } } else { /* Otherwise fall back to packet counting and assume that the * first two packets are the headers. */ - if (dec->packetno == 0) - res = celt_dec_chain_parse_header (dec, buf); - else if (dec->packetno == 1) - res = celt_dec_chain_parse_comments (dec, buf); - else if (dec->packetno <= 1 + dec->header.extra_headers) + if (dec->packetno == 0) { + GST_DEBUG_OBJECT (dec, "counted streamheader"); + res = gst_celt_dec_parse_header (dec, buf); + gst_audio_decoder_finish_frame (bdec, NULL, 1); + } else if (dec->packetno == 1) { + GST_DEBUG_OBJECT (dec, "counted vorbiscomments"); + res = gst_celt_dec_parse_comments (dec, buf); + gst_audio_decoder_finish_frame (bdec, NULL, 1); + } else if (dec->packetno <= 1 + dec->header.extra_headers) { + GST_DEBUG_OBJECT (dec, "counted extra header"); + gst_audio_decoder_finish_frame (bdec, NULL, 1); res = GST_FLOW_OK; - else - res = celt_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), - GST_BUFFER_DURATION (buf)); + } else { + res = gst_celt_dec_parse_data (dec, buf); + } } done: dec->packetno++; - gst_buffer_unref (buf); - gst_object_unref (dec); - return res; } - -static GstStateChangeReturn -celt_dec_change_state (GstElement * element, GstStateChange transition) -{ - GstStateChangeReturn ret; - GstCeltDec *dec = GST_CELT_DEC (element); - - switch (transition) { - case GST_STATE_CHANGE_NULL_TO_READY: - case GST_STATE_CHANGE_READY_TO_PAUSED: - case GST_STATE_CHANGE_PAUSED_TO_PLAYING: - default: - break; - } - - ret = parent_class->change_state (element, transition); - if (ret != GST_STATE_CHANGE_SUCCESS) - return ret; - - switch (transition) { - case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - gst_celt_dec_reset (dec); - break; - case GST_STATE_CHANGE_READY_TO_NULL: - break; - default: - break; - } - - return ret; -} diff --git a/ext/celt/gstceltdec.h b/ext/celt/gstceltdec.h index b6b49605df..9baf719d90 100644 --- a/ext/celt/gstceltdec.h +++ b/ext/celt/gstceltdec.h @@ -22,6 +22,7 @@ #define __GST_CELT_DEC_H__ #include +#include #include #include @@ -42,22 +43,15 @@ typedef struct _GstCeltDec GstCeltDec; typedef struct _GstCeltDecClass GstCeltDecClass; struct _GstCeltDec { - GstElement element; - - /* pads */ - GstPad *sinkpad; - GstPad *srcpad; + GstAudioDecoder element; CELTDecoder *state; CELTMode *mode; CELTHeader header; gint frame_size; - GstClockTime frame_duration; guint64 packetno; - GstSegment segment; /* STREAM LOCK */ - gint64 granulepos; /* -1 = needs to be set from current time */ gboolean discont; GstBuffer *streamheader; @@ -66,7 +60,7 @@ struct _GstCeltDec { }; struct _GstCeltDecClass { - GstElementClass parent_class; + GstAudioDecoderClass parent_class; }; GType gst_celt_dec_get_type (void); From 714ced7d19324b18e26eac2ee91224b198dd4ba7 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 16 Nov 2011 18:35:29 +0000 Subject: [PATCH 17/25] opusdec: let the base class handle all timing --- ext/opus/gstopusdec.c | 33 +++++---------------------------- ext/opus/gstopusdec.h | 1 - 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/ext/opus/gstopusdec.c b/ext/opus/gstopusdec.c index d4cdabdded..624ac81e42 100644 --- a/ext/opus/gstopusdec.c +++ b/ext/opus/gstopusdec.c @@ -120,8 +120,6 @@ gst_opus_dec_reset (GstOpusDec * dec) dec->state = NULL; } - dec->next_ts = 0; - gst_buffer_replace (&dec->streamheader, NULL); gst_buffer_replace (&dec->vorbiscomment, NULL); } @@ -213,8 +211,7 @@ gst_opus_dec_setup_from_peer_caps (GstOpusDec * dec) } static GstFlowReturn -opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, - GstClockTime timestamp, GstClockTime duration) +opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf) { GstFlowReturn res = GST_FLOW_OK; gint size; @@ -240,8 +237,6 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, size = GST_BUFFER_SIZE (buf); GST_DEBUG_OBJECT (dec, "received buffer of size %u", size); - - /* copy timestamp */ } else { /* concealment data, pass NULL as the bits parameters */ GST_DEBUG_OBJECT (dec, "creating concealment data"); @@ -276,20 +271,6 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf, } GST_DEBUG_OBJECT (dec, "decoded %d samples", n); - if (GST_CLOCK_TIME_IS_VALID (timestamp)) { - GST_BUFFER_TIMESTAMP (outbuf) = timestamp; - } else { - GST_BUFFER_TIMESTAMP (outbuf) = dec->next_ts; - } - - GST_BUFFER_DURATION (outbuf) = - gst_util_uint64_scale (n, GST_SECOND, dec->sample_rate); - dec->next_ts = GST_BUFFER_TIMESTAMP (outbuf) + GST_BUFFER_DURATION (outbuf); - - GST_LOG_OBJECT (dec, "pushing buffer with ts=%" GST_TIME_FORMAT ", dur=%" - GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)), - GST_TIME_ARGS (GST_BUFFER_DURATION (outbuf))); - res = gst_audio_decoder_finish_frame (GST_AUDIO_DECODER (dec), outbuf, 1); if (res != GST_FLOW_OK) @@ -392,8 +373,7 @@ gst_opus_dec_handle_frame (GstAudioDecoder * adec, GstBuffer * buf) gst_audio_decoder_finish_frame (adec, NULL, 1); res = GST_FLOW_OK; } else { - res = opus_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), - GST_BUFFER_DURATION (buf)); + res = opus_dec_chain_parse_data (dec, buf); } } else { /* Otherwise fall back to packet counting and assume that the @@ -405,8 +385,7 @@ gst_opus_dec_handle_frame (GstAudioDecoder * adec, GstBuffer * buf) res = gst_opus_dec_parse_header (dec, buf); gst_audio_decoder_finish_frame (adec, NULL, 1); } else { - res = opus_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), - GST_BUFFER_DURATION (buf)); + res = opus_dec_chain_parse_data (dec, buf); } break; case 1: @@ -415,14 +394,12 @@ gst_opus_dec_handle_frame (GstAudioDecoder * adec, GstBuffer * buf) res = gst_opus_dec_parse_comments (dec, buf); gst_audio_decoder_finish_frame (adec, NULL, 1); } else { - res = opus_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), - GST_BUFFER_DURATION (buf)); + res = opus_dec_chain_parse_data (dec, buf); } break; default: { - res = opus_dec_chain_parse_data (dec, buf, GST_BUFFER_TIMESTAMP (buf), - GST_BUFFER_DURATION (buf)); + res = opus_dec_chain_parse_data (dec, buf); break; } } diff --git a/ext/opus/gstopusdec.h b/ext/opus/gstopusdec.h index 38dd2799ad..9e78330e1f 100644 --- a/ext/opus/gstopusdec.h +++ b/ext/opus/gstopusdec.h @@ -47,7 +47,6 @@ struct _GstOpusDec { OpusDecoder *state; guint64 packetno; - GstClockTime next_ts; GstBuffer *streamheader; GstBuffer *vorbiscomment; From f3f9e4b9786d00299e0c6909180c4cd326489bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 16 Nov 2011 10:38:49 -0800 Subject: [PATCH 18/25] h264parse: Implement ::sink_get_caps to allow stream-format conversion again Just proxying the downstream caps will prevent h264parse from accepting a different stream-format than what is supported downstream, although it could convert to a different stream-format. --- gst/videoparsers/gsth264parse.c | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c index 2127125f40..6fdad115ba 100644 --- a/gst/videoparsers/gsth264parse.c +++ b/gst/videoparsers/gsth264parse.c @@ -91,6 +91,7 @@ static void gst_h264_parse_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static gboolean gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps); +static GstCaps *gst_h264_parse_get_caps (GstBaseParse * parse); static GstFlowReturn gst_h264_parse_chain (GstPad * pad, GstBuffer * buffer); static void @@ -138,6 +139,7 @@ gst_h264_parse_class_init (GstH264ParseClass * klass) parse_class->pre_push_frame = GST_DEBUG_FUNCPTR (gst_h264_parse_pre_push_frame); parse_class->set_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_set_caps); + parse_class->get_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_get_caps); } static void @@ -1336,6 +1338,38 @@ refuse_caps: } } +static GstCaps * +gst_h264_parse_get_caps (GstBaseParse * parse) +{ + GstCaps *peercaps; + GstCaps *res; + + peercaps = gst_pad_get_allowed_caps (GST_BASE_PARSE_SRC_PAD (parse)); + if (peercaps) { + guint i, n; + + peercaps = gst_caps_make_writable (peercaps); + n = gst_caps_get_size (peercaps); + for (i = 0; i < n; i++) { + GstStructure *s = gst_caps_get_structure (peercaps, i); + gst_structure_remove_field (s, "alignment"); + gst_structure_remove_field (s, "stream-format"); + } + + res = + gst_caps_intersect_full (peercaps, + gst_pad_get_pad_template_caps (GST_BASE_PARSE_SRC_PAD (parse)), + GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (peercaps); + } else { + res = + gst_caps_copy (gst_pad_get_pad_template_caps (GST_BASE_PARSE_SRC_PAD + (parse))); + } + + return res; +} + static GstFlowReturn gst_h264_parse_chain (GstPad * pad, GstBuffer * buffer) { From af8771ef2fda4f2374f80ca23a637d85f1a766f6 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 16 Nov 2011 18:43:53 +0000 Subject: [PATCH 19/25] opusenc: fix constrained-vbr property name typo --- ext/opus/gstopusenc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opus/gstopusenc.c b/ext/opus/gstopusenc.c index 4be63cb882..838e7826f6 100644 --- a/ext/opus/gstopusenc.c +++ b/ext/opus/gstopusenc.c @@ -226,7 +226,7 @@ gst_opus_enc_class_init (GstOpusEncClass * klass) "Constant bit rate", DEFAULT_CBR, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_CONSTRAINED_VBR, - g_param_spec_boolean ("constrained-cbr", "Constrained VBR", + g_param_spec_boolean ("constrained-vbr", "Constrained VBR", "Constrained VBR", DEFAULT_CONSTRAINED_VBR, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_COMPLEXITY, From 15b5fecf303b2cd76116b90b5d85142f42f83538 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 16 Nov 2011 18:49:03 +0000 Subject: [PATCH 20/25] opusenc: do not include variable fields in caps Those can vary from one packet to the next, so have no reason to be in the caps. --- ext/opus/gstopusenc.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ext/opus/gstopusenc.c b/ext/opus/gstopusenc.c index 838e7826f6..f632a72056 100644 --- a/ext/opus/gstopusenc.c +++ b/ext/opus/gstopusenc.c @@ -93,9 +93,7 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS ("audio/x-opus, " - "rate = (int) { 8000, 12000, 16000, 24000, 48000 }, " - "channels = (int) [ 1, 2 ], " "frame-size = (int) [ 2, 60 ]") + GST_STATIC_CAPS ("audio/x-opus") ); #define DEFAULT_AUDIO TRUE @@ -695,10 +693,7 @@ gst_opus_enc_handle_frame (GstAudioEncoder * benc, GstBuffer * buf) buf2 = gst_opus_enc_create_metadata_buffer (enc); /* mark and put on caps */ - caps = - gst_caps_new_simple ("audio/x-opus", "rate", G_TYPE_INT, - enc->sample_rate, "channels", G_TYPE_INT, enc->n_channels, "frame-size", - G_TYPE_INT, enc->frame_size, NULL); + caps = gst_caps_from_string ("audio/x-opus"); caps = _gst_caps_set_buffer_array (caps, "streamheader", buf1, buf2, NULL); /* negotiate with these caps */ From ec089662de70902e5751738573c453c57c5aac81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 16 Nov 2011 10:45:52 -0800 Subject: [PATCH 21/25] mpeg4videoparse: Don't require parsed=false on the sinkpad caps --- gst/mpeg4videoparse/mpeg4videoparse.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gst/mpeg4videoparse/mpeg4videoparse.c b/gst/mpeg4videoparse/mpeg4videoparse.c index 3040b61ab6..ad7833d082 100644 --- a/gst/mpeg4videoparse/mpeg4videoparse.c +++ b/gst/mpeg4videoparse/mpeg4videoparse.c @@ -45,8 +45,7 @@ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS ("video/mpeg, " - "mpegversion = (int) 4, " - "parsed = (boolean) false, " "systemstream = (boolean) false") + "mpegversion = (int) 4, " "systemstream = (boolean) false") ); /* Properties */ From d746cd7d67346b7e3de716d1eb6ed164c7cff8ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 16 Nov 2011 10:57:55 -0800 Subject: [PATCH 22/25] mpeg4videoparse: Change rank to PRIMARY+1 --- gst/mpeg4videoparse/mpeg4videoparse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst/mpeg4videoparse/mpeg4videoparse.c b/gst/mpeg4videoparse/mpeg4videoparse.c index ad7833d082..4f3c7e3c1d 100644 --- a/gst/mpeg4videoparse/mpeg4videoparse.c +++ b/gst/mpeg4videoparse/mpeg4videoparse.c @@ -610,7 +610,7 @@ plugin_init (GstPlugin * plugin) GST_DEBUG_CATEGORY_INIT (mpeg4v_parse_debug, "mpeg4videoparse", 0, "MPEG-4 video parser"); - if (!gst_element_register (plugin, "mpeg4videoparse", GST_RANK_SECONDARY, + if (!gst_element_register (plugin, "mpeg4videoparse", GST_RANK_PRIMARY + 1, gst_mpeg4vparse_get_type ())) return FALSE; From 94534a0b831bf629f95cd5bf52840c5df6b41948 Mon Sep 17 00:00:00 2001 From: Stefan Sauer Date: Mon, 14 Nov 2011 11:05:44 +0100 Subject: [PATCH 23/25] facedetect: code maintenance Update example to be useful. Draw colored ellipses instead of unicolor circles. Reflow code for more local variables. Improve parameter descriptions. --- ext/opencv/gstfacedetect.c | 72 +++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/ext/opencv/gstfacedetect.c b/ext/opencv/gstfacedetect.c index 49fa481660..61669b6185 100644 --- a/ext/opencv/gstfacedetect.c +++ b/ext/opencv/gstfacedetect.c @@ -51,7 +51,7 @@ * * Example launch line * |[ - * gst-launch-0.10 videotestsrc ! decodebin ! ffmpegcolorspace ! facedetect ! ffmpegcolorspace ! xvimagesink + * gst-launch-0.10 autovideosrc ! decodebin2 ! colorspace ! facedetect ! colorspace ! xvimagesink * ]| * */ @@ -225,7 +225,7 @@ gst_facedetect_class_init (GstfacedetectClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_SCALE_FACTOR, g_param_spec_double ("scale-factor", "Scale factor", - "Factor by which the windows is scaled after each scan", + "Factor by which the frame is scaled after each object scan", 1.1, 10.0, DEFAULT_SCALE_FACTOR, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_MIN_NEIGHBORS, @@ -234,13 +234,13 @@ gst_facedetect_class_init (GstfacedetectClass * klass) "an object", 0, G_MAXINT, DEFAULT_MIN_NEIGHBORS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_MIN_SIZE_WIDTH, - g_param_spec_int ("min-size-width", "Minimum size width", - "Minimum window width size", 0, G_MAXINT, DEFAULT_MIN_SIZE_WIDTH, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_param_spec_int ("min-size-width", "Minimum face width", + "Minimum area width to be recognized as a face", 0, G_MAXINT, + DEFAULT_MIN_SIZE_WIDTH, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_MIN_SIZE_HEIGHT, - g_param_spec_int ("min-size-height", "Minimum size height", - "Minimum window height size", 0, G_MAXINT, DEFAULT_MIN_SIZE_HEIGHT, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_param_spec_int ("min-size-height", "Minimum face height", + "Minimum area height to be recognized as a face", 0, G_MAXINT, + DEFAULT_MIN_SIZE_HEIGHT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } /* initialize the new element @@ -389,18 +389,16 @@ static GstFlowReturn gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf, IplImage * img) { - Gstfacedetect *filter; - CvSeq *faces; - int i; - - filter = GST_FACEDETECT (base); - - cvCvtColor (img, filter->cvGray, CV_RGB2GRAY); - cvClearMemStorage (filter->cvStorage); + Gstfacedetect *filter = GST_FACEDETECT (base); if (filter->cvCascade) { GstMessage *msg = NULL; GValue facelist = { 0 }; + CvSeq *faces; + gint i; + + cvCvtColor (img, filter->cvGray, CV_RGB2GRAY); + cvClearMemStorage (filter->cvStorage); faces = cvHaarDetectObjects (filter->cvGray, filter->cvCascade, @@ -419,36 +417,44 @@ gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf, for (i = 0; i < (faces ? faces->total : 0); i++) { CvRect *r = (CvRect *) cvGetSeqElem (faces, i); GValue value = { 0 }; - GstStructure *s = gst_structure_new ("face", "x", G_TYPE_UINT, r->x, "y", G_TYPE_UINT, r->y, "width", G_TYPE_UINT, r->width, "height", G_TYPE_UINT, r->height, NULL); - GstMessage *m = gst_message_new_element (GST_OBJECT (filter), s); + GST_LOG_OBJECT (filter, "%2d/%2d: x,y = %4u,%4u: w.h = %4u,%4u", i, + faces->total, r->x, r->y, r->width, r->height); g_value_init (&value, GST_TYPE_STRUCTURE); gst_value_set_structure (&value, s); gst_value_list_append_value (&facelist, &value); g_value_unset (&value); - - gst_element_post_message (GST_ELEMENT (filter), m); - - if (filter->display) { - if (gst_buffer_is_writable (buf)) { + } + if (filter->display) { + if (gst_buffer_is_writable (buf)) { + /* draw colored circles for each face */ + for (i = 0; i < (faces ? faces->total : 0); i++) { + CvRect *r = (CvRect *) cvGetSeqElem (faces, i); CvPoint center; - int radius; - center.x = cvRound ((r->x + r->width * 0.5)); - center.y = cvRound ((r->y + r->height * 0.5)); - radius = cvRound ((r->width + r->height) * 0.25); - cvCircle (img, center, radius, CV_RGB (255, 32, 32), 3, 8, 0); - } else { - GST_DEBUG_OBJECT (filter, "Buffer is not writable, not drawing " - "circles for faces"); - } - } + CvSize axes; + gdouble w = r->width * 0.5; + gdouble h = r->height * 0.6; /* tweak for face form */ + gint cb = 255 - ((i & 3) << 7); + gint cg = 255 - ((i & 12) << 5); + gint cr = 255 - ((i & 48) << 3); + center.x = cvRound ((r->x + w)); + center.y = cvRound ((r->y + h)); + axes.width = w; + axes.height = h; + cvEllipse (img, center, axes, 0.0, 0.0, 360.0, CV_RGB (cr, cg, cb), + 3, 8, 0); + } + } else { + GST_LOG_OBJECT (filter, "Buffer is not writable, not drawing " + "circles for faces"); + } } if (msg) { From a857c905901ae08be197277f007e690d4003125c Mon Sep 17 00:00:00 2001 From: Stefan Sauer Date: Wed, 16 Nov 2011 20:51:27 +0100 Subject: [PATCH 24/25] facedetect: Gstfacedetect -> GstFacedetect --- ext/opencv/gstfacedetect.c | 22 +++++++++++----------- ext/opencv/gstfacedetect.h | 12 ++++++------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ext/opencv/gstfacedetect.c b/ext/opencv/gstfacedetect.c index 61669b6185..57901704be 100644 --- a/ext/opencv/gstfacedetect.c +++ b/ext/opencv/gstfacedetect.c @@ -141,7 +141,7 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_STATIC_CAPS ("video/x-raw-rgb") ); -GST_BOILERPLATE (Gstfacedetect, gst_facedetect, GstOpencvVideoFilter, +GST_BOILERPLATE (GstFacedetect, gst_facedetect, GstOpencvVideoFilter, GST_TYPE_OPENCV_VIDEO_FILTER); static void gst_facedetect_set_property (GObject * object, guint prop_id, @@ -155,13 +155,13 @@ static gboolean gst_facedetect_set_caps (GstOpencvVideoFilter * transform, static GstFlowReturn gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf, IplImage * img); -static void gst_facedetect_load_profile (Gstfacedetect * filter); +static void gst_facedetect_load_profile (GstFacedetect * filter); /* Clean up */ static void gst_facedetect_finalize (GObject * obj) { - Gstfacedetect *filter = GST_FACEDETECT (obj); + GstFacedetect *filter = GST_FACEDETECT (obj); if (filter->cvGray) { cvReleaseImage (&filter->cvGray); @@ -196,7 +196,7 @@ gst_facedetect_base_init (gpointer gclass) /* initialize the facedetect's class */ static void -gst_facedetect_class_init (GstfacedetectClass * klass) +gst_facedetect_class_init (GstFacedetectClass * klass) { GObjectClass *gobject_class; GstOpencvVideoFilterClass *gstopencvbasefilter_class; @@ -249,7 +249,7 @@ gst_facedetect_class_init (GstfacedetectClass * klass) * initialize instance structure */ static void -gst_facedetect_init (Gstfacedetect * filter, GstfacedetectClass * gclass) +gst_facedetect_init (GstFacedetect * filter, GstFacedetectClass * gclass) { filter->profile = g_strdup (DEFAULT_PROFILE); filter->display = TRUE; @@ -268,7 +268,7 @@ static void gst_facedetect_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { - Gstfacedetect *filter = GST_FACEDETECT (object); + GstFacedetect *filter = GST_FACEDETECT (object); switch (prop_id) { case PROP_PROFILE: @@ -304,7 +304,7 @@ static void gst_facedetect_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - Gstfacedetect *filter = GST_FACEDETECT (object); + GstFacedetect *filter = GST_FACEDETECT (object); switch (prop_id) { case PROP_PROFILE: @@ -342,7 +342,7 @@ gst_facedetect_set_caps (GstOpencvVideoFilter * transform, gint in_width, gint in_height, gint in_depth, gint in_channels, gint out_width, gint out_height, gint out_depth, gint out_channels) { - Gstfacedetect *filter; + GstFacedetect *filter; filter = GST_FACEDETECT (transform); @@ -361,7 +361,7 @@ gst_facedetect_set_caps (GstOpencvVideoFilter * transform, gint in_width, } static GstMessage * -gst_facedetect_message_new (Gstfacedetect * filter, GstBuffer * buf) +gst_facedetect_message_new (GstFacedetect * filter, GstBuffer * buf) { GstBaseTransform *trans = GST_BASE_TRANSFORM_CAST (filter); GstStructure *s; @@ -389,7 +389,7 @@ static GstFlowReturn gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf, IplImage * img) { - Gstfacedetect *filter = GST_FACEDETECT (base); + GstFacedetect *filter = GST_FACEDETECT (base); if (filter->cvCascade) { GstMessage *msg = NULL; @@ -469,7 +469,7 @@ gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf, static void -gst_facedetect_load_profile (Gstfacedetect * filter) +gst_facedetect_load_profile (GstFacedetect * filter) { filter->cvCascade = (CvHaarClassifierCascade *) cvLoad (filter->profile, 0, 0, 0); diff --git a/ext/opencv/gstfacedetect.h b/ext/opencv/gstfacedetect.h index a5a0b49f4f..cb66e50c32 100644 --- a/ext/opencv/gstfacedetect.h +++ b/ext/opencv/gstfacedetect.h @@ -59,17 +59,17 @@ G_BEGIN_DECLS #define GST_TYPE_FACEDETECT \ (gst_facedetect_get_type()) #define GST_FACEDETECT(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_FACEDETECT,Gstfacedetect)) + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_FACEDETECT,GstFacedetect)) #define GST_FACEDETECT_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_FACEDETECT,GstfacedetectClass)) + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_FACEDETECT,GstFacedetectClass)) #define GST_IS_FACEDETECT(obj) \ (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_FACEDETECT)) #define GST_IS_FACEDETECT_CLASS(klass) \ (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_FACEDETECT)) -typedef struct _Gstfacedetect Gstfacedetect; -typedef struct _GstfacedetectClass GstfacedetectClass; +typedef struct _GstFacedetect GstFacedetect; +typedef struct _GstFacedetectClass GstFacedetectClass; -struct _Gstfacedetect +struct _GstFacedetect { GstOpencvVideoFilter element; @@ -87,7 +87,7 @@ struct _Gstfacedetect CvMemStorage *cvStorage; }; -struct _GstfacedetectClass +struct _GstFacedetectClass { GstOpencvVideoFilterClass parent_class; }; From fefa1df8b9115404c94ec2aa2e9d1ff8ca79d34c Mon Sep 17 00:00:00 2001 From: Stefan Sauer Date: Wed, 16 Nov 2011 20:53:13 +0100 Subject: [PATCH 25/25] facedetect: detect face features Also detect mouth, nose and eyes. Drop faces that don't have them. Fixes leaking the cascades. Adds more docs. --- ext/opencv/gstfacedetect.c | 331 +++++++++++++++++++++++++++++++------ ext/opencv/gstfacedetect.h | 11 +- 2 files changed, 287 insertions(+), 55 deletions(-) diff --git a/ext/opencv/gstfacedetect.c b/ext/opencv/gstfacedetect.c index 57901704be..0b952e37e2 100644 --- a/ext/opencv/gstfacedetect.c +++ b/ext/opencv/gstfacedetect.c @@ -3,6 +3,7 @@ * Copyright (C) 2005 Thomas Vander Stichele * Copyright (C) 2005 Ronald S. Bultje * Copyright (C) 2008 Michael Sheldon + * Copyright (C) 2011 Stefan Sauer * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -48,14 +49,27 @@ * * Performs face detection on videos and images. * + * The image is scaled down multiple times using the GstFacedetect::scale-factor + * until the size is <= GstFacedetect::min-size-width or + * GstFacedetect::min-size-height. + * * * Example launch line * |[ * gst-launch-0.10 autovideosrc ! decodebin2 ! colorspace ! facedetect ! colorspace ! xvimagesink - * ]| + * ]| Detect and show faces + * |[ + * gst-launch-0.10 autovideosrc ! video/x-raw-yuv,width=320,height=240 ! colorspace ! facedetect min-size-width=60 min-size-height=60 ! colorspace ! xvimagesink + * ]| Detect large faces on a smaller image + * * */ +/* FIXME: development version of OpenCV has CV_HAAR_FIND_BIGGEST_OBJECT which + * we might want to use if available + * see https://code.ros.org/svn/opencv/trunk/opencv/modules/objdetect/src/haar.cpp + */ + #ifdef HAVE_CONFIG_H # include #endif @@ -67,7 +81,10 @@ GST_DEBUG_CATEGORY_STATIC (gst_facedetect_debug); #define GST_CAT_DEFAULT gst_facedetect_debug -#define DEFAULT_PROFILE "/usr/share/opencv/haarcascades/haarcascade_frontalface_default.xml" +#define DEFAULT_FACE_PROFILE "/usr/share/opencv/haarcascades/haarcascade_frontalface_default.xml" +#define DEFAULT_NOSE_PROFILE "/usr/share/opencv/haarcascades/haarcascade_mcs_nose.xml" +#define DEFAULT_MOUTH_PROFILE "/usr/share/opencv/haarcascades/haarcascade_mcs_mouth.xml" +#define DEFAULT_EYES_PROFILE "/usr/share/opencv/haarcascades/haarcascade_mcs_eyepair_small.xml" #define DEFAULT_SCALE_FACTOR 1.1 #define DEFAULT_FLAGS 0 #define DEFAULT_MIN_NEIGHBORS 3 @@ -85,7 +102,10 @@ enum { PROP_0, PROP_DISPLAY, - PROP_PROFILE, + PROP_FACE_PROFILE, + PROP_NOSE_PROFILE, + PROP_MOUTH_PROFILE, + PROP_EYES_PROFILE, PROP_SCALE_FACTOR, PROP_MIN_NEIGHBORS, PROP_FLAGS, @@ -155,7 +175,8 @@ static gboolean gst_facedetect_set_caps (GstOpencvVideoFilter * transform, static GstFlowReturn gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf, IplImage * img); -static void gst_facedetect_load_profile (GstFacedetect * filter); +static CvHaarClassifierCascade *gst_facedetect_load_profile (GstFacedetect * + filter, gchar * profile); /* Clean up */ static void @@ -163,14 +184,24 @@ gst_facedetect_finalize (GObject * obj) { GstFacedetect *filter = GST_FACEDETECT (obj); - if (filter->cvGray) { + if (filter->cvGray) cvReleaseImage (&filter->cvGray); - } - if (filter->cvStorage) { + if (filter->cvStorage) cvReleaseMemStorage (&filter->cvStorage); - } - g_free (filter->profile); + g_free (filter->face_profile); + g_free (filter->nose_profile); + g_free (filter->mouth_profile); + g_free (filter->eyes_profile); + + if (filter->cvFaceDetect) + cvReleaseHaarClassifierCascade (&filter->cvFaceDetect); + if (filter->cvNoseDetect) + cvReleaseHaarClassifierCascade (&filter->cvNoseDetect); + if (filter->cvMouthDetect) + cvReleaseHaarClassifierCascade (&filter->cvMouthDetect); + if (filter->cvEyesDetect) + cvReleaseHaarClassifierCascade (&filter->cvEyesDetect); G_OBJECT_CLASS (parent_class)->finalize (obj); } @@ -215,10 +246,24 @@ gst_facedetect_class_init (GstFacedetectClass * klass) g_param_spec_boolean ("display", "Display", "Sets whether the detected faces should be highlighted in the output", TRUE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_PROFILE, - g_param_spec_string ("profile", "Profile", + + g_object_class_install_property (gobject_class, PROP_FACE_PROFILE, + g_param_spec_string ("profile", "Face profile", "Location of Haar cascade file to use for face detection", - DEFAULT_PROFILE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + DEFAULT_FACE_PROFILE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_NOSE_PROFILE, + g_param_spec_string ("nose-profile", "Nose profile", + "Location of Haar cascade file to use for nose detection", + DEFAULT_NOSE_PROFILE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_MOUTH_PROFILE, + g_param_spec_string ("mouth-profile", "Mouth profile", + "Location of Haar cascade file to use for mouth detection", + DEFAULT_MOUTH_PROFILE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_EYES_PROFILE, + g_param_spec_string ("eyes-profile", "Eyes profile", + "Location of Haar cascade file to use for eye-pair detection", + DEFAULT_EYES_PROFILE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_FLAGS, g_param_spec_flags ("flags", "Flags", "Flags to cvHaarDetectObjects", GST_TYPE_OPENCV_FACE_DETECT_FLAGS, DEFAULT_FLAGS, @@ -244,21 +289,29 @@ gst_facedetect_class_init (GstFacedetectClass * klass) } /* initialize the new element - * instantiate pads and add them to element - * set pad calback functions * initialize instance structure */ static void gst_facedetect_init (GstFacedetect * filter, GstFacedetectClass * gclass) { - filter->profile = g_strdup (DEFAULT_PROFILE); + filter->face_profile = g_strdup (DEFAULT_FACE_PROFILE); + filter->nose_profile = g_strdup (DEFAULT_NOSE_PROFILE); + filter->mouth_profile = g_strdup (DEFAULT_MOUTH_PROFILE); + filter->eyes_profile = g_strdup (DEFAULT_EYES_PROFILE); filter->display = TRUE; filter->scale_factor = DEFAULT_SCALE_FACTOR; filter->min_neighbors = DEFAULT_MIN_NEIGHBORS; filter->flags = DEFAULT_FLAGS; filter->min_size_width = DEFAULT_MIN_SIZE_WIDTH; filter->min_size_height = DEFAULT_MIN_SIZE_HEIGHT; - gst_facedetect_load_profile (filter); + filter->cvFaceDetect = + gst_facedetect_load_profile (filter, filter->face_profile); + filter->cvNoseDetect = + gst_facedetect_load_profile (filter, filter->nose_profile); + filter->cvMouthDetect = + gst_facedetect_load_profile (filter, filter->mouth_profile); + filter->cvEyesDetect = + gst_facedetect_load_profile (filter, filter->eyes_profile); gst_opencv_video_filter_set_in_place (GST_OPENCV_VIDEO_FILTER_CAST (filter), TRUE); @@ -271,10 +324,37 @@ gst_facedetect_set_property (GObject * object, guint prop_id, GstFacedetect *filter = GST_FACEDETECT (object); switch (prop_id) { - case PROP_PROFILE: - g_free (filter->profile); - filter->profile = g_value_dup_string (value); - gst_facedetect_load_profile (filter); + case PROP_FACE_PROFILE: + g_free (filter->face_profile); + if (filter->cvFaceDetect) + cvReleaseHaarClassifierCascade (&filter->cvFaceDetect); + filter->face_profile = g_value_dup_string (value); + filter->cvFaceDetect = + gst_facedetect_load_profile (filter, filter->face_profile); + break; + case PROP_NOSE_PROFILE: + g_free (filter->nose_profile); + if (filter->cvNoseDetect) + cvReleaseHaarClassifierCascade (&filter->cvNoseDetect); + filter->nose_profile = g_value_dup_string (value); + filter->cvNoseDetect = + gst_facedetect_load_profile (filter, filter->nose_profile); + break; + case PROP_MOUTH_PROFILE: + g_free (filter->mouth_profile); + if (filter->cvMouthDetect) + cvReleaseHaarClassifierCascade (&filter->cvMouthDetect); + filter->mouth_profile = g_value_dup_string (value); + filter->cvMouthDetect = + gst_facedetect_load_profile (filter, filter->mouth_profile); + break; + case PROP_EYES_PROFILE: + g_free (filter->eyes_profile); + if (filter->cvEyesDetect) + cvReleaseHaarClassifierCascade (&filter->cvEyesDetect); + filter->eyes_profile = g_value_dup_string (value); + filter->cvEyesDetect = + gst_facedetect_load_profile (filter, filter->eyes_profile); break; case PROP_DISPLAY: filter->display = g_value_get_boolean (value); @@ -307,8 +387,17 @@ gst_facedetect_get_property (GObject * object, guint prop_id, GstFacedetect *filter = GST_FACEDETECT (object); switch (prop_id) { - case PROP_PROFILE: - g_value_set_string (value, filter->profile); + case PROP_FACE_PROFILE: + g_value_set_string (value, filter->face_profile); + break; + case PROP_NOSE_PROFILE: + g_value_set_string (value, filter->nose_profile); + break; + case PROP_MOUTH_PROFILE: + g_value_set_string (value, filter->mouth_profile); + break; + case PROP_EYES_PROFILE: + g_value_set_string (value, filter->eyes_profile); break; case PROP_DISPLAY: g_value_set_boolean (value, filter->display); @@ -391,17 +480,27 @@ gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf, { GstFacedetect *filter = GST_FACEDETECT (base); - if (filter->cvCascade) { + if (filter->cvFaceDetect) { GstMessage *msg = NULL; GValue facelist = { 0 }; CvSeq *faces; + CvSeq *mouth, *nose, *eyes; gint i; + gboolean do_display = FALSE; + + if (filter->display) { + if (gst_buffer_is_writable (buf)) { + do_display = TRUE; + } else { + GST_LOG_OBJECT (filter, "Buffer is not writable, not drawing faces."); + } + } cvCvtColor (img, filter->cvGray, CV_RGB2GRAY); cvClearMemStorage (filter->cvStorage); faces = - cvHaarDetectObjects (filter->cvGray, filter->cvCascade, + cvHaarDetectObjects (filter->cvGray, filter->cvFaceDetect, filter->cvStorage, filter->scale_factor, filter->min_neighbors, filter->flags, cvSize (filter->min_size_width, filter->min_size_height) #if (CV_MAJOR_VERSION >= 2) && (CV_MINOR_VERSION >= 2) @@ -417,43 +516,167 @@ gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf, for (i = 0; i < (faces ? faces->total : 0); i++) { CvRect *r = (CvRect *) cvGetSeqElem (faces, i); GValue value = { 0 }; - GstStructure *s = gst_structure_new ("face", + GstStructure *s; + guint mw = filter->min_size_width / 8; + guint mh = filter->min_size_height / 8; + guint rnx, rny, rnw, rnh; + guint rmx, rmy, rmw, rmh; + guint rex, rey, rew, reh; + gboolean have_nose, have_mouth, have_eyes; + + /* detect face features */ + + rnx = r->x + r->width / 4; + rny = r->y + r->height / 4; + rnw = r->width / 2; + rnh = r->height / 2; + cvSetImageROI (filter->cvGray, cvRect (rnx, rny, rnw, rnh)); + nose = + cvHaarDetectObjects (filter->cvGray, filter->cvNoseDetect, + filter->cvStorage, filter->scale_factor, filter->min_neighbors, + filter->flags, cvSize (mw, mh) +#if (CV_MAJOR_VERSION >= 2) && (CV_MINOR_VERSION >= 2) + , cvSize (mw + 2, mh + 2) +#endif + ); + have_nose = (nose && nose->total); + cvResetImageROI (filter->cvGray); + + rmx = r->x; + rmy = r->y + r->height / 2; + rmw = r->width; + rmh = r->height / 2; + cvSetImageROI (filter->cvGray, cvRect (rmx, rmy, rmw, rmh)); + mouth = + cvHaarDetectObjects (filter->cvGray, filter->cvMouthDetect, + filter->cvStorage, filter->scale_factor, filter->min_neighbors, + filter->flags, cvSize (mw, mh) +#if (CV_MAJOR_VERSION >= 2) && (CV_MINOR_VERSION >= 2) + , cvSize (mw + 2, mh + 2) +#endif + ); + have_mouth = (mouth && mouth->total); + cvResetImageROI (filter->cvGray); + + rex = r->x; + rey = r->y; + rew = r->width; + reh = r->height / 2; + cvSetImageROI (filter->cvGray, cvRect (rex, rey, rew, reh)); + eyes = + cvHaarDetectObjects (filter->cvGray, filter->cvEyesDetect, + filter->cvStorage, filter->scale_factor, filter->min_neighbors, + filter->flags, cvSize (mw, mh) +#if (CV_MAJOR_VERSION >= 2) && (CV_MINOR_VERSION >= 2) + , cvSize (mw + 2, mh + 2) +#endif + ); + have_eyes = (eyes && eyes->total); + cvResetImageROI (filter->cvGray); + + GST_LOG_OBJECT (filter, + "%2d/%2d: x,y = %4u,%4u: w.h = %4u,%4u : features(e,n,m) = %d,%d,%d", + i, faces->total, r->x, r->y, r->width, r->height, + have_eyes, have_nose, have_mouth); + + /* ignore 'face' where we don't fix mount/nose/eyes ? */ + if (!(have_eyes && have_nose && have_mouth)) + continue; + + s = gst_structure_new ("face", "x", G_TYPE_UINT, r->x, "y", G_TYPE_UINT, r->y, "width", G_TYPE_UINT, r->width, "height", G_TYPE_UINT, r->height, NULL); - - GST_LOG_OBJECT (filter, "%2d/%2d: x,y = %4u,%4u: w.h = %4u,%4u", i, - faces->total, r->x, r->y, r->width, r->height); + if (nose && nose->total) { + CvRect *sr = (CvRect *) cvGetSeqElem (nose, 0); + GST_LOG_OBJECT (filter, "nose/%d: x,y = %4u,%4u: w.h = %4u,%4u", + nose->total, rnx + sr->x, rny + sr->y, sr->width, sr->height); + gst_structure_set (s, + "nose->x", G_TYPE_UINT, rnx + sr->x, + "nose->y", G_TYPE_UINT, rny + sr->y, + "nose->width", G_TYPE_UINT, sr->width, + "nose->height", G_TYPE_UINT, sr->height, NULL); + } + if (mouth && mouth->total) { + CvRect *sr = (CvRect *) cvGetSeqElem (mouth, 0); + GST_LOG_OBJECT (filter, "mouth/%d: x,y = %4u,%4u: w.h = %4u,%4u", + mouth->total, rmx + sr->x, rmy + sr->y, sr->width, sr->height); + gst_structure_set (s, + "mouth->x", G_TYPE_UINT, rmx + sr->x, + "mouth->y", G_TYPE_UINT, rmy + sr->y, + "mouth->width", G_TYPE_UINT, sr->width, + "mouth->height", G_TYPE_UINT, sr->height, NULL); + } + if (eyes && eyes->total) { + CvRect *sr = (CvRect *) cvGetSeqElem (eyes, 0); + GST_LOG_OBJECT (filter, "eyes/%d: x,y = %4u,%4u: w.h = %4u,%4u", + eyes->total, rex + sr->x, rey + sr->y, sr->width, sr->height); + gst_structure_set (s, + "eyes->x", G_TYPE_UINT, rex + sr->x, + "eyes->y", G_TYPE_UINT, rey + sr->y, + "eyes->width", G_TYPE_UINT, sr->width, + "eyes->height", G_TYPE_UINT, sr->height, NULL); + } g_value_init (&value, GST_TYPE_STRUCTURE); gst_value_set_structure (&value, s); gst_value_list_append_value (&facelist, &value); g_value_unset (&value); - } - if (filter->display) { - if (gst_buffer_is_writable (buf)) { - /* draw colored circles for each face */ - for (i = 0; i < (faces ? faces->total : 0); i++) { - CvRect *r = (CvRect *) cvGetSeqElem (faces, i); - CvPoint center; - CvSize axes; - gdouble w = r->width * 0.5; - gdouble h = r->height * 0.6; /* tweak for face form */ - gint cb = 255 - ((i & 3) << 7); - gint cg = 255 - ((i & 12) << 5); - gint cr = 255 - ((i & 48) << 3); - center.x = cvRound ((r->x + w)); - center.y = cvRound ((r->y + h)); + if (do_display) { + CvPoint center; + CvSize axes; + gdouble w, h; + gint cb = 255 - ((i & 3) << 7); + gint cg = 255 - ((i & 12) << 5); + gint cr = 255 - ((i & 48) << 3); + + w = r->width / 2; + h = r->height / 2; + center.x = cvRound ((r->x + w)); + center.y = cvRound ((r->y + h)); + axes.width = w; + axes.height = h * 1.25; /* tweak for face form */ + cvEllipse (img, center, axes, 0.0, 0.0, 360.0, CV_RGB (cr, cg, cb), + 3, 8, 0); + + if (nose && nose->total) { + CvRect *sr = (CvRect *) cvGetSeqElem (nose, 0); + + w = sr->width / 2; + h = sr->height / 2; + center.x = cvRound ((rnx + sr->x + w)); + center.y = cvRound ((rny + sr->y + h)); axes.width = w; + axes.height = h * 1.25; /* tweak for nose form */ + cvEllipse (img, center, axes, 0.0, 0.0, 360.0, CV_RGB (cr, cg, cb), + 1, 8, 0); + } + if (mouth && mouth->total) { + CvRect *sr = (CvRect *) cvGetSeqElem (mouth, 0); + + w = sr->width / 2; + h = sr->height / 2; + center.x = cvRound ((rmx + sr->x + w)); + center.y = cvRound ((rmy + sr->y + h)); + axes.width = w * 1.5; /* tweak for mouth form */ axes.height = h; cvEllipse (img, center, axes, 0.0, 0.0, 360.0, CV_RGB (cr, cg, cb), - 3, 8, 0); + 1, 8, 0); + } + if (eyes && eyes->total) { + CvRect *sr = (CvRect *) cvGetSeqElem (eyes, 0); + + w = sr->width / 2; + h = sr->height / 2; + center.x = cvRound ((rex + sr->x + w)); + center.y = cvRound ((rey + sr->y + h)); + axes.width = w * 1.5; /* tweak for eyes form */ + axes.height = h; + cvEllipse (img, center, axes, 0.0, 0.0, 360.0, CV_RGB (cr, cg, cb), + 1, 8, 0); } - } else { - GST_LOG_OBJECT (filter, "Buffer is not writable, not drawing " - "circles for faces"); } } @@ -468,14 +691,16 @@ gst_facedetect_transform_ip (GstOpencvVideoFilter * base, GstBuffer * buf, } -static void -gst_facedetect_load_profile (GstFacedetect * filter) +static CvHaarClassifierCascade * +gst_facedetect_load_profile (GstFacedetect * filter, gchar * profile) { - filter->cvCascade = - (CvHaarClassifierCascade *) cvLoad (filter->profile, 0, 0, 0); - if (!filter->cvCascade) { - GST_WARNING ("Couldn't load Haar classifier cascade: %s.", filter->profile); + CvHaarClassifierCascade *cascade; + + if (!(cascade = (CvHaarClassifierCascade *) cvLoad (profile, 0, 0, 0))) { + GST_WARNING_OBJECT (filter, "Couldn't load Haar classifier cascade: %s.", + profile); } + return cascade; } diff --git a/ext/opencv/gstfacedetect.h b/ext/opencv/gstfacedetect.h index cb66e50c32..90fe2c6836 100644 --- a/ext/opencv/gstfacedetect.h +++ b/ext/opencv/gstfacedetect.h @@ -3,6 +3,7 @@ * Copyright (C) 2005 Thomas Vander Stichele * Copyright (C) 2005 Ronald S. Bultje * Copyright (C) 2008 Michael Sheldon + * Copyright (C) 2011 Stefan Sauer * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -75,7 +76,10 @@ struct _GstFacedetect gboolean display; - gchar *profile; + gchar *face_profile; + gchar *nose_profile; + gchar *mouth_profile; + gchar *eyes_profile; gdouble scale_factor; gint min_neighbors; gint flags; @@ -83,7 +87,10 @@ struct _GstFacedetect gint min_size_height; IplImage *cvGray; - CvHaarClassifierCascade *cvCascade; + CvHaarClassifierCascade *cvFaceDetect; + CvHaarClassifierCascade *cvNoseDetect; + CvHaarClassifierCascade *cvMouthDetect; + CvHaarClassifierCascade *cvEyesDetect; CvMemStorage *cvStorage; };