diff --git a/ext/opus/Makefile.am b/ext/opus/Makefile.am index 88845a3cab..cb0a9b338a 100644 --- a/ext/opus/Makefile.am +++ b/ext/opus/Makefile.am @@ -1,6 +1,6 @@ plugin_LTLIBRARIES = libgstopus.la -libgstopus_la_SOURCES = gstopus.c gstopusdec.c gstopusenc.c gstopusparse.c gstopusheader.c +libgstopus_la_SOURCES = gstopus.c gstopusdec.c gstopusenc.c gstopusparse.c gstopusheader.c gstopuscommon.c libgstopus_la_CFLAGS = \ -DGST_USE_UNSTABLE_API \ $(GST_PLUGINS_BASE_CFLAGS) \ @@ -15,4 +15,4 @@ libgstopus_la_LIBADD = \ libgstopus_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) $(LIBM) libgstopus_la_LIBTOOLFLAGS = --tag=disable-static -noinst_HEADERS = gstopusenc.h gstopusdec.h gstopusparse.h gstopusheader.h +noinst_HEADERS = gstopusenc.h gstopusdec.h gstopusparse.h gstopusheader.h gstopuscommon.h diff --git a/ext/opus/gstopuscommon.c b/ext/opus/gstopuscommon.c new file mode 100644 index 0000000000..fc3e0376b9 --- /dev/null +++ b/ext/opus/gstopuscommon.c @@ -0,0 +1,72 @@ +/* GStreamer + * Copyright (C) 2009 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "gstopuscommon.h" + +/* http://www.xiph.org/vorbis/doc/Vorbis_I_spec.html#x1-800004.3.9 */ +/* copy of the same structure in the vorbis plugin */ +const GstAudioChannelPosition gst_opus_channel_positions[][8] = { + { /* Mono */ + GST_AUDIO_CHANNEL_POSITION_FRONT_MONO}, + { /* Stereo */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT}, + { /* Stereo + Centre */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT}, + { /* Quadraphonic */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT, + GST_AUDIO_CHANNEL_POSITION_REAR_LEFT, + GST_AUDIO_CHANNEL_POSITION_REAR_RIGHT, + }, + { /* Stereo + Centre + rear stereo */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT, + GST_AUDIO_CHANNEL_POSITION_REAR_LEFT, + GST_AUDIO_CHANNEL_POSITION_REAR_RIGHT, + }, + { /* Full 5.1 Surround */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT, + GST_AUDIO_CHANNEL_POSITION_REAR_LEFT, + GST_AUDIO_CHANNEL_POSITION_REAR_RIGHT, + GST_AUDIO_CHANNEL_POSITION_LFE, + }, + { /* 6.1 Surround, in Vorbis spec since 2010-01-13 */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT, + GST_AUDIO_CHANNEL_POSITION_SIDE_LEFT, + GST_AUDIO_CHANNEL_POSITION_SIDE_RIGHT, + GST_AUDIO_CHANNEL_POSITION_REAR_CENTER, + GST_AUDIO_CHANNEL_POSITION_LFE}, + { /* 7.1 Surround, in Vorbis spec since 2010-01-13 */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT, + GST_AUDIO_CHANNEL_POSITION_SIDE_LEFT, + GST_AUDIO_CHANNEL_POSITION_SIDE_RIGHT, + GST_AUDIO_CHANNEL_POSITION_REAR_LEFT, + GST_AUDIO_CHANNEL_POSITION_REAR_RIGHT, + GST_AUDIO_CHANNEL_POSITION_LFE}, +}; diff --git a/ext/opus/gstopuscommon.h b/ext/opus/gstopuscommon.h new file mode 100644 index 0000000000..96a303e303 --- /dev/null +++ b/ext/opus/gstopuscommon.h @@ -0,0 +1,33 @@ +/* GStreamer Opus Encoder + * Copyright (C) 2009 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + + +#ifndef __GST_OPUS_COMMON_H__ +#define __GST_OPUS_COMMON_H__ + +#include +#include + +G_BEGIN_DECLS + +extern const GstAudioChannelPosition gst_opus_channel_positions[][8]; + +G_END_DECLS + +#endif /* __GST_OPUS_COMMON_H__ */ diff --git a/ext/opus/gstopusdec.c b/ext/opus/gstopusdec.c index 4e1a3a4e07..3a63349675 100644 --- a/ext/opus/gstopusdec.c +++ b/ext/opus/gstopusdec.c @@ -41,9 +41,11 @@ # include "config.h" #endif +#include #include #include #include "gstopusheader.h" +#include "gstopuscommon.h" #include "gstopusdec.h" GST_DEBUG_CATEGORY_STATIC (opusdec_debug); @@ -54,9 +56,9 @@ GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS ("audio/x-raw, " - "format = (string) { S16LE }, " + "format = (string) { " GST_AUDIO_NE (S16) " }, " "rate = (int) { 8000, 12000, 16000, 24000, 48000 }, " - "channels = (int) [ 1, 2 ] ") + "channels = (int) [ 1, 8 ] ") ); static GstStaticPadTemplate opus_dec_sink_factory = @@ -68,6 +70,19 @@ GST_STATIC_PAD_TEMPLATE ("sink", G_DEFINE_TYPE (GstOpusDec, gst_opus_dec, GST_TYPE_AUDIO_DECODER); +#define DB_TO_LINEAR(x) pow (10., (x) / 20.) + +#define DEFAULT_USE_INBAND_FEC FALSE +#define DEFAULT_APPLY_GAIN TRUE + +enum +{ + PROP_0, + PROP_USE_INBAND_FEC, + PROP_APPLY_GAIN +}; + + static GstFlowReturn gst_opus_dec_parse_header (GstOpusDec * dec, GstBuffer * buf); static gboolean gst_opus_dec_start (GstAudioDecoder * dec); @@ -76,16 +91,26 @@ static GstFlowReturn gst_opus_dec_handle_frame (GstAudioDecoder * dec, GstBuffer * buffer); static gboolean gst_opus_dec_set_format (GstAudioDecoder * bdec, GstCaps * caps); +static void gst_opus_dec_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); +static void gst_opus_dec_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); + static void gst_opus_dec_class_init (GstOpusDecClass * klass) { + GObjectClass *gobject_class; GstAudioDecoderClass *adclass; GstElementClass *element_class; + gobject_class = (GObjectClass *) klass; adclass = (GstAudioDecoderClass *) klass; element_class = (GstElementClass *) klass; + gobject_class->set_property = gst_opus_dec_set_property; + gobject_class->get_property = gst_opus_dec_get_property; + adclass->start = GST_DEBUG_FUNCPTR (gst_opus_dec_start); adclass->stop = GST_DEBUG_FUNCPTR (gst_opus_dec_stop); adclass->handle_frame = GST_DEBUG_FUNCPTR (gst_opus_dec_handle_frame); @@ -99,6 +124,15 @@ gst_opus_dec_class_init (GstOpusDecClass * klass) "Codec/Decoder/Audio", "decode opus streams to audio", "Sebastian Dröge "); + g_object_class_install_property (gobject_class, PROP_USE_INBAND_FEC, + g_param_spec_boolean ("use-inband-fec", "Use in-band FEC", + "Use forward error correction if available", DEFAULT_USE_INBAND_FEC, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_APPLY_GAIN, + g_param_spec_boolean ("apply-gain", "Apply gain", + "Apply gain if any is specified in the header", DEFAULT_APPLY_GAIN, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); GST_DEBUG_CATEGORY_INIT (opusdec_debug, "opusdec", 0, "opus decoding element"); @@ -109,14 +143,17 @@ gst_opus_dec_reset (GstOpusDec * dec) { dec->packetno = 0; if (dec->state) { - opus_decoder_destroy (dec->state); + opus_multistream_decoder_destroy (dec->state); dec->state = NULL; } gst_buffer_replace (&dec->streamheader, NULL); gst_buffer_replace (&dec->vorbiscomment, NULL); + gst_buffer_replace (&dec->last_buffer, NULL); + dec->primed = FALSE; dec->pre_skip = 0; + dec->r128_gain = 0; } static void @@ -124,6 +161,8 @@ gst_opus_dec_init (GstOpusDec * dec) { dec->sample_rate = 0; dec->n_channels = 0; + dec->use_inband_fec = FALSE; + dec->apply_gain = DEFAULT_APPLY_GAIN; gst_opus_dec_reset (dec); } @@ -138,6 +177,11 @@ gst_opus_dec_start (GstAudioDecoder * dec) /* we know about concealment */ gst_audio_decoder_set_plc_aware (dec, TRUE); + if (odec->use_inband_fec) { + gst_audio_decoder_set_latency (dec, 2 * GST_MSECOND + GST_MSECOND / 2, + 120 * GST_MSECOND); + } + return TRUE; } @@ -151,15 +195,111 @@ gst_opus_dec_stop (GstAudioDecoder * dec) return TRUE; } +static double +gst_opus_dec_get_r128_gain (gint16 r128_gain) +{ + return r128_gain / (double) (1 << 8); +} + +static double +gst_opus_dec_get_r128_volume (gint16 r128_gain) +{ + return DB_TO_LINEAR (gst_opus_dec_get_r128_gain (r128_gain)); +} + static GstFlowReturn gst_opus_dec_parse_header (GstOpusDec * dec, GstBuffer * buf) { - g_return_val_if_fail (gst_opus_header_is_header (buf, "OpusHead", 8), - GST_FLOW_ERROR); - g_return_val_if_fail (GST_BUFFER_SIZE (buf) >= 19, GST_FLOW_ERROR); + const guint8 *data; + GstCaps *caps; + GstStructure *s; + const GstAudioChannelPosition *pos = NULL; - dec->pre_skip = GST_READ_UINT16_LE (GST_BUFFER_DATA (buf) + 10); - GST_INFO_OBJECT (dec, "Found pre-skip of %u samples", dec->pre_skip); + g_return_val_if_fail (gst_opus_header_is_id_header (buf), GST_FLOW_ERROR); + + data = gst_buffer_map (buf, NULL, NULL, GST_MAP_READ); + + g_return_val_if_fail (dec->n_channels != data[9], GST_FLOW_ERROR); + + dec->n_channels = data[9]; + dec->pre_skip = GST_READ_UINT16_LE (data + 10); + dec->r128_gain = GST_READ_UINT16_LE (data + 14); + dec->r128_gain_volume = gst_opus_dec_get_r128_volume (dec->r128_gain); + GST_INFO_OBJECT (dec, + "Found pre-skip of %u samples, R128 gain %d (volume %f)", + dec->pre_skip, dec->r128_gain, dec->r128_gain_volume); + + dec->channel_mapping_family = data[18]; + if (dec->channel_mapping_family == 0) { + /* implicit mapping */ + GST_INFO_OBJECT (dec, "Channel mapping family 0, implicit mapping"); + dec->n_streams = dec->n_stereo_streams = 1; + dec->channel_mapping[0] = 0; + dec->channel_mapping[1] = 1; + } else { + dec->n_streams = data[19]; + dec->n_stereo_streams = data[20]; + memcpy (dec->channel_mapping, data + 21, dec->n_channels); + + if (dec->channel_mapping_family == 1) { + GST_INFO_OBJECT (dec, "Channel mapping family 1, Vorbis mapping"); + switch (dec->n_channels) { + case 1: + case 2: + /* nothing */ + break; + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + pos = gst_opus_channel_positions[dec->n_channels - 1]; + break; + default:{ + gint i; + GstAudioChannelPosition *posn = + g_new (GstAudioChannelPosition, dec->n_channels); + + GST_ELEMENT_WARNING (GST_ELEMENT (dec), STREAM, DECODE, + (NULL), ("Using NONE channel layout for more than 8 channels")); + + for (i = 0; i < dec->n_channels; i++) + posn[i] = GST_AUDIO_CHANNEL_POSITION_NONE; + + pos = posn; + } + } + } else { + GST_INFO_OBJECT (dec, "Channel mapping family %d", + dec->channel_mapping_family); + } + } + + /* negotiate width with downstream */ + caps = gst_pad_get_allowed_caps (GST_AUDIO_DECODER_SRC_PAD (dec)); + s = gst_caps_get_structure (caps, 0); + gst_structure_fixate_field_nearest_int (s, "rate", 48000); + gst_structure_get_int (s, "rate", &dec->sample_rate); + gst_structure_fixate_field_nearest_int (s, "channels", dec->n_channels); + gst_structure_get_int (s, "channels", &dec->n_channels); + + GST_INFO_OBJECT (dec, "Negotiated %d channels, %d Hz", dec->n_channels, + dec->sample_rate); + + if (pos) { + gst_audio_set_channel_positions (gst_caps_get_structure (caps, 0), pos); + } + + if (dec->n_channels > 8) { + g_free ((GstAudioChannelPosition *) pos); + } + + GST_INFO_OBJECT (dec, "Setting src caps to %" GST_PTR_FORMAT, caps); + gst_pad_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), caps); + gst_caps_unref (caps); + + gst_buffer_unmap (buf, (guint8 *) data, -1); return GST_FLOW_OK; } @@ -171,50 +311,8 @@ gst_opus_dec_parse_comments (GstOpusDec * dec, GstBuffer * buf) return GST_FLOW_OK; } -static void -gst_opus_dec_setup_from_peer_caps (GstOpusDec * dec) -{ - GstPad *srcpad, *peer; - GstStructure *s; - GstCaps *caps; - const GstCaps *template_caps; - const GstCaps *peer_caps; - - srcpad = GST_AUDIO_DECODER_SRC_PAD (dec); - peer = gst_pad_get_peer (srcpad); - - if (peer) { - template_caps = gst_pad_get_pad_template_caps (srcpad); - peer_caps = gst_pad_get_caps (peer); - GST_DEBUG_OBJECT (dec, "Peer caps: %" GST_PTR_FORMAT, peer_caps); - caps = gst_caps_intersect (template_caps, peer_caps); - gst_pad_fixate_caps (peer, caps); - GST_DEBUG_OBJECT (dec, "Fixated caps: %" GST_PTR_FORMAT, caps); - - s = gst_caps_get_structure (caps, 0); - if (!gst_structure_get_int (s, "channels", &dec->n_channels)) { - dec->n_channels = 2; - GST_WARNING_OBJECT (dec, "Failed to get channels, using default %d", - dec->n_channels); - } else { - GST_DEBUG_OBJECT (dec, "Got channels %d", dec->n_channels); - } - if (!gst_structure_get_int (s, "rate", &dec->sample_rate)) { - dec->sample_rate = 48000; - GST_WARNING_OBJECT (dec, "Failed to get rate, using default %d", - dec->sample_rate); - } else { - GST_DEBUG_OBJECT (dec, "Got sample rate %d", dec->sample_rate); - } - - gst_audio_decoder_set_outcaps (GST_AUDIO_DECODER (dec), caps); - } else { - GST_WARNING_OBJECT (dec, "Failed to get src pad peer"); - } -} - static GstFlowReturn -opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf) +opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buffer) { GstFlowReturn res = GST_FLOW_OK; gsize size, out_size; @@ -224,42 +322,51 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf) int n, err; int samples; unsigned int packet_size; + GstBuffer *buf; if (dec->state == NULL) { - gst_opus_dec_setup_from_peer_caps (dec); - GST_DEBUG_OBJECT (dec, "Creating decoder with %d channels, %d Hz", dec->n_channels, dec->sample_rate); - dec->state = opus_decoder_create (dec->sample_rate, dec->n_channels, &err); + dec->state = opus_multistream_decoder_create (dec->sample_rate, + dec->n_channels, dec->n_streams, dec->n_stereo_streams, + dec->channel_mapping, &err); if (!dec->state || err != OPUS_OK) goto creation_failed; } + if (buffer) { + GST_DEBUG_OBJECT (dec, "Received buffer of size %u", + gst_buffer_get_size (buffer)); + } else { + GST_DEBUG_OBJECT (dec, "Received missing buffer"); + } + + /* if using in-band FEC, we introdude one extra frame's delay as we need + to potentially wait for next buffer to decode a missing buffer */ + if (dec->use_inband_fec && !dec->primed) { + GST_DEBUG_OBJECT (dec, "First buffer received in FEC mode, early out"); + goto done; + } + + /* That's the buffer we'll be sending to the opus decoder. */ + buf = dec->use_inband_fec && dec->last_buffer ? dec->last_buffer : buffer; + if (buf) { data = gst_buffer_map (buf, &size, NULL, GST_MAP_READ); - GST_DEBUG_OBJECT (dec, "received buffer of size %u", size); + GST_DEBUG_OBJECT (dec, "Using buffer of size %u", size); } else { /* concealment data, pass NULL as the bits parameters */ - GST_DEBUG_OBJECT (dec, "creating concealment data"); + GST_DEBUG_OBJECT (dec, "Using NULL buffer"); data = NULL; size = 0; } - if (data) { - samples = - opus_packet_get_samples_per_frame (data, - dec->sample_rate) * opus_packet_get_nb_frames (data, size); - packet_size = samples * dec->n_channels * 2; - GST_DEBUG_OBJECT (dec, "bandwidth %d", opus_packet_get_bandwidth (data)); - GST_DEBUG_OBJECT (dec, "samples %d", samples); - } else { - /* use maximum size (120 ms) as we do now know in advance how many samples - will be returned */ - samples = 120 * dec->sample_rate / 1000; - } - + /* use maximum size (120 ms) as the number of returned samples is + not constant over the stream. */ + samples = 120 * dec->sample_rate / 1000; packet_size = samples * dec->n_channels * 2; + outbuf = gst_buffer_new_and_alloc (packet_size); if (!outbuf) { goto buffer_failed; @@ -267,39 +374,81 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf) out_data = (gint16 *) gst_buffer_map (outbuf, &out_size, NULL, GST_MAP_WRITE); - n = opus_decode (dec->state, data, size, out_data, samples, 0); + if (dec->use_inband_fec) { + if (dec->last_buffer) { + /* normal delayed decode */ + n = opus_multistream_decode (dec->state, data, size, out_data, samples, + 0); + } else { + /* FEC reconstruction decode */ + n = opus_multistream_decode (dec->state, data, size, out_data, samples, + 1); + } + } else { + /* normal decode */ + n = opus_multistream_decode (dec->state, data, size, out_data, samples, 0); + } gst_buffer_unmap (buf, data, size); gst_buffer_unmap (outbuf, out_data, out_size); + if (n < 0) { GST_ELEMENT_ERROR (dec, STREAM, DECODE, ("Decoding error: %d", n), (NULL)); return GST_FLOW_ERROR; } GST_DEBUG_OBJECT (dec, "decoded %d samples", n); - GST_BUFFER_SIZE (outbuf) = n * 2 * dec->n_channels; + gst_buffer_set_size (outbuf, n * 2 * dec->n_channels); /* Skip any samples that need skipping */ if (dec->pre_skip > 0) { guint scaled_pre_skip = dec->pre_skip * dec->sample_rate / 48000; guint skip = scaled_pre_skip > n ? n : scaled_pre_skip; guint scaled_skip = skip * 48000 / dec->sample_rate; - GST_BUFFER_SIZE (outbuf) -= skip * 2 * dec->n_channels; - GST_BUFFER_DATA (outbuf) += skip * 2 * dec->n_channels; + + gst_buffer_resize (outbuf, skip * 2 * dec->n_channels, -1); dec->pre_skip -= scaled_skip; GST_INFO_OBJECT (dec, "Skipping %u samples (%u at 48000 Hz, %u left to skip)", skip, scaled_skip, dec->pre_skip); - if (GST_BUFFER_SIZE (outbuf) == 0) { + if (gst_buffer_get_size (outbuf) == 0) { gst_buffer_unref (outbuf); outbuf = NULL; } } + /* Apply gain */ + /* Would be better off leaving this to a volume element, as this is + a naive conversion that does too many int/float conversions. + However, we don't have control over the pipeline... + So make it optional if the user program wants to use a volume, + but do it by default so the correct volume goes out by default */ + if (dec->apply_gain && outbuf && dec->r128_gain) { + gsize rsize; + unsigned int i, nsamples; + double volume = dec->r128_gain_volume; + gint16 *samples = + (gint16 *) gst_buffer_map (outbuf, &rsize, NULL, GST_MAP_READWRITE); + + GST_DEBUG_OBJECT (dec, "Applying gain: volume %f", volume); + nsamples = rsize / 2; + for (i = 0; i < nsamples; ++i) { + int sample = (int) (samples[i] * volume + 0.5); + samples[i] = sample < -32768 ? -32768 : sample > 32767 ? 32767 : sample; + } + gst_buffer_unmap (outbuf, samples, rsize); + } + res = gst_audio_decoder_finish_frame (GST_AUDIO_DECODER (dec), outbuf, 1); if (res != GST_FLOW_OK) GST_DEBUG_OBJECT (dec, "flow: %s", gst_flow_get_name (res)); +done: + if (dec->use_inband_fec) { + gst_buffer_replace (&dec->last_buffer, buffer); + dec->primed = TRUE; + } + return res; creation_failed: @@ -436,3 +585,41 @@ gst_opus_dec_handle_frame (GstAudioDecoder * adec, GstBuffer * buf) return res; } + +static void +gst_opus_dec_get_property (GObject * object, guint prop_id, GValue * value, + GParamSpec * pspec) +{ + GstOpusDec *dec = GST_OPUS_DEC (object); + + switch (prop_id) { + case PROP_USE_INBAND_FEC: + g_value_set_boolean (value, dec->use_inband_fec); + break; + case PROP_APPLY_GAIN: + g_value_set_boolean (value, dec->apply_gain); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_opus_dec_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstOpusDec *dec = GST_OPUS_DEC (object); + + switch (prop_id) { + case PROP_USE_INBAND_FEC: + dec->use_inband_fec = g_value_get_boolean (value); + break; + case PROP_APPLY_GAIN: + dec->apply_gain = g_value_get_boolean (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} diff --git a/ext/opus/gstopusdec.h b/ext/opus/gstopusdec.h index eee27dc554..3ccfa26969 100644 --- a/ext/opus/gstopusdec.h +++ b/ext/opus/gstopusdec.h @@ -23,7 +23,7 @@ #include #include -#include +#include G_BEGIN_DECLS @@ -44,7 +44,7 @@ typedef struct _GstOpusDecClass GstOpusDecClass; struct _GstOpusDec { GstAudioDecoder element; - OpusDecoder *state; + OpusMSDecoder *state; guint64 packetno; @@ -54,6 +54,19 @@ struct _GstOpusDec { int sample_rate; int n_channels; guint32 pre_skip; + gint16 r128_gain; + + guint8 n_streams; + guint8 n_stereo_streams; + guint8 channel_mapping_family; + guint8 channel_mapping[256]; + + gboolean apply_gain; + double r128_gain_volume; + + gboolean use_inband_fec; + GstBuffer *last_buffer; + gboolean primed; }; struct _GstOpusDecClass { diff --git a/ext/opus/gstopusenc.c b/ext/opus/gstopusenc.c index 9dec51ee52..b7e282c95c 100644 --- a/ext/opus/gstopusenc.c +++ b/ext/opus/gstopusenc.c @@ -49,6 +49,7 @@ #include #include #include "gstopusheader.h" +#include "gstopuscommon.h" #include "gstopusenc.h" GST_DEBUG_CATEGORY_STATIC (opusenc_debug); @@ -177,9 +178,6 @@ static gint64 gst_opus_enc_get_latency (GstOpusEnc * enc); static GstFlowReturn gst_opus_enc_encode (GstOpusEnc * enc, GstBuffer * buffer); -static GstFlowReturn gst_opus_enc_encode (GstOpusEnc * enc, GstBuffer * buf); -static gint64 gst_opus_enc_get_latency (GstOpusEnc * enc); - #define gst_opus_enc_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstOpusEnc, gst_opus_enc, GST_TYPE_AUDIO_ENCODER, G_IMPLEMENT_INTERFACE (GST_TYPE_TAG_SETTER, NULL); @@ -199,11 +197,11 @@ gst_opus_enc_class_init (GstOpusEncClass * klass) gobject_class->set_property = gst_opus_enc_set_property; gobject_class->get_property = gst_opus_enc_get_property; - gst_element_class_add_pad_template (element_class, + gst_element_class_add_pad_template (gstelement_class, gst_static_pad_template_get (&src_factory)); - gst_element_class_add_pad_template (element_class, + gst_element_class_add_pad_template (gstelement_class, gst_static_pad_template_get (&sink_factory)); - gst_element_class_set_details_simple (element_class, "Opus audio encoder", + gst_element_class_set_details_simple (gstelement_class, "Opus audio encoder", "Codec/Encoder/Audio", "Encodes audio in Opus format", "Sebastian Dröge "); @@ -290,7 +288,7 @@ gst_opus_enc_finalize (GObject * object) } static void -gst_opus_enc_init (GstOpusEnc * enc, GstOpusEncClass * klass) +gst_opus_enc_init (GstOpusEnc * enc) { GstAudioEncoder *benc = GST_AUDIO_ENCODER (enc); @@ -324,7 +322,7 @@ gst_opus_enc_start (GstAudioEncoder * benc) GstOpusEnc *enc = GST_OPUS_ENC (benc); GST_DEBUG_OBJECT (enc, "start"); - enc->tags = gst_tag_list_new (); + enc->tags = gst_tag_list_new_empty (); enc->header_sent = FALSE; return TRUE; @@ -338,7 +336,7 @@ gst_opus_enc_stop (GstAudioEncoder * benc) GST_DEBUG_OBJECT (enc, "stop"); enc->header_sent = FALSE; if (enc->state) { - opus_encoder_destroy (enc->state); + opus_multistream_encoder_destroy (enc->state); enc->state = NULL; } gst_tag_list_free (enc->tags); @@ -402,6 +400,82 @@ gst_opus_enc_get_frame_samples (GstOpusEnc * enc) return frame_samples; } +static void +gst_opus_enc_setup_channel_mapping (GstOpusEnc * enc, const GstAudioInfo * info) +{ +#define MAPS(idx,pos) (GST_AUDIO_INFO_POSITION (info, (idx)) == GST_AUDIO_CHANNEL_POSITION_##pos) + + int n; + + GST_DEBUG_OBJECT (enc, "Setting up channel mapping for %d channels", + enc->n_channels); + + /* Start by setting up a default trivial mapping */ + for (n = 0; n < 255; ++n) + enc->channel_mapping[n] = n; + + /* For one channel, use the basic RTP mapping */ + if (enc->n_channels == 1) { + GST_INFO_OBJECT (enc, "Mono, trivial RTP mapping"); + enc->channel_mapping_family = 0; + enc->channel_mapping[0] = 0; + return; + } + + /* For two channels, use the basic RTP mapping if the channels are + mapped as left/right. */ + if (enc->n_channels == 2) { + if (MAPS (0, FRONT_LEFT) && MAPS (1, FRONT_RIGHT)) { + GST_INFO_OBJECT (enc, "Stereo, canonical mapping"); + enc->channel_mapping_family = 0; + /* The channel mapping is implicit for family 0, that's why we do not + attempt to create one for right/left - this will be mapped to the + Vorbis mapping below. */ + } else { + GST_DEBUG_OBJECT (enc, "Stereo, but not canonical mapping, continuing"); + } + } + + /* For channels between 1 and 8, we use the Vorbis mapping if we can + find a permutation that matches it. Mono will have been taken care + of earlier, but this code also handles it. */ + if (enc->n_channels >= 1 && enc->n_channels <= 8) { + GST_DEBUG_OBJECT (enc, + "In range for the Vorbis mapping, checking channel positions"); + for (n = 0; n < enc->n_channels; ++n) { + GstAudioChannelPosition pos = GST_AUDIO_INFO_POSITION (info, n); + int c; + + GST_DEBUG_OBJECT (enc, "Channel %d has position %d", n, pos); + for (c = 0; c < enc->n_channels; ++c) { + if (gst_opus_channel_positions[enc->n_channels - 1][c] == pos) { + GST_DEBUG_OBJECT (enc, "Found in Vorbis mapping as channel %d", c); + break; + } + } + if (c == enc->n_channels) { + /* We did not find that position, so use undefined */ + GST_WARNING_OBJECT (enc, + "Position %d not found in Vorbis mapping, using unknown mapping", + pos); + enc->channel_mapping_family = 255; + return; + } + GST_DEBUG_OBJECT (enc, "Mapping output channel %d to %d", c, n); + enc->channel_mapping[c] = n; + } + GST_INFO_OBJECT (enc, "Permutation found, using Vorbis mapping"); + enc->channel_mapping_family = 1; + return; + } + + /* For other cases, we use undefined, with the default trivial mapping */ + GST_WARNING_OBJECT (enc, "Unknown mapping"); + enc->channel_mapping_family = 255; + +#undef MAPS +} + static gboolean gst_opus_enc_set_format (GstAudioEncoder * benc, GstAudioInfo * info) { @@ -413,12 +487,13 @@ gst_opus_enc_set_format (GstAudioEncoder * benc, GstAudioInfo * info) enc->n_channels = GST_AUDIO_INFO_CHANNELS (info); enc->sample_rate = GST_AUDIO_INFO_RATE (info); + gst_opus_enc_setup_channel_mapping (enc, info); GST_DEBUG_OBJECT (benc, "Setup with %d channels, %d Hz", enc->n_channels, enc->sample_rate); /* handle reconfigure */ if (enc->state) { - opus_encoder_destroy (enc->state); + opus_multistream_encoder_destroy (enc->state); enc->state = NULL; } if (!gst_opus_enc_setup (enc)) @@ -441,29 +516,30 @@ gst_opus_enc_setup (GstOpusEnc * enc) GST_DEBUG_OBJECT (enc, "setup"); - enc->setup = FALSE; - - enc->state = opus_encoder_create (enc->sample_rate, enc->n_channels, + enc->state = + opus_multistream_encoder_create (enc->sample_rate, enc->n_channels, + (enc->n_channels + 1) / 2, enc->n_channels / 2, enc->channel_mapping, enc->audio_or_voip ? OPUS_APPLICATION_AUDIO : OPUS_APPLICATION_VOIP, &error); if (!enc->state || error != OPUS_OK) goto encoder_creation_failed; - opus_encoder_ctl (enc->state, OPUS_SET_BITRATE (enc->bitrate), 0); - opus_encoder_ctl (enc->state, OPUS_SET_BANDWIDTH (enc->bandwidth), 0); - opus_encoder_ctl (enc->state, OPUS_SET_VBR (!enc->cbr), 0); - opus_encoder_ctl (enc->state, OPUS_SET_VBR_CONSTRAINT (enc->constrained_vbr), + opus_multistream_encoder_ctl (enc->state, OPUS_SET_BITRATE (enc->bitrate), 0); + opus_multistream_encoder_ctl (enc->state, OPUS_SET_BANDWIDTH (enc->bandwidth), 0); - opus_encoder_ctl (enc->state, OPUS_SET_COMPLEXITY (enc->complexity), 0); - opus_encoder_ctl (enc->state, OPUS_SET_INBAND_FEC (enc->inband_fec), 0); - opus_encoder_ctl (enc->state, OPUS_SET_DTX (enc->dtx), 0); - opus_encoder_ctl (enc->state, + opus_multistream_encoder_ctl (enc->state, OPUS_SET_VBR (!enc->cbr), 0); + opus_multistream_encoder_ctl (enc->state, + OPUS_SET_VBR_CONSTRAINT (enc->constrained_vbr), 0); + opus_multistream_encoder_ctl (enc->state, + OPUS_SET_COMPLEXITY (enc->complexity), 0); + opus_multistream_encoder_ctl (enc->state, + OPUS_SET_INBAND_FEC (enc->inband_fec), 0); + opus_multistream_encoder_ctl (enc->state, OPUS_SET_DTX (enc->dtx), 0); + opus_multistream_encoder_ctl (enc->state, OPUS_SET_PACKET_LOSS_PERC (enc->packet_loss_percentage), 0); GST_LOG_OBJECT (enc, "we have frame size %d", enc->frame_size); - enc->setup = TRUE; - return TRUE; encoder_creation_failed: @@ -501,7 +577,7 @@ gst_opus_enc_sink_event (GstAudioEncoder * benc, GstEvent * event) static GstFlowReturn gst_opus_enc_encode (GstOpusEnc * enc, GstBuffer * buf) { - guint8 *bdata, *data, *mdata = NULL; + guint8 *bdata = NULL, *data, *mdata = NULL; gsize bsize, size; gsize bytes = enc->frame_samples * enc->n_channels * 2; gint ret = GST_FLOW_OK; @@ -535,17 +611,17 @@ gst_opus_enc_encode (GstOpusEnc * enc, GstBuffer * buf) gsize out_size; GstBuffer *outbuf; - outbuf = gst_buffer_new_and_alloc (enc->max_payload_size); + outbuf = gst_buffer_new_and_alloc (enc->max_payload_size * enc->n_channels); if (!outbuf) goto done; GST_DEBUG_OBJECT (enc, "encoding %d samples (%d bytes)", - enc->frame_samples); + enc->frame_samples, (int) bytes); out_data = gst_buffer_map (outbuf, &out_size, NULL, GST_MAP_WRITE); encoded_size = - opus_encode (enc->state, (const gint16 *) data, enc->frame_samples, - out_data, enc->max_payload_size); + opus_multistream_encode (enc->state, (const gint16 *) data, + enc->frame_samples, out_data, enc->max_payload_size * enc->n_channels); gst_buffer_unmap (outbuf, out_data, out_size); if (encoded_size < 0) { @@ -555,12 +631,13 @@ gst_opus_enc_encode (GstOpusEnc * enc, GstBuffer * buf) } else if (encoded_size > enc->max_payload_size) { GST_WARNING_OBJECT (enc, "Encoded size %d is higher than max payload size (%d bytes)", - outsize, enc->max_payload_size); + out_size, enc->max_payload_size); ret = GST_FLOW_ERROR; goto done; } - GST_BUFFER_SIZE (outbuf) = outsize; + GST_DEBUG_OBJECT (enc, "Output packet is %u bytes", encoded_size); + gst_buffer_set_size (outbuf, encoded_size); ret = gst_audio_encoder_finish_frame (GST_AUDIO_ENCODER (enc), outbuf, @@ -601,7 +678,8 @@ gst_opus_enc_handle_frame (GstAudioEncoder * benc, GstBuffer * buf) enc->headers = NULL; gst_opus_header_create_caps (&caps, &enc->headers, enc->n_channels, - enc->sample_rate, gst_tag_setter_get_tag_list (GST_TAG_SETTER (enc))); + enc->sample_rate, enc->channel_mapping_family, enc->channel_mapping, + gst_tag_setter_get_tag_list (GST_TAG_SETTER (enc))); /* negotiate with these caps */ @@ -613,7 +691,7 @@ gst_opus_enc_handle_frame (GstAudioEncoder * benc, GstBuffer * buf) } GST_DEBUG_OBJECT (enc, "received buffer %p of %u bytes", buf, - buf ? GST_BUFFER_SIZE (buf) : 0); + buf ? gst_buffer_get_size (buf) : 0); ret = gst_opus_enc_encode (enc, buf); @@ -684,7 +762,7 @@ gst_opus_enc_set_property (GObject * object, guint prop_id, g_mutex_lock (enc->property_lock); \ enc->prop = g_value_get_##type (value); \ if (enc->state) { \ - opus_encoder_ctl (enc->state, OPUS_SET_##ctl (enc->prop)); \ + opus_multistream_encoder_ctl (enc->state, OPUS_SET_##ctl (enc->prop)); \ } \ g_mutex_unlock (enc->property_lock); \ } while(0) @@ -710,7 +788,7 @@ gst_opus_enc_set_property (GObject * object, guint prop_id, /* this one has an opposite meaning to the opus ctl... */ g_mutex_lock (enc->property_lock); enc->cbr = g_value_get_boolean (value); - opus_encoder_ctl (enc->state, OPUS_SET_VBR (!enc->cbr)); + opus_multistream_encoder_ctl (enc->state, OPUS_SET_VBR (!enc->cbr)); g_mutex_unlock (enc->property_lock); break; case PROP_CONSTRAINED_VBR: diff --git a/ext/opus/gstopusenc.h b/ext/opus/gstopusenc.h index 772f6f4cc7..8c2c3c6e82 100644 --- a/ext/opus/gstopusenc.h +++ b/ext/opus/gstopusenc.h @@ -26,7 +26,7 @@ #include #include -#include +#include G_BEGIN_DECLS @@ -50,7 +50,7 @@ typedef struct _GstOpusEncClass GstOpusEncClass; struct _GstOpusEnc { GstAudioEncoder element; - OpusEncoder *state; + OpusMSEncoder *state; /* Locks those properties which may be changed at play time */ GMutex *property_lock; @@ -72,12 +72,14 @@ struct _GstOpusEnc { gint n_channels; gint sample_rate; - gboolean setup; gboolean header_sent; GSList *headers; GstTagList *tags; + + guint8 channel_mapping_family; + guint8 channel_mapping[256]; }; struct _GstOpusEncClass { diff --git a/ext/opus/gstopusheader.c b/ext/opus/gstopusheader.c index 3551055840..4f95363336 100644 --- a/ext/opus/gstopusheader.c +++ b/ext/opus/gstopusheader.c @@ -27,7 +27,8 @@ #include "gstopusheader.h" static GstBuffer * -gst_opus_enc_create_id_buffer (gint nchannels, gint sample_rate) +gst_opus_enc_create_id_buffer (gint nchannels, gint sample_rate, + guint8 channel_mapping_family, const guint8 * channel_mapping) { GstBuffer *buffer; GstByteWriter bw; @@ -38,10 +39,15 @@ gst_opus_enc_create_id_buffer (gint nchannels, gint sample_rate) gst_byte_writer_put_data (&bw, (const guint8 *) "OpusHead", 8); gst_byte_writer_put_uint8 (&bw, 0); /* version number */ gst_byte_writer_put_uint8 (&bw, nchannels); - gst_byte_writer_put_uint16_le (&bw, 0); /* pre-skip *//* TODO: endianness ? */ + gst_byte_writer_put_uint16_le (&bw, 0); /* pre-skip */ gst_byte_writer_put_uint32_le (&bw, sample_rate); - gst_byte_writer_put_uint16_le (&bw, 0); /* output gain *//* TODO: endianness ? */ - gst_byte_writer_put_uint8 (&bw, 0); /* channel mapping *//* TODO: what is this ? */ + gst_byte_writer_put_uint16_le (&bw, 0); /* output gain */ + gst_byte_writer_put_uint8 (&bw, channel_mapping_family); + if (channel_mapping_family > 0) { + gst_byte_writer_put_uint8 (&bw, (nchannels + 1) / 2); + gst_byte_writer_put_uint8 (&bw, nchannels / 2); + gst_byte_writer_put_data (&bw, channel_mapping, nchannels); + } buffer = gst_byte_writer_reset_and_get_buffer (&bw); @@ -62,7 +68,7 @@ gst_opus_enc_create_metadata_buffer (const GstTagList * tags) if (tags == NULL) { /* FIXME: better fix chain of callers to not write metadata at all, * if there is none */ - empty_tags = gst_tag_list_new (); + empty_tags = gst_tag_list_new_empty (); tags = empty_tags; } comments = @@ -136,23 +142,11 @@ _gst_caps_set_buffer_array (GstCaps * caps, const gchar * field, } void -gst_opus_header_create_caps (GstCaps ** caps, GSList ** headers, gint nchannels, - gint sample_rate, const GstTagList * tags) +gst_opus_header_create_caps_from_headers (GstCaps ** caps, GSList ** headers, + GstBuffer * buf1, GstBuffer * buf2) { - GstBuffer *buf1, *buf2; - g_return_if_fail (caps); g_return_if_fail (headers && !*headers); - g_return_if_fail (nchannels > 0); - g_return_if_fail (sample_rate >= 0); /* 0 -> unset */ - - /* Opus streams in Ogg begin with two headers; the initial header (with - most of the codec setup parameters) which is mandated by the Ogg - bitstream spec. The second header holds any comment fields. */ - - /* create header buffers */ - buf1 = gst_opus_enc_create_id_buffer (nchannels, sample_rate); - buf2 = gst_opus_enc_create_metadata_buffer (tags); /* mark and put on caps */ *caps = gst_caps_from_string ("audio/x-opus"); @@ -162,9 +156,87 @@ gst_opus_header_create_caps (GstCaps ** caps, GSList ** headers, gint nchannels, *headers = g_slist_prepend (*headers, buf1); } +void +gst_opus_header_create_caps (GstCaps ** caps, GSList ** headers, gint nchannels, + gint sample_rate, guint8 channel_mapping_family, + const guint8 * channel_mapping, const GstTagList * tags) +{ + GstBuffer *buf1, *buf2; + + g_return_if_fail (caps); + g_return_if_fail (headers && !*headers); + g_return_if_fail (nchannels > 0); + g_return_if_fail (sample_rate >= 0); /* 0 -> unset */ + g_return_if_fail (channel_mapping_family == 0 || channel_mapping); + + /* Opus streams in Ogg begin with two headers; the initial header (with + most of the codec setup parameters) which is mandated by the Ogg + bitstream spec. The second header holds any comment fields. */ + + /* create header buffers */ + buf1 = + gst_opus_enc_create_id_buffer (nchannels, sample_rate, + channel_mapping_family, channel_mapping); + buf2 = gst_opus_enc_create_metadata_buffer (tags); + + gst_opus_header_create_caps_from_headers (caps, headers, buf1, buf2); +} + gboolean gst_opus_header_is_header (GstBuffer * buf, const char *magic, guint magic_size) { - return (GST_BUFFER_SIZE (buf) >= magic_size - && !memcmp (magic, GST_BUFFER_DATA (buf), magic_size)); + return (gst_buffer_get_size (buf) >= magic_size + && !gst_buffer_memcmp (buf, 0, magic, magic_size)); +} + +gboolean +gst_opus_header_is_id_header (GstBuffer * buf) +{ + gsize size = gst_buffer_get_size (buf); + guint8 *data = NULL; + guint8 channels, channel_mapping_family, n_streams, n_stereo_streams; + gboolean ret = FALSE; + + if (size < 19) + goto beach; + if (!gst_opus_header_is_header (buf, "OpusHead", 8)) + goto beach; + + data = gst_buffer_map (buf, &size, NULL, GST_MAP_READ); + + channels = data[9]; + + if (channels == 0) + goto beach; + + channel_mapping_family = data[18]; + + if (channel_mapping_family == 0) { + if (channels > 2) + goto beach; + } else { + channels = data[9]; + if (size < 21 + channels) + goto beach; + n_streams = data[19]; + n_stereo_streams = data[20]; + if (n_streams == 0) + goto beach; + if (n_stereo_streams > n_streams) + goto beach; + if (n_streams + n_stereo_streams > 255) + goto beach; + } + ret = TRUE; + +beach: + if (data) + gst_buffer_unmap (buf, data, size); + return ret; +} + +gboolean +gst_opus_header_is_comment_header (GstBuffer * buf) +{ + return gst_opus_header_is_header (buf, "OpusTags", 8); } diff --git a/ext/opus/gstopusheader.h b/ext/opus/gstopusheader.h index 4594264ccd..3b2cfc265f 100644 --- a/ext/opus/gstopusheader.h +++ b/ext/opus/gstopusheader.h @@ -25,8 +25,16 @@ G_BEGIN_DECLS -extern void gst_opus_header_create_caps (GstCaps **caps, GSList **headers, gint nchannels, gint sample_rate, const GstTagList *tags); -extern gboolean gst_opus_header_is_header (GstBuffer * buf, const char *magic, guint magic_size); +extern void gst_opus_header_create_caps_from_headers (GstCaps **caps, GSList **headers, + GstBuffer *id_header, GstBuffer *comment_header); +extern void gst_opus_header_create_caps (GstCaps **caps, GSList **headers, + gint nchannels, gint sample_rate, + guint8 channel_mapping_family, const guint8 *channel_mapping, + const GstTagList *tags); +extern gboolean gst_opus_header_is_header (GstBuffer * buf, + const char *magic, guint magic_size); +extern gboolean gst_opus_header_is_id_header (GstBuffer * buf); +extern gboolean gst_opus_header_is_comment_header (GstBuffer * buf); G_END_DECLS