From bfdda8e09a11f565ff935f4f5c4e0267e72b7554 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 23 Nov 2011 11:58:54 +0000 Subject: [PATCH 1/7] opusdec: add in-band FEC support This allows reconstruction of lost packets if FEC info is included in the next packet, at the cost of extra latency. Since we do not know if the stream has FEC (and this can change at runtime), we always incur the latency, even if we never lose any frame, or see any FEC information. Off by default. --- ext/opus/gstopusdec.c | 123 ++++++++++++++++++++++++++++++++++++------ ext/opus/gstopusdec.h | 4 ++ 2 files changed, 110 insertions(+), 17 deletions(-) diff --git a/ext/opus/gstopusdec.c b/ext/opus/gstopusdec.c index 89eec6941e..7d70fea184 100644 --- a/ext/opus/gstopusdec.c +++ b/ext/opus/gstopusdec.c @@ -67,6 +67,14 @@ GST_STATIC_PAD_TEMPLATE ("sink", GST_STATIC_CAPS ("audio/x-opus") ); +#define DEFAULT_USE_INBAND_FEC FALSE + +enum +{ + PROP_0, + PROP_USE_INBAND_FEC +}; + GST_BOILERPLATE (GstOpusDec, gst_opus_dec, GstAudioDecoder, GST_TYPE_AUDIO_DECODER); @@ -78,6 +86,11 @@ static GstFlowReturn gst_opus_dec_handle_frame (GstAudioDecoder * dec, GstBuffer * buffer); static gboolean gst_opus_dec_set_format (GstAudioDecoder * bdec, GstCaps * caps); +static void gst_opus_dec_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); +static void gst_opus_dec_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); + static void gst_opus_dec_base_init (gpointer g_class) @@ -97,17 +110,27 @@ gst_opus_dec_base_init (gpointer g_class) static void gst_opus_dec_class_init (GstOpusDecClass * klass) { + GObjectClass *gobject_class; GstAudioDecoderClass *adclass; GstElementClass *gstelement_class; + gobject_class = (GObjectClass *) klass; adclass = (GstAudioDecoderClass *) klass; gstelement_class = (GstElementClass *) klass; + gobject_class->set_property = gst_opus_dec_set_property; + gobject_class->get_property = gst_opus_dec_get_property; + adclass->start = GST_DEBUG_FUNCPTR (gst_opus_dec_start); adclass->stop = GST_DEBUG_FUNCPTR (gst_opus_dec_stop); adclass->handle_frame = GST_DEBUG_FUNCPTR (gst_opus_dec_handle_frame); adclass->set_format = GST_DEBUG_FUNCPTR (gst_opus_dec_set_format); + g_object_class_install_property (gobject_class, PROP_USE_INBAND_FEC, + g_param_spec_boolean ("use-inband-fec", "Use in-band FEC", + "Use forward error correction if available", DEFAULT_USE_INBAND_FEC, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + GST_DEBUG_CATEGORY_INIT (opusdec_debug, "opusdec", 0, "opus decoding element"); } @@ -123,6 +146,8 @@ gst_opus_dec_reset (GstOpusDec * dec) gst_buffer_replace (&dec->streamheader, NULL); gst_buffer_replace (&dec->vorbiscomment, NULL); + gst_buffer_replace (&dec->last_buffer, NULL); + dec->primed = FALSE; dec->pre_skip = 0; } @@ -132,6 +157,7 @@ gst_opus_dec_init (GstOpusDec * dec, GstOpusDecClass * g_class) { dec->sample_rate = 0; dec->n_channels = 0; + dec->use_inband_fec = FALSE; gst_opus_dec_reset (dec); } @@ -146,6 +172,11 @@ gst_opus_dec_start (GstAudioDecoder * dec) /* we know about concealment */ gst_audio_decoder_set_plc_aware (dec, TRUE); + if (odec->use_inband_fec) { + gst_audio_decoder_set_latency (dec, 2 * GST_MSECOND + GST_MSECOND / 2, + 120 * GST_MSECOND); + } + return TRUE; } @@ -222,7 +253,7 @@ gst_opus_dec_setup_from_peer_caps (GstOpusDec * dec) } static GstFlowReturn -opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf) +opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buffer) { GstFlowReturn res = GST_FLOW_OK; gint size; @@ -232,6 +263,7 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf) int n, err; int samples; unsigned int packet_size; + GstBuffer *buf; if (dec->state == NULL) { gst_opus_dec_setup_from_peer_caps (dec); @@ -243,30 +275,37 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf) goto creation_failed; } + if (buffer) { + GST_DEBUG_OBJECT (dec, "Received buffer of size %u", + GST_BUFFER_SIZE (buffer)); + } else { + GST_DEBUG_OBJECT (dec, "Received missing buffer"); + } + + /* if using in-band FEC, we introdude one extra frame's delay as we need + to potentially wait for next buffer to decode a missing buffer */ + if (dec->use_inband_fec && !dec->primed) { + GST_DEBUG_OBJECT (dec, "First buffer received in FEC mode, early out"); + goto done; + } + + /* That's the buffer we'll be sending to the opus decoder. */ + buf = dec->use_inband_fec && dec->last_buffer ? dec->last_buffer : buffer; + if (buf) { data = GST_BUFFER_DATA (buf); size = GST_BUFFER_SIZE (buf); - - GST_DEBUG_OBJECT (dec, "received buffer of size %u", size); + GST_DEBUG_OBJECT (dec, "Using buffer of size %u", size); } else { /* concealment data, pass NULL as the bits parameters */ - GST_DEBUG_OBJECT (dec, "creating concealment data"); + GST_DEBUG_OBJECT (dec, "Using NULL buffer"); data = NULL; size = 0; } - if (data) { - samples = - opus_packet_get_samples_per_frame (data, - dec->sample_rate) * opus_packet_get_nb_frames (data, size); - packet_size = samples * dec->n_channels * 2; - GST_DEBUG_OBJECT (dec, "bandwidth %d", opus_packet_get_bandwidth (data)); - GST_DEBUG_OBJECT (dec, "samples %d", samples); - } else { - /* use maximum size (120 ms) as we do now know in advance how many samples - will be returned */ - samples = 120 * dec->sample_rate / 1000; - } + /* use maximum size (120 ms) as the number of returned samples is + not constant over the stream. */ + samples = 120 * dec->sample_rate / 1000; packet_size = samples * dec->n_channels * 2; res = gst_pad_alloc_buffer_and_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), @@ -280,7 +319,19 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf) out_data = (gint16 *) GST_BUFFER_DATA (outbuf); - n = opus_decode (dec->state, data, size, out_data, samples, 0); + if (dec->use_inband_fec) { + if (dec->last_buffer) { + /* normal delayed decode */ + n = opus_decode (dec->state, data, size, out_data, samples, 0); + } else { + /* FEC reconstruction decode */ + n = opus_decode (dec->state, data, size, out_data, samples, 1); + } + } else { + /* normal decode */ + n = opus_decode (dec->state, data, size, out_data, samples, 0); + } + if (n < 0) { GST_ELEMENT_ERROR (dec, STREAM, DECODE, ("Decoding error: %d", n), (NULL)); return GST_FLOW_ERROR; @@ -311,6 +362,12 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buf) if (res != GST_FLOW_OK) GST_DEBUG_OBJECT (dec, "flow: %s", gst_flow_get_name (res)); +done: + if (dec->use_inband_fec) { + gst_buffer_replace (&dec->last_buffer, buffer); + dec->primed = TRUE; + } + return res; creation_failed: @@ -437,3 +494,35 @@ gst_opus_dec_handle_frame (GstAudioDecoder * adec, GstBuffer * buf) return res; } + +static void +gst_opus_dec_get_property (GObject * object, guint prop_id, GValue * value, + GParamSpec * pspec) +{ + GstOpusDec *dec = GST_OPUS_DEC (object); + + switch (prop_id) { + case PROP_USE_INBAND_FEC: + g_value_set_boolean (value, dec->use_inband_fec); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_opus_dec_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstOpusDec *dec = GST_OPUS_DEC (object); + + switch (prop_id) { + case PROP_USE_INBAND_FEC: + dec->use_inband_fec = g_value_get_boolean (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} diff --git a/ext/opus/gstopusdec.h b/ext/opus/gstopusdec.h index eee27dc554..081e575247 100644 --- a/ext/opus/gstopusdec.h +++ b/ext/opus/gstopusdec.h @@ -54,6 +54,10 @@ struct _GstOpusDec { int sample_rate; int n_channels; guint32 pre_skip; + + gboolean use_inband_fec; + GstBuffer *last_buffer; + gboolean primed; }; struct _GstOpusDecClass { From 5c8812f58c5d0f41d29e4ec4bb647a1151050a74 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 23 Nov 2011 13:22:12 +0000 Subject: [PATCH 2/7] opusdec: implement replay gain It would ideally be better to leave this to a rgvolume element, but we don't control the pipeline. So do it by default, and allow disabling it via a property, so the correct volume should always be output. --- ext/opus/gstopusdec.c | 55 +++++++++++++++++++++++++++++++++++++++++-- ext/opus/gstopusdec.h | 4 ++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/ext/opus/gstopusdec.c b/ext/opus/gstopusdec.c index 7d70fea184..e85a6b7471 100644 --- a/ext/opus/gstopusdec.c +++ b/ext/opus/gstopusdec.c @@ -41,6 +41,7 @@ # include "config.h" #endif +#include #include #include #include "gstopusheader.h" @@ -67,12 +68,16 @@ GST_STATIC_PAD_TEMPLATE ("sink", GST_STATIC_CAPS ("audio/x-opus") ); +#define DB_TO_LINEAR(x) pow (10., (x) / 20.) + #define DEFAULT_USE_INBAND_FEC FALSE +#define DEFAULT_APPLY_GAIN TRUE enum { PROP_0, - PROP_USE_INBAND_FEC + PROP_USE_INBAND_FEC, + PROP_APPLY_GAIN }; GST_BOILERPLATE (GstOpusDec, gst_opus_dec, GstAudioDecoder, @@ -131,6 +136,11 @@ gst_opus_dec_class_init (GstOpusDecClass * klass) "Use forward error correction if available", DEFAULT_USE_INBAND_FEC, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_APPLY_GAIN, + g_param_spec_boolean ("apply-gain", "Apply gain", + "Apply gain if any is specified in the header", DEFAULT_APPLY_GAIN, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + GST_DEBUG_CATEGORY_INIT (opusdec_debug, "opusdec", 0, "opus decoding element"); } @@ -150,6 +160,7 @@ gst_opus_dec_reset (GstOpusDec * dec) dec->primed = FALSE; dec->pre_skip = 0; + dec->r128_gain = 0; } static void @@ -158,6 +169,7 @@ gst_opus_dec_init (GstOpusDec * dec, GstOpusDecClass * g_class) dec->sample_rate = 0; dec->n_channels = 0; dec->use_inband_fec = FALSE; + dec->apply_gain = DEFAULT_APPLY_GAIN; gst_opus_dec_reset (dec); } @@ -190,6 +202,18 @@ gst_opus_dec_stop (GstAudioDecoder * dec) return TRUE; } +static double +gst_opus_dec_get_r128_gain (gint16 r128_gain) +{ + return r128_gain / (double) (1 << 8); +} + +static double +gst_opus_dec_get_r128_volume (gint16 r128_gain) +{ + return DB_TO_LINEAR (gst_opus_dec_get_r128_gain (r128_gain)); +} + static GstFlowReturn gst_opus_dec_parse_header (GstOpusDec * dec, GstBuffer * buf) { @@ -198,7 +222,11 @@ gst_opus_dec_parse_header (GstOpusDec * dec, GstBuffer * buf) g_return_val_if_fail (GST_BUFFER_SIZE (buf) >= 19, GST_FLOW_ERROR); dec->pre_skip = GST_READ_UINT16_LE (GST_BUFFER_DATA (buf) + 10); - GST_INFO_OBJECT (dec, "Found pre-skip of %u samples", dec->pre_skip); + dec->r128_gain = GST_READ_UINT16_LE (GST_BUFFER_DATA (buf) + 14); + dec->r128_gain_volume = gst_opus_dec_get_r128_volume (dec->r128_gain); + GST_INFO_OBJECT (dec, + "Found pre-skip of %u samples, R128 gain %d (volume %f)", + dec->pre_skip, dec->r128_gain, dec->r128_gain_volume); return GST_FLOW_OK; } @@ -357,6 +385,23 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buffer) } } + /* Apply gain */ + /* Would be better off leaving this to a volume element, as this is + a naive conversion that does too many int/float conversions. + However, we don't have control over the pipeline... + So make it optional if the user program wants to use a volume, + but do it by default so the correct volume goes out by default */ + if (dec->apply_gain && outbuf && dec->r128_gain) { + unsigned int i, nsamples = GST_BUFFER_SIZE (outbuf) / 2; + double volume = dec->r128_gain_volume; + gint16 *samples = (gint16 *) GST_BUFFER_DATA (outbuf); + GST_DEBUG_OBJECT (dec, "Applying gain: volume %f", volume); + for (i = 0; i < nsamples; ++i) { + int sample = (int) (samples[i] * volume + 0.5); + samples[i] = sample < -32768 ? -32768 : sample > 32767 ? 32767 : sample; + } + } + res = gst_audio_decoder_finish_frame (GST_AUDIO_DECODER (dec), outbuf, 1); if (res != GST_FLOW_OK) @@ -505,6 +550,9 @@ gst_opus_dec_get_property (GObject * object, guint prop_id, GValue * value, case PROP_USE_INBAND_FEC: g_value_set_boolean (value, dec->use_inband_fec); break; + case PROP_APPLY_GAIN: + g_value_set_boolean (value, dec->apply_gain); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -521,6 +569,9 @@ gst_opus_dec_set_property (GObject * object, guint prop_id, case PROP_USE_INBAND_FEC: dec->use_inband_fec = g_value_get_boolean (value); break; + case PROP_APPLY_GAIN: + dec->apply_gain = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; diff --git a/ext/opus/gstopusdec.h b/ext/opus/gstopusdec.h index 081e575247..b074787d7f 100644 --- a/ext/opus/gstopusdec.h +++ b/ext/opus/gstopusdec.h @@ -54,6 +54,10 @@ struct _GstOpusDec { int sample_rate; int n_channels; guint32 pre_skip; + gint16 r128_gain; + + gboolean apply_gain; + double r128_gain_volume; gboolean use_inband_fec; GstBuffer *last_buffer; From d1d8cbad95611203718b5d93c1f929528e978cb5 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 23 Nov 2011 16:36:54 +0000 Subject: [PATCH 3/7] opusenc: remove useless setup field --- ext/opus/gstopusenc.c | 4 ---- ext/opus/gstopusenc.h | 1 - 2 files changed, 5 deletions(-) diff --git a/ext/opus/gstopusenc.c b/ext/opus/gstopusenc.c index 29b9faf756..71e19b045a 100644 --- a/ext/opus/gstopusenc.c +++ b/ext/opus/gstopusenc.c @@ -458,8 +458,6 @@ gst_opus_enc_setup (GstOpusEnc * enc) GST_DEBUG_OBJECT (enc, "setup"); - enc->setup = FALSE; - enc->state = opus_encoder_create (enc->sample_rate, enc->n_channels, enc->audio_or_voip ? OPUS_APPLICATION_AUDIO : OPUS_APPLICATION_VOIP, &error); @@ -479,8 +477,6 @@ gst_opus_enc_setup (GstOpusEnc * enc) GST_LOG_OBJECT (enc, "we have frame size %d", enc->frame_size); - enc->setup = TRUE; - return TRUE; encoder_creation_failed: diff --git a/ext/opus/gstopusenc.h b/ext/opus/gstopusenc.h index 772f6f4cc7..58c617c5a1 100644 --- a/ext/opus/gstopusenc.h +++ b/ext/opus/gstopusenc.h @@ -72,7 +72,6 @@ struct _GstOpusEnc { gint n_channels; gint sample_rate; - gboolean setup; gboolean header_sent; GSList *headers; From e4ae3e89faa4b5fe09319ffdee1e696b5c8e8448 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 23 Nov 2011 17:32:03 +0000 Subject: [PATCH 4/7] opusdec: shuffle supported sample rates to favor 48000 --- ext/opus/gstopusdec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opus/gstopusdec.c b/ext/opus/gstopusdec.c index e85a6b7471..7ac2bda279 100644 --- a/ext/opus/gstopusdec.c +++ b/ext/opus/gstopusdec.c @@ -55,7 +55,7 @@ GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS ("audio/x-raw-int, " - "rate = (int) { 8000, 12000, 16000, 24000, 48000 }, " + "rate = (int) { 48000, 24000, 16000, 12000, 8000 }, " "channels = (int) [ 1, 2 ], " "endianness = (int) BYTE_ORDER, " "signed = (boolean) true, " "width = (int) 16, " "depth = (int) 16") From 670c365400b4babd4b907f807bbc52de0b7232a4 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Wed, 23 Nov 2011 17:49:58 +0000 Subject: [PATCH 5/7] opus: switch to multistream API It's very similar to the basic API, and is a superset ot it, which will allow encoding and decoding more than 2 channels. --- ext/opus/gstopusdec.c | 23 ++++++++++++++++++----- ext/opus/gstopusdec.h | 6 ++++-- ext/opus/gstopusenc.c | 40 +++++++++++++++++++++++++--------------- ext/opus/gstopusenc.h | 4 ++-- 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/ext/opus/gstopusdec.c b/ext/opus/gstopusdec.c index 7ac2bda279..f8b39ba08c 100644 --- a/ext/opus/gstopusdec.c +++ b/ext/opus/gstopusdec.c @@ -150,7 +150,7 @@ gst_opus_dec_reset (GstOpusDec * dec) { dec->packetno = 0; if (dec->state) { - opus_decoder_destroy (dec->state); + opus_multistream_decoder_destroy (dec->state); dec->state = NULL; } @@ -228,6 +228,16 @@ gst_opus_dec_parse_header (GstOpusDec * dec, GstBuffer * buf) "Found pre-skip of %u samples, R128 gain %d (volume %f)", dec->pre_skip, dec->r128_gain, dec->r128_gain_volume); + dec->channel_mapping_family = GST_BUFFER_DATA (buf)[18]; + if (dec->channel_mapping_family != 0) { + GST_ELEMENT_ERROR (dec, STREAM, DECODE, + ("Decoding error: unsupported channel nmapping family %d", + dec->channel_mapping_family), (NULL)); + return GST_FLOW_ERROR; + } + dec->channel_mapping[0] = 0; + dec->channel_mapping[1] = 1; + return GST_FLOW_OK; } @@ -298,7 +308,8 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buffer) GST_DEBUG_OBJECT (dec, "Creating decoder with %d channels, %d Hz", dec->n_channels, dec->sample_rate); - dec->state = opus_decoder_create (dec->sample_rate, dec->n_channels, &err); + dec->state = opus_multistream_decoder_create (dec->sample_rate, + dec->n_channels, 1, 1, dec->channel_mapping, &err); if (!dec->state || err != OPUS_OK) goto creation_failed; } @@ -350,14 +361,16 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buffer) if (dec->use_inband_fec) { if (dec->last_buffer) { /* normal delayed decode */ - n = opus_decode (dec->state, data, size, out_data, samples, 0); + n = opus_multistream_decode (dec->state, data, size, out_data, samples, + 0); } else { /* FEC reconstruction decode */ - n = opus_decode (dec->state, data, size, out_data, samples, 1); + n = opus_multistream_decode (dec->state, data, size, out_data, samples, + 1); } } else { /* normal decode */ - n = opus_decode (dec->state, data, size, out_data, samples, 0); + n = opus_multistream_decode (dec->state, data, size, out_data, samples, 0); } if (n < 0) { diff --git a/ext/opus/gstopusdec.h b/ext/opus/gstopusdec.h index b074787d7f..aa04b814d9 100644 --- a/ext/opus/gstopusdec.h +++ b/ext/opus/gstopusdec.h @@ -23,7 +23,7 @@ #include #include -#include +#include G_BEGIN_DECLS @@ -44,7 +44,7 @@ typedef struct _GstOpusDecClass GstOpusDecClass; struct _GstOpusDec { GstAudioDecoder element; - OpusDecoder *state; + OpusMSDecoder *state; guint64 packetno; @@ -55,6 +55,8 @@ struct _GstOpusDec { int n_channels; guint32 pre_skip; gint16 r128_gain; + guint8 channel_mapping_family; + guint8 channel_mapping[256]; gboolean apply_gain; double r128_gain_volume; diff --git a/ext/opus/gstopusenc.c b/ext/opus/gstopusenc.c index 71e19b045a..93e00aa38f 100644 --- a/ext/opus/gstopusenc.c +++ b/ext/opus/gstopusenc.c @@ -355,7 +355,7 @@ gst_opus_enc_stop (GstAudioEncoder * benc) GST_DEBUG_OBJECT (enc, "stop"); enc->header_sent = FALSE; if (enc->state) { - opus_encoder_destroy (enc->state); + opus_multistream_encoder_destroy (enc->state); enc->state = NULL; } gst_tag_list_free (enc->tags); @@ -435,7 +435,7 @@ gst_opus_enc_set_format (GstAudioEncoder * benc, GstAudioInfo * info) /* handle reconfigure */ if (enc->state) { - opus_encoder_destroy (enc->state); + opus_multistream_encoder_destroy (enc->state); enc->state = NULL; } if (!gst_opus_enc_setup (enc)) @@ -455,24 +455,34 @@ static gboolean gst_opus_enc_setup (GstOpusEnc * enc) { int error = OPUS_OK; + unsigned char mapping[256]; + int n; GST_DEBUG_OBJECT (enc, "setup"); - enc->state = opus_encoder_create (enc->sample_rate, enc->n_channels, + for (n = 0; n < enc->n_channels; ++n) + mapping[n] = n; + + enc->state = + opus_multistream_encoder_create (enc->sample_rate, enc->n_channels, + (enc->n_channels + 1) / 2, enc->n_channels / 2, mapping, enc->audio_or_voip ? OPUS_APPLICATION_AUDIO : OPUS_APPLICATION_VOIP, &error); if (!enc->state || error != OPUS_OK) goto encoder_creation_failed; - opus_encoder_ctl (enc->state, OPUS_SET_BITRATE (enc->bitrate), 0); - opus_encoder_ctl (enc->state, OPUS_SET_BANDWIDTH (enc->bandwidth), 0); - opus_encoder_ctl (enc->state, OPUS_SET_VBR (!enc->cbr), 0); - opus_encoder_ctl (enc->state, OPUS_SET_VBR_CONSTRAINT (enc->constrained_vbr), + opus_multistream_encoder_ctl (enc->state, OPUS_SET_BITRATE (enc->bitrate), 0); + opus_multistream_encoder_ctl (enc->state, OPUS_SET_BANDWIDTH (enc->bandwidth), 0); - opus_encoder_ctl (enc->state, OPUS_SET_COMPLEXITY (enc->complexity), 0); - opus_encoder_ctl (enc->state, OPUS_SET_INBAND_FEC (enc->inband_fec), 0); - opus_encoder_ctl (enc->state, OPUS_SET_DTX (enc->dtx), 0); - opus_encoder_ctl (enc->state, + opus_multistream_encoder_ctl (enc->state, OPUS_SET_VBR (!enc->cbr), 0); + opus_multistream_encoder_ctl (enc->state, + OPUS_SET_VBR_CONSTRAINT (enc->constrained_vbr), 0); + opus_multistream_encoder_ctl (enc->state, + OPUS_SET_COMPLEXITY (enc->complexity), 0); + opus_multistream_encoder_ctl (enc->state, + OPUS_SET_INBAND_FEC (enc->inband_fec), 0); + opus_multistream_encoder_ctl (enc->state, OPUS_SET_DTX (enc->dtx), 0); + opus_multistream_encoder_ctl (enc->state, OPUS_SET_PACKET_LOSS_PERC (enc->packet_loss_percentage), 0); GST_LOG_OBJECT (enc, "we have frame size %d", enc->frame_size); @@ -557,8 +567,8 @@ gst_opus_enc_encode (GstOpusEnc * enc, GstBuffer * buf) enc->frame_samples); outsize = - opus_encode (enc->state, (const gint16 *) data, enc->frame_samples, - GST_BUFFER_DATA (outbuf), enc->max_payload_size); + opus_multistream_encode (enc->state, (const gint16 *) data, + enc->frame_samples, GST_BUFFER_DATA (outbuf), enc->max_payload_size); if (outsize < 0) { GST_ERROR_OBJECT (enc, "Encoding failed: %d", outsize); @@ -694,7 +704,7 @@ gst_opus_enc_set_property (GObject * object, guint prop_id, g_mutex_lock (enc->property_lock); \ enc->prop = g_value_get_##type (value); \ if (enc->state) { \ - opus_encoder_ctl (enc->state, OPUS_SET_##ctl (enc->prop)); \ + opus_multistream_encoder_ctl (enc->state, OPUS_SET_##ctl (enc->prop)); \ } \ g_mutex_unlock (enc->property_lock); \ } while(0) @@ -720,7 +730,7 @@ gst_opus_enc_set_property (GObject * object, guint prop_id, /* this one has an opposite meaning to the opus ctl... */ g_mutex_lock (enc->property_lock); enc->cbr = g_value_get_boolean (value); - opus_encoder_ctl (enc->state, OPUS_SET_VBR (!enc->cbr)); + opus_multistream_encoder_ctl (enc->state, OPUS_SET_VBR (!enc->cbr)); g_mutex_unlock (enc->property_lock); break; case PROP_CONSTRAINED_VBR: diff --git a/ext/opus/gstopusenc.h b/ext/opus/gstopusenc.h index 58c617c5a1..8304e82556 100644 --- a/ext/opus/gstopusenc.h +++ b/ext/opus/gstopusenc.h @@ -26,7 +26,7 @@ #include #include -#include +#include G_BEGIN_DECLS @@ -50,7 +50,7 @@ typedef struct _GstOpusEncClass GstOpusEncClass; struct _GstOpusEnc { GstAudioEncoder element; - OpusEncoder *state; + OpusMSEncoder *state; /* Locks those properties which may be changed at play time */ GMutex *property_lock; From d38f4b8a09e83db60264d22f827e957b357ef74e Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Thu, 24 Nov 2011 13:29:56 +0000 Subject: [PATCH 6/7] opus: multichannel support --- ext/opus/Makefile.am | 4 +- ext/opus/gstopuscommon.c | 72 ++++++++++++++++++++ ext/opus/gstopuscommon.h | 33 +++++++++ ext/opus/gstopusdec.c | 141 +++++++++++++++++++++++---------------- ext/opus/gstopusdec.h | 3 + ext/opus/gstopusenc.c | 100 +++++++++++++++++++++++---- ext/opus/gstopusenc.h | 3 + ext/opus/gstopusheader.c | 92 ++++++++++++++++++++----- ext/opus/gstopusheader.h | 12 +++- 9 files changed, 369 insertions(+), 91 deletions(-) create mode 100644 ext/opus/gstopuscommon.c create mode 100644 ext/opus/gstopuscommon.h diff --git a/ext/opus/Makefile.am b/ext/opus/Makefile.am index 88845a3cab..cb0a9b338a 100644 --- a/ext/opus/Makefile.am +++ b/ext/opus/Makefile.am @@ -1,6 +1,6 @@ plugin_LTLIBRARIES = libgstopus.la -libgstopus_la_SOURCES = gstopus.c gstopusdec.c gstopusenc.c gstopusparse.c gstopusheader.c +libgstopus_la_SOURCES = gstopus.c gstopusdec.c gstopusenc.c gstopusparse.c gstopusheader.c gstopuscommon.c libgstopus_la_CFLAGS = \ -DGST_USE_UNSTABLE_API \ $(GST_PLUGINS_BASE_CFLAGS) \ @@ -15,4 +15,4 @@ libgstopus_la_LIBADD = \ libgstopus_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) $(LIBM) libgstopus_la_LIBTOOLFLAGS = --tag=disable-static -noinst_HEADERS = gstopusenc.h gstopusdec.h gstopusparse.h gstopusheader.h +noinst_HEADERS = gstopusenc.h gstopusdec.h gstopusparse.h gstopusheader.h gstopuscommon.h diff --git a/ext/opus/gstopuscommon.c b/ext/opus/gstopuscommon.c new file mode 100644 index 0000000000..fc3e0376b9 --- /dev/null +++ b/ext/opus/gstopuscommon.c @@ -0,0 +1,72 @@ +/* GStreamer + * Copyright (C) 2009 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "gstopuscommon.h" + +/* http://www.xiph.org/vorbis/doc/Vorbis_I_spec.html#x1-800004.3.9 */ +/* copy of the same structure in the vorbis plugin */ +const GstAudioChannelPosition gst_opus_channel_positions[][8] = { + { /* Mono */ + GST_AUDIO_CHANNEL_POSITION_FRONT_MONO}, + { /* Stereo */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT}, + { /* Stereo + Centre */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT}, + { /* Quadraphonic */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT, + GST_AUDIO_CHANNEL_POSITION_REAR_LEFT, + GST_AUDIO_CHANNEL_POSITION_REAR_RIGHT, + }, + { /* Stereo + Centre + rear stereo */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT, + GST_AUDIO_CHANNEL_POSITION_REAR_LEFT, + GST_AUDIO_CHANNEL_POSITION_REAR_RIGHT, + }, + { /* Full 5.1 Surround */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT, + GST_AUDIO_CHANNEL_POSITION_REAR_LEFT, + GST_AUDIO_CHANNEL_POSITION_REAR_RIGHT, + GST_AUDIO_CHANNEL_POSITION_LFE, + }, + { /* 6.1 Surround, in Vorbis spec since 2010-01-13 */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT, + GST_AUDIO_CHANNEL_POSITION_SIDE_LEFT, + GST_AUDIO_CHANNEL_POSITION_SIDE_RIGHT, + GST_AUDIO_CHANNEL_POSITION_REAR_CENTER, + GST_AUDIO_CHANNEL_POSITION_LFE}, + { /* 7.1 Surround, in Vorbis spec since 2010-01-13 */ + GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT, + GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER, + GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT, + GST_AUDIO_CHANNEL_POSITION_SIDE_LEFT, + GST_AUDIO_CHANNEL_POSITION_SIDE_RIGHT, + GST_AUDIO_CHANNEL_POSITION_REAR_LEFT, + GST_AUDIO_CHANNEL_POSITION_REAR_RIGHT, + GST_AUDIO_CHANNEL_POSITION_LFE}, +}; diff --git a/ext/opus/gstopuscommon.h b/ext/opus/gstopuscommon.h new file mode 100644 index 0000000000..96a303e303 --- /dev/null +++ b/ext/opus/gstopuscommon.h @@ -0,0 +1,33 @@ +/* GStreamer Opus Encoder + * Copyright (C) 2009 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + + +#ifndef __GST_OPUS_COMMON_H__ +#define __GST_OPUS_COMMON_H__ + +#include +#include + +G_BEGIN_DECLS + +extern const GstAudioChannelPosition gst_opus_channel_positions[][8]; + +G_END_DECLS + +#endif /* __GST_OPUS_COMMON_H__ */ diff --git a/ext/opus/gstopusdec.c b/ext/opus/gstopusdec.c index f8b39ba08c..58283973a2 100644 --- a/ext/opus/gstopusdec.c +++ b/ext/opus/gstopusdec.c @@ -45,6 +45,7 @@ #include #include #include "gstopusheader.h" +#include "gstopuscommon.h" #include "gstopusdec.h" GST_DEBUG_CATEGORY_STATIC (opusdec_debug); @@ -56,7 +57,7 @@ GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS ("audio/x-raw-int, " "rate = (int) { 48000, 24000, 16000, 12000, 8000 }, " - "channels = (int) [ 1, 2 ], " + "channels = (int) [ 1, 8 ], " "endianness = (int) BYTE_ORDER, " "signed = (boolean) true, " "width = (int) 16, " "depth = (int) 16") ); @@ -217,26 +218,91 @@ gst_opus_dec_get_r128_volume (gint16 r128_gain) static GstFlowReturn gst_opus_dec_parse_header (GstOpusDec * dec, GstBuffer * buf) { - g_return_val_if_fail (gst_opus_header_is_header (buf, "OpusHead", 8), - GST_FLOW_ERROR); - g_return_val_if_fail (GST_BUFFER_SIZE (buf) >= 19, GST_FLOW_ERROR); + const guint8 *data = GST_BUFFER_DATA (buf); + GstCaps *caps; + GstStructure *s; + const GstAudioChannelPosition *pos = NULL; - dec->pre_skip = GST_READ_UINT16_LE (GST_BUFFER_DATA (buf) + 10); - dec->r128_gain = GST_READ_UINT16_LE (GST_BUFFER_DATA (buf) + 14); + g_return_val_if_fail (gst_opus_header_is_id_header (buf), GST_FLOW_ERROR); + g_return_val_if_fail (dec->n_channels != data[9], GST_FLOW_ERROR); + + dec->n_channels = data[9]; + dec->pre_skip = GST_READ_UINT16_LE (data + 10); + dec->r128_gain = GST_READ_UINT16_LE (data + 14); dec->r128_gain_volume = gst_opus_dec_get_r128_volume (dec->r128_gain); GST_INFO_OBJECT (dec, "Found pre-skip of %u samples, R128 gain %d (volume %f)", dec->pre_skip, dec->r128_gain, dec->r128_gain_volume); - dec->channel_mapping_family = GST_BUFFER_DATA (buf)[18]; - if (dec->channel_mapping_family != 0) { - GST_ELEMENT_ERROR (dec, STREAM, DECODE, - ("Decoding error: unsupported channel nmapping family %d", - dec->channel_mapping_family), (NULL)); - return GST_FLOW_ERROR; + dec->channel_mapping_family = data[18]; + if (dec->channel_mapping_family == 0) { + /* implicit mapping */ + GST_INFO_OBJECT (dec, "Channel mapping family 0, implicit mapping"); + dec->n_streams = dec->n_stereo_streams = 1; + dec->channel_mapping[0] = 0; + dec->channel_mapping[1] = 1; + } else { + dec->n_streams = data[19]; + dec->n_stereo_streams = data[20]; + memcpy (dec->channel_mapping, data + 21, dec->n_channels); + + if (dec->channel_mapping_family == 1) { + GST_INFO_OBJECT (dec, "Channel mapping family 1, Vorbis mapping"); + switch (dec->n_channels) { + case 1: + case 2: + /* nothing */ + break; + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + pos = gst_opus_channel_positions[dec->n_channels - 1]; + break; + default:{ + gint i; + GstAudioChannelPosition *posn = + g_new (GstAudioChannelPosition, dec->n_channels); + + GST_ELEMENT_WARNING (GST_ELEMENT (dec), STREAM, DECODE, + (NULL), ("Using NONE channel layout for more than 8 channels")); + + for (i = 0; i < dec->n_channels; i++) + posn[i] = GST_AUDIO_CHANNEL_POSITION_NONE; + + pos = posn; + } + } + } else { + GST_INFO_OBJECT (dec, "Channel mapping family %d", + dec->channel_mapping_family); + } } - dec->channel_mapping[0] = 0; - dec->channel_mapping[1] = 1; + + /* negotiate width with downstream */ + caps = gst_pad_get_allowed_caps (GST_AUDIO_DECODER_SRC_PAD (dec)); + s = gst_caps_get_structure (caps, 0); + gst_structure_fixate_field_nearest_int (s, "rate", 48000); + gst_structure_get_int (s, "rate", &dec->sample_rate); + gst_structure_fixate_field_nearest_int (s, "channels", dec->n_channels); + gst_structure_get_int (s, "channels", &dec->n_channels); + + GST_INFO_OBJECT (dec, "Negotiated %d channels, %d Hz", dec->n_channels, + dec->sample_rate); + + if (pos) { + gst_audio_set_channel_positions (gst_caps_get_structure (caps, 0), pos); + } + + if (dec->n_channels > 8) { + g_free ((GstAudioChannelPosition *) pos); + } + + GST_INFO_OBJECT (dec, "Setting src caps to %" GST_PTR_FORMAT, caps); + gst_pad_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), caps); + gst_caps_unref (caps); return GST_FLOW_OK; } @@ -248,48 +314,6 @@ gst_opus_dec_parse_comments (GstOpusDec * dec, GstBuffer * buf) return GST_FLOW_OK; } -static void -gst_opus_dec_setup_from_peer_caps (GstOpusDec * dec) -{ - GstPad *srcpad, *peer; - GstStructure *s; - GstCaps *caps; - const GstCaps *template_caps; - const GstCaps *peer_caps; - - srcpad = GST_AUDIO_DECODER_SRC_PAD (dec); - peer = gst_pad_get_peer (srcpad); - - if (peer) { - template_caps = gst_pad_get_pad_template_caps (srcpad); - peer_caps = gst_pad_get_caps (peer); - GST_DEBUG_OBJECT (dec, "Peer caps: %" GST_PTR_FORMAT, peer_caps); - caps = gst_caps_intersect (template_caps, peer_caps); - gst_pad_fixate_caps (peer, caps); - GST_DEBUG_OBJECT (dec, "Fixated caps: %" GST_PTR_FORMAT, caps); - - s = gst_caps_get_structure (caps, 0); - if (!gst_structure_get_int (s, "channels", &dec->n_channels)) { - dec->n_channels = 2; - GST_WARNING_OBJECT (dec, "Failed to get channels, using default %d", - dec->n_channels); - } else { - GST_DEBUG_OBJECT (dec, "Got channels %d", dec->n_channels); - } - if (!gst_structure_get_int (s, "rate", &dec->sample_rate)) { - dec->sample_rate = 48000; - GST_WARNING_OBJECT (dec, "Failed to get rate, using default %d", - dec->sample_rate); - } else { - GST_DEBUG_OBJECT (dec, "Got sample rate %d", dec->sample_rate); - } - - gst_pad_set_caps (GST_AUDIO_DECODER_SRC_PAD (dec), caps); - } else { - GST_WARNING_OBJECT (dec, "Failed to get src pad peer"); - } -} - static GstFlowReturn opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buffer) { @@ -304,12 +328,11 @@ opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buffer) GstBuffer *buf; if (dec->state == NULL) { - gst_opus_dec_setup_from_peer_caps (dec); - GST_DEBUG_OBJECT (dec, "Creating decoder with %d channels, %d Hz", dec->n_channels, dec->sample_rate); dec->state = opus_multistream_decoder_create (dec->sample_rate, - dec->n_channels, 1, 1, dec->channel_mapping, &err); + dec->n_channels, dec->n_streams, dec->n_stereo_streams, + dec->channel_mapping, &err); if (!dec->state || err != OPUS_OK) goto creation_failed; } diff --git a/ext/opus/gstopusdec.h b/ext/opus/gstopusdec.h index aa04b814d9..3ccfa26969 100644 --- a/ext/opus/gstopusdec.h +++ b/ext/opus/gstopusdec.h @@ -55,6 +55,9 @@ struct _GstOpusDec { int n_channels; guint32 pre_skip; gint16 r128_gain; + + guint8 n_streams; + guint8 n_stereo_streams; guint8 channel_mapping_family; guint8 channel_mapping[256]; diff --git a/ext/opus/gstopusenc.c b/ext/opus/gstopusenc.c index 93e00aa38f..6dad531156 100644 --- a/ext/opus/gstopusenc.c +++ b/ext/opus/gstopusenc.c @@ -49,6 +49,7 @@ #include #include #include "gstopusheader.h" +#include "gstopuscommon.h" #include "gstopusenc.h" GST_DEBUG_CATEGORY_STATIC (opusenc_debug); @@ -116,8 +117,8 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS ("audio/x-raw-int, " - "rate = (int) { 8000, 12000, 16000, 24000, 48000 }, " - "channels = (int) [ 1, 2 ], " + "rate = (int) { 48000, 24000, 16000, 12000, 8000 }, " + "channels = (int) [ 1, 8 ], " "endianness = (int) BYTE_ORDER, " "signed = (boolean) TRUE, " "width = (int) 16, " "depth = (int) 16") ); @@ -419,6 +420,82 @@ gst_opus_enc_get_frame_samples (GstOpusEnc * enc) return frame_samples; } +static void +gst_opus_enc_setup_channel_mapping (GstOpusEnc * enc, const GstAudioInfo * info) +{ +#define MAPS(idx,pos) (GST_AUDIO_INFO_POSITION (info, (idx)) == GST_AUDIO_CHANNEL_POSITION_##pos) + + int n; + + GST_DEBUG_OBJECT (enc, "Setting up channel mapping for %d channels", + enc->n_channels); + + /* Start by setting up a default trivial mapping */ + for (n = 0; n < 255; ++n) + enc->channel_mapping[n] = n; + + /* For one channel, use the basic RTP mapping */ + if (enc->n_channels == 1) { + GST_INFO_OBJECT (enc, "Mono, trivial RTP mapping"); + enc->channel_mapping_family = 0; + enc->channel_mapping[0] = 0; + return; + } + + /* For two channels, use the basic RTP mapping if the channels are + mapped as left/right. */ + if (enc->n_channels == 2) { + if (MAPS (0, FRONT_LEFT) && MAPS (1, FRONT_RIGHT)) { + GST_INFO_OBJECT (enc, "Stereo, canonical mapping"); + enc->channel_mapping_family = 0; + /* The channel mapping is implicit for family 0, that's why we do not + attempt to create one for right/left - this will be mapped to the + Vorbis mapping below. */ + } else { + GST_DEBUG_OBJECT (enc, "Stereo, but not canonical mapping, continuing"); + } + } + + /* For channels between 1 and 8, we use the Vorbis mapping if we can + find a permutation that matches it. Mono will have been taken care + of earlier, but this code also handles it. */ + if (enc->n_channels >= 1 && enc->n_channels <= 8) { + GST_DEBUG_OBJECT (enc, + "In range for the Vorbis mapping, checking channel positions"); + for (n = 0; n < enc->n_channels; ++n) { + GstAudioChannelPosition pos = GST_AUDIO_INFO_POSITION (info, n); + int c; + + GST_DEBUG_OBJECT (enc, "Channel %d has position %d", n, pos); + for (c = 0; c < enc->n_channels; ++c) { + if (gst_opus_channel_positions[enc->n_channels - 1][c] == pos) { + GST_DEBUG_OBJECT (enc, "Found in Vorbis mapping as channel %d", c); + break; + } + } + if (c == enc->n_channels) { + /* We did not find that position, so use undefined */ + GST_WARNING_OBJECT (enc, + "Position %d not found in Vorbis mapping, using unknown mapping", + pos); + enc->channel_mapping_family = 255; + return; + } + GST_DEBUG_OBJECT (enc, "Mapping output channel %d to %d", c, n); + enc->channel_mapping[c] = n; + } + GST_INFO_OBJECT (enc, "Permutation found, using Vorbis mapping"); + enc->channel_mapping_family = 1; + return; + } + + /* For other cases, we use undefined, with the default trivial mapping */ + GST_WARNING_OBJECT (enc, "Unknown mapping"); + enc->channel_mapping_family = 255; + +#undef MAPS +} + static gboolean gst_opus_enc_set_format (GstAudioEncoder * benc, GstAudioInfo * info) { @@ -430,6 +507,7 @@ gst_opus_enc_set_format (GstAudioEncoder * benc, GstAudioInfo * info) enc->n_channels = GST_AUDIO_INFO_CHANNELS (info); enc->sample_rate = GST_AUDIO_INFO_RATE (info); + gst_opus_enc_setup_channel_mapping (enc, info); GST_DEBUG_OBJECT (benc, "Setup with %d channels, %d Hz", enc->n_channels, enc->sample_rate); @@ -455,17 +533,12 @@ static gboolean gst_opus_enc_setup (GstOpusEnc * enc) { int error = OPUS_OK; - unsigned char mapping[256]; - int n; GST_DEBUG_OBJECT (enc, "setup"); - for (n = 0; n < enc->n_channels; ++n) - mapping[n] = n; - enc->state = opus_multistream_encoder_create (enc->sample_rate, enc->n_channels, - (enc->n_channels + 1) / 2, enc->n_channels / 2, mapping, + (enc->n_channels + 1) / 2, enc->n_channels / 2, enc->channel_mapping, enc->audio_or_voip ? OPUS_APPLICATION_AUDIO : OPUS_APPLICATION_VOIP, &error); if (!enc->state || error != OPUS_OK) @@ -557,18 +630,19 @@ gst_opus_enc_encode (GstOpusEnc * enc, GstBuffer * buf) GstBuffer *outbuf; ret = gst_pad_alloc_buffer_and_set_caps (GST_AUDIO_ENCODER_SRC_PAD (enc), - GST_BUFFER_OFFSET_NONE, enc->max_payload_size, + GST_BUFFER_OFFSET_NONE, enc->max_payload_size * enc->n_channels, GST_PAD_CAPS (GST_AUDIO_ENCODER_SRC_PAD (enc)), &outbuf); if (GST_FLOW_OK != ret) goto done; GST_DEBUG_OBJECT (enc, "encoding %d samples (%d bytes)", - enc->frame_samples); + enc->frame_samples, (int) bytes); outsize = opus_multistream_encode (enc->state, (const gint16 *) data, - enc->frame_samples, GST_BUFFER_DATA (outbuf), enc->max_payload_size); + enc->frame_samples, GST_BUFFER_DATA (outbuf), + enc->max_payload_size * enc->n_channels); if (outsize < 0) { GST_ERROR_OBJECT (enc, "Encoding failed: %d", outsize); @@ -582,6 +656,7 @@ gst_opus_enc_encode (GstOpusEnc * enc, GstBuffer * buf) goto done; } + GST_DEBUG_OBJECT (enc, "Output packet is %u bytes", outsize); GST_BUFFER_SIZE (outbuf) = outsize; ret = @@ -621,7 +696,8 @@ gst_opus_enc_handle_frame (GstAudioEncoder * benc, GstBuffer * buf) enc->headers = NULL; gst_opus_header_create_caps (&caps, &enc->headers, enc->n_channels, - enc->sample_rate, gst_tag_setter_get_tag_list (GST_TAG_SETTER (enc))); + enc->sample_rate, enc->channel_mapping_family, enc->channel_mapping, + gst_tag_setter_get_tag_list (GST_TAG_SETTER (enc))); /* negotiate with these caps */ diff --git a/ext/opus/gstopusenc.h b/ext/opus/gstopusenc.h index 8304e82556..8c2c3c6e82 100644 --- a/ext/opus/gstopusenc.h +++ b/ext/opus/gstopusenc.h @@ -77,6 +77,9 @@ struct _GstOpusEnc { GSList *headers; GstTagList *tags; + + guint8 channel_mapping_family; + guint8 channel_mapping[256]; }; struct _GstOpusEncClass { diff --git a/ext/opus/gstopusheader.c b/ext/opus/gstopusheader.c index 3551055840..0379c909f4 100644 --- a/ext/opus/gstopusheader.c +++ b/ext/opus/gstopusheader.c @@ -27,7 +27,8 @@ #include "gstopusheader.h" static GstBuffer * -gst_opus_enc_create_id_buffer (gint nchannels, gint sample_rate) +gst_opus_enc_create_id_buffer (gint nchannels, gint sample_rate, + guint8 channel_mapping_family, const guint8 * channel_mapping) { GstBuffer *buffer; GstByteWriter bw; @@ -41,7 +42,12 @@ gst_opus_enc_create_id_buffer (gint nchannels, gint sample_rate) gst_byte_writer_put_uint16_le (&bw, 0); /* pre-skip *//* TODO: endianness ? */ gst_byte_writer_put_uint32_le (&bw, sample_rate); gst_byte_writer_put_uint16_le (&bw, 0); /* output gain *//* TODO: endianness ? */ - gst_byte_writer_put_uint8 (&bw, 0); /* channel mapping *//* TODO: what is this ? */ + gst_byte_writer_put_uint8 (&bw, channel_mapping_family); + if (channel_mapping_family > 0) { + gst_byte_writer_put_uint8 (&bw, (nchannels + 1) / 2); + gst_byte_writer_put_uint8 (&bw, nchannels / 2); + gst_byte_writer_put_data (&bw, channel_mapping, nchannels); + } buffer = gst_byte_writer_reset_and_get_buffer (&bw); @@ -136,23 +142,11 @@ _gst_caps_set_buffer_array (GstCaps * caps, const gchar * field, } void -gst_opus_header_create_caps (GstCaps ** caps, GSList ** headers, gint nchannels, - gint sample_rate, const GstTagList * tags) +gst_opus_header_create_caps_from_headers (GstCaps ** caps, GSList ** headers, + GstBuffer * buf1, GstBuffer * buf2) { - GstBuffer *buf1, *buf2; - g_return_if_fail (caps); g_return_if_fail (headers && !*headers); - g_return_if_fail (nchannels > 0); - g_return_if_fail (sample_rate >= 0); /* 0 -> unset */ - - /* Opus streams in Ogg begin with two headers; the initial header (with - most of the codec setup parameters) which is mandated by the Ogg - bitstream spec. The second header holds any comment fields. */ - - /* create header buffers */ - buf1 = gst_opus_enc_create_id_buffer (nchannels, sample_rate); - buf2 = gst_opus_enc_create_metadata_buffer (tags); /* mark and put on caps */ *caps = gst_caps_from_string ("audio/x-opus"); @@ -162,9 +156,75 @@ gst_opus_header_create_caps (GstCaps ** caps, GSList ** headers, gint nchannels, *headers = g_slist_prepend (*headers, buf1); } +void +gst_opus_header_create_caps (GstCaps ** caps, GSList ** headers, gint nchannels, + gint sample_rate, guint8 channel_mapping_family, + const guint8 * channel_mapping, const GstTagList * tags) +{ + GstBuffer *buf1, *buf2; + + g_return_if_fail (caps); + g_return_if_fail (headers && !*headers); + g_return_if_fail (nchannels > 0); + g_return_if_fail (sample_rate >= 0); /* 0 -> unset */ + g_return_if_fail (channel_mapping_family == 0 || channel_mapping); + + /* Opus streams in Ogg begin with two headers; the initial header (with + most of the codec setup parameters) which is mandated by the Ogg + bitstream spec. The second header holds any comment fields. */ + + /* create header buffers */ + buf1 = + gst_opus_enc_create_id_buffer (nchannels, sample_rate, + channel_mapping_family, channel_mapping); + buf2 = gst_opus_enc_create_metadata_buffer (tags); + + gst_opus_header_create_caps_from_headers (caps, headers, buf1, buf2); +} + gboolean gst_opus_header_is_header (GstBuffer * buf, const char *magic, guint magic_size) { return (GST_BUFFER_SIZE (buf) >= magic_size && !memcmp (magic, GST_BUFFER_DATA (buf), magic_size)); } + +gboolean +gst_opus_header_is_id_header (GstBuffer * buf) +{ + gsize size = GST_BUFFER_SIZE (buf); + const guint8 *data = GST_BUFFER_DATA (buf); + guint8 channels, channel_mapping_family, n_streams, n_stereo_streams; + + if (size < 19) + return FALSE; + if (!gst_opus_header_is_header (buf, "OpusHead", 8)) + return FALSE; + channels = data[9]; + if (channels == 0) + return FALSE; + channel_mapping_family = data[18]; + if (channel_mapping_family == 0) { + if (channels > 2) + return FALSE; + } else { + channels = data[9]; + if (size < 21 + channels) + return FALSE; + n_streams = data[19]; + n_stereo_streams = data[20]; + if (n_streams == 0) + return FALSE; + if (n_stereo_streams > n_streams) + return FALSE; + if (n_streams + n_stereo_streams > 255) + return FALSE; + } + return TRUE; +} + +gboolean +gst_opus_header_is_comment_header (GstBuffer * buf) +{ + return gst_opus_header_is_header (buf, "OpusTags", 8); +} diff --git a/ext/opus/gstopusheader.h b/ext/opus/gstopusheader.h index 4594264ccd..3b2cfc265f 100644 --- a/ext/opus/gstopusheader.h +++ b/ext/opus/gstopusheader.h @@ -25,8 +25,16 @@ G_BEGIN_DECLS -extern void gst_opus_header_create_caps (GstCaps **caps, GSList **headers, gint nchannels, gint sample_rate, const GstTagList *tags); -extern gboolean gst_opus_header_is_header (GstBuffer * buf, const char *magic, guint magic_size); +extern void gst_opus_header_create_caps_from_headers (GstCaps **caps, GSList **headers, + GstBuffer *id_header, GstBuffer *comment_header); +extern void gst_opus_header_create_caps (GstCaps **caps, GSList **headers, + gint nchannels, gint sample_rate, + guint8 channel_mapping_family, const guint8 *channel_mapping, + const GstTagList *tags); +extern gboolean gst_opus_header_is_header (GstBuffer * buf, + const char *magic, guint magic_size); +extern gboolean gst_opus_header_is_id_header (GstBuffer * buf); +extern gboolean gst_opus_header_is_comment_header (GstBuffer * buf); G_END_DECLS From 32e9de842fc963a4af4d9a7148f7e3c909ab96c8 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Thu, 24 Nov 2011 13:38:59 +0000 Subject: [PATCH 7/7] opus: pre-skip and output gain are little endian, remove reminder note --- ext/opus/gstopusheader.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opus/gstopusheader.c b/ext/opus/gstopusheader.c index 0379c909f4..1aa3b0d01b 100644 --- a/ext/opus/gstopusheader.c +++ b/ext/opus/gstopusheader.c @@ -39,9 +39,9 @@ gst_opus_enc_create_id_buffer (gint nchannels, gint sample_rate, gst_byte_writer_put_data (&bw, (const guint8 *) "OpusHead", 8); gst_byte_writer_put_uint8 (&bw, 0); /* version number */ gst_byte_writer_put_uint8 (&bw, nchannels); - gst_byte_writer_put_uint16_le (&bw, 0); /* pre-skip *//* TODO: endianness ? */ + gst_byte_writer_put_uint16_le (&bw, 0); /* pre-skip */ gst_byte_writer_put_uint32_le (&bw, sample_rate); - gst_byte_writer_put_uint16_le (&bw, 0); /* output gain *//* TODO: endianness ? */ + gst_byte_writer_put_uint16_le (&bw, 0); /* output gain */ gst_byte_writer_put_uint8 (&bw, channel_mapping_family); if (channel_mapping_family > 0) { gst_byte_writer_put_uint8 (&bw, (nchannels + 1) / 2);