From 8ca40fa86f09d84a2e6d27a045db092b2986b06c Mon Sep 17 00:00:00 2001 From: Ilya Konstantinov Date: Tue, 24 Feb 2015 05:57:24 +0200 Subject: [PATCH] osxaudiosrc: iOS resampling causes stuttering Fixes stuttering audio when iOS AU is resampling. To make AU resample, one has to request a rate that differs from AVAudioSession's sampleRate. The resampling itself is not the culprit, but rather our API misuse. AudioUnitRender modifies the mDataByteSize members with the actual read bytes count. Therefore, they must be reinitialized before each AudioUnitRender. (The buffers themselves can be preallocated.) The "stutter" was caused by one AudioUnitRender making the buffer too small for other AudioUnitRender invocations, making them fail with -50 (paramErr). By way of luck, when AU didn't resample, all AudioUnitRender invocations read the same number of bytes. (This patch addresses some non-interleaved audio concerns, but at this moment the elements do not support non-interleaved audio and non-interleaved is untested.) https://bugzilla.gnome.org/show_bug.cgi?id=744922 --- sys/osxaudio/gstosxaudiosrc.c | 11 +++++++++++ sys/osxaudio/gstosxcoreaudio.c | 18 ++++++++--------- sys/osxaudio/gstosxcoreaudio.h | 1 + sys/osxaudio/gstosxcoreaudiocommon.c | 29 ++++++++++++++++------------ sys/osxaudio/gstosxcoreaudiocommon.h | 2 +- 5 files changed, 38 insertions(+), 23 deletions(-) diff --git a/sys/osxaudio/gstosxaudiosrc.c b/sys/osxaudio/gstosxaudiosrc.c index a17988251f..dea94012f2 100644 --- a/sys/osxaudio/gstosxaudiosrc.c +++ b/sys/osxaudio/gstosxaudiosrc.c @@ -384,8 +384,16 @@ gst_osx_audio_src_io_proc (GstOsxAudioRingBuffer * buf, gint writeseg; gint len; gint remaining; + UInt32 n; gint offset = 0; + /* Previous invoke of AudioUnitRender changed mDataByteSize into + * number of bytes actually read. Reset the members. */ + for (n = 0; n < buf->core_audio->recBufferList->mNumberBuffers; ++n) { + buf->core_audio->recBufferList->mBuffers[n].mDataByteSize = + buf->core_audio->recBufferSize; + } + status = AudioUnitRender (buf->core_audio->audiounit, ioActionFlags, inTimeStamp, inBusNumber, inNumberFrames, buf->core_audio->recBufferList); @@ -394,6 +402,9 @@ gst_osx_audio_src_io_proc (GstOsxAudioRingBuffer * buf, return status; } + /* TODO: To support non-interleaved audio, go over all mBuffers, + * not just the first one. */ + remaining = buf->core_audio->recBufferList->mBuffers[0].mDataByteSize; while (remaining) { diff --git a/sys/osxaudio/gstosxcoreaudio.c b/sys/osxaudio/gstosxcoreaudio.c index c2dcd17c11..a29a58d482 100644 --- a/sys/osxaudio/gstosxcoreaudio.c +++ b/sys/osxaudio/gstosxcoreaudio.c @@ -126,9 +126,11 @@ gst_core_audio_initialize (GstCoreAudio * core_audio, if (core_audio->is_src) { /* create AudioBufferList needed for recording */ + core_audio->recBufferSize = frame_size * format.mBytesPerFrame; core_audio->recBufferList = - buffer_list_alloc (format.mChannelsPerFrame, - frame_size * format.mBytesPerFrame); + buffer_list_alloc (format.mChannelsPerFrame, core_audio->recBufferSize, + /* Currently always TRUE (i.e. interleaved) */ + !(format.mFormatFlags & kAudioFormatFlagIsNonInterleaved)); } /* Initialize the AudioUnit */ @@ -141,10 +143,8 @@ gst_core_audio_initialize (GstCoreAudio * core_audio, return TRUE; error: - if (core_audio->is_src && core_audio->recBufferList) { - buffer_list_free (core_audio->recBufferList); - core_audio->recBufferList = NULL; - } + buffer_list_free (core_audio->recBufferList); + core_audio->recBufferList = NULL; return FALSE; } @@ -153,10 +153,8 @@ gst_core_audio_unitialize (GstCoreAudio * core_audio) { AudioUnitUninitialize (core_audio->audiounit); - if (core_audio->recBufferList) { - buffer_list_free (core_audio->recBufferList); - core_audio->recBufferList = NULL; - } + buffer_list_free (core_audio->recBufferList); + core_audio->recBufferList = NULL; } void diff --git a/sys/osxaudio/gstosxcoreaudio.h b/sys/osxaudio/gstosxcoreaudio.h index a9d701c49a..7ebcba2f75 100644 --- a/sys/osxaudio/gstosxcoreaudio.h +++ b/sys/osxaudio/gstosxcoreaudio.h @@ -90,6 +90,7 @@ struct _GstCoreAudio /* For LPCM in/out */ AudioUnit audiounit; + UInt32 recBufferSize; /* AudioUnitRender clobbers mDataByteSize */ AudioBufferList *recBufferList; #ifndef HAVE_IOS diff --git a/sys/osxaudio/gstosxcoreaudiocommon.c b/sys/osxaudio/gstosxcoreaudiocommon.c index badb40cefb..4484431417 100644 --- a/sys/osxaudio/gstosxcoreaudiocommon.c +++ b/sys/osxaudio/gstosxcoreaudiocommon.c @@ -150,18 +150,21 @@ gst_core_audio_io_proc_stop (GstCoreAudio * core_audio) } AudioBufferList * -buffer_list_alloc (int channels, int size) +buffer_list_alloc (UInt32 channels, UInt32 size, gboolean interleaved) { AudioBufferList *list; - int total_size; - int n; + gsize list_size; + UInt32 num_buffers, n; - total_size = sizeof (AudioBufferList) + 1 * sizeof (AudioBuffer); - list = (AudioBufferList *) g_malloc (total_size); + num_buffers = interleaved ? 1 : channels; + list_size = G_STRUCT_OFFSET (AudioBufferList, mBuffers[num_buffers]); + list = (AudioBufferList *) g_malloc (list_size); - list->mNumberBuffers = 1; - for (n = 0; n < (int) list->mNumberBuffers; ++n) { - list->mBuffers[n].mNumberChannels = channels; + list->mNumberBuffers = num_buffers; + for (n = 0; n < num_buffers; ++n) { + /* See http://lists.apple.com/archives/coreaudio-api/2015/Feb/msg00027.html */ + list->mBuffers[n].mNumberChannels = interleaved ? channels : 1; + /* AudioUnitRender will keep overwriting mDataByteSize */ list->mBuffers[n].mDataByteSize = size; list->mBuffers[n].mData = g_malloc (size); } @@ -172,11 +175,13 @@ buffer_list_alloc (int channels, int size) void buffer_list_free (AudioBufferList * list) { - int n; + UInt32 n; - for (n = 0; n < (int) list->mNumberBuffers; ++n) { - if (list->mBuffers[n].mData) - g_free (list->mBuffers[n].mData); + if (list == NULL) + return; + + for (n = 0; n < list->mNumberBuffers; ++n) { + g_free (list->mBuffers[n].mData); } g_free (list); diff --git a/sys/osxaudio/gstosxcoreaudiocommon.h b/sys/osxaudio/gstosxcoreaudiocommon.h index e9fbb8e7fd..228a1646fd 100644 --- a/sys/osxaudio/gstosxcoreaudiocommon.h +++ b/sys/osxaudio/gstosxcoreaudiocommon.h @@ -40,7 +40,7 @@ gboolean gst_core_audio_io_proc_start (GstCoreAudio * core_a gboolean gst_core_audio_io_proc_stop (GstCoreAudio * core_audio); -AudioBufferList * buffer_list_alloc (int channels, int size); +AudioBufferList * buffer_list_alloc (UInt32 channels, UInt32 size, gboolean interleaved); void buffer_list_free (AudioBufferList * list);