From 07cf7b2a297f55434f2d0bba43780b6e9848f8e3 Mon Sep 17 00:00:00 2001 From: Dario Marino Saccavino Date: Wed, 18 Jan 2023 14:42:38 +0000 Subject: [PATCH] wasapi2: Add option to monitor loopback device's mute state When loopback recording from a render device, the wasapi2src element captures audio even if the device is muted. This change adds the 'loopback-silence-on-device-mute' property that, when set to `true`, causes wasapi2src to inject silence in the pipeline when the device is muted. Fixes: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/1306 Part-of: --- .../sys/wasapi2/gstwasapi2client.cpp | 220 ++++++++++++++++-- .../sys/wasapi2/gstwasapi2client.h | 2 + .../sys/wasapi2/gstwasapi2ringbuffer.cpp | 18 +- .../sys/wasapi2/gstwasapi2ringbuffer.h | 3 + .../sys/wasapi2/gstwasapi2src.c | 53 +++++ .../sys/wasapi2/gstwasapi2util.h | 1 + 6 files changed, 281 insertions(+), 16 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2client.cpp b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2client.cpp index 72bcc8393e..9917ebe57b 100644 --- a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2client.cpp +++ b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2client.cpp @@ -95,15 +95,29 @@ static void gst_wasapi2_client_on_device_activated (GstWasapi2Client * client, IAudioClient * audio_client); +static void +gst_wasapi2_client_on_endpoint_volume_activated (GstWasapi2Client * client, + IAudioEndpointVolume * audio_endpoint_volume); + +static void +gst_wasapi2_client_set_endpoint_muted (GstWasapi2Client * client, + gboolean muted); + /* *INDENT-OFF* */ class GstWasapiDeviceActivator : public RuntimeClass, FtmBase, IActivateAudioInterfaceCompletionHandler> { public: + typedef enum { + WASAPI_IFACE_AUDIO_CLIENT, + WASAPI_IFACE_AUDIO_ENDPOINT_VOLUME, + } WasapiInterface; + GstWasapiDeviceActivator () { g_weak_ref_init (&listener_, nullptr); + interface_to_activate_ = WASAPI_IFACE_AUDIO_CLIENT; } ~GstWasapiDeviceActivator () @@ -112,7 +126,9 @@ public: } HRESULT - RuntimeClassInitialize (GstWasapi2Client * listener, gpointer dispatcher) + RuntimeClassInitialize (GstWasapi2Client * listener, + gpointer dispatcher, + WasapiInterface interface_to_activate) { if (!listener) return E_INVALIDARG; @@ -129,6 +145,8 @@ public: GST_INFO("Main UI dispatcher is available"); } + interface_to_activate_ = interface_to_activate; + return S_OK; } @@ -136,6 +154,7 @@ public: (IActivateAudioInterfaceAsyncOperation *async_op) { ComPtr audio_client; + ComPtr audio_endpoint_volume; HRESULT hr = S_OK; HRESULT hr_async_op = S_OK; ComPtr audio_interface; @@ -162,15 +181,34 @@ public: goto done; } - hr = audio_interface.As (&audio_client); - if (!gst_wasapi2_result (hr)) { - GST_ERROR_OBJECT (client, "Failed to get IAudioClient3 interface"); - goto done; + switch (interface_to_activate_) { + case WASAPI_IFACE_AUDIO_CLIENT: + hr = audio_interface.As (&audio_client); + if (!gst_wasapi2_result (hr)) { + GST_ERROR_OBJECT (client, "Failed to get IAudioClient3 interface"); + goto done; + } + break; + case WASAPI_IFACE_AUDIO_ENDPOINT_VOLUME: + hr = audio_interface.As (&audio_endpoint_volume); + if (!gst_wasapi2_result (hr)) { + GST_ERROR_OBJECT (client, "Failed to get IAudioEndpointVolume interface"); + goto done; + } + break; } done: /* Should call this method anyway, listener will wait this event */ - gst_wasapi2_client_on_device_activated (client, audio_client.Get()); + switch (interface_to_activate_) { + case WASAPI_IFACE_AUDIO_CLIENT: + gst_wasapi2_client_on_device_activated (client, audio_client.Get()); + break; + case WASAPI_IFACE_AUDIO_ENDPOINT_VOLUME: + gst_wasapi2_client_on_endpoint_volume_activated (client, audio_endpoint_volume.Get()); + break; + } + gst_object_unref (client); /* return S_OK anyway, but listener can know it's succeeded or not * by passed IAudioClient handle via gst_wasapi2_client_on_device_activated @@ -192,16 +230,27 @@ public: ComPtr async_op; HRESULT async_hr = S_OK; PROPVARIANT activate_params = {}; + IID iid = {}; + + switch (interface_to_activate_) { + case WASAPI_IFACE_AUDIO_CLIENT: + iid = __uuidof (IAudioClient); + break; + case WASAPI_IFACE_AUDIO_ENDPOINT_VOLUME: + iid = __uuidof (IAudioEndpointVolume); + break; + } + if (params) { activate_params.vt = VT_BLOB; activate_params.blob.cbSize = sizeof(GST_AUDIOCLIENT_ACTIVATION_PARAMS); activate_params.blob.pBlobData = (BYTE *) params; async_hr = ActivateAudioInterfaceAsync (device_id.c_str (), - __uuidof(IAudioClient), &activate_params, this, &async_op); + iid, &activate_params, this, &async_op); } else { async_hr = ActivateAudioInterfaceAsync (device_id.c_str (), - __uuidof(IAudioClient), nullptr, this, &async_op); + iid, nullptr, this, &async_op); } /* for debugging */ @@ -234,6 +283,46 @@ public: private: GWeakRef listener_; ComPtr dispatcher_; + WasapiInterface interface_to_activate_; +}; + +class GstWasapiEndpointVolumeCallback + : public RuntimeClass, FtmBase, + IAudioEndpointVolumeCallback> +{ +public: + GstWasapiEndpointVolumeCallback () + { + g_weak_ref_init (&client_, nullptr); + } + + ~GstWasapiEndpointVolumeCallback () + { + g_weak_ref_set (&client_, nullptr); + } + + HRESULT + RuntimeClassInitialize (GstWasapi2Client * client) + { + if (!client) + return E_INVALIDARG; + g_weak_ref_set (&client_, client); + return S_OK; + } + + STDMETHOD(OnNotify) + (AUDIO_VOLUME_NOTIFICATION_DATA * notify) + { + GstWasapi2Client *client = (GstWasapi2Client *) g_weak_ref_get (&client_); + if (client) { + gst_wasapi2_client_set_endpoint_muted (client, !!notify->bMuted); + gst_object_unref (client); + } + return S_OK; + } + +private: + GWeakRef client_; }; /* *INDENT-ON* */ @@ -273,7 +362,11 @@ struct _GstWasapi2Client guint target_pid; IAudioClient *audio_client; - GstWasapiDeviceActivator *activator; + + GMutex endpoint_volume_lock; + IAudioEndpointVolume *audio_endpoint_volume; + GstWasapiEndpointVolumeCallback *endpoint_volume_callback; + gint is_endpoint_muted; GstCaps *supported_caps; @@ -382,6 +475,8 @@ gst_wasapi2_client_init (GstWasapi2Client * self) g_cond_init (&self->init_cond); self->activate_state = GST_WASAPI2_CLIENT_ACTIVATE_INIT; + g_mutex_init (&self->endpoint_volume_lock); + self->context = g_main_context_new (); self->loop = g_main_loop_new (self->context, FALSE); } @@ -432,6 +527,8 @@ gst_wasapi2_client_finalize (GObject * object) g_mutex_clear (&self->init_lock); g_cond_clear (&self->init_cond); + g_mutex_clear (&self->endpoint_volume_lock); + G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -534,6 +631,55 @@ gst_wasapi2_client_on_device_activated (GstWasapi2Client * self, g_mutex_unlock (&self->init_lock); } +static void +gst_wasapi2_client_on_endpoint_volume_activated (GstWasapi2Client * self, + IAudioEndpointVolume * audio_endpoint_volume) +{ + GST_INFO_OBJECT (self, "Audio Endpoint Volume activated"); + + if (audio_endpoint_volume) { + HRESULT hr; + ComPtr < GstWasapiEndpointVolumeCallback > callback; + + g_mutex_lock (&self->endpoint_volume_lock); + audio_endpoint_volume->AddRef (); + self->audio_endpoint_volume = audio_endpoint_volume; + + hr = MakeAndInitialize < GstWasapiEndpointVolumeCallback > (&callback, + self); + if (!gst_wasapi2_result (hr)) { + GST_WARNING_OBJECT (self, + "Could not create endpoint volume callback object"); + } else { + hr = audio_endpoint_volume->RegisterControlChangeNotify (callback.Get ()); + if (!gst_wasapi2_result (hr)) { + GST_WARNING_OBJECT (self, + "Failed to register endpoint volume callback"); + } else { + BOOL initially_muted = FALSE; + + self->endpoint_volume_callback = callback.Detach (); + + hr = audio_endpoint_volume->GetMute (&initially_muted); + if (gst_wasapi2_result (hr)) { + gst_wasapi2_client_set_endpoint_muted (self, !!initially_muted); + } + } + } + g_mutex_unlock (&self->endpoint_volume_lock); + } else { + GST_WARNING_OBJECT (self, "IAudioEndpointVolume is unavailable"); + } +} + +static void +gst_wasapi2_client_set_endpoint_muted (GstWasapi2Client * self, gboolean muted) +{ + GST_DEBUG_OBJECT (self, "Audio Endpoint Volume: muted=%d", muted); + + g_atomic_int_set (&self->is_endpoint_muted, muted); +} + /* *INDENT-OFF* */ static std::string convert_wstring_to_string (const std::wstring &wstr) @@ -581,7 +727,8 @@ gst_wasapi2_client_get_default_device_id (GstWasapi2Client * self) static gboolean gst_wasapi2_client_activate_async (GstWasapi2Client * self, - GstWasapiDeviceActivator * activator) + GstWasapiDeviceActivator * activator, + GstWasapiDeviceActivator * endpoint_volume_activator) { /* *INDENT-OFF* */ ComPtr device_info_static; @@ -866,6 +1013,20 @@ activate: goto failed; } + /* activate the endpoint volume interface */ + if (endpoint_volume_activator) { + if (use_default_device) { + GST_INFO_OBJECT (self, + "Endpoint volume monitoring for the default device is not implemented."); + } else { + hr = endpoint_volume_activator->ActivateDeviceAsync + (target_device_id_wstring, nullptr); + if (!gst_wasapi2_result (hr)) { + GST_WARNING_OBJECT (self, "Failed to activate device"); + } + } + } + g_mutex_lock (&self->lock); if (self->activate_state == GST_WASAPI2_CLIENT_ACTIVATE_INIT) self->activate_state = GST_WASAPI2_CLIENT_ACTIVATE_WAIT; @@ -905,10 +1066,11 @@ gst_wasapi2_client_thread_func (GstWasapi2Client * self) GSource *source; HRESULT hr; /* *INDENT-OFF* */ - ComPtr activator; + ComPtr client_activator; + ComPtr endpoint_volume_activator; - hr = MakeAndInitialize (&activator, - self, self->dispatcher); + hr = MakeAndInitialize (&client_activator, + self, self->dispatcher, GstWasapiDeviceActivator::WASAPI_IFACE_AUDIO_CLIENT); /* *INDENT-ON* */ if (!gst_wasapi2_result (hr)) { @@ -917,7 +1079,17 @@ gst_wasapi2_client_thread_func (GstWasapi2Client * self) goto run_loop; } - gst_wasapi2_client_activate_async (self, activator.Get ()); + /* Initialize audio endpoint volume activator */ + hr = MakeAndInitialize < GstWasapiDeviceActivator > + (&endpoint_volume_activator, self, self->dispatcher, + GstWasapiDeviceActivator::WASAPI_IFACE_AUDIO_ENDPOINT_VOLUME); + if (!gst_wasapi2_result (hr)) { + GST_WARNING_OBJECT (self, + "Could not create endpoint volume activator object"); + } + + gst_wasapi2_client_activate_async (self, client_activator.Get (), + endpoint_volume_activator.Get ()); if (!self->dispatcher) { /* In case that dispatcher is unavailable, wait activation synchroniously */ @@ -948,9 +1120,19 @@ run_loop: GST_WASAPI2_CLEAR_COM (self->audio_client); + g_mutex_lock (&self->endpoint_volume_lock); + if (self->audio_endpoint_volume && self->endpoint_volume_callback) { + self->audio_endpoint_volume-> + UnregisterControlChangeNotify (self->endpoint_volume_callback); + } + GST_WASAPI2_CLEAR_COM (self->endpoint_volume_callback); + GST_WASAPI2_CLEAR_COM (self->audio_endpoint_volume); + g_mutex_unlock (&self->endpoint_volume_lock); + /* Reset explicitly to ensure that it happens before * RoInitializeWrapper dtor is called */ - activator.Reset (); + client_activator.Reset (); + endpoint_volume_activator.Reset (); GST_DEBUG_OBJECT (self, "Exit thread function"); @@ -1097,3 +1279,11 @@ gst_wasapi2_client_get_handle (GstWasapi2Client * client) return client->audio_client; } + +gboolean +gst_wasapi2_client_is_endpoint_muted (GstWasapi2Client * client) +{ + g_return_val_if_fail (GST_IS_WASAPI2_CLIENT (client), FALSE); + + return g_atomic_int_get (&client->is_endpoint_muted); +} diff --git a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2client.h b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2client.h index 4c2ee2134e..4bec42e133 100644 --- a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2client.h +++ b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2client.h @@ -79,6 +79,8 @@ gboolean gst_wasapi2_client_ensure_activation (GstWasapi2Client * clie IAudioClient * gst_wasapi2_client_get_handle (GstWasapi2Client * client); +gboolean gst_wasapi2_client_is_endpoint_muted (GstWasapi2Client * client); + GstCaps * gst_wasapi2_client_get_caps (GstWasapi2Client * client); G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2ringbuffer.cpp b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2ringbuffer.cpp index db3bf3bcf1..8069d333a0 100644 --- a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2ringbuffer.cpp +++ b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2ringbuffer.cpp @@ -176,6 +176,8 @@ struct _GstWasapi2RingBuffer gboolean mute_changed; gboolean volume_changed; + gboolean monitor_device_mute; + GstCaps *supported_caps; }; @@ -462,6 +464,7 @@ gst_wasapi2_ring_buffer_read (GstWasapi2RingBuffer * self) gint segment; guint8 *readptr; gint len; + gboolean is_device_muted; if (!capture_client) { GST_ERROR_OBJECT (self, "IAudioCaptureClient is not available"); @@ -475,6 +478,9 @@ gst_wasapi2_ring_buffer_read (GstWasapi2RingBuffer * self) goto out; } + is_device_muted = g_atomic_int_get (&self->monitor_device_mute) && + gst_wasapi2_client_is_endpoint_muted (self->client); + to_read_bytes = to_read * GST_AUDIO_INFO_BPF (info); GST_LOG_OBJECT (self, "Reading %d frames offset at %" G_GUINT64_FORMAT @@ -539,7 +545,8 @@ gst_wasapi2_ring_buffer_read (GstWasapi2RingBuffer * self) if (len > to_read_bytes) len = to_read_bytes; - if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) == AUDCLNT_BUFFERFLAGS_SILENT) { + if (((flags & AUDCLNT_BUFFERFLAGS_SILENT) == AUDCLNT_BUFFERFLAGS_SILENT) || + is_device_muted) { gst_audio_format_info_fill_silence (ringbuffer->spec.info.finfo, readptr + self->segoffset, len); } else { @@ -1483,3 +1490,12 @@ gst_wasapi2_ring_buffer_get_volume (GstWasapi2RingBuffer * buf, gfloat * volume) return hr; } + +void +gst_wasapi2_ring_buffer_set_device_mute_monitoring (GstWasapi2RingBuffer * buf, + gboolean value) +{ + g_return_if_fail (GST_IS_WASAPI2_RING_BUFFER (buf)); + + g_atomic_int_set (&buf->monitor_device_mute, value); +} diff --git a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2ringbuffer.h b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2ringbuffer.h index f8d91d8fe2..a87378697e 100644 --- a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2ringbuffer.h +++ b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2ringbuffer.h @@ -51,6 +51,9 @@ HRESULT gst_wasapi2_ring_buffer_set_volume (GstWasapi2RingBuffer HRESULT gst_wasapi2_ring_buffer_get_volume (GstWasapi2RingBuffer * buf, gfloat * volume); +void gst_wasapi2_ring_buffer_set_device_mute_monitoring (GstWasapi2RingBuffer * buf, + gboolean value); + G_END_DECLS #endif /* __GST_WASAPI2_RING_BUFFER_H__ */ diff --git a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2src.c b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2src.c index 0b612eb5ef..454706b8f2 100644 --- a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2src.c +++ b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2src.c @@ -119,6 +119,7 @@ gst_wasapi2_src_loopback_mode_get_type (void) #define DEFAULT_VOLUME 1.0 #define DEFAULT_LOOPBACK FALSE #define DEFAULT_LOOPBACK_MODE GST_WASAPI2_SRC_LOOPBACK_DEFAULT +#define DEFAULT_LOOPBACK_SILENCE_ON_DEVICE_MUTE FALSE enum { @@ -131,6 +132,7 @@ enum PROP_LOOPBACK, PROP_LOOPBACK_MODE, PROP_LOOPBACK_TARGET_PID, + PROP_LOOPBACK_SILENCE_ON_DEVICE_MUTE, }; struct _GstWasapi2Src @@ -146,6 +148,7 @@ struct _GstWasapi2Src gboolean loopback; GstWasapi2SrcLoopbackMode loopback_mode; guint loopback_pid; + gboolean loopback_silence_on_device_mute; gboolean mute_changed; gboolean volume_changed; @@ -168,6 +171,8 @@ static void gst_wasapi2_src_set_mute (GstWasapi2Src * self, gboolean mute); static gboolean gst_wasapi2_src_get_mute (GstWasapi2Src * self); static void gst_wasapi2_src_set_volume (GstWasapi2Src * self, gdouble volume); static gdouble gst_wasapi2_src_get_volume (GstWasapi2Src * self); +static void gst_wasapi2_src_set_silence_on_mute (GstWasapi2Src * self, + gboolean value); #define gst_wasapi2_src_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstWasapi2Src, gst_wasapi2_src, @@ -274,6 +279,22 @@ gst_wasapi2_src_class_init (GstWasapi2SrcClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } + /** + * GstWasapi2Src:loopback-silence-on-device-mute: + * + * When loopback recording, if the device is muted, inject silence in the pipeline + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, + PROP_LOOPBACK_SILENCE_ON_DEVICE_MUTE, + g_param_spec_boolean ("loopback-silence-on-device-mute", + "Loopback Silence On Device Mute", + "When loopback recording, if the device is muted, inject silence in the pipeline", + DEFAULT_LOOPBACK_SILENCE_ON_DEVICE_MUTE, + GST_PARAM_MUTABLE_PLAYING | G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS)); + gst_element_class_add_static_pad_template (element_class, &src_template); gst_element_class_set_static_metadata (element_class, "Wasapi2Src", "Source/Audio/Hardware", @@ -304,6 +325,8 @@ gst_wasapi2_src_init (GstWasapi2Src * self) self->volume = DEFAULT_VOLUME; self->low_latency = DEFAULT_LOW_LATENCY; self->loopback = DEFAULT_LOOPBACK; + self->loopback_silence_on_device_mute = + DEFAULT_LOOPBACK_SILENCE_ON_DEVICE_MUTE; } static void @@ -348,6 +371,9 @@ gst_wasapi2_src_set_property (GObject * object, guint prop_id, case PROP_LOOPBACK_TARGET_PID: self->loopback_pid = g_value_get_uint (value); break; + case PROP_LOOPBACK_SILENCE_ON_DEVICE_MUTE: + gst_wasapi2_src_set_silence_on_mute (self, g_value_get_boolean (value)); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -382,6 +408,9 @@ gst_wasapi2_src_get_property (GObject * object, guint prop_id, case PROP_LOOPBACK_TARGET_PID: g_value_set_uint (value, self->loopback_pid); break; + case PROP_LOOPBACK_SILENCE_ON_DEVICE_MUTE: + g_value_set_boolean (value, self->loopback_silence_on_device_mute); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -489,6 +518,11 @@ gst_wasapi2_src_create_ringbuffer (GstAudioBaseSrc * src) self->loopback_pid); g_free (name); + if (self->loopback) { + gst_wasapi2_ring_buffer_set_device_mute_monitoring (GST_WASAPI2_RING_BUFFER + (ringbuffer), self->loopback_silence_on_device_mute); + } + return ringbuffer; } @@ -608,3 +642,22 @@ gst_wasapi2_src_get_volume (GstWasapi2Src * self) return volume; } + +static void +gst_wasapi2_src_set_silence_on_mute (GstWasapi2Src * self, gboolean value) +{ + GstAudioBaseSrc *bsrc = GST_AUDIO_BASE_SRC_CAST (self); + + GST_OBJECT_LOCK (self); + + self->loopback_silence_on_device_mute = value; + + if (self->loopback && bsrc->ringbuffer) { + GstWasapi2RingBuffer *ringbuffer = + GST_WASAPI2_RING_BUFFER (bsrc->ringbuffer); + + gst_wasapi2_ring_buffer_set_device_mute_monitoring (ringbuffer, value); + } + + GST_OBJECT_UNLOCK (self); +} diff --git a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2util.h b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2util.h index 91c8fee5a4..f3bb08fd31 100644 --- a/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2util.h +++ b/subprojects/gst-plugins-bad/sys/wasapi2/gstwasapi2util.h @@ -25,6 +25,7 @@ #include #include #include +#include G_BEGIN_DECLS