From ba4a260c07fb359cc6efd2002cbe7e63d971b0e5 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 11 Dec 2024 17:44:20 -0300 Subject: [PATCH] souphttpsrc: Add the notion of "retry-backoff" So that the user can force waits between retries Part-of: --- .../docs/gst_plugins_cache.json | 28 ++++ .../ext/soup/gstsouphttpsrc.c | 137 ++++++++++++++++-- .../ext/soup/gstsouphttpsrc.h | 14 +- 3 files changed, 164 insertions(+), 15 deletions(-) diff --git a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json index 9e51f67cc5..f856f950ab 100644 --- a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json +++ b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json @@ -24502,6 +24502,34 @@ "type": "gint", "writable": true }, + "retry-backoff-factor": { + "blurb": "Exponential retry backoff factor in seconds", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "1.79769e+308", + "min": "0", + "mutable": "null", + "readable": true, + "type": "gdouble", + "writable": true + }, + "retry-backoff-max": { + "blurb": "Maximum backoff delay in seconds", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "60", + "max": "1.79769e+308", + "min": "0", + "mutable": "null", + "readable": true, + "type": "gdouble", + "writable": true + }, "ssl-ca-file": { "blurb": "Location of a SSL anchor CA file to use", "conditionally-available": false, diff --git a/subprojects/gst-plugins-good/ext/soup/gstsouphttpsrc.c b/subprojects/gst-plugins-good/ext/soup/gstsouphttpsrc.c index d7155dd181..7292b1a3cd 100644 --- a/subprojects/gst-plugins-good/ext/soup/gstsouphttpsrc.c +++ b/subprojects/gst-plugins-good/ext/soup/gstsouphttpsrc.c @@ -202,6 +202,8 @@ enum PROP_RETRIES, PROP_METHOD, PROP_TLS_INTERACTION, + PROP_RETRY_BACKOFF_FACTOR, + PROP_RETRY_BACKOFF_MAX, }; enum @@ -224,6 +226,8 @@ static guint gst_soup_http_src_signals[LAST_SIGNAL] = { 0 }; #define DEFAULT_TLS_INTERACTION NULL #define DEFAULT_TIMEOUT 15 #define DEFAULT_RETRIES 3 +#define DEFAULT_RETRY_BACKOFF_FACTOR 0.0 +#define DEFAULT_RETRY_BACKOFF_MAX 60.0 #define DEFAULT_SOUP_METHOD NULL #define GROW_BLOCKSIZE_LIMIT 1 @@ -513,6 +517,39 @@ gst_soup_http_src_class_init (GstSoupHTTPSrcClass * klass) "The HTTP method to use (GET, HEAD, OPTIONS, etc)", DEFAULT_SOUP_METHOD, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** + * GstSoupHTTPSrc:retry-backoff-factor: + * + * A backoff factor to apply between attempts after the second try + * (most errors are resolved immediately by a second try without a delay). + * souphttpsrc will sleep for: + * + * ``` + * {backoff factor} * (2 ** ({number of previous retries})) + * `` + * + * seconds + * + * Since: 1.26 + */ + g_object_class_install_property (gobject_class, PROP_RETRY_BACKOFF_FACTOR, + g_param_spec_double ("retry-backoff-factor", "Backoff Factor", + "Exponential retry backoff factor in seconds", 0.0, G_MAXDOUBLE, + DEFAULT_RETRY_BACKOFF_FACTOR, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + /** + * GstSoupHTTPSrc:retry-backoff-max: + * + * Maximum retry backoff delay in seconds + * + * Since: 1.26 + */ + g_object_class_install_property (gobject_class, PROP_RETRY_BACKOFF_MAX, + g_param_spec_double ("retry-backoff-max", "Maximum retry Backoff delay", + "Maximum backoff delay in seconds", 0.0, G_MAXDOUBLE, + DEFAULT_RETRY_BACKOFF_MAX, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); /** * GstSoupHTTPSrc::accept-certificate: * @souphttpsrc: a #GstSoupHTTPSrc @@ -562,7 +599,7 @@ gst_soup_http_src_class_init (GstSoupHTTPSrcClass * klass) static void gst_soup_http_src_reset (GstSoupHTTPSrc * src) { - src->retry_count = 0; + src->retry.count = 0; src->have_size = FALSE; src->got_headers = FALSE; src->headers_ret = GST_FLOW_OK; @@ -588,6 +625,14 @@ gst_soup_http_src_reset (GstSoupHTTPSrc * src) src->iradio_url = NULL; } +static void +gst_soup_http_src_init_retry_params (GstSoupHTTPSrc * src) +{ + src->retry.max = DEFAULT_RETRIES; + src->retry.backoff_factor = DEFAULT_RETRY_BACKOFF_FACTOR; + src->retry.backoff_max = DEFAULT_RETRY_BACKOFF_MAX; +} + static void gst_soup_http_src_init (GstSoupHTTPSrc * src) { @@ -617,7 +662,6 @@ gst_soup_http_src_init (GstSoupHTTPSrc * src) src->ssl_use_system_ca_file = DEFAULT_SSL_USE_SYSTEM_CA_FILE; src->tls_database = DEFAULT_TLS_DATABASE; src->tls_interaction = DEFAULT_TLS_INTERACTION; - src->max_retries = DEFAULT_RETRIES; src->method = DEFAULT_SOUP_METHOD; src->minimum_blocksize = gst_base_src_get_blocksize (GST_BASE_SRC_CAST (src)); proxy = g_getenv ("http_proxy"); @@ -630,6 +674,7 @@ gst_soup_http_src_init (GstSoupHTTPSrc * src) gst_base_src_set_automatic_eos (GST_BASE_SRC (src), FALSE); gst_soup_http_src_reset (src); + gst_soup_http_src_init_retry_params (src); } static void @@ -785,7 +830,9 @@ gst_soup_http_src_set_property (GObject * object, guint prop_id, src->tls_interaction = g_value_dup_object (value); break; case PROP_RETRIES: - src->max_retries = g_value_get_int (value); + GST_OBJECT_LOCK (src); + src->retry.max = g_value_get_int (value); + GST_OBJECT_UNLOCK (src); break; case PROP_METHOD: g_free (src->method); @@ -802,6 +849,16 @@ gst_soup_http_src_set_property (GObject * object, guint prop_id, src->ssl_use_system_ca_file = g_value_get_boolean (value); } break; + case PROP_RETRY_BACKOFF_FACTOR: + GST_OBJECT_LOCK (src); + src->retry.backoff_factor = g_value_get_double (value); + GST_OBJECT_UNLOCK (src); + break; + case PROP_RETRY_BACKOFF_MAX: + GST_OBJECT_LOCK (src); + src->retry.backoff_max = g_value_get_double (value); + GST_OBJECT_UNLOCK (src); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -881,7 +938,9 @@ gst_soup_http_src_get_property (GObject * object, guint prop_id, g_value_set_object (value, src->tls_interaction); break; case PROP_RETRIES: - g_value_set_int (value, src->max_retries); + GST_OBJECT_LOCK (src); + g_value_set_int (value, src->retry.max); + GST_OBJECT_UNLOCK (src); break; case PROP_METHOD: g_value_set_string (value, src->method); @@ -894,6 +953,16 @@ gst_soup_http_src_get_property (GObject * object, guint prop_id, if (gst_soup_loader_get_api_version () == 2) g_value_set_boolean (value, src->ssl_use_system_ca_file); break; + case PROP_RETRY_BACKOFF_FACTOR: + GST_OBJECT_LOCK (src); + g_value_set_double (value, src->retry.backoff_factor); + GST_OBJECT_UNLOCK (src); + break; + case PROP_RETRY_BACKOFF_MAX: + GST_OBJECT_LOCK (src); + g_value_set_double (value, src->retry.backoff_max); + GST_OBJECT_UNLOCK (src); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1657,6 +1726,10 @@ gst_soup_http_src_parse_status (SoupMessage * msg, GstSoupHTTPSrc * src) } /* SOUP_STATUS_IS_TRANSPORT_ERROR was replaced with GError in libsoup-3.0 */ + GST_OBJECT_LOCK (src); + Retry retry = src->retry; + GST_OBJECT_UNLOCK (src); + #if !defined(LINK_SOUP) || LINK_SOUP == 2 if (SOUP_STATUS_IS_TRANSPORT_ERROR (status_code)) { switch (status_code) { @@ -1675,7 +1748,7 @@ gst_soup_http_src_parse_status (SoupMessage * msg, GstSoupHTTPSrc * src) _("Secure connection setup failed.")); return GST_FLOW_ERROR; case SOUP_STATUS_IO_ERROR: - if (src->max_retries == -1 || src->retry_count < src->max_retries) + if (retry.max == -1 || retry.count < retry.max) return GST_FLOW_CUSTOM_ERROR; SOUP_HTTP_SRC_ERROR (src, msg, RESOURCE, READ, _("A network error occurred, or the server closed the connection " @@ -1698,10 +1771,10 @@ gst_soup_http_src_parse_status (SoupMessage * msg, GstSoupHTTPSrc * src) switch (status_code) { case SOUP_STATUS_SERVICE_UNAVAILABLE: case SOUP_STATUS_INTERNAL_SERVER_ERROR: - if (src->max_retries == -1 || src->retry_count < src->max_retries) { + if (retry.max == -1 || retry.count < retry.max) { return GST_FLOW_CUSTOM_ERROR; } - /* Error out when max_retries is reached. */ + /* Error out when retry_params.max is reached. */ break; default: break; @@ -1950,12 +2023,38 @@ gst_soup_http_src_do_request (GstSoupHTTPSrc * src, const gchar * method) GstFlowReturn ret; SoupMessageHeaders *request_headers; - if (src->max_retries != -1 && src->retry_count > src->max_retries) { + GST_OBJECT_LOCK (src); + if (src->retry.count > src->retry.max) { GST_DEBUG_OBJECT (src, "Max retries reached"); + GST_OBJECT_UNLOCK (src); + return GST_FLOW_ERROR; } - src->retry_count++; + src->retry.count++; + Retry retry = src->retry; + GST_OBJECT_UNLOCK (src); + + if (retry.count > 1 && retry.backoff_factor > 0.0) { + gdouble backoff = MIN (retry.backoff_factor * (1 << (retry.count - 1)), + retry.backoff_max); + + if (backoff > 0.0) { + gint64 end_time = g_get_monotonic_time () + (backoff * G_USEC_PER_SEC); + gint64 remaining_time = end_time - g_get_monotonic_time (); + + while (!g_cancellable_is_cancelled (src->cancellable) + && remaining_time > 0) { + GST_INFO_OBJECT (src, "Backoff for %.2f seconds before retry %d", + backoff, retry.count); + g_mutex_lock (&src->retry.lock); + g_cond_wait_until (&src->retry.cond, &src->retry.lock, end_time); + g_mutex_unlock (&src->retry.lock); + + remaining_time = end_time - g_get_monotonic_time (); + } + } + } /* EOS immediately if we have an empty segment */ if (src->request_position == src->stop_position) return GST_FLOW_EOS; @@ -2183,7 +2282,9 @@ gst_soup_http_src_read_buffer (GstSoupHTTPSrc * src, GstBuffer ** outbuf) gst_soup_http_src_update_position (src, res.nbytes); /* Got some data, reset retry counter */ - src->retry_count = 0; + GST_OBJECT_LOCK (src); + src->retry.count = 0; + GST_OBJECT_UNLOCK (src); gst_soup_http_src_check_update_blocksize (src, res.nbytes); @@ -2339,7 +2440,9 @@ done: } if (ret == GST_FLOW_FLUSHING) { - src->retry_count = 0; + GST_OBJECT_LOCK (src); + src->retry.count = 0; + GST_OBJECT_UNLOCK (src); } return ret; @@ -2369,10 +2472,15 @@ gst_soup_http_src_stop (GstBaseSrc * bsrc) gst_soup_http_src_stream_clear (src); - if (src->keep_alive && !src->msg && !src->session_is_shared) + if (src->keep_alive && !src->msg && !src->session_is_shared) { g_cancellable_cancel (src->cancellable); - else + + g_mutex_lock (&src->retry.lock); + g_cond_broadcast (&src->retry.cond); + g_mutex_unlock (&src->retry.lock); + } else { gst_soup_http_src_session_close (src); + } gst_soup_http_src_reset (src); return TRUE; @@ -2432,6 +2540,9 @@ gst_soup_http_src_unlock (GstBaseSrc * bsrc) GST_DEBUG_OBJECT (src, "unlock()"); g_cancellable_cancel (src->cancellable); + g_mutex_lock (&src->retry.lock); + g_cond_broadcast (&src->retry.cond); + g_mutex_unlock (&src->retry.lock); return TRUE; } diff --git a/subprojects/gst-plugins-good/ext/soup/gstsouphttpsrc.h b/subprojects/gst-plugins-good/ext/soup/gstsouphttpsrc.h index 89705943bf..25e7f558e0 100644 --- a/subprojects/gst-plugins-good/ext/soup/gstsouphttpsrc.h +++ b/subprojects/gst-plugins-good/ext/soup/gstsouphttpsrc.h @@ -45,6 +45,16 @@ typedef enum { GST_SOUP_HTTP_SRC_SESSION_IO_STATUS_CANCELLED, } GstSoupHTTPSrcSessionIOStatus; +typedef struct { + gint max; + gint count; /* Number of retries since we received data */ + + gdouble backoff_factor; + gdouble backoff_max; + GMutex lock; + GCond cond; +} Retry; + /* opaque from here, implementation detail */ typedef struct _GstSoupSession GstSoupSession; @@ -66,8 +76,6 @@ struct _GstSoupHTTPSrc { gboolean session_is_shared; GstSoupSession *external_session; /* Shared via GstContext */ SoupMessage *msg; /* Request message. */ - gint retry_count; /* Number of retries since we received data */ - gint max_retries; /* Maximum number of retries */ gchar *method; /* HTTP method */ GstFlowReturn headers_ret; @@ -124,6 +132,8 @@ struct _GstSoupHTTPSrc { GstEvent *http_headers_event; gint64 last_socket_read_time; + + Retry retry; }; struct _GstSoupHTTPSrcClass {