diff --git a/gst-libs/gst/rtsp/gstrtspconnection.c b/gst-libs/gst/rtsp/gstrtspconnection.c
index 073852282c..0c1f22b990 100644
--- a/gst-libs/gst/rtsp/gstrtspconnection.c
+++ b/gst-libs/gst/rtsp/gstrtspconnection.c
@@ -2482,18 +2482,30 @@ gst_rtsp_connection_next_timeout (GstRTSPConnection * conn, GTimeVal * timeout)
   gdouble elapsed;
   glong sec;
   gulong usec;
+  gint ctimeout;
 
   g_return_val_if_fail (conn != NULL, GST_RTSP_EINVAL);
   g_return_val_if_fail (timeout != NULL, GST_RTSP_EINVAL);
 
-  /* Because we should act before the timeout we timeout 5
-   * seconds in advance. */
-  elapsed = g_timer_elapsed (conn->timer, &usec) + 5;
-  if (elapsed >= conn->timeout) {
+  ctimeout = conn->timeout;
+  if (ctimeout >= 20) {
+    /* Because we should act before the timeout we timeout 5
+     * seconds in advance. */
+    ctimeout -= 5;
+  } else if (ctimeout >= 5) {
+    /* else timeout 20% earlier */
+    ctimeout -= ctimeout / 5;
+  } else if (ctimeout >= 1) {
+    /* else timeout 1 second earlier */
+    ctimeout -= 1;
+  }
+
+  elapsed = g_timer_elapsed (conn->timer, &usec);
+  if (elapsed >= ctimeout) {
     sec = 0;
     usec = 0;
   } else {
-    sec = conn->timeout - elapsed;
+    sec = ctimeout - elapsed;
     if (usec <= G_USEC_PER_SEC)
       usec = G_USEC_PER_SEC - usec;
     else