diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c
index c9581b08d4..48ef879b3b 100644
--- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c
+++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c
@@ -4167,7 +4167,8 @@ session_cleanup (const gchar * key, RTPSource * source, ReportData * data)
         /* this is an internal source that is not using our suggested ssrc.
          * since there must be another source using this ssrc, we can remove
          * this one instead of making it a receiver forever */
-        if (source->ssrc != sess->suggested_ssrc) {
+        if (source->ssrc != sess->suggested_ssrc
+            && source->media_ssrc != sess->suggested_ssrc) {
           rtp_source_mark_bye (source, "timed out");
           /* do not schedule bye here, since we are inside the RTCP timeout
            * processing and scheduling bye will interfere with SR/RR sending */
diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c
index cb12644039..0a0d858c4b 100644
--- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c
+++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c
@@ -825,6 +825,7 @@ rtp_source_update_send_caps (RTPSource * src, GstCaps * caps)
   GstStructure *s;
   guint val;
   gint ival;
+  guint ssrc, rtx_ssrc = -1;
   gboolean rtx;
 
   /* nothing changed, return */
@@ -833,7 +834,17 @@ rtp_source_update_send_caps (RTPSource * src, GstCaps * caps)
 
   s = gst_caps_get_structure (caps, 0);
 
-  rtx = (gst_structure_get_uint (s, "rtx-ssrc", &val) && val == src->ssrc);
+  if (!gst_structure_get_uint (s, "ssrc", &ssrc))
+    return;
+  gst_structure_get_uint (s, "rtx-ssrc", &rtx_ssrc);
+
+  if (src->ssrc != ssrc && src->ssrc != rtx_ssrc) {
+    GST_WARNING ("got ssrc %u/%u that doesn't match with this source's ssrc %u",
+        ssrc, rtx_ssrc, src->ssrc);
+    return;
+  }
+
+  rtx = (rtx_ssrc == src->ssrc);
 
   if (gst_structure_get_int (s, rtx ? "rtx-payload" : "payload", &ival))
     src->payload = ival;
@@ -859,6 +870,12 @@ rtp_source_update_send_caps (RTPSource * src, GstCaps * caps)
       src->seqnum_offset);
 
   gst_caps_replace (&src->send_caps, caps);
+
+  if (rtx) {
+    src->media_ssrc = ssrc;
+  } else {
+    src->media_ssrc = -1;
+  }
 }
 
 /**
diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h
index a2723050ee..e23d53700d 100644
--- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h
+++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h
@@ -137,6 +137,9 @@ struct _RTPSource {
   /*< private >*/
   guint32       ssrc;
 
+  /* If not -1 then this is the SSRC of the corresponding media RTPSource */
+  guint32       media_ssrc;
+
   guint16       generation;
   GHashTable    *reported_in_sr_of;     /* set of SSRCs */