diff --git a/gst/rtpmanager/rtpsession.c b/gst/rtpmanager/rtpsession.c
index 3214de2d5b..e65719cef8 100644
--- a/gst/rtpmanager/rtpsession.c
+++ b/gst/rtpmanager/rtpsession.c
@@ -2274,7 +2274,7 @@ rtp_session_process_rtcp (RTPSession * sess, GstBuffer * buffer,
   update_arrival_stats (sess, &arrival, FALSE, buffer, current_time, -1,
       ntpnstime);
 
-  if (sess->sent_bye)
+  if (sess->source->sent_bye)
     goto ignore;
 
   /* start processing the compound packet */
@@ -2516,7 +2516,6 @@ rtp_session_schedule_bye_locked (RTPSession * sess, const gchar * reason,
   INIT_AVG (sess->stats.avg_rtcp_packet_size, 100);
   sess->stats.bye_members = 1;
   sess->first_rtcp = TRUE;
-  sess->sent_bye = FALSE;
   sess->allow_early = TRUE;
 
   /* reschedule transmission */
@@ -2604,7 +2603,7 @@ rtp_session_next_timeout (RTPSession * sess, GstClockTime current_time)
   }
 
   if (sess->source->marked_bye) {
-    if (sess->sent_bye) {
+    if (sess->source->sent_bye) {
       GST_DEBUG ("we sent BYE already");
       interval = GST_CLOCK_TIME_NONE;
     } else if (sess->stats.active_sources >= 50) {
@@ -3094,7 +3093,7 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time,
       /* generate BYE instead */
       GST_DEBUG ("generating BYE message");
       session_bye (sess, &data);
-      sess->sent_bye = TRUE;
+      own->sent_bye = TRUE;
     } else {
       /* loop over all known sources and do something */
       g_hash_table_foreach (sess->ssrcs[sess->mask_idx],
@@ -3133,7 +3132,6 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time,
     g_hash_table_insert (sess->ssrcs[sess->mask_idx],
         GINT_TO_POINTER (own->ssrc), own);
 
-    sess->sent_bye = FALSE;
     sess->change_ssrc = FALSE;
     notify = TRUE;
     GST_DEBUG ("changed our SSRC to %08x", own->ssrc);
@@ -3165,7 +3163,7 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time,
       GST_DEBUG ("%p, sending RTCP packet, avg size %u, %u", &sess->stats,
           sess->stats.avg_rtcp_packet_size, packet_size);
       result =
-          sess->callbacks.send_rtcp (sess, own, data.rtcp, sess->sent_bye,
+          sess->callbacks.send_rtcp (sess, own, data.rtcp, own->sent_bye,
           sess->send_rtcp_user_data);
     } else {
       GST_DEBUG ("freeing packet callback: %p"
diff --git a/gst/rtpmanager/rtpsession.h b/gst/rtpmanager/rtpsession.h
index a85fa3eaef..4dbd8a453d 100644
--- a/gst/rtpmanager/rtpsession.h
+++ b/gst/rtpmanager/rtpsession.h
@@ -213,8 +213,6 @@ struct _RTPSession {
 
   GstClockTime  next_early_rtcp_time;
 
-  gboolean      sent_bye;
-
   RTPSessionCallbacks   callbacks;
   gpointer              process_rtp_user_data;
   gpointer              send_rtp_user_data;
diff --git a/gst/rtpmanager/rtpsource.c b/gst/rtpmanager/rtpsource.c
index b54910fe01..d426cd7253 100644
--- a/gst/rtpmanager/rtpsource.c
+++ b/gst/rtpmanager/rtpsource.c
@@ -223,6 +223,7 @@ rtp_source_reset (RTPSource * src)
   if (src->bye_reason)
     g_free (src->bye_reason);
   src->bye_reason = NULL;
+  src->sent_bye = FALSE;
 
   src->stats.cycles = -1;
   src->stats.jitter = 0;
diff --git a/gst/rtpmanager/rtpsource.h b/gst/rtpmanager/rtpsource.h
index 82d010a6df..34b9d3b9d4 100644
--- a/gst/rtpmanager/rtpsource.h
+++ b/gst/rtpmanager/rtpsource.h
@@ -147,6 +147,7 @@ struct _RTPSource {
 
   gboolean      marked_bye;
   gchar        *bye_reason;
+  gboolean      sent_bye;
 
   GSocketAddress *rtp_from;
   GSocketAddress *rtcp_from;