diff --git a/ext/srtp/gstsrtpenc.c b/ext/srtp/gstsrtpenc.c
index ddd5661126..e057efbdb8 100644
--- a/ext/srtp/gstsrtpenc.c
+++ b/ext/srtp/gstsrtpenc.c
@@ -379,6 +379,8 @@ check_new_stream_locked (GstSrtpEnc * filter, guint32 ssrc)
   else
     ret = srtp_add_stream (filter->session, &policy);
 
+  filter->first_session = FALSE;
+
   if (HAS_CRYPTO (filter))
     gst_buffer_unmap (filter->key, &map);