diff --git a/tests/check/elements/adaptive_demux_common.c b/tests/check/elements/adaptive_demux_common.c
index ab55795707..a8fcb1e7c5 100644
--- a/tests/check/elements/adaptive_demux_common.c
+++ b/tests/check/elements/adaptive_demux_common.c
@@ -175,10 +175,12 @@ gst_adaptive_demux_test_check_received_data (GstAdaptiveDemuxTestEngine *
       (guint64) gst_buffer_get_size (buffer),
       testOutputStreamData->expected_size, stream->segment_start);
 
-  fail_unless (stream->total_received_size +
-      stream->segment_received_size +
-      gst_buffer_get_size (buffer) <= testOutputStreamData->expected_size,
-      "Received unexpected data, please check what segments are being downloaded");
+  /* Only verify after seeking */
+  if (testData->seek_event && testData->seeked)
+    fail_unless (stream->total_received_size +
+        stream->segment_received_size +
+        gst_buffer_get_size (buffer) <= testOutputStreamData->expected_size,
+        "Received unexpected data, please check what segments are being downloaded");
 
   streamOffset = stream->segment_start + stream->segment_received_size;
   if (testOutputStreamData->expected_data) {
@@ -357,6 +359,7 @@ testSeekAdaptiveDemuxSendsData (GstAdaptiveDemuxTestEngine * engine,
         g_cond_wait (&testData->test_task_state_cond,
             &testData->test_task_state_lock);
       }
+      testData->seeked = TRUE;
       g_mutex_unlock (&testData->test_task_state_lock);
       /* we can continue now, but this buffer will be rejected by AppSink
        * because it is in flushing mode
diff --git a/tests/check/elements/adaptive_demux_common.h b/tests/check/elements/adaptive_demux_common.h
index c71e6a367d..448671b81f 100644
--- a/tests/check/elements/adaptive_demux_common.h
+++ b/tests/check/elements/adaptive_demux_common.h
@@ -113,6 +113,7 @@ typedef struct _GstAdaptiveDemuxTestCase
    */
   guint64 threshold_for_seek;
   GstEvent *seek_event;
+  gboolean seeked;
 
   gpointer signal_context;
 } GstAdaptiveDemuxTestCase;