diff --git a/gst/hls/gsthlsdemux.c b/gst/hls/gsthlsdemux.c
index 63e729a44e..2e7b2fe58a 100644
--- a/gst/hls/gsthlsdemux.c
+++ b/gst/hls/gsthlsdemux.c
@@ -330,7 +330,7 @@ gst_hls_demux_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
       GstSeekType start_type, stop_type;
       gint64 start, stop;
       GList *walk;
-      GstClockTime current_pos, target_pos;
+      GstClockTime position, current_pos, target_pos;
       gint current_sequence;
       GstM3U8MediaFile *file;
 
@@ -397,8 +397,8 @@ gst_hls_demux_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
       GST_M3U8_CLIENT_LOCK (demux->client);
       GST_DEBUG_OBJECT (demux, "seeking to sequence %d", current_sequence);
       demux->client->sequence = current_sequence;
-      gst_m3u8_client_get_current_position (demux->client, &demux->position);
-      demux->position_shift = start - demux->position;
+      gst_m3u8_client_get_current_position (demux->client, &position);
+      demux->position_shift = start - position;
       demux->need_segment = TRUE;
       GST_M3U8_CLIENT_UNLOCK (demux->client);
 
@@ -663,8 +663,10 @@ gst_hls_demux_stream_loop (GstHLSDemux * demux)
   g_object_unref (fragment);
 
   if (demux->need_segment) {
-    GstClockTime start = demux->position + demux->position_shift;
     GstSegment segment;
+    GstClockTime start = GST_BUFFER_PTS (buf);
+
+    start += demux->position_shift;
     /* And send a newsegment */
     GST_DEBUG_OBJECT (demux, "Sending new-segment. segment start:%"
         GST_TIME_FORMAT, GST_TIME_ARGS (start));
@@ -676,9 +678,6 @@ gst_hls_demux_stream_loop (GstHLSDemux * demux)
     demux->position_shift = 0;
   }
 
-  if (GST_CLOCK_TIME_IS_VALID (GST_BUFFER_DURATION (buf)))
-    demux->position += GST_BUFFER_DURATION (buf);
-
   ret = gst_pad_push (demux->srcpad, buf);
   if (ret != GST_FLOW_OK)
     goto error_pushing;
@@ -754,7 +753,6 @@ gst_hls_demux_reset (GstHLSDemux * demux, gboolean dispose)
   }
   g_queue_clear (demux->queue);
 
-  demux->position = 0;
   demux->position_shift = 0;
   demux->need_segment = TRUE;
 }
@@ -877,7 +875,6 @@ gst_hls_demux_cache_fragments (GstHLSDemux * demux)
       demux->client->sequence -= demux->fragments_cache;
     else
       demux->client->sequence = 0;
-    gst_m3u8_client_get_current_position (demux->client, &demux->position);
     GST_M3U8_CLIENT_UNLOCK (demux->client);
   } else {
     GstClockTime duration = gst_m3u8_client_get_duration (demux->client);
diff --git a/gst/hls/gsthlsdemux.h b/gst/hls/gsthlsdemux.h
index 28470f88a9..9af84880b3 100644
--- a/gst/hls/gsthlsdemux.h
+++ b/gst/hls/gsthlsdemux.h
@@ -85,7 +85,6 @@ struct _GstHLSDemux
   gboolean cancelled;
 
   /* Position in the stream */
-  GstClockTime position;
   GstClockTime position_shift;
   gboolean need_segment;
 };