diff --git a/gst-libs/gst/video/gstvideodecoder.c b/gst-libs/gst/video/gstvideodecoder.c
index 788ac21a03..4292a9c636 100644
--- a/gst-libs/gst/video/gstvideodecoder.c
+++ b/gst-libs/gst/video/gstvideodecoder.c
@@ -352,6 +352,7 @@ struct _GstVideoDecoderPrivate
   GstClockTime last_timestamp_out;
   /* incoming pts - dts */
   GstClockTime pts_delta;
+  gboolean reordered_output;
 
   /* reverse playback */
   /* collect input */
@@ -1620,6 +1621,7 @@ gst_video_decoder_reset (GstVideoDecoder * decoder, gboolean full)
       gst_tag_list_unref (priv->tags);
     priv->tags = NULL;
     priv->tags_changed = FALSE;
+    priv->reordered_output = FALSE;
   }
 
   priv->discont = TRUE;
@@ -2174,13 +2176,51 @@ gst_video_decoder_prepare_finish_frame (GstVideoDecoder *
 
     /* and set if needed;
      * valid delta means we have reasonable DTS input */
-    if (!GST_CLOCK_TIME_IS_VALID (frame->pts) && !seen_none &&
+    /* also, if we ended up reordered, means this approach is conflicting
+     * with some sparse existing PTS, and so it does not work out */
+    if (!priv->reordered_output &&
+        !GST_CLOCK_TIME_IS_VALID (frame->pts) && !seen_none &&
         GST_CLOCK_TIME_IS_VALID (priv->pts_delta)) {
       frame->pts = min_ts + priv->pts_delta;
       GST_DEBUG_OBJECT (decoder,
           "no valid PTS, using oldest DTS %" GST_TIME_FORMAT,
           GST_TIME_ARGS (frame->pts));
     }
+
+    /* some more maintenance, ts2 holds PTS */
+    min_ts = GST_CLOCK_TIME_NONE;
+    seen_none = FALSE;
+    for (l = priv->frames; l; l = l->next) {
+      GstVideoCodecFrame *tmp = l->data;
+
+      if (!GST_CLOCK_TIME_IS_VALID (tmp->abidata.ABI.ts2)) {
+        seen_none = TRUE;
+        continue;
+      }
+
+      if (!GST_CLOCK_TIME_IS_VALID (min_ts) || tmp->abidata.ABI.ts2 < min_ts) {
+        min_ts = tmp->abidata.ABI.ts2;
+        oframe = tmp;
+      }
+    }
+    /* save a ts if needed */
+    if (oframe && oframe != frame) {
+      oframe->abidata.ABI.ts2 = frame->abidata.ABI.ts2;
+    }
+
+    /* if we detected reordered output, then PTS are void,
+     * however those were obtained; bogus input, subclass etc */
+    if (priv->reordered_output && !seen_none) {
+      GST_DEBUG_OBJECT (decoder, "invaliding PTS");
+      frame->pts = GST_CLOCK_TIME_NONE;
+    }
+
+    if (!GST_CLOCK_TIME_IS_VALID (frame->pts) && !seen_none) {
+      frame->pts = min_ts;
+      GST_DEBUG_OBJECT (decoder,
+          "no valid PTS, using oldest PTS %" GST_TIME_FORMAT,
+          GST_TIME_ARGS (frame->pts));
+    }
   }
 
 
@@ -2202,6 +2242,7 @@ gst_video_decoder_prepare_finish_frame (GstVideoDecoder *
           "decreasing timestamp (%" GST_TIME_FORMAT " < %"
           GST_TIME_FORMAT ")",
           GST_TIME_ARGS (frame->pts), GST_TIME_ARGS (priv->last_timestamp_out));
+      priv->reordered_output = TRUE;
     }
   }
 
@@ -2590,9 +2631,11 @@ gst_video_decoder_decode_frame (GstVideoDecoder * decoder,
   frame->pts = GST_BUFFER_PTS (frame->input_buffer);
   frame->dts = GST_BUFFER_DTS (frame->input_buffer);
   frame->duration = GST_BUFFER_DURATION (frame->input_buffer);
-  frame->abidata.ABI.ts = frame->dts;
 
   /* For keyframes, PTS = DTS */
+  /* FIXME upstream can be quite wrong about the keyframe aspect,
+   * so we could be going off here as well,
+   * maybe let subclass decide if it really is/was a keyframe */
   if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame)) {
     if (!GST_CLOCK_TIME_IS_VALID (frame->pts)) {
       frame->pts = frame->dts;
@@ -2605,6 +2648,9 @@ gst_video_decoder_decode_frame (GstVideoDecoder * decoder,
     }
   }
 
+  frame->abidata.ABI.ts = frame->dts;
+  frame->abidata.ABI.ts2 = frame->pts;
+
   GST_LOG_OBJECT (decoder, "PTS %" GST_TIME_FORMAT ", DTS %" GST_TIME_FORMAT,
       GST_TIME_ARGS (frame->pts), GST_TIME_ARGS (frame->dts));
   GST_LOG_OBJECT (decoder, "dist %d", frame->distance_from_sync);
diff --git a/gst-libs/gst/video/gstvideoutils.h b/gst-libs/gst/video/gstvideoutils.h
index a3ff4f9e35..63833e236e 100644
--- a/gst-libs/gst/video/gstvideoutils.h
+++ b/gst-libs/gst/video/gstvideoutils.h
@@ -249,6 +249,7 @@ struct _GstVideoCodecFrame
   union {
     struct {
       GstClockTime ts;
+      GstClockTime ts2;
     } ABI;
     void         *padding[GST_PADDING_LARGE];
   } abidata;