diff --git a/gst-libs/gst/video/gstvideodecoder.c b/gst-libs/gst/video/gstvideodecoder.c
index 1a5a0165d3..808e7ac930 100644
--- a/gst-libs/gst/video/gstvideodecoder.c
+++ b/gst-libs/gst/video/gstvideodecoder.c
@@ -1169,7 +1169,9 @@ gst_video_decoder_src_query (GstPad * pad, GstObject * parent, GstQuery * query)
 
         GST_OBJECT_LOCK (dec);
         min_latency += dec->priv->min_latency;
-        if (max_latency != GST_CLOCK_TIME_NONE) {
+        if (dec->priv->max_latency == GST_CLOCK_TIME_NONE) {
+          max_latency = GST_CLOCK_TIME_NONE;
+        } else if (max_latency != GST_CLOCK_TIME_NONE) {
           max_latency += dec->priv->max_latency;
         }
         GST_OBJECT_UNLOCK (dec);
diff --git a/gst-libs/gst/video/gstvideoencoder.c b/gst-libs/gst/video/gstvideoencoder.c
index 67b426d686..933fc6c4f3 100644
--- a/gst-libs/gst/video/gstvideoencoder.c
+++ b/gst-libs/gst/video/gstvideoencoder.c
@@ -1043,8 +1043,10 @@ gst_video_encoder_src_query (GstPad * pad, GstObject * parent, GstQuery * query)
 
         GST_OBJECT_LOCK (enc);
         min_latency += priv->min_latency;
-        if (max_latency != GST_CLOCK_TIME_NONE) {
-          max_latency += priv->max_latency;
+        if (enc->priv->max_latency == GST_CLOCK_TIME_NONE) {
+          max_latency = GST_CLOCK_TIME_NONE;
+        } else if (max_latency != GST_CLOCK_TIME_NONE) {
+          max_latency += enc->priv->max_latency;
         }
         GST_OBJECT_UNLOCK (enc);