diff --git a/ext/raw1394/gstdv1394src.c b/ext/raw1394/gstdv1394src.c
index 41a57e7f66..09755ba53b 100644
--- a/ext/raw1394/gstdv1394src.c
+++ b/ext/raw1394/gstdv1394src.c
@@ -980,7 +980,7 @@ gst_dv1394src_query (GstBaseSrc * basesrc, GstQuery * query)
   switch (GST_QUERY_TYPE (query)) {
     case GST_QUERY_LATENCY:
     {
-      gst_query_set_latency (query, TRUE, GST_SECOND / 25, GST_CLOCK_TIME_NONE);
+      gst_query_set_latency (query, TRUE, GST_SECOND / 25, GST_SECOND / 25);
     }
       break;
     default:
diff --git a/ext/vpx/gstvp8enc.c b/ext/vpx/gstvp8enc.c
index 5fca8f6b10..083c5902f1 100644
--- a/ext/vpx/gstvp8enc.c
+++ b/ext/vpx/gstvp8enc.c
@@ -1500,6 +1500,7 @@ gst_vp8_enc_set_format (GstVideoEncoder * video_encoder,
   GstVideoInfo *info = &state->info;
   GstVideoCodecState *output_state;
   gchar *profile_str;
+  GstClockTime latency;
 
   encoder = GST_VP8_ENC (video_encoder);
   GST_DEBUG_OBJECT (video_encoder, "set_format");
@@ -1681,13 +1682,16 @@ gst_vp8_enc_set_format (GstVideoEncoder * video_encoder,
   }
 
   if (GST_VIDEO_INFO_FPS_D (info) == 0 || GST_VIDEO_INFO_FPS_N (info) == 0) {
-    gst_video_encoder_set_latency (video_encoder, 0, GST_CLOCK_TIME_NONE);
+    /* FIXME: Assume 25fps for unknown framerates. Better than reporting
+     * that we introduce no latency while we actually do
+     */
+    latency = gst_util_uint64_scale (encoder->cfg.g_lag_in_frames,
+        1 * GST_SECOND, 25);
   } else {
-    gst_video_encoder_set_latency (video_encoder, 0,
-        gst_util_uint64_scale (encoder->cfg.g_lag_in_frames,
-            GST_VIDEO_INFO_FPS_D (info) * GST_SECOND,
-            GST_VIDEO_INFO_FPS_N (info)));
+    latency = gst_util_uint64_scale (encoder->cfg.g_lag_in_frames,
+        GST_VIDEO_INFO_FPS_D (info) * GST_SECOND, GST_VIDEO_INFO_FPS_N (info));
   }
+  gst_video_encoder_set_latency (video_encoder, latency, latency);
   encoder->inited = TRUE;
 
   /* Store input state */
diff --git a/ext/vpx/gstvp9enc.c b/ext/vpx/gstvp9enc.c
index 84b4b2228e..7a18e6102e 100644
--- a/ext/vpx/gstvp9enc.c
+++ b/ext/vpx/gstvp9enc.c
@@ -1476,6 +1476,7 @@ gst_vp9_enc_set_format (GstVideoEncoder * video_encoder,
   GstVideoInfo *info = &state->info;
   GstVideoCodecState *output_state;
   gchar *profile_str;
+  GstClockTime latency;
 
   encoder = GST_VP9_ENC (video_encoder);
   GST_DEBUG_OBJECT (video_encoder, "set_format");
@@ -1660,13 +1661,16 @@ gst_vp9_enc_set_format (GstVideoEncoder * video_encoder,
   }
 
   if (GST_VIDEO_INFO_FPS_D (info) == 0 || GST_VIDEO_INFO_FPS_N (info) == 0) {
-    gst_video_encoder_set_latency (video_encoder, 0, GST_CLOCK_TIME_NONE);
+    /* FIXME: Assume 25fps for unknown framerates. Better than reporting
+     * that we introduce no latency while we actually do
+     */
+    latency = gst_util_uint64_scale (encoder->cfg.g_lag_in_frames,
+        1 * GST_SECOND, 25);
   } else {
-    gst_video_encoder_set_latency (video_encoder, 0,
-        gst_util_uint64_scale (encoder->cfg.g_lag_in_frames,
-            GST_VIDEO_INFO_FPS_D (info) * GST_SECOND,
-            GST_VIDEO_INFO_FPS_N (info)));
+    latency = gst_util_uint64_scale (encoder->cfg.g_lag_in_frames,
+        GST_VIDEO_INFO_FPS_D (info) * GST_SECOND, GST_VIDEO_INFO_FPS_N (info));
   }
+  gst_video_encoder_set_latency (video_encoder, latency, latency);
   encoder->inited = TRUE;
 
   /* Store input state */
diff --git a/gst/interleave/interleave.c b/gst/interleave/interleave.c
index 2a2fd6e4b1..1d36654e99 100644
--- a/gst/interleave/interleave.c
+++ b/gst/interleave/interleave.c
@@ -1079,15 +1079,17 @@ gst_interleave_src_query_latency (GstInterleave * self, GstQuery * query)
         if (res) {
           gst_query_parse_latency (peerquery, &live_cur, &min_cur, &max_cur);
 
-          if (min_cur > min)
-            min = min_cur;
+          if (live_cur) {
+            if (min_cur > min)
+              min = min_cur;
 
-          if (max_cur != GST_CLOCK_TIME_NONE &&
-              ((max != GST_CLOCK_TIME_NONE && max_cur > max) ||
-                  (max == GST_CLOCK_TIME_NONE)))
-            max = max_cur;
+            if (max == GST_CLOCK_TIME_NONE)
+              max = max_cur;
+            else if (max_cur < max)
+              max = max_cur;
 
-          live = live || live_cur;
+            live = TRUE;
+          }
         }
 
         gst_query_unref (peerquery);
diff --git a/gst/rtsp/gstrtpdec.c b/gst/rtsp/gstrtpdec.c
index b59aa5dfda..e24927bf49 100644
--- a/gst/rtsp/gstrtpdec.c
+++ b/gst/rtsp/gstrtpdec.c
@@ -372,6 +372,7 @@ gst_rtp_dec_query_src (GstPad * pad, GstObject * parent, GstQuery * query)
     case GST_QUERY_LATENCY:
     {
       /* we pretend to be live with a 3 second latency */
+      /* FIXME: Do we really have infinite maximum latency? */
       gst_query_set_latency (query, TRUE, 3 * GST_SECOND, -1);
       res = TRUE;
       break;
diff --git a/gst/videomixer/videomixer2.c b/gst/videomixer/videomixer2.c
index abb18f8ab9..e3956458af 100644
--- a/gst/videomixer/videomixer2.c
+++ b/gst/videomixer/videomixer2.c
@@ -1472,15 +1472,17 @@ gst_videomixer2_query_latency (GstVideoMixer2 * mix, GstQuery * query)
         if (res) {
           gst_query_parse_latency (peerquery, &live_cur, &min_cur, &max_cur);
 
-          if (min_cur > min)
-            min = min_cur;
+          if (live_cur) {
+            if (min_cur > min)
+              min = min_cur;
 
-          if (max_cur != GST_CLOCK_TIME_NONE &&
-              ((max != GST_CLOCK_TIME_NONE && max_cur > max) ||
-                  (max == GST_CLOCK_TIME_NONE)))
-            max = max_cur;
+            if (max == GST_CLOCK_TIME_NONE)
+              max = max_cur;
+            else if (max_cur < max)
+              max = max_cur;
 
-          live = live || live_cur;
+            live = TRUE;
+          }
         }
 
         gst_query_unref (peerquery);