FaSTTUBe/GST-Tensordecoder-ov_ep
37
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

videorate: Refactor so upstream/downstream time domains are properly decoupled

Browse Source 
Refactor the videorate element to properly separate upstream and downstream
time domains when the `rate` property is set.

The previous implementation had issues with time domain conversions when
changing rates, especially during seeks after which we were basically confusing
input and output timestamps.

This also fixes rate changes as we are now tracking the wanted input timestamps
and not confusing them with the output, so this way we do not drop buffers when
the rate is changed while playing, meaning that the related tests have been
fixed

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/9013>
This commit is contained in:
Thibault Saunier 2025-04-11 13:33:05 -04:00 committed by GStreamer Marge Bot
parent 242b4fa931
commit 51d43529c3
9 changed files with 355 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

422
subprojects/gst-plugins-base/gst/videorate/gstvideorate.c
View File

@ -76,6 +76,7 @@
*/
#include "gst/gstcaps.h"
#include "gst/gstclock.h"
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
@ -573,6 +574,86 @@ gst_video_rate_transform_caps (GstBaseTransform * trans,
return ret;
}
static GstClockTime
gst_videorate_rate_compute_input_frame_duration (GstVideoRate * videorate,
GstBuffer * buf)
{
if (videorate->from_rate_numerator) {
return gst_util_uint64_scale (1,
videorate->from_rate_denominator * GST_SECOND,
videorate->from_rate_numerator);
} else if (buf && GST_BUFFER_DURATION (buf)) {
return GST_BUFFER_DURATION (buf);
} else if (videorate->prevbuf && GST_BUFFER_DURATION (videorate->prevbuf)) {
return GST_BUFFER_DURATION (videorate->prevbuf);
} else if (videorate->to_rate_numerator) {
GST_WARNING_OBJECT (videorate,
"No duration available for input frame, using output framerate");
return gst_util_uint64_scale (1,
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
}
g_warning ("No duration available for input frame");
return 0;
}
static gboolean
gst_video_rate_compute_best_input_ts (GstVideoRate * videorate, GstBuffer * buf,
GstClockTime * best_ts)
{
if (videorate->segment.rate >= 0) {
g_assert (GST_CLOCK_TIME_IS_VALID (videorate->next_output_ts));
g_assert (GST_CLOCK_TIME_IS_VALID (videorate->base_input_ts));
g_assert (GST_CLOCK_TIME_IS_VALID (videorate->base_output_ts));
g_assert (videorate->next_output_ts >= videorate->base_output_ts);
*best_ts =
videorate->base_input_ts + ((videorate->next_output_ts -
videorate->base_output_ts) * videorate->rate);
GST_LOG_OBJECT (videorate,
"best: %" GST_TIMEP_FORMAT
" = base_input_ts %" GST_TIMEP_FORMAT " + ((next_output_ts %"
GST_TIMEP_FORMAT " - base_output_ts %" GST_TIMEP_FORMAT ") * rate %f)",
best_ts, &videorate->base_input_ts, &videorate->next_output_ts,
&videorate->base_output_ts, videorate->rate);
return TRUE;
}
if (!GST_CLOCK_TIME_IS_VALID (videorate->next_output_ts)) {
GST_ERROR_OBJECT (videorate, "next_output_ts is invalid");
return FALSE;
}
GstClockTime input_frame_duration =
gst_videorate_rate_compute_input_frame_duration (videorate, buf);
GstClockTime output_frame_duration;
if (videorate->to_rate_numerator) {
output_frame_duration = gst_util_uint64_scale (1,
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
} else {
output_frame_duration = input_frame_duration;
}
*best_ts =
videorate->base_input_ts - ((videorate->base_output_ts -
(videorate->next_output_ts + output_frame_duration))
* videorate->rate) - input_frame_duration;
GST_LOG_OBJECT (videorate,
"best: %" GST_TIMEP_FORMAT
" = base_input_ts %" GST_TIMEP_FORMAT
" - ((base_output_ts %" GST_TIMEP_FORMAT
" - (next_output_ts %" GST_TIMEP_FORMAT " + output_frame_duration))"
" * rate %f) - input_frame_duration",
best_ts, &videorate->base_input_ts,
&videorate->base_output_ts, &videorate->next_output_ts, videorate->rate);
return TRUE;
}
static GstCaps *
gst_video_rate_fixate_caps (GstBaseTransform * trans,
GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
@ -624,12 +705,20 @@ gst_video_rate_setcaps (GstBaseTransform * trans, GstCaps * in_caps,
/* out_frame_count is scaled by the frame rate caps when calculating next_ts.
* when the frame rate caps change, we must update base_ts and reset
* out_frame_count */
if (videorate->to_rate_numerator) {
videorate->base_ts +=
if (videorate->to_rate_numerator && videorate->out_frame_count) {
gst_video_rate_compute_best_input_ts (videorate, NULL,
&videorate->base_input_ts);
videorate->base_output_ts +=
gst_util_uint64_scale (videorate->out_frame_count +
(videorate->segment.rate < 0.0 ? 1 : 0),
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
GST_LOG_OBJECT (videorate, "base_input_ts %" GST_TIME_FORMAT
" base_output_ts %" GST_TIME_FORMAT,
GST_TIME_ARGS (videorate->base_input_ts),
GST_TIME_ARGS (videorate->base_output_ts));
}
videorate->out_frame_count = 0;
videorate->to_rate_numerator = rate_numerator;
@ -663,12 +752,13 @@ gst_video_rate_reset (GstVideoRate * videorate, gboolean on_flush)
videorate->in = 0;
videorate->out = 0;
videorate->base_ts = 0;
videorate->base_input_ts = 0;
videorate->base_output_ts = 0;
videorate->out_frame_count = 0;
videorate->drop = 0;
videorate->dup = 0;
videorate->next_ts = GST_CLOCK_TIME_NONE;
videorate->next_end_ts = GST_CLOCK_TIME_NONE;
videorate->next_output_ts = GST_CLOCK_TIME_NONE;
videorate->next_output_end_ts = GST_CLOCK_TIME_NONE;
videorate->last_ts = GST_CLOCK_TIME_NONE;
videorate->discont = TRUE;
videorate->average = 0;
@ -730,23 +820,23 @@ gst_video_rate_push_buffer (GstVideoRate * videorate, GstBuffer * outbuf,
GST_BUFFER_FLAG_UNSET (outbuf, GST_BUFFER_FLAG_GAP);
/* this is the timestamp we put on the buffer */
push_ts = videorate->next_ts;
push_ts = videorate->next_output_ts;
videorate->out++;
videorate->out_frame_count++;
if (videorate->segment.rate < 0.0) {
videorate->next_end_ts = push_ts;
videorate->next_output_end_ts = push_ts;
if (videorate->to_rate_numerator) {
/* interpolate next expected timestamp in the segment */
GstClockTimeDiff next_ts = videorate->base_ts -
GstClockTimeDiff next_ts = videorate->base_output_ts -
gst_util_uint64_scale (videorate->out_frame_count + 1,
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
if (next_ts < 0 || next_ts < videorate->segment.start)
videorate->next_ts = GST_CLOCK_TIME_NONE;
videorate->next_output_ts = GST_CLOCK_TIME_NONE;
else
videorate->next_ts = next_ts;
videorate->next_output_ts = next_ts;
GST_BUFFER_DURATION (outbuf) =
gst_util_uint64_scale (videorate->out_frame_count,
@ -756,19 +846,19 @@ gst_video_rate_push_buffer (GstVideoRate * videorate, GstBuffer * outbuf,
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
} else if (next_intime != GST_CLOCK_TIME_NONE) {
videorate->next_ts = next_intime;
videorate->next_output_ts = next_intime;
} else {
GST_FIXME_OBJECT (videorate, "No next intime for reverse playback");
}
} else {
videorate->next_end_ts = GST_CLOCK_TIME_NONE;
videorate->next_output_end_ts = GST_CLOCK_TIME_NONE;
if (videorate->to_rate_numerator) {
/* interpolate next expected timestamp in the segment */
videorate->next_ts = videorate->base_ts +
videorate->next_output_ts = videorate->base_output_ts +
gst_util_uint64_scale (videorate->out_frame_count,
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
GST_BUFFER_DURATION (outbuf) = videorate->next_ts - push_ts;
GST_BUFFER_DURATION (outbuf) = videorate->next_output_ts - push_ts;
} else if (!invalid_duration) {
/* There must always be a valid duration on prevbuf if rate > 0,
* it is ensured in the transform_ip function */
@ -776,7 +866,7 @@ gst_video_rate_push_buffer (GstVideoRate * videorate, GstBuffer * outbuf,
g_assert (GST_BUFFER_DURATION_IS_VALID (outbuf));
g_assert (GST_BUFFER_DURATION (outbuf) != 0);
videorate->next_ts
videorate->next_output_ts
= GST_BUFFER_PTS (outbuf) + GST_BUFFER_DURATION (outbuf);
}
}
@ -883,7 +973,7 @@ gst_video_rate_swap_prev (GstVideoRate * videorate, GstBuffer * buffer,
while (!first_buf && buffer && gst_video_rate_past_buffer (videorate)) {
GST_LOG_OBJECT (videorate,
"next %" GST_TIME_FORMAT " vs latest %"
GST_TIME_FORMAT, GST_TIME_ARGS (videorate->next_ts),
GST_TIME_FORMAT, GST_TIME_ARGS (videorate->next_output_ts),
GST_TIME_ARGS (videorate->prev_ts));
gst_video_rate_flush_prev (videorate,
@ -903,11 +993,11 @@ static gboolean
gst_video_rate_check_duplicate_to_close_segment (GstVideoRate * videorate,
GstClockTime last_input_ts, gboolean is_first)
{
GstClockTime next_stream_time = videorate->next_ts;
GstClockTime next_stream_time = videorate->next_output_ts;
GstClockTime max_closing_segment_duplication_duration =
videorate->max_closing_segment_duplication_duration;
if (!GST_CLOCK_TIME_IS_VALID (videorate->next_ts))
if (!GST_CLOCK_TIME_IS_VALID (videorate->next_output_ts))
return FALSE;
if (videorate->segment.rate > 0.0) {
@ -921,10 +1011,10 @@ gst_video_rate_check_duplicate_to_close_segment (GstVideoRate * videorate,
return FALSE;
if (GST_CLOCK_TIME_IS_VALID (max_closing_segment_duplication_duration)) {
if (last_input_ts > videorate->next_ts)
if (last_input_ts > videorate->next_output_ts)
return TRUE;
return (videorate->next_ts - last_input_ts <
return (videorate->next_output_ts - last_input_ts <
max_closing_segment_duplication_duration);
}
@ -942,10 +1032,10 @@ gst_video_rate_check_duplicate_to_close_segment (GstVideoRate * videorate,
return FALSE;
if (GST_CLOCK_TIME_IS_VALID (max_closing_segment_duplication_duration)) {
if (last_input_ts < videorate->next_ts)
if (last_input_ts < videorate->next_output_ts)
return TRUE;
return (last_input_ts - videorate->next_ts <
return (last_input_ts - videorate->next_output_ts <
max_closing_segment_duplication_duration);
}
@ -1060,7 +1150,7 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event)
switch (GST_EVENT_TYPE (event)) {
case GST_EVENT_SEGMENT:
{
GstSegment segment;
GstSegment input_segment, segment;
gint seqnum;
GstCaps *rolled_back_caps;
@ -1068,6 +1158,7 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event)
if (segment.format != GST_FORMAT_TIME)
goto format_error;
input_segment = segment;
segment.start = (gint64) (segment.start / videorate->rate);
segment.position = (gint64) (segment.position / videorate->rate);
if (GST_CLOCK_TIME_IS_VALID (segment.stop))
@ -1117,22 +1208,29 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event)
}
/* Convert next_ts and last_ts to new segment. */
videorate->next_ts =
videorate->next_output_ts =
convert_position (&videorate->segment, &segment,
videorate->next_ts);
if (videorate->next_ts == -1)
videorate->last_ts = -1;
videorate->last_ts =
convert_position (&videorate->segment, &segment,
videorate->last_ts);
videorate->next_output_ts);
if (videorate->next_ts != -1)
videorate->base_ts = videorate->next_ts;
else if (segment.rate < 0)
videorate->base_ts = segment.stop;
else
videorate->base_ts = segment.start;
if (!GST_CLOCK_TIME_IS_VALID (videorate->next_output_ts)) {
videorate->last_ts = GST_CLOCK_TIME_NONE;
} else {
videorate->last_ts =
convert_position (&videorate->segment, &segment,
videorate->last_ts);
}
if (GST_CLOCK_TIME_IS_VALID (videorate->next_output_ts)) {
videorate->base_output_ts = videorate->next_output_ts;
videorate->base_input_ts =
videorate->next_output_ts * videorate->rate;
} else if (segment.rate < 0) {
videorate->base_input_ts = input_segment.stop;
videorate->base_output_ts = segment.stop;
} else {
videorate->base_input_ts = input_segment.start;
videorate->base_output_ts = segment.start;
}
videorate->out_frame_count = 0;
gst_buffer_replace (&videorate->prevbuf, NULL);
@ -1140,7 +1238,7 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event)
gst_segment_copy_into (&segment, &videorate->segment);
GST_DEBUG_OBJECT (videorate, "updated segment: %" GST_SEGMENT_FORMAT,
&videorate->segment);
&segment);
seqnum = gst_event_get_seqnum (event);
gst_event_unref (event);
event = gst_event_new_segment (&segment);
@ -1177,11 +1275,11 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event)
MIN (videorate->max_closing_segment_duplication_duration,
duration);
end_ts = videorate->next_ts + duration;
end_ts = videorate->next_output_ts + duration;
while (res == GST_FLOW_OK && ((videorate->segment.rate > 0.0
&& GST_CLOCK_TIME_IS_VALID (videorate->segment.stop)
&& GST_CLOCK_TIME_IS_VALID (videorate->next_ts)
&& videorate->next_ts < end_ts)
&& GST_CLOCK_TIME_IS_VALID (videorate->next_output_ts)
&& videorate->next_output_ts < end_ts)
|| count < 1)) {
res =
gst_video_rate_flush_prev (videorate, count > 0,
@ -1300,21 +1398,19 @@ gst_video_rate_src_event (GstBaseTransform * trans, GstEvent * event)
"account. Timestamp: %" GST_TIME_FORMAT " -> %" GST_TIME_FORMAT
" - diff %" G_GINT64_FORMAT "-> %" G_GINT64_FORMAT,
GST_TIME_ARGS (timestamp),
GST_TIME_ARGS (videorate->base_ts + ((timestamp -
videorate->base_ts) * videorate->rate)), diff,
GST_TIME_ARGS (videorate->base_output_ts + ((timestamp -
videorate->base_output_ts) * videorate->rate)), diff,
(GstClockTimeDiff) (diff * videorate->rate));
if (videorate->segment.rate < 0.0) {
timestamp =
(videorate->segment.stop - videorate->base_ts) -
((videorate->segment.stop - videorate->base_ts -
(videorate->segment.stop - videorate->base_output_ts) -
((videorate->segment.stop - videorate->base_output_ts -
timestamp) * videorate->rate);
} else if (videorate->base_ts > timestamp) {
timestamp = videorate->base_ts;
} else {
timestamp =
videorate->base_ts + ((timestamp -
videorate->base_ts) * videorate->rate);
videorate->base_output_ts + ((timestamp -
videorate->base_output_ts) * videorate->rate);
}
diff *= videorate->rate;
@ -1668,7 +1764,7 @@ gst_video_rate_switch_mode_if_needed (GstVideoRate * videorate)
gst_video_rate_swap_prev (videorate, NULL, 0, NULL);
} else {
/* enable regular mode */
videorate->next_ts = GST_CLOCK_TIME_NONE;
videorate->next_output_ts = GST_CLOCK_TIME_NONE;
skip = TRUE;
}
@ -1702,11 +1798,13 @@ gst_video_rate_do_max_duplicate (GstVideoRate * videorate, GstBuffer * buffer,
videorate->discont = TRUE;
if (videorate->segment.rate < 0.0) {
videorate->base_ts -= prevtime - intime;
videorate->base_output_ts -= (prevtime - intime) * videorate->rate;
videorate->base_input_ts = prevtime - intime;
} else {
videorate->base_ts += intime - prevtime;
videorate->base_output_ts += (intime - prevtime) * videorate->rate;
videorate->base_input_ts += intime - prevtime;
}
videorate->next_ts = intime;
videorate->next_output_ts = intime / videorate->rate;
/* Swap in new buffer and get rid of old buffer so that starting with
* the next input buffer we output from the new position */
gst_video_rate_swap_prev (videorate, buffer, intime, NULL);
@ -1717,7 +1815,7 @@ gst_video_rate_do_max_duplicate (GstVideoRate * videorate, GstBuffer * buffer,
}
static gboolean
gst_video_rate_apply_pending_rate (GstVideoRate * videorate)
gst_video_rate_apply_pending_rate (GstVideoRate * videorate, GstBuffer * buffer)
{
gboolean ret = FALSE;
@ -1726,14 +1824,26 @@ gst_video_rate_apply_pending_rate (GstVideoRate * videorate)
goto done;
ret = TRUE;
if (videorate->segment.rate < 0)
videorate->base_ts -= gst_util_uint64_scale (videorate->out_frame_count,
if (videorate->segment.rate < 0) {
GstClockTime prev_base_input_ts = videorate->base_input_ts;
if (!gst_video_rate_compute_best_input_ts (videorate, buffer,
&videorate->base_input_ts)) {
GST_ERROR_OBJECT (videorate, "Could not update base input TS");
videorate->base_input_ts = prev_base_input_ts;
}
videorate->base_output_ts -=
gst_util_uint64_scale (videorate->out_frame_count,
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
else
videorate->base_ts += gst_util_uint64_scale (videorate->out_frame_count,
} else {
gst_video_rate_compute_best_input_ts (videorate, buffer,
&videorate->base_input_ts);
videorate->base_output_ts +=
gst_util_uint64_scale (videorate->out_frame_count,
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
}
videorate->rate = videorate->pending_rate;
videorate->out_frame_count = 0;
@ -1743,6 +1853,50 @@ done:
return ret;
}
static void
gst_video_rate_compute_next_ts (GstVideoRate * videorate, GstClockTime in_ts,
gboolean skip)
{
if (GST_CLOCK_TIME_IS_VALID (videorate->next_output_ts))
return;
if (videorate->skip_to_first || skip) {
/* new buffer, we expect to output a buffer that matches the first
* timestamp in the segment */
/* FIXME: Take into account videorate->rate */
videorate->base_output_ts = in_ts;
videorate->base_input_ts = in_ts;
videorate->next_output_ts = in_ts;
videorate->out_frame_count = 0;
return;
}
if (videorate->segment.rate >= 0.0) {
videorate->next_output_ts = videorate->segment.start;
return;
}
/* Reverse playback */
if (videorate->to_rate_numerator) {
GstClockTime frame_duration = gst_util_uint64_scale (1,
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
videorate->next_output_ts = videorate->segment.stop;
if (videorate->next_output_ts > frame_duration)
videorate->next_output_ts =
MAX (videorate->segment.start,
videorate->next_output_ts - frame_duration);
else
videorate->next_output_ts = videorate->segment.start;
} else {
/* What else can we do? */
videorate->next_output_ts = in_ts;
}
}
static gboolean
gst_video_rate_past_buffer (GstVideoRate * videorate)
{
@ -1750,16 +1904,15 @@ gst_video_rate_past_buffer (GstVideoRate * videorate)
/* A new rate may have been set which must be considered here for
timestamp calculation */
gst_video_rate_apply_pending_rate (videorate);
/* On variable framerate we cannot push upfront */
if (videorate->force_variable_rate)
return FALSE;
if (videorate->segment.rate < 0.0) {
next_ts = videorate->next_ts;
next_ts = videorate->next_output_ts;
if (!GST_CLOCK_TIME_IS_VALID (videorate->next_ts))
if (!GST_CLOCK_TIME_IS_VALID (videorate->next_output_ts))
return FALSE;
if (videorate->to_rate_numerator) {
@ -1771,18 +1924,23 @@ gst_video_rate_past_buffer (GstVideoRate * videorate)
next_end_ts = next_ts + GST_BUFFER_DURATION (videorate->prevbuf);
}
next_ts = videorate->base_ts - ((videorate->base_ts -
next_ts) * videorate->rate);
if (videorate->base_ts > next_end_ts)
next_end_ts = videorate->base_ts - ((videorate->base_ts -
next_end_ts) * videorate->rate);
else
next_end_ts = videorate->base_ts;
next_ts =
MAX (0,
GST_CLOCK_DIFF (((videorate->base_output_ts -
next_ts) * videorate->rate), videorate->base_output_ts));
if (videorate->base_output_ts > next_end_ts) {
next_end_ts =
MAX (0,
GST_CLOCK_DIFF ((videorate->base_output_ts -
next_end_ts) * videorate->rate, videorate->base_output_ts));
} else {
next_end_ts = videorate->base_output_ts;
}
return next_end_ts >= videorate->prev_ts;
} else {
next_ts = videorate->base_ts + ((videorate->next_ts -
videorate->base_ts) * videorate->rate);
next_ts = videorate->base_output_ts + ((videorate->next_output_ts -
videorate->base_output_ts) * videorate->rate);
return next_ts <= videorate->prev_ts;
}
}
@ -1831,9 +1989,11 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
if (videorate->average_period > 0)
return gst_video_rate_trans_ip_max_avg (videorate, buffer);
gst_video_rate_apply_pending_rate (videorate);
in_ts = GST_BUFFER_PTS (buffer);
in_dur = GST_BUFFER_DURATION (buffer);
if (videorate->segment.rate >= 0.0)
gst_video_rate_compute_next_ts (videorate, in_ts, skip);
gst_video_rate_apply_pending_rate (videorate, buffer);
if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (in_ts))) {
/* For reverse playback, we need all input timestamps as we can't
@ -1870,45 +2030,17 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
gst_video_rate_swap_prev (videorate, buffer, in_ts, NULL);
videorate->in++;
if (!GST_CLOCK_TIME_IS_VALID (videorate->next_ts)) {
/* new buffer, we expect to output a buffer that matches the first
* timestamp in the segment */
if (videorate->skip_to_first || skip) {
videorate->base_ts = in_ts;
videorate->next_ts = in_ts;
videorate->out_frame_count = 0;
} else {
if (videorate->segment.rate < 0.0) {
if (videorate->to_rate_numerator) {
GstClockTime frame_duration = gst_util_uint64_scale (1,
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
videorate->next_ts = videorate->segment.stop;
if (videorate->next_ts > frame_duration)
videorate->next_ts =
MAX (videorate->segment.start,
videorate->next_ts - frame_duration);
else
videorate->next_ts = videorate->segment.start;
} else {
/* What else can we do? */
videorate->next_ts = in_ts;
}
} else {
videorate->next_ts = videorate->segment.start;
}
}
}
if (videorate->segment.rate < 0.0)
gst_video_rate_compute_next_ts (videorate, in_ts, skip);
/* In drop-only mode we can already decide here if we should output the
* current frame or drop it because it's coming earlier than our minimum
* allowed frame period. This also keeps latency down to 0 frames
*/
if (videorate->drop_only) {
if ((videorate->segment.rate > 0.0 && in_ts >= videorate->next_ts) ||
(videorate->segment.rate < 0.0 && in_ts <= videorate->next_ts)) {
if ((videorate->segment.rate > 0.0 && in_ts >= videorate->next_output_ts)
|| (videorate->segment.rate < 0.0
&& in_ts <= videorate->next_output_ts)) {
res = gst_video_rate_push_buffer (videorate,
gst_buffer_ref (buffer), FALSE, GST_CLOCK_TIME_NONE, FALSE);
} else {
@ -1927,7 +2059,7 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
GST_LOG_OBJECT (videorate,
"BEGINNING prev buf %" GST_TIME_FORMAT " new buf %" GST_TIME_FORMAT
" outgoing ts %" GST_TIME_FORMAT, GST_TIME_ARGS (prev_ts),
GST_TIME_ARGS (in_ts), GST_TIME_ARGS (videorate->next_ts));
GST_TIME_ARGS (in_ts), GST_TIME_ARGS (videorate->next_output_ts));
videorate->in++;
@ -1950,9 +2082,10 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
/* got 2 buffers, see which one is the best */
do {
GstClockTime next_ts;
GstClockTime best_input_ts;
if (gst_video_rate_apply_pending_rate (videorate))
if (gst_video_rate_apply_pending_rate (videorate, buffer))
goto done;
if (videorate->segment.rate < 0.0) {
@ -1970,56 +2103,33 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
}
if (videorate->segment.rate < 0.0) {
GstClockTime next_end_ts;
GstClockTime prev_endtime;
GstClockTime in_endtime;
next_ts = videorate->next_ts;
if (!GST_CLOCK_TIME_IS_VALID (next_ts)) {
GST_DEBUG_OBJECT (videorate, "Already reached segment start,"
"ignoring buffer");
if (!gst_video_rate_compute_best_input_ts (videorate, buffer,
&best_input_ts)) {
GST_DEBUG_OBJECT (videorate,
"Already reached segment start," "ignoring buffer");
break;
}
GstClockTime best_input_end_ts =
best_input_ts +
gst_videorate_rate_compute_input_frame_duration (videorate, buffer);
GstClockTime prev_endtime =
prev_ts + GST_BUFFER_DURATION (videorate->prevbuf);
GstClockTime in_endtime = in_ts + GST_BUFFER_DURATION (buffer);
prev_endtime = prev_ts + GST_BUFFER_DURATION (videorate->prevbuf);
in_endtime = in_ts + GST_BUFFER_DURATION (buffer);
if (videorate->to_rate_numerator) {
GstClockTime frame_duration = gst_util_uint64_scale (1,
videorate->to_rate_denominator * GST_SECOND,
videorate->to_rate_numerator);
next_end_ts = next_ts + frame_duration;
} else {
next_end_ts = next_ts + GST_BUFFER_DURATION (videorate->prevbuf);
}
next_ts = videorate->base_ts - (
(videorate->base_ts - next_ts) * videorate->rate);
if (videorate->base_ts > next_end_ts)
next_end_ts =
videorate->base_ts - ((videorate->base_ts -
next_end_ts) * videorate->rate);
else
next_end_ts = videorate->base_ts;
diff1 = GST_CLOCK_DIFF (next_end_ts, prev_endtime);
diff2 = GST_CLOCK_DIFF (in_endtime, next_end_ts);
diff1 = GST_CLOCK_DIFF (best_input_end_ts, prev_endtime);
diff2 = GST_CLOCK_DIFF (in_endtime, best_input_end_ts);
diff1 *= (1.0 - videorate->new_pref);
diff2 *= videorate->new_pref;
GST_LOG_OBJECT (videorate,
"diff with prev %" GST_STIME_FORMAT " diff with new %"
GST_STIME_FORMAT " outgoing ts %" GST_TIME_FORMAT,
GST_STIME_ARGS (diff1), GST_STIME_ARGS (diff2),
GST_TIME_ARGS (next_end_ts));
} else {
next_ts =
videorate->base_ts + ((videorate->next_ts -
videorate->base_ts) * videorate->rate);
if (!gst_video_rate_compute_best_input_ts (videorate, buffer,
&best_input_ts)) {
g_error
("Computing best input ts should always work on forward playback");
}
diff1 = GST_CLOCK_DIFF (prev_ts, next_ts);
diff2 = GST_CLOCK_DIFF (next_ts, in_ts);
diff1 = GST_CLOCK_DIFF (prev_ts, best_input_ts);
diff2 = GST_CLOCK_DIFF (best_input_ts, in_ts);
diff1 *= videorate->new_pref;
diff2 *= (1.0 - videorate->new_pref);
@ -2027,7 +2137,8 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
"diff with prev %" GST_STIME_FORMAT " diff with new %"
GST_STIME_FORMAT " outgoing ts %" GST_TIME_FORMAT,
GST_STIME_ARGS (diff1), GST_STIME_ARGS (diff2),
GST_TIME_ARGS (next_ts));
GST_TIME_ARGS (best_input_ts));
}
/* output first one when its the best */
@ -2036,8 +2147,8 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
count++;
/* on error the _flush function posted a warning already */
if ((r = gst_video_rate_flush_prev (videorate,
count > 1, in_ts, FALSE)) != GST_FLOW_OK) {
if ((r = gst_video_rate_flush_prev (videorate, count > 1, in_ts,
FALSE)) != GST_FLOW_OK) {
res = r;
goto done;
}
@ -2063,14 +2174,14 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
GST_LOG_OBJECT (videorate,
"new is best, old never used, drop, outgoing ts %"
GST_TIME_FORMAT, GST_TIME_ARGS (videorate->next_ts));
GST_TIME_FORMAT, GST_TIME_ARGS (videorate->next_output_ts));
}
GST_LOG_OBJECT (videorate,
"END, putting new in old, diff1 %" GST_STIME_FORMAT
", diff2 %" GST_STIME_FORMAT ", next_ts %" GST_TIME_FORMAT
", in %" G_GUINT64_FORMAT ", out %" G_GUINT64_FORMAT ", drop %"
G_GUINT64_FORMAT ", dup %" G_GUINT64_FORMAT, GST_STIME_ARGS (diff1),
GST_STIME_ARGS (diff2), GST_TIME_ARGS (videorate->next_ts),
GST_STIME_ARGS (diff2), GST_TIME_ARGS (videorate->next_output_ts),
videorate->in, videorate->out, videorate->drop, videorate->dup);
/* swap in new one when it's the best */
@ -2096,10 +2207,13 @@ invalid_buffer:
}
}
static gboolean
gst_video_rate_start (GstBaseTransform * trans)
{
gst_video_rate_reset (GST_VIDEO_RATE (trans), FALSE);
GstVideoRate *videorate = GST_VIDEO_RATE (trans);
gst_video_rate_reset (videorate, FALSE);
return TRUE;
}

31
subprojects/gst-plugins-base/gst/videorate/gstvideorate.h
View File

@ -41,18 +41,35 @@ struct _GstVideoRate
/* video state */
gint from_rate_numerator, from_rate_denominator;
gint to_rate_numerator, to_rate_denominator;
guint64 next_ts; /* Timestamp of next buffer to output */
guint64 next_end_ts; /* Timestamp of end of next buffer to output */
guint64 next_output_ts; /* Timestamp of next buffer to output */
guint64 next_output_end_ts; /* Timestamp of end of next buffer to output */
GstBuffer *prevbuf;
gboolean prevbuf_pushed;
guint64 prev_ts; /* Previous buffer timestamp */
guint64 prev_end_ts; /* Previous buffer end timestamp */
guint64 out_frame_count; /* number of frames output since the beginning
* of the segment or the last frame rate caps
* change, whichever was later */
guint64 base_ts; /* used in next_ts calculation after a
* frame rate caps change */
/* number of frames output since:
* - the beginning of the segment
* - the last frame rate caps change
* - the last change of the rate
* whichever happenned last */
guint64 out_frame_count;
/* Same logic as out_frame_count but for the input frames received and used or dropped */
guint64 in_frame_count;
/* Timestamp of the buffer for the first output buffer
* as represented by `out_frame_count` */
guint64 base_output_ts;
/* Timestamp of the buffer for the first input buffer
* as represented by `in_frame_count` */
guint64 base_input_ts;
/* Next frame to output is a GST_BUFFER_FLAG_DISCONT */
gboolean discont;
guint64 last_ts; /* Timestamp of last input buffer */
guint64 average_period;

2
subprojects/gst-plugins-base/tests/validate/videorate/change_rate_reverse_playback.validatetest
View File

@ -42,8 +42,6 @@ set-property, playback-time=-1, target-element-name=videorate, property-name=rat
crank-clock, repeat=10
wait, on-clock=true
# Source is now EOS, videorate is filling up the segment with last buffer
checkpoint, text="Filling up segment with last buffer"
crank-clock, repeat=10
stop, on-message=eos

3
subprojects/gst-plugins-base/tests/validate/videorate/change_rate_reverse_playback/flow-expectations/log-videorate-sink-expected
View File

@ -66,7 +66,4 @@ buffer: content-id=49, pts=0:00:00.300000000, dur=0:00:00.100000000, flags=disco
buffer: content-id=50, pts=0:00:00.200000000, dur=0:00:00.100000000, flags=discont
buffer: content-id=51, pts=0:00:00.100000000, dur=0:00:00.100000000, flags=discont
buffer: content-id=52, pts=0:00:00.000000000, dur=0:00:00.100000000, flags=discont
CHECKPOINT: Filling up segment with last buffer
event eos: (no structure)

39
subprojects/gst-plugins-base/tests/validate/videorate/change_rate_reverse_playback/flow-expectations/log-videorate-src-expected
View File

@ -46,25 +46,22 @@ buffer: content-id=16, pts=0:00:01.800000000, dur=0:00:00.100000000, flags=gap
CHECKPOINT: Setting videorate.rate=2.0
buffer: content-id=34, pts=0:00:01.700000000, dur=0:00:00.100000000
buffer: content-id=36, pts=0:00:01.600000000, dur=0:00:00.100000000
buffer: content-id=38, pts=0:00:01.500000000, dur=0:00:00.100000000
buffer: content-id=40, pts=0:00:01.400000000, dur=0:00:00.100000000
buffer: content-id=42, pts=0:00:01.300000000, dur=0:00:00.100000000
buffer: content-id=44, pts=0:00:01.200000000, dur=0:00:00.100000000
buffer: content-id=46, pts=0:00:01.100000000, dur=0:00:00.100000000
buffer: content-id=48, pts=0:00:01.000000000, dur=0:00:00.100000000
buffer: content-id=50, pts=0:00:00.900000000, dur=0:00:00.100000000
buffer: content-id=52, pts=0:00:00.800000000, dur=0:00:00.100000000
CHECKPOINT: Filling up segment with last buffer
buffer: content-id=52, pts=0:00:00.700000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=52, pts=0:00:00.600000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=52, pts=0:00:00.500000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=52, pts=0:00:00.400000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=52, pts=0:00:00.300000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=52, pts=0:00:00.200000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=52, pts=0:00:00.100000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=52, pts=0:00:00.000000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=16, pts=0:00:01.700000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=18, pts=0:00:01.600000000, dur=0:00:00.100000000
buffer: content-id=20, pts=0:00:01.500000000, dur=0:00:00.100000000
buffer: content-id=22, pts=0:00:01.400000000, dur=0:00:00.100000000
buffer: content-id=24, pts=0:00:01.300000000, dur=0:00:00.100000000
buffer: content-id=26, pts=0:00:01.200000000, dur=0:00:00.100000000
buffer: content-id=28, pts=0:00:01.100000000, dur=0:00:00.100000000
buffer: content-id=30, pts=0:00:01.000000000, dur=0:00:00.100000000
buffer: content-id=32, pts=0:00:00.900000000, dur=0:00:00.100000000
buffer: content-id=34, pts=0:00:00.800000000, dur=0:00:00.100000000
buffer: content-id=36, pts=0:00:00.700000000, dur=0:00:00.100000000
buffer: content-id=38, pts=0:00:00.600000000, dur=0:00:00.100000000
buffer: content-id=40, pts=0:00:00.500000000, dur=0:00:00.100000000
buffer: content-id=42, pts=0:00:00.400000000, dur=0:00:00.100000000
buffer: content-id=44, pts=0:00:00.300000000, dur=0:00:00.100000000
buffer: content-id=46, pts=0:00:00.200000000, dur=0:00:00.100000000
buffer: content-id=48, pts=0:00:00.100000000, dur=0:00:00.100000000
buffer: content-id=50, pts=0:00:00.000000000, dur=0:00:00.100000000
event eos: (no structure)

2
subprojects/gst-plugins-base/tests/validate/videorate/change_rate_while_playing.validatetest
View File

@ -3,7 +3,7 @@ meta,
ignore-eos=true,
args = {
# We just want each frame to be different, and we just check their content by 'id'
"videotestsrc ! video/x-raw,format=I420,framerate=10/1,width=320,height=240 ! videorate name=videorate ! fakesink sync=true qos=true",
"videotestsrc ! video/x-raw,format=I420,framerate=10/1,width=320,height=240 ! videorate name=videorate drop-out-of-segment=true ! fakesink sync=true qos=true",
},
configs = {
"$(validateflow), pad=videorate:src, buffers-checksum=as-id, ignored-event-types={ tag }",

4
subprojects/gst-plugins-base/tests/validate/videorate/change_rate_while_playing/flow-expectations/log-videorate-sink-expected
View File

@ -25,11 +25,11 @@ CHECKPOINT: Setting videorate.rate=0.1
buffer: content-id=12, pts=0:00:01.000000000, dur=0:00:00.100000000
buffer: content-id=13, pts=0:00:01.100000000, dur=0:00:00.100000000
buffer: content-id=14, pts=0:00:01.200000000, dur=0:00:00.100000000
buffer: content-id=15, pts=0:00:01.300000000, dur=0:00:00.100000000
buffer: content-id=16, pts=0:00:01.400000000, dur=0:00:00.100000000
CHECKPOINT: Setting videorate.rate=2.0
buffer: content-id=15, pts=0:00:01.300000000, dur=0:00:00.100000000
buffer: content-id=16, pts=0:00:01.400000000, dur=0:00:00.100000000
buffer: content-id=17, pts=0:00:01.500000000, dur=0:00:00.100000000
buffer: content-id=18, pts=0:00:01.600000000, dur=0:00:00.100000000
buffer: content-id=19, pts=0:00:01.700000000, dur=0:00:00.100000000

56
subprojects/gst-plugins-base/tests/validate/videorate/change_rate_while_playing/flow-expectations/log-videorate-src-expected
View File

@ -23,35 +23,35 @@ buffer: content-id=11, pts=0:00:01.100000000, dur=0:00:00.100000000
CHECKPOINT: Setting videorate.rate=0.1
buffer: content-id=14, pts=0:00:01.200000000, dur=0:00:00.100000000
buffer: content-id=14, pts=0:00:01.300000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=14, pts=0:00:01.400000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=14, pts=0:00:01.500000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=14, pts=0:00:01.600000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=14, pts=0:00:01.700000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=15, pts=0:00:01.800000000, dur=0:00:00.100000000
buffer: content-id=15, pts=0:00:01.900000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=15, pts=0:00:02.000000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=15, pts=0:00:02.100000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=15, pts=0:00:02.200000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=15, pts=0:00:02.300000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=15, pts=0:00:02.400000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=15, pts=0:00:02.500000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=15, pts=0:00:02.600000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=15, pts=0:00:02.700000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=16, pts=0:00:02.800000000, dur=0:00:00.100000000
buffer: content-id=16, pts=0:00:02.900000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=16, pts=0:00:03.000000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=16, pts=0:00:03.100000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=11, pts=0:00:01.200000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=12, pts=0:00:01.300000000, dur=0:00:00.100000000
buffer: content-id=12, pts=0:00:01.400000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=12, pts=0:00:01.500000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=12, pts=0:00:01.600000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=12, pts=0:00:01.700000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=12, pts=0:00:01.800000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=12, pts=0:00:01.900000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=12, pts=0:00:02.000000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=12, pts=0:00:02.100000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=12, pts=0:00:02.200000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=13, pts=0:00:02.300000000, dur=0:00:00.100000000
buffer: content-id=13, pts=0:00:02.400000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=13, pts=0:00:02.500000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=13, pts=0:00:02.600000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=13, pts=0:00:02.700000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=13, pts=0:00:02.800000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=13, pts=0:00:02.900000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=13, pts=0:00:03.000000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=13, pts=0:00:03.100000000, dur=0:00:00.100000000, flags=gap
CHECKPOINT: Setting videorate.rate=2.0
buffer: content-id=34, pts=0:00:03.200000000, dur=0:00:00.100000000
buffer: content-id=36, pts=0:00:03.300000000, dur=0:00:00.100000000
buffer: content-id=38, pts=0:00:03.400000000, dur=0:00:00.100000000
buffer: content-id=40, pts=0:00:03.500000000, dur=0:00:00.100000000
buffer: content-id=41, pts=0:00:03.600000000, dur=0:00:00.100000000
buffer: content-id=41, pts=0:00:03.700000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=41, pts=0:00:03.800000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=41, pts=0:00:03.900000000, dur=0:00:00.100000000, flags=gap
buffer: content-id=13, pts=0:00:03.200000000, dur=0:00:00.100000000
buffer: content-id=15, pts=0:00:03.300000000, dur=0:00:00.100000000
buffer: content-id=17, pts=0:00:03.400000000, dur=0:00:00.100000000
buffer: content-id=19, pts=0:00:03.500000000, dur=0:00:00.100000000
buffer: content-id=21, pts=0:00:03.600000000, dur=0:00:00.100000000
buffer: content-id=23, pts=0:00:03.700000000, dur=0:00:00.100000000
buffer: content-id=25, pts=0:00:03.800000000, dur=0:00:00.100000000
buffer: content-id=27, pts=0:00:03.900000000, dur=0:00:00.100000000
event eos: (no structure)

15
subprojects/gst-plugins-base/tests/validate/videorate/reverse.10_to_1fps.validatetest
View File

@ -7,10 +7,23 @@ seek, start=0.0, stop=5.0, flags=accurate+flush, rate=-1.0
# First buffer is display as fast as possible
crank-clock, expected-elapsed-time=0.0
crank-clock, repeat=4, expected-elapsed-time=1.0
check-position, expected-position=5.0
crank-clock, expected-elapsed-time=1.0
check-position, expected-position=4.0
crank-clock, expected-elapsed-time=1.0
check-position, expected-position=3.0
crank-clock, expected-elapsed-time=1.0
check-position, expected-position=2.0
crank-clock, expected-elapsed-time=1.0
check-position, expected-position=1.0
# Waiting 1 second on EOS
crank-clock, expected-elapsed-time=1.0
check-position, expected-position=0.0
seek, start=5.0, stop=10.0, flags=accurate+flush, rate=-1.0, on-message=eos