ges: Set framerate caps filter on the last time effect

The responsibility to change the framerate to the one
requested on the VideoTrack should always be the last time effect in it.

This introduces a new `capsfilter` at the end of video effect that
will be in passthrough most of the time.
t show

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/9013>

subprojects/gst-editing-services/ges/ges-clip.c

@@ -1928,6 +1928,31 @@ ges_clip_has_scale_effect (GESClip * clip)
   return clip->priv->nb_scale_effects > 0;
 }
 
+GstElement *
+ges_clip_get_last_time_effect_filter (GESClip * clip, GESTrack * track)
+{
+  GList *topeffects = ges_clip_get_top_effects (clip);
+  for (GList * tmp = topeffects; tmp; tmp = tmp->next) {
+    GESTrackElement *effect = tmp->data;
+
+    if (GES_IS_TIME_EFFECT (effect)
+        && ges_track_element_get_track (effect) == track) {
+      GstElement *bin = ges_track_element_get_element (effect);
+      GstElement *element =
+          gst_bin_get_by_name (GST_BIN (bin), "___ges__effectcapsfilter");
+
+      if (!element) {
+        GST_ERROR_OBJECT (clip, "Last timeeffect doesn't have a capsfilter ");
+      }
+
+      return element;
+    }
+  }
+
+  g_list_free_full (topeffects, gst_object_unref);
+  return NULL;
+}
+
 static gboolean
 _remove_child (GESContainer * container, GESTimelineElement * element)
 {

subprojects/gst-editing-services/ges/ges-effect-asset.c

@@ -228,7 +228,8 @@ get_pad_from_elements_with_request_pad (GstElement * effect,
 
 static gboolean
 ghost_pad (GstElement * effect, const gchar * bin_desc, GstPad * pad,
-    gint n_pad, const gchar * converter_str, GError ** error)
+    gint n_pad, const gchar * converter_str, gboolean add_caps_filter,
+    GError ** error)
 {
   gchar *name;
   GstPad *peer, *ghosted;
@@ -236,14 +237,26 @@ ghost_pad (GstElement * effect, const gchar * bin_desc, GstPad * pad,
   GstElement *converter;
 
   if (!converter_str) {
+    g_assert (!add_caps_filter);
     ghosted = pad;
     goto ghost;
   }
 
-  converter = gst_parse_bin_from_description_full (converter_str, TRUE, NULL,
+  gchar *converter_str_with_capsfilter = NULL;
+  if (add_caps_filter) {
+    converter_str_with_capsfilter =
+        g_strdup_printf
+        ("bin.( name=___ges__converter_bin %s ! capsfilter name=___ges__effectcapsfilter )",
+        converter_str);
+  }
+  converter =
+      gst_parse_bin_from_description_full (converter_str_with_capsfilter ?
+      converter_str_with_capsfilter : converter_str, TRUE, NULL,
       GST_PARSE_FLAG_NO_SINGLE_ELEMENT_BINS | GST_PARSE_FLAG_PLACE_IN_BIN,
       error);
 
+  g_free (converter_str_with_capsfilter);
+
   if (!converter) {
     GST_ERROR_OBJECT (effect, "Could not create converter '%s'", converter_str);
     return FALSE;
@@ -291,6 +304,7 @@ ges_effect_from_description (const gchar * bin_desc, GESTrackType type,
   gint n_sink = 0;
   GstPad *srcpad = NULL;
   GstCaps *valid_caps = NULL;
+  gboolean add_caps_filter = FALSE;
   const gchar *converter_str = NULL;
   GList *tmp, *sinkpads = NULL, *elems_with_reqsink = NULL,
       *elems_with_reqsrc = NULL;
@@ -307,6 +321,8 @@ ges_effect_from_description (const gchar * bin_desc, GESTrackType type,
   if (type == GES_TRACK_TYPE_VIDEO) {
     valid_caps = gst_caps_from_string ("video/x-raw(ANY)");
     converter_str = "videoconvert";
+
+    add_caps_filter = TRUE;
   } else if (type == GES_TRACK_TYPE_AUDIO) {
     valid_caps = gst_caps_from_string ("audio/x-raw(ANY)");
     converter_str = "audioconvert ! audioresample ! audioconvert";
@@ -336,12 +352,14 @@ ges_effect_from_description (const gchar * bin_desc, GESTrackType type,
   }
 
   for (tmp = sinkpads; tmp; tmp = tmp->next) {
-    if (!ghost_pad (effect, bin_desc, tmp->data, n_sink, converter_str, error))
+    if (!ghost_pad (effect, bin_desc, tmp->data, n_sink, converter_str, FALSE,
+            error))
       goto err;
     n_sink++;
   }
 
-  if (!ghost_pad (effect, bin_desc, srcpad, 0, converter_str, error))
+  if (!ghost_pad (effect, bin_desc, srcpad, 0, converter_str, add_caps_filter,
+          error))
     goto err;
 
 done:

subprojects/gst-editing-services/ges/ges-internal.h

@@ -457,6 +457,7 @@ G_GNUC_INTERNAL gboolean          ges_clip_can_set_track_of_child (GESClip * cli
 G_GNUC_INTERNAL gboolean          ges_clip_can_set_time_property_of_child (GESClip * clip, GESTrackElement * child, GObject * prop_object, GParamSpec * pspec, const GValue * value, GError ** error);
 G_GNUC_INTERNAL GstClockTime      ges_clip_duration_limit_with_new_children_inpoints (GESClip * clip, GHashTable * child_inpoints);
 G_GNUC_INTERNAL GstClockTime      ges_clip_get_core_internal_time_from_timeline_time (GESClip * clip, GstClockTime timeline_time, gboolean * no_core, GError ** error);
+G_GNUC_INTERNAL GstElement*       ges_clip_get_last_time_effect_filter           (GESClip * clip, GESTrack *track);
 G_GNUC_INTERNAL void              ges_clip_empty_from_track       (GESClip * clip, GESTrack * track);
 G_GNUC_INTERNAL void              ges_clip_set_add_error          (GESClip * clip, GError * error);
 G_GNUC_INTERNAL void              ges_clip_take_add_error         (GESClip * clip, GError ** error);

subprojects/gst-editing-services/ges/ges-track-element.c

@@ -1172,8 +1172,11 @@ ges_track_element_add_children_props (GESTrackElement * self,
       case GST_ITERATOR_OK:
       {
         GstElement *child = g_value_get_object (&item);
-        ges_track_element_add_child_props (self, child, wanted_categories,
-            blacklist, whitelist);
+
+        if (!g_str_has_prefix (GST_OBJECT_NAME (child), "___ges__")) {
+          ges_track_element_add_child_props (self, child, wanted_categories,
+              blacklist, whitelist);
+        }
         g_value_reset (&item);
         break;
       }

subprojects/gst-editing-services/ges/gstframepositioner.c

@@ -122,6 +122,25 @@ gst_compositor_operator_get_type_and_default_value (int *default_operator_value)
   return operator_gtype;
 }
 
+static GstElement *
+get_last_time_effect_filter (GstFramePositioner * self)
+{
+  if (!self->track_source) {
+    GST_DEBUG_OBJECT (self, "No track src");
+    return FALSE;
+  }
+
+  GESTimelineElement *parent = GES_TIMELINE_ELEMENT_PARENT (self->track_source);
+
+  if (!parent || !GES_IS_CLIP (parent)) {
+    GST_DEBUG_OBJECT (self, "Not in a clip");
+    return NULL;
+  }
+
+  GESTrack *track = ges_track_element_get_track (self->track_source);
+  return ges_clip_get_last_time_effect_filter (GES_CLIP (parent), track);
+}
+
 static gboolean
 scales_downstream (GstFramePositioner * self)
 {
@@ -324,10 +343,6 @@ gst_frame_positioner_update_properties (GstFramePositioner * pos,
         pos->track_width, "height", G_TYPE_INT, pos->track_height, NULL);
   }
 
-  if (pos->fps_n != -1)
-    gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION, pos->fps_n,
-        pos->fps_d, NULL);
-
   if (pos->par_n != -1)
     gst_caps_set_simple (caps, "pixel-aspect-ratio", GST_TYPE_FRACTION,
         pos->par_n, pos->par_d, NULL);
@@ -373,6 +388,26 @@ done:
         "height", G_TYPE_INT, pos->natural_height, NULL);
   }
 
+  if (pos->fps_n != -1) {
+    GstElement *timeffect_capsfilter = get_last_time_effect_filter (pos);
+
+    if (timeffect_capsfilter) {
+      GstCaps *timeffect_caps =
+          gst_caps_new_simple ("video/x-raw", "framerate", GST_TYPE_FRACTION,
+          pos->fps_n, pos->fps_d, NULL);
+      gst_caps_set_features_simple (timeffect_caps,
+          gst_caps_features_new_any ());
+
+      GST_DEBUG_OBJECT (pos,
+          "Setting framerate on source last timne effect capsfilter");
+      g_object_set (timeffect_capsfilter, "caps", timeffect_caps, NULL);
+    } else {
+      GST_DEBUG_OBJECT (pos, "Setting framerate on source capsfilter");
+      gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION, pos->fps_n,
+          pos->fps_d, NULL);
+    }
+  }
+
   GST_DEBUG_OBJECT (pos, "setting caps %" GST_PTR_FORMAT, caps);
 
   g_object_set (pos->capsfilter, "caps", caps, NULL);
@@ -508,6 +543,37 @@ gst_frame_positioner_dispose (GObject * object)
   G_OBJECT_CLASS (gst_frame_positioner_parent_class)->dispose (object);
 }
 
+static GstStateChangeReturn
+gst_frame_positioner_change_state (GstElement * element,
+    GstStateChange transition)
+{
+  GstFramePositioner *framepositioner = GST_FRAME_POSITIONNER (element);
+  if (transition == GST_STATE_CHANGE_READY_TO_PAUSED) {
+    gboolean track_mixing = TRUE;
+
+    if (framepositioner->current_track)
+      track_mixing = ges_track_get_mixing (framepositioner->current_track);
+    gst_frame_positioner_update_properties (framepositioner, track_mixing, 0,
+        0);
+  }
+
+  GstStateChangeReturn res =
+      GST_ELEMENT_CLASS (gst_frame_positioner_parent_class)->change_state
+      (element, transition);
+
+  if (transition == GST_STATE_CHANGE_PAUSED_TO_READY) {
+    GstElement *timeffect_capsfilter =
+        get_last_time_effect_filter (framepositioner);
+
+    if (timeffect_capsfilter) {
+      g_object_set (timeffect_capsfilter, "caps", NULL, NULL);
+    }
+  }
+
+  return res;
+}
+
+
 static void
 gst_frame_positioner_class_init (GstFramePositionerClass * klass)
 {
@@ -518,6 +584,7 @@ gst_frame_positioner_class_init (GstFramePositionerClass * klass)
   guint n_pspecs = PROP_LAST;
 
   GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+  GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
   GstBaseTransformClass *base_transform_class =
       GST_BASE_TRANSFORM_CLASS (klass);
 
@@ -532,6 +599,7 @@ gst_frame_positioner_class_init (GstFramePositionerClass * klass)
   gobject_class->set_property = gst_frame_positioner_set_property;
   gobject_class->get_property = gst_frame_positioner_get_property;
   gobject_class->dispose = gst_frame_positioner_dispose;
+  element_class->change_state = gst_frame_positioner_change_state;
   base_transform_class->transform_ip =
       GST_DEBUG_FUNCPTR (gst_frame_positioner_transform_ip);
 

subprojects/gst-integration-testsuites/ges/scenarios/check_last_time_effect_framerate_filter.validatetest

@@ -0,0 +1,19 @@
+meta,
+    tool = "ges-launch-$(gst_api_version)",
+    handles-states=true,
+    overrides={
+        "change-severity, issue-id=\"event::segment-has-wrong-start\", new-severity=\"warning\"",
+    },
+    args={
+        "-t", "video",
+        "+clip", "testbin://video,caps=\"video/x-raw,framerate=60/1,format=I420\"", "d=1.0",
+        "+effect", "videorate rate=0.1 name=first",
+        "+effect", "videorate rate=2.0 name=last",
+        "--videosink=fakesink name=videosink",
+    },
+    configs={
+        "$(validateflow), pad=last:src, caps-properties={ framerate }, logged-event-types = { caps }",
+        "$(validateflow), pad=first:src, caps-properties={ framerate }, logged-event-types = { caps }",
+    }
+
+play

subprojects/gst-integration-testsuites/ges/scenarios/check_last_time_effect_framerate_filter/flow-expectations/log-first-src-expected

@@ -0,0 +1 @@
+event caps: video/x-raw, framerate=(fraction)60/1;

subprojects/gst-integration-testsuites/ges/scenarios/check_last_time_effect_framerate_filter/flow-expectations/log-last-src-expected

@@ -0,0 +1 @@
+event caps: video/x-raw, framerate=(fraction)30/1;