From bad6b7e60a180db2c0c2d556135d38e263f19fab Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 20 Jun 2023 16:01:37 -0400 Subject: [PATCH] ges: Implement a `gesvideoscale` bin This allows user to be in control of when scaling happens in the pipeline. It can be plugged as an effect and scaling will be forced to happen in the effect instead of in the compositor. Without this, it would not be possible for users to, for example, crop a video source before scaling to the target source size. Part-of: --- girs/GES-1.0.gir | 21 ++- .../gst-editing-services/ges/ges-clip.c | 34 ++++ .../gst-editing-services/ges/ges-effect.c | 21 ++- .../gst-editing-services/ges/ges-internal.h | 1 + .../ges/ges-smart-video-mixer.c | 9 +- subprojects/gst-editing-services/ges/ges.c | 7 +- .../gst-editing-services/ges/gesvideoscale.c | 174 ++++++++++++++++++ .../ges/gstframepositioner.c | 27 ++- .../gst-editing-services/ges/meson.build | 1 + .../scenarios/videoscale_effect.validatetest | 37 ++++ .../testsuites/ges.testslist | 1 + 11 files changed, 320 insertions(+), 13 deletions(-) create mode 100644 subprojects/gst-editing-services/ges/gesvideoscale.c create mode 100644 subprojects/gst-integration-testsuites/ges/scenarios/videoscale_effect.validatetest diff --git a/girs/GES-1.0.gir b/girs/GES-1.0.gir index b5151e374b..aac093e295 100644 --- a/girs/GES-1.0.gir +++ b/girs/GES-1.0.gir @@ -3536,10 +3536,23 @@ and [clip management](http://pitivi.org/manual/usingclips.html). - Currently we only support effects with N sinkpads and one single srcpad. -Apart from `gesaudiomixer` and `gescompositor` which can be used as effects -and where sinkpads will be requested as needed based on the timeline topology -GES will always request at most one sinkpad per effect (when required). + Any GStreamer filter can be used as effects in GES. The only restriction we +have is that effects element should have a single [sinkpad](GST_PAD_SINK) +(which will be requested if necessary) and a single [srcpad](GST_PAD_SRC). + +Note that `gesaudiomixer` and `gescompositor` can be used as effects even +though they can have several sinkpads. + +## GES specific effects: + +* **`gesvideoscale`**: GES implements a specific scaling bin that allows + specifying where scaling will happen inside the chain of effects. By + default scaling can happen either in the source (if the source doesn't have + a specific size, like `videotestsrc` or [mixing](ges_track_set_mixing) has + been disabled) or in the mixing element otherwise, when adding that element + as an effect, GES guarantees that the scaling will happen in it. This can + be useful for example if you want to crop the video before scaling or apply + rounding corners to the video after scaling, etc... > Note: GES always adds converters (`audioconvert ! audioresample ! > audioconvert` for audio effects and `videoconvert` for video effects) to diff --git a/subprojects/gst-editing-services/ges/ges-clip.c b/subprojects/gst-editing-services/ges/ges-clip.c index 777b9f9555..f7b0332f1b 100644 --- a/subprojects/gst-editing-services/ges/ges-clip.c +++ b/subprojects/gst-editing-services/ges/ges-clip.c @@ -250,6 +250,7 @@ struct _GESClipPrivate gboolean allow_any_remove; + gint nb_scale_effects; gboolean use_effect_priority; guint32 effect_priority; GError *add_error; @@ -1638,6 +1639,7 @@ _add_child (GESContainer * container, GESTimelineElement * element) GESTimeline *timeline = GES_TIMELINE_ELEMENT_TIMELINE (container); GESClipPrivate *priv = self->priv; GESAsset *asset, *creator_asset; + gboolean adding_scale_effect = FALSE; gboolean prev_prevent = priv->prevent_duration_limit_update; gboolean prev_prevent_outpoint = priv->prevent_children_outpoint_update; GList *tmp; @@ -1775,6 +1777,14 @@ _add_child (GESContainer * container, GESTimelineElement * element) new_prio = MAX (new_prio, _PRIORITY (tmp->data) + 1); } } + + if (GES_IS_EFFECT (element)) { + GESAsset *asset = ges_extractable_get_asset (GES_EXTRACTABLE (element)); + const gchar *bindesc = ges_asset_get_id (asset); + + adding_scale_effect = !strstr (bindesc, "gesvideoscale"); + } + /* make sure higher than core */ for (tmp = container->children; tmp; tmp = tmp->next) { if (_IS_CORE_CHILD (tmp->data)) @@ -1817,9 +1827,16 @@ _add_child (GESContainer * container, GESTimelineElement * element) _update_active_for_track (self, track_el); priv->nb_effects++; + GST_DEBUG_OBJECT (self, "Adding %ith effect: %" GES_FORMAT " Priority %i", priv->nb_effects, GES_ARGS (element), new_prio); + if (adding_scale_effect) { + GST_DEBUG_OBJECT (self, "Adding scaling effect to clip " + "%" GES_FORMAT, GES_ARGS (self)); + priv->nb_scale_effects += 1; + } + /* changing priorities, and updating their offset */ priv->prevent_resort = TRUE; priv->setting_priority = TRUE; @@ -1900,6 +1917,12 @@ ges_clip_set_remove_error (GESClip * clip, GError * error) priv->remove_error = error; } +gboolean +ges_clip_has_scale_effect (GESClip * clip) +{ + return clip->priv->nb_scale_effects > 0; +} + static gboolean _remove_child (GESContainer * container, GESTimelineElement * element) { @@ -1961,6 +1984,17 @@ _remove_child (GESContainer * container, GESTimelineElement * element) * relative priorities */ /* height may have changed */ _compute_height (container); + + if (GES_IS_EFFECT (element)) { + GESAsset *asset = ges_extractable_get_asset (GES_EXTRACTABLE (element)); + const gchar *bindesc = ges_asset_get_id (asset); + + if (bindesc && !strstr (bindesc, "gesvideoscale")) { + GST_DEBUG_OBJECT (self, "Removing scaling effect to clip " + "%" GES_FORMAT, GES_ARGS (self)); + priv->nb_scale_effects -= 1; + } + } } /* duration-limit updated in _child_removed */ return TRUE; diff --git a/subprojects/gst-editing-services/ges/ges-effect.c b/subprojects/gst-editing-services/ges/ges-effect.c index fb6bf782ee..b26c1c91ca 100644 --- a/subprojects/gst-editing-services/ges/ges-effect.c +++ b/subprojects/gst-editing-services/ges/ges-effect.c @@ -23,10 +23,23 @@ * @short_description: adds an effect build from a parse-launch style bin * description to a stream in a GESSourceClip or a GESLayer * - * Currently we only support effects with N sinkpads and one single srcpad. - * Apart from `gesaudiomixer` and `gescompositor` which can be used as effects - * and where sinkpads will be requested as needed based on the timeline topology - * GES will always request at most one sinkpad per effect (when required). + * Any GStreamer filter can be used as effects in GES. The only restriction we + * have is that effects element should have a single [sinkpad](GST_PAD_SINK) + * (which will be requested if necessary) and a single [srcpad](GST_PAD_SRC). + * + * Note that `gesaudiomixer` and `gescompositor` can be used as effects even + * though they can have several sinkpads. + * + * ## GES specific effects: + * + * * **`gesvideoscale`**: GES implements a specific scaling bin that allows + * specifying where scaling will happen inside the chain of effects. By + * default scaling can happen either in the source (if the source doesn't have + * a specific size, like `videotestsrc` or [mixing](ges_track_set_mixing) has + * been disabled) or in the mixing element otherwise, when adding that element + * as an effect, GES guarantees that the scaling will happen in it. This can + * be useful for example if you want to crop the video before scaling or apply + * rounding corners to the video after scaling, etc... * * > Note: GES always adds converters (`audioconvert ! audioresample ! * > audioconvert` for audio effects and `videoconvert` for video effects) to diff --git a/subprojects/gst-editing-services/ges/ges-internal.h b/subprojects/gst-editing-services/ges/ges-internal.h index 785b2f8ddb..2cb5eb0c50 100644 --- a/subprojects/gst-editing-services/ges/ges-internal.h +++ b/subprojects/gst-editing-services/ges/ges-internal.h @@ -462,6 +462,7 @@ G_GNUC_INTERNAL void ges_clip_set_add_error (GESClip * cli G_GNUC_INTERNAL void ges_clip_take_add_error (GESClip * clip, GError ** error); G_GNUC_INTERNAL void ges_clip_set_remove_error (GESClip * clip, GError * error); G_GNUC_INTERNAL void ges_clip_take_remove_error (GESClip * clip, GError ** error); +G_GNUC_INTERNAL gboolean ges_clip_has_scale_effect (GESClip * clip); /**************************************************** * GESLayer * diff --git a/subprojects/gst-editing-services/ges/ges-smart-video-mixer.c b/subprojects/gst-editing-services/ges/ges-smart-video-mixer.c index f71bb24c88..3a8b9d608c 100644 --- a/subprojects/gst-editing-services/ges/ges-smart-video-mixer.c +++ b/subprojects/gst-editing-services/ges/ges-smart-video-mixer.c @@ -240,8 +240,13 @@ set_pad_properties_from_composition_meta (GstPad * mixer_pad, g_object_set (mixer_pad, "alpha", meta->alpha * transalpha, NULL); } - g_object_set (mixer_pad, "xpos", meta->posx, "ypos", - meta->posy, "width", meta->width, "height", meta->height, NULL); + g_object_set (mixer_pad, "xpos", meta->posx, "ypos", meta->posy, NULL); + + if (meta->width >= 0) + g_object_set (mixer_pad, "width", meta->width, NULL); + + if (meta->height >= 0) + g_object_set (mixer_pad, "height", meta->height, NULL); if (self->ABI.abi.has_operator) g_object_set (mixer_pad, "operator", meta->operator, NULL); diff --git a/subprojects/gst-editing-services/ges/ges.c b/subprojects/gst-editing-services/ges/ges.c index d34c4efd79..a2d8c0b5e6 100644 --- a/subprojects/gst-editing-services/ges/ges.c +++ b/subprojects/gst-editing-services/ges/ges.c @@ -54,6 +54,7 @@ G_LOCK_DEFINE_STATIC (init_lock); * between init/deinit */ static GThread *initialized_thread = NULL; +extern GType ges_video_scale_get_type (void); #ifndef GST_DISABLE_GST_DEBUG static gpointer @@ -96,9 +97,10 @@ ges_init_pre (GOptionContext * context, GOptionGroup * group, gpointer data, return TRUE; } + static gboolean -ges_init_post (GOptionContext * context, GOptionGroup * group, gpointer data, - GError ** error) +ges_init_post (GOptionContext * context, GOptionGroup * group, + gpointer data, GError ** error) { GESUriClipAssetClass *uriasset_klass = NULL; GstElementFactory *nlecomposition_factory = NULL; @@ -154,6 +156,7 @@ ges_init_post (GOptionContext * context, GOptionGroup * group, gpointer data, ges_asset_cache_init (); + gst_element_register (NULL, "gesvideoscale", 0, ges_video_scale_get_type ()); gst_element_register (NULL, "gesaudiomixer", 0, GES_TYPE_SMART_ADDER); gst_element_register (NULL, "gescompositor", 0, GES_TYPE_SMART_MIXER); gst_element_register (NULL, "framepositioner", 0, GST_TYPE_FRAME_POSITIONNER); diff --git a/subprojects/gst-editing-services/ges/gesvideoscale.c b/subprojects/gst-editing-services/ges/gesvideoscale.c new file mode 100644 index 0000000000..3ae0810c30 --- /dev/null +++ b/subprojects/gst-editing-services/ges/gesvideoscale.c @@ -0,0 +1,174 @@ +/* GStreamer + * Copyright (C) 2023 Thibault Saunier + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Suite 500, + * Boston, MA 02110-1335, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "ges-frame-composition-meta.h" + +typedef struct _GESVideoScale GESVideoScale; +typedef struct +{ + GstBinClass parent_class; +} GESVideoScaleClass; + +struct _GESVideoScale +{ + GstBin parent; + + GstPad *sink; + GstElement *capsfilter; + + gint width, height; +}; + +/* *INDENT-OFF* */ +static GstStaticPadTemplate gst_video_scale_sink_template = + GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("ANY") + ); + +static GstStaticPadTemplate gst_video_scale_src_template = + GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("ANY") + ); + +GES_DECLARE_TYPE (VideoScale, video_scale, VIDEO_SCALE) +G_DEFINE_TYPE (GESVideoScale, ges_video_scale, GST_TYPE_BIN); +/* *INDENT-ON* */ + +static void +set_dimension (GESVideoScale * self, gint width, gint height) +{ + GstCaps *caps = gst_caps_new_simple ("video/x-raw", + "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, + NULL); + + if (width >= 0) + gst_caps_set_simple (caps, "width", G_TYPE_INT, width, NULL); + if (height >= 0) + gst_caps_set_simple (caps, "height", G_TYPE_INT, height, NULL); + + gst_caps_set_features (caps, 0, gst_caps_features_new_any ()); + g_object_set (self->capsfilter, "caps", caps, NULL); + gst_caps_unref (caps); + + GST_OBJECT_LOCK (self); + self->width = width; + self->height = height; + GST_OBJECT_UNLOCK (self); +} + +static GstFlowReturn +chain (GstPad * pad, GESVideoScale * self, GstBuffer * buffer) +{ + GESFrameCompositionMeta *meta; + + meta = + (GESFrameCompositionMeta *) gst_buffer_get_meta (buffer, + ges_frame_composition_meta_api_get_type ()); + + if (meta) { + GST_OBJECT_LOCK (self); + if (meta->height != self->height || meta->width != self->width) { + GST_OBJECT_UNLOCK (self); + + set_dimension (self, meta->width, meta->height); + } else { + GST_OBJECT_UNLOCK (self); + } + + meta->height = meta->width = -1; + } + + return gst_proxy_pad_chain_default (pad, GST_OBJECT (self), buffer); +} + +static GstStateChangeReturn +change_state (GstElement * element, GstStateChange transition) +{ + GESVideoScale *self = GES_VIDEO_SCALE (element); + GstStateChangeReturn res = + ((GstElementClass *) ges_video_scale_parent_class)->change_state (element, + transition); + + if (transition == GST_STATE_CHANGE_PAUSED_TO_READY) { + GST_OBJECT_LOCK (self); + self->width = 0; + self->height = 0; + GST_OBJECT_UNLOCK (self); + } + + return res; +} + +static void +ges_video_scale_init (GESVideoScale * self) +{ + GstPad *pad; + GstElement *scale; + GstPadTemplate *template = + gst_static_pad_template_get (&gst_video_scale_sink_template); + + scale = gst_element_factory_make ("videoscale", NULL); + g_object_set (scale, "add-borders", FALSE, NULL); + self->capsfilter = gst_element_factory_make ("capsfilter", NULL); + + gst_bin_add_many (GST_BIN (self), scale, self->capsfilter, NULL); + gst_element_link (scale, self->capsfilter); + + self->sink = + gst_ghost_pad_new_from_template ("sink", scale->sinkpads->data, template); + gst_pad_set_chain_function (self->sink, (GstPadChainFunction) chain); + gst_element_add_pad (GST_ELEMENT (self), self->sink); + gst_object_unref (template); + + template = gst_static_pad_template_get (&gst_video_scale_src_template); + pad = + gst_ghost_pad_new_from_template ("src", self->capsfilter->srcpads->data, + template); + gst_element_add_pad (GST_ELEMENT (self), pad); + gst_object_unref (template); +} + +static void +ges_video_scale_class_init (GESVideoScaleClass * klass) +{ + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + + gst_element_class_set_static_metadata (element_class, + "VideoScale", + "Video/Filter", + "Scaling element usable as a GES effect", + "Thibault Saunier "); + + gst_element_class_add_static_pad_template (element_class, + &gst_video_scale_sink_template); + gst_element_class_add_static_pad_template (element_class, + &gst_video_scale_src_template); + + element_class->change_state = change_state; +} diff --git a/subprojects/gst-editing-services/ges/gstframepositioner.c b/subprojects/gst-editing-services/ges/gstframepositioner.c index 34783baf1f..57c7357358 100644 --- a/subprojects/gst-editing-services/ges/gstframepositioner.c +++ b/subprojects/gst-editing-services/ges/gstframepositioner.c @@ -113,6 +113,24 @@ gst_compositor_operator_get_type_and_default_value (int *default_operator_value) return operator_gtype; } +static gboolean +scales_downstream (GstFramePositioner * self) +{ + if (self->scale_in_compositor) + return TRUE; + + if (!self->track_source) + return self->scale_in_compositor; + + GESTimelineElement *parent = GES_TIMELINE_ELEMENT_PARENT (self->track_source); + + if (!parent || !GES_IS_CLIP (parent)) { + return self->scale_in_compositor; + } + + return ges_clip_has_scale_effect (GES_CLIP (parent)); +} + static void _weak_notify_cb (GstFramePositioner * pos, GObject * old) { @@ -274,7 +292,7 @@ gst_frame_positioner_update_properties (GstFramePositioner * pos, caps = gst_caps_from_string ("video/x-raw(ANY)"); if (pos->track_width && pos->track_height && - (!track_mixing || !pos->scale_in_compositor)) { + (!track_mixing || !scales_downstream (pos))) { gst_caps_set_simple (caps, "width", G_TYPE_INT, pos->track_width, "height", G_TYPE_INT, pos->track_height, NULL); } @@ -321,6 +339,13 @@ gst_frame_positioner_update_properties (GstFramePositioner * pos, reposition_properties (pos, old_track_width, old_track_height); done: + if (scales_downstream (pos) && pos->natural_width && pos->natural_height) { + GST_DEBUG_OBJECT (pos, + "Forcing natural width in source make downstream scaling work"); + gst_caps_set_simple (caps, "width", G_TYPE_INT, pos->natural_width, + "height", G_TYPE_INT, pos->natural_height, NULL); + } + GST_DEBUG_OBJECT (pos, "setting caps %" GST_PTR_FORMAT, caps); g_object_set (pos->capsfilter, "caps", caps, NULL); diff --git a/subprojects/gst-editing-services/ges/meson.build b/subprojects/gst-editing-services/ges/meson.build index 147517947a..7fd16e6764 100644 --- a/subprojects/gst-editing-services/ges/meson.build +++ b/subprojects/gst-editing-services/ges/meson.build @@ -67,6 +67,7 @@ ges_sources = files([ 'ges-structure-parser.c', 'ges-marker-list.c', 'ges-discoverer-manager.c', + 'gesvideoscale.c', 'gstframepositioner.c' ]) diff --git a/subprojects/gst-integration-testsuites/ges/scenarios/videoscale_effect.validatetest b/subprojects/gst-integration-testsuites/ges/scenarios/videoscale_effect.validatetest new file mode 100644 index 0000000000..e745647d00 --- /dev/null +++ b/subprojects/gst-integration-testsuites/ges/scenarios/videoscale_effect.validatetest @@ -0,0 +1,37 @@ +set-globals, media_file="$(test_dir)/../../medias/defaults/matroska/timed_frames_video_only_1fps.mkv" +meta, + tool = "ges-launch-$(gst_api_version)", + handles-states=true, + seek=true, + needs_preroll=true, + args = { + --track-types, video, + --video-caps, "video/x-raw, format=(string)I420, width=(int)1080, height=(int)720, framerate=(fraction)1/1", + --videosink, "$(videosink) name=videosink", + } + +# Add a clip and check that the first frame is displayed +add-clip, name=clip, asset-id="file://$(media_file)", layer-priority=0, type=GESUriClip, name=(string)theclip +set-child-properties, width=100, height=100, posx=10, posy=10, element-name=theclip +container-add-child, container-name=theclip, asset-id="videocrop name=videocrop", child-type=(string)GESEffect, child-name=crop; +set-child-properties, bottom=200, element-name=crop +container-add-child, container-name=theclip, asset-id="gesvideoscale name=videoscale", child-type=(string)GESEffect; + +pause + +# Checking that the 'framepositioner' is forcing caps to the clip "natual size" +check-properties, gesvideourisource0-capsfilter::caps=(GstCaps)"video/x-raw(ANY),framerate=(fraction)1/1,width=(int)1080,height=(int)720" +check-properties, videoscale::parent::parent::priority=3 +check-properties, videocrop::parent::parent::priority=4 +check-current-pad-caps, target-element-name=videocrop, pad=sink, expected-caps=[video/x-raw,width=1080,height=720] +check-current-pad-caps, target-element-name=videocrop, pad=src, expected-caps=[video/x-raw,width=1080,height=520] +check-current-pad-caps, target-element-name=videoscale, pad=sink, expected-caps=[video/x-raw,width=1080,height=520] +check-current-pad-caps, target-element-name=videoscale, pad=src, expected-caps=[video/x-raw,width=100,height=100] + +check-properties, + gessmartmixer0-compositor.sink_0::xpos=10, + gessmartmixer0-compositor.sink_0::ypos=10, + gessmartmixer0-compositor.sink_0::width=-1, + gessmartmixer0-compositor.sink_0::height=-1 + +stop diff --git a/subprojects/gst-integration-testsuites/testsuites/ges.testslist b/subprojects/gst-integration-testsuites/testsuites/ges.testslist index 2850961ed4..c064ac5e3c 100644 --- a/subprojects/gst-integration-testsuites/testsuites/ges.testslist +++ b/subprojects/gst-integration-testsuites/testsuites/ges.testslist @@ -760,3 +760,4 @@ ges.test.edit_deeply_nested_timeline_too_short ges.test.play_deeply_nested_back_to_back ges.test.play_two_nested_back_to_back ges.test.seek_on_stack_change_on_internal_subtimeline +ges.test.videoscale_effect