From 530ae2cbb03c117425cdbff3721279903c4a66bd Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 3 Feb 2015 00:11:06 +1100 Subject: [PATCH] glupload/download/convert: provide transform_caps functions Allows finer grain decisions about formats and features at each stage of the pipeline. Also provide propose_allocation for glupload besed on the supported methods. --- ext/gl/gstglimagesink.c | 120 ++++++++---- ext/gl/gstglmixer.c | 202 ++++++++++---------- ext/gl/gstgltestsrc.c | 87 +++++++++ gst-libs/gst/gl/gstglcolorconvert.c | 23 +++ gst-libs/gst/gl/gstglcolorconvert.h | 4 + gst-libs/gst/gl/gstgldownload.c | 71 +++++++ gst-libs/gst/gl/gstgldownload.h | 4 + gst-libs/gst/gl/gstglfilter.c | 279 ++++++++++++---------------- gst-libs/gst/gl/gstglupload.c | 224 ++++++++++++++++++++++ gst-libs/gst/gl/gstglupload.h | 23 ++- gst-libs/gst/gl/gstglutils.c | 25 +++ gst-libs/gst/gl/gstglutils.h | 2 + 12 files changed, 747 insertions(+), 317 deletions(-) diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c index 7373fcd2f3..36c3db427b 100644 --- a/ext/gl/gstglimagesink.c +++ b/ext/gl/gstglimagesink.c @@ -139,6 +139,8 @@ gst_glimage_sink_change_state (GstElement * element, GstStateChange transition); static void gst_glimage_sink_get_times (GstBaseSink * bsink, GstBuffer * buf, GstClockTime * start, GstClockTime * end); static gboolean gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps); +static GstCaps *gst_glimage_sink_get_caps (GstBaseSink * bsink, + GstCaps * filter); static GstFlowReturn gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf); static GstFlowReturn gst_glimage_sink_show_frame (GstVideoSink * bsink, @@ -353,6 +355,7 @@ gst_glimage_sink_class_init (GstGLImageSinkClass * klass) gstelement_class->set_context = gst_glimage_sink_set_context; gstbasesink_class->query = GST_DEBUG_FUNCPTR (gst_glimage_sink_query); gstbasesink_class->set_caps = gst_glimage_sink_set_caps; + gstbasesink_class->get_caps = gst_glimage_sink_get_caps; gstbasesink_class->get_times = gst_glimage_sink_get_times; gstbasesink_class->prepare = gst_glimage_sink_prepare; gstbasesink_class->propose_allocation = gst_glimage_sink_propose_allocation; @@ -764,6 +767,80 @@ gst_glimage_sink_get_times (GstBaseSink * bsink, GstBuffer * buf, } } +/* copies the given caps */ +static GstCaps * +gst_glimage_sink_caps_remove_format_info (GstCaps * caps) +{ + GstStructure *st; + GstCapsFeatures *f; + gint i, n; + GstCaps *res; + + res = gst_caps_new_empty (); + + n = gst_caps_get_size (caps); + for (i = 0; i < n; i++) { + st = gst_caps_get_structure (caps, i); + f = gst_caps_get_features (caps, i); + + /* If this is already expressed by the existing caps + * skip this structure */ + if (i > 0 && gst_caps_is_subset_structure_full (res, st, f)) + continue; + + st = gst_structure_copy (st); + /* Only remove format info for the cases when we can actually convert */ + if (!gst_caps_features_is_any (f) + && gst_caps_features_is_equal (f, + GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) + gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", + NULL); + gst_structure_remove_fields (st, "width", "height", NULL); + + gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); + } + + return res; +} + +static GstCaps * +gst_glimage_sink_get_caps (GstBaseSink * bsink, GstCaps * filter) +{ + GstGLImageSink *gl_sink = GST_GLIMAGE_SINK (bsink); + GstCaps *tmp = NULL; + GstCaps *result = NULL; + + tmp = gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA"); + + result = gst_glimage_sink_caps_remove_format_info (tmp); + gst_caps_unref (tmp); + tmp = result; + GST_DEBUG_OBJECT (bsink, "remove format returned caps %" GST_PTR_FORMAT, tmp); + + result = + gst_gl_color_convert_transform_caps (gl_sink->context, GST_PAD_SRC, tmp, + NULL); + gst_caps_unref (tmp); + tmp = result; + + result = + gst_gl_upload_transform_caps (gl_sink->context, GST_PAD_SRC, tmp, NULL); + gst_caps_unref (tmp); + tmp = result; + GST_DEBUG_OBJECT (bsink, "transfer returned caps %" GST_PTR_FORMAT, tmp); + + if (filter) { + result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + } else { + result = tmp; + } + + GST_DEBUG_OBJECT (bsink, "returning caps: %" GST_PTR_FORMAT, result); + + return result; +} + static gboolean gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) { @@ -1083,11 +1160,6 @@ gst_glimage_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) GstCaps *caps; guint size; gboolean need_pool; - GstStructure *gl_context; - gchar *platform, *gl_apis; - gpointer handle; - GstAllocator *allocator = NULL; - GstAllocationParams params; if (!_ensure_gl_setup (glimage_sink)) return FALSE; @@ -1140,45 +1212,11 @@ gst_glimage_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) gst_object_unref (pool); } - /* we also support various metadata */ - gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); + gst_gl_upload_propose_allocation (glimage_sink->upload, NULL, query); + if (glimage_sink->context->gl_vtable->FenceSync) gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); - gl_apis = - gst_gl_api_to_string (gst_gl_context_get_gl_api (glimage_sink->context)); - platform = - gst_gl_platform_to_string (gst_gl_context_get_gl_platform - (glimage_sink->context)); - handle = (gpointer) gst_gl_context_get_gl_context (glimage_sink->context); - - gl_context = - gst_structure_new ("GstVideoGLTextureUploadMeta", "gst.gl.GstGLContext", - GST_GL_TYPE_CONTEXT, glimage_sink->context, "gst.gl.context.handle", - G_TYPE_POINTER, handle, "gst.gl.context.type", G_TYPE_STRING, platform, - "gst.gl.context.apis", G_TYPE_STRING, gl_apis, NULL); - gst_query_add_allocation_meta (query, - GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, gl_context); - - g_free (gl_apis); - g_free (platform); - gst_structure_free (gl_context); - - gst_allocation_params_init (¶ms); - - allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR); - gst_query_add_allocation_param (query, allocator, ¶ms); - gst_object_unref (allocator); - -#if GST_GL_HAVE_PLATFORM_EGL - if (gst_gl_context_check_feature (glimage_sink->context, - "EGL_KHR_image_base")) { - allocator = gst_allocator_find (GST_EGL_IMAGE_MEMORY_TYPE); - gst_query_add_allocation_param (query, allocator, ¶ms); - gst_object_unref (allocator); - } -#endif - return TRUE; /* ERRORS */ diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index a122f8f278..57135a662e 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -117,6 +117,46 @@ gst_gl_mixer_pad_finalize (GObject * object) G_OBJECT_CLASS (gst_gl_mixer_pad_parent_class)->finalize (object); } +static void +_init_upload (GstGLMixer * mix, GstGLMixerPad * pad) +{ + GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad); + + if (!pad->upload) { + GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad)); + GstCaps *upload_caps = gst_caps_copy (in_caps); + GstCapsFeatures *gl_features = + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + + pad->upload = gst_gl_upload_new (mix->context); + + gst_caps_set_features (upload_caps, 0, + gst_caps_features_copy (gl_features)); + gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps); + gst_caps_unref (in_caps); + + if (!pad->convert) { + GstVideoInfo gl_info; + GstCaps *gl_caps; + + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vaggpad->info), + GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); + gl_caps = gst_video_info_to_caps (&gl_info); + gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features)); + + pad->convert = gst_gl_color_convert_new (mix->context); + + gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps); + gst_caps_unref (gl_caps); + } + + gst_caps_unref (upload_caps); + gst_caps_features_free (gl_features); + } +} + static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) @@ -153,7 +193,7 @@ _negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) } static gboolean -gst_gl_mixer_propose_allocation (GstGLMixer * mix, +gst_gl_mixer_propose_allocation (GstGLMixer * mix, GstGLMixerPad * pad, GstQuery * decide_query, GstQuery * query) { GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); @@ -163,11 +203,6 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, guint size = 0; gboolean need_pool; GError *error = NULL; - GstStructure *gl_context; - gchar *platform, *gl_apis; - gpointer handle; - GstAllocator *allocator = NULL; - GstAllocationParams params; gst_query_parse_allocation (query, &caps, &need_pool); @@ -229,32 +264,12 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, } /* we also support various metadata */ - gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); if (mix->context->gl_vtable->FenceSync) gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); - gl_apis = gst_gl_api_to_string (gst_gl_context_get_gl_api (mix->context)); - platform = - gst_gl_platform_to_string (gst_gl_context_get_gl_platform (mix->context)); - handle = (gpointer) gst_gl_context_get_gl_context (mix->context); + _init_upload (mix, pad); - gl_context = - gst_structure_new ("GstVideoGLTextureUploadMeta", "gst.gl.GstGLContext", - GST_GL_TYPE_CONTEXT, mix->context, "gst.gl.context.handle", - G_TYPE_POINTER, handle, "gst.gl.context.type", G_TYPE_STRING, platform, - "gst.gl.context.apis", G_TYPE_STRING, gl_apis, NULL); - gst_query_add_allocation_meta (query, - GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, gl_context); - - g_free (gl_apis); - g_free (platform); - gst_structure_free (gl_context); - - gst_allocation_params_init (¶ms); - - allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR); - gst_query_add_allocation_param (query, allocator, ¶ms); - gst_object_unref (allocator); + gst_gl_upload_propose_allocation (pad->upload, decide_query, query); return TRUE; @@ -306,26 +321,9 @@ static GstCaps * gst_gl_mixer_set_caps_features (const GstCaps * caps, const gchar * feature_name) { - GstCaps *tmp = gst_caps_copy (caps); - guint n = gst_caps_get_size (tmp); - guint i = 0; - - for (i = 0; i < n; i++) { - GstCapsFeatures *features = gst_caps_get_features (tmp, i); - if (features) { - guint n_f = gst_caps_features_get_size (features); - guint j = 0; - for (j = 0; j < n_f; j++) { - gst_caps_features_remove_id (features, - gst_caps_features_get_nth_id (features, j)); - } - } - - gst_caps_features_add (features, feature_name); - gst_caps_set_simple (tmp, "format", G_TYPE_STRING, "RGBA", NULL); - } - - return tmp; + GstCaps *ret = gst_gl_caps_replace_all_caps_features (caps, feature_name); + gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL); + return ret; } /* copies the given caps */ @@ -367,30 +365,25 @@ gst_gl_mixer_caps_remove_format_info (GstCaps * caps) GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) { - GstCaps *result = NULL; - GstCaps *glcaps = gst_gl_mixer_set_caps_features (caps, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY); -#if GST_GL_HAVE_PLATFORM_EGL - GstCaps *eglcaps = gst_gl_mixer_set_caps_features (caps, - GST_CAPS_FEATURE_MEMORY_EGL_IMAGE); -#endif - GstCaps *uploadcaps = gst_gl_mixer_set_caps_features (caps, - GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META); - GstCaps *raw_caps = - gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)); + GstCaps *result, *tmp, *gl_caps; - result = gst_caps_new_empty (); + gl_caps = gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA"); - result = gst_caps_merge (result, glcaps); -#if GST_GL_HAVE_PLATFORM_EGL - result = gst_caps_merge (result, eglcaps); -#endif - result = gst_caps_merge (result, uploadcaps); - result = gst_caps_merge (result, raw_caps); + result = + gst_gl_color_convert_transform_caps (mix->context, GST_PAD_SRC, gl_caps, + NULL); + tmp = result; + GST_DEBUG_OBJECT (mix, "convert returned caps %" GST_PTR_FORMAT, tmp); - result = gst_caps_merge (result, gst_gl_mixer_caps_remove_format_info (caps)); + result = gst_gl_mixer_caps_remove_format_info (tmp); + gst_caps_unref (tmp); + tmp = result; + GST_DEBUG_OBJECT (mix, "remove format returned caps %" GST_PTR_FORMAT, tmp); - GST_DEBUG_OBJECT (mix, "returning %" GST_PTR_FORMAT, result); + result = gst_gl_upload_transform_caps (mix->context, GST_PAD_SRC, tmp, NULL); + gst_caps_unref (tmp); + tmp = result; + GST_DEBUG_OBJECT (mix, "transfer returned caps %" GST_PTR_FORMAT, tmp); return result; } @@ -398,7 +391,7 @@ gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) static GstCaps * gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) { - GstCaps *srccaps; + GstCaps *sinkcaps; GstCaps *template_caps; GstCaps *filtered_caps; GstCaps *returned_caps; @@ -406,17 +399,18 @@ gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) template_caps = gst_pad_get_pad_template_caps (pad); - srccaps = gst_pad_get_current_caps (pad); - if (srccaps == NULL) { + sinkcaps = gst_pad_get_current_caps (pad); + if (sinkcaps == NULL) { had_current_caps = FALSE; - srccaps = template_caps; + sinkcaps = template_caps; } else { - srccaps = gst_caps_merge (srccaps, gst_gl_mixer_update_caps (mix, srccaps)); + sinkcaps = + gst_caps_merge (sinkcaps, gst_gl_mixer_update_caps (mix, sinkcaps)); } - filtered_caps = srccaps; + filtered_caps = sinkcaps; if (filter) - filtered_caps = gst_caps_intersect (srccaps, filter); + filtered_caps = gst_caps_intersect (sinkcaps, filter); returned_caps = gst_caps_intersect (filtered_caps, template_caps); if (filter) @@ -436,6 +430,7 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, gboolean ret = FALSE; GstGLMixer *mix = GST_GL_MIXER (agg); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); + GstGLMixerPad *pad = GST_GL_MIXER_PAD (bpad); GST_TRACE ("QUERY %" GST_PTR_FORMAT, query); @@ -481,7 +476,7 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, decide_query); /* pass the query to the propose_allocation vmethod if any */ - ret = gst_gl_mixer_propose_allocation (mix, decide_query, query); + ret = gst_gl_mixer_propose_allocation (mix, pad, decide_query, query); if (decide_query) gst_query_unref (decide_query); @@ -721,7 +716,8 @@ gst_gl_mixer_src_activate_mode (GstAggregator * aggregator, GstPadMode mode, static gboolean gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) { - GstCaps *filter, *current_caps, *retcaps; + GstGLMixer *mix = GST_GL_MIXER (agg); + GstCaps *filter, *current_caps, *retcaps, *gl_caps; gst_query_parse_caps (query, &filter); @@ -729,12 +725,30 @@ gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) if (current_caps == NULL) current_caps = gst_pad_get_pad_template_caps (agg->srcpad); + /* convert from current caps to GLMemory caps */ + gl_caps = + gst_caps_merge (gst_caps_merge (gst_gl_mixer_set_caps_features + (current_caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY), + gst_gl_mixer_set_caps_features (current_caps, + GST_CAPS_FEATURE_MEMORY_EGL_IMAGE)), + gst_gl_mixer_set_caps_features (current_caps, + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META)); + retcaps = + gst_gl_download_transform_caps (mix->context, GST_PAD_SINK, current_caps, + NULL); + retcaps = gst_caps_merge (gl_caps, retcaps); + gst_caps_unref (current_caps); + current_caps = retcaps; + retcaps = gst_gl_mixer_caps_remove_format_info (current_caps); gst_caps_unref (current_caps); - if (filter) - retcaps = + if (filter) { + current_caps = gst_caps_intersect_full (filter, retcaps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (retcaps); + retcaps = current_caps; + } gst_query_set_caps_result (query, retcaps); gst_caps_unref (retcaps); @@ -1054,8 +1068,6 @@ _default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (frame->pad); GstGLMixerPad *pad = frame->pad; GstBuffer *uploaded_buf, *gl_buffer; - GstCaps *gl_caps; - GstCapsFeatures *gl_features; GstVideoInfo gl_info; GstVideoFrame gl_frame; GstGLSyncMeta *sync_meta; @@ -1064,34 +1076,8 @@ _default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, GST_VIDEO_FORMAT_RGBA, GST_VIDEO_INFO_WIDTH (&vaggpad->info), GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); - gl_features = - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); - gl_caps = gst_video_info_to_caps (&gl_info); - gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features)); - - if (!pad->upload) { - GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad)); - GstCaps *upload_caps = gst_caps_copy (in_caps); - - pad->upload = gst_gl_upload_new (mix->context); - - gst_caps_set_features (upload_caps, 0, - gst_caps_features_copy (gl_features)); - gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps); - - if (!pad->convert) { - pad->convert = gst_gl_color_convert_new (mix->context); - - gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps); - } - - gst_caps_unref (upload_caps); - gst_caps_unref (in_caps); - } - - gst_caps_features_free (gl_features); - gst_caps_unref (gl_caps); + _init_upload (mix, pad); sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); if (sync_meta) diff --git a/ext/gl/gstgltestsrc.c b/ext/gl/gstgltestsrc.c index 6b10a2cdab..dcbbd1e9f3 100644 --- a/ext/gl/gstgltestsrc.c +++ b/ext/gl/gstgltestsrc.c @@ -93,6 +93,7 @@ static void gst_gl_test_src_get_property (GObject * object, guint prop_id, static void gst_gl_test_src_dispose (GObject * object); static gboolean gst_gl_test_src_setcaps (GstBaseSrc * bsrc, GstCaps * caps); +static GstCaps *gst_gl_test_src_getcaps (GstBaseSrc * bsrc, GstCaps * filter); static GstCaps *gst_gl_test_src_fixate (GstBaseSrc * bsrc, GstCaps * caps); static gboolean gst_gl_test_src_is_seekable (GstBaseSrc * psrc); @@ -189,6 +190,7 @@ gst_gl_test_src_class_init (GstGLTestSrcClass * klass) element_class->set_context = gst_gl_test_src_set_context; gstbasesrc_class->set_caps = gst_gl_test_src_setcaps; + gstbasesrc_class->get_caps = gst_gl_test_src_getcaps; gstbasesrc_class->is_seekable = gst_gl_test_src_is_seekable; gstbasesrc_class->do_seek = gst_gl_test_src_do_seek; gstbasesrc_class->query = gst_gl_test_src_query; @@ -470,6 +472,91 @@ wrong_caps: } } +/* copies the given caps */ +static GstCaps * +gst_gl_test_src_caps_remove_format_info (GstCaps * caps) +{ + GstStructure *st; + GstCapsFeatures *f; + gint i, n; + GstCaps *res; + + res = gst_caps_new_empty (); + + n = gst_caps_get_size (caps); + for (i = 0; i < n; i++) { + st = gst_caps_get_structure (caps, i); + f = gst_caps_get_features (caps, i); + + /* If this is already expressed by the existing caps + * skip this structure */ + if (i > 0 && gst_caps_is_subset_structure_full (res, st, f)) + continue; + + st = gst_structure_copy (st); + /* Only remove format info for the cases when we can actually convert */ + if (!gst_caps_features_is_any (f) + && gst_caps_features_is_equal (f, + GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) + gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", + NULL); + gst_structure_remove_fields (st, "width", "height", NULL); + + gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); + } + + return res; +} + +static GstCaps * +gst_gl_test_src_set_caps_features (const GstCaps * caps, + const gchar * feature_name) +{ + GstCaps *ret = gst_gl_caps_replace_all_caps_features (caps, feature_name); + gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL); + return ret; +} + +static GstCaps * +gst_gl_test_src_getcaps (GstBaseSrc * bsrc, GstCaps * filter) +{ + GstGLTestSrc *src = GST_GL_TEST_SRC (bsrc); + GstCaps *tmp = NULL; + GstCaps *result = NULL; + GstCaps *gl_caps; + GstCaps *caps = + gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA"); + + tmp = gst_gl_test_src_caps_remove_format_info (caps); + GST_DEBUG_OBJECT (bsrc, "remove format returned caps %" GST_PTR_FORMAT, tmp); + + gl_caps = + gst_caps_merge (gst_caps_merge (gst_gl_test_src_set_caps_features (tmp, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY), + gst_gl_test_src_set_caps_features (tmp, + GST_CAPS_FEATURE_MEMORY_EGL_IMAGE)), + gst_gl_test_src_set_caps_features (tmp, + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META)); + result = + gst_gl_download_transform_caps (src->context, GST_PAD_SINK, tmp, NULL); + result = gst_caps_merge (gl_caps, result); + + gst_caps_unref (tmp); + tmp = result; + GST_DEBUG_OBJECT (bsrc, "transfer returned caps %" GST_PTR_FORMAT, tmp); + + if (filter) { + result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + } else { + result = tmp; + } + + GST_DEBUG_OBJECT (bsrc, "returning caps: %" GST_PTR_FORMAT, result); + + return result; +} + static void gst_gl_test_src_set_context (GstElement * element, GstContext * context) { diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c index 53e8c9d89e..2594e2e9a6 100644 --- a/gst-libs/gst/gl/gstglcolorconvert.c +++ b/gst-libs/gst/gl/gstglcolorconvert.c @@ -631,6 +631,29 @@ gst_gl_color_convert_set_caps (GstGLColorConvert * convert, return ret; } +GstCaps * +gst_gl_color_convert_transform_caps (GstGLContext * convert, + GstPadDirection direction, GstCaps * caps, GstCaps * filter) +{ + GstCaps *templ, *result, *tmp; + + templ = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, GST_GL_COLOR_CONVERT_FORMATS)); + + tmp = gst_caps_intersect_full (caps, templ, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (templ); + + if (filter) { + result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + } else { + result = tmp; + } + + return result; +} + /** * gst_gl_color_convert_perform: * @convert: a #GstGLColorConvert diff --git a/gst-libs/gst/gl/gstglcolorconvert.h b/gst-libs/gst/gl/gstglcolorconvert.h index a6a5232e10..748ac90ff7 100644 --- a/gst-libs/gst/gl/gstglcolorconvert.h +++ b/gst-libs/gst/gl/gstglcolorconvert.h @@ -99,6 +99,10 @@ struct _GstGLColorConvertClass GstGLColorConvert * gst_gl_color_convert_new (GstGLContext * context); +GstCaps * gst_gl_color_convert_transform_caps (GstGLContext * convert, + GstPadDirection direction, + GstCaps * caps, + GstCaps * filter); gboolean gst_gl_color_convert_set_caps (GstGLColorConvert * convert, GstCaps * in_caps, GstCaps * out_caps); diff --git a/gst-libs/gst/gl/gstgldownload.c b/gst-libs/gst/gl/gstgldownload.c index a5d5b624ce..b5ec02b9bf 100644 --- a/gst-libs/gst/gl/gstgldownload.c +++ b/gst-libs/gst/gl/gstgldownload.c @@ -180,6 +180,77 @@ gst_gl_download_set_format (GstGLDownload * download, GstVideoInfo * out_info) GST_OBJECT_UNLOCK (download); } +static GstCaps * +_set_caps_features (const GstCaps * caps, const gchar * feature_name) +{ + GstCaps *tmp = gst_caps_copy (caps); + guint n = gst_caps_get_size (tmp); + guint i = 0; + + for (i = 0; i < n; i++) { + GstCapsFeatures *features = gst_caps_get_features (tmp, i); + if (features) { + guint n_f = gst_caps_features_get_size (features); + guint j = 0; + for (j = 0; j < n_f; j++) { + gst_caps_features_remove_id (features, + gst_caps_features_get_nth_id (features, j)); + } + } + + gst_caps_features_add (features, feature_name); + } + + return tmp; +} + +GstCaps * +gst_gl_download_transform_caps (GstGLContext * context, + GstPadDirection direction, GstCaps * caps, GstCaps * filter) +{ + GstCaps *gl_templ, *templ, *result, *tmp; + + templ = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)); + gl_templ = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, GST_GL_COLOR_CONVERT_FORMATS)); + + if (direction == GST_PAD_SRC) { + tmp = gst_caps_intersect_full (caps, templ, GST_CAPS_INTERSECT_FIRST); + result = _set_caps_features (tmp, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + gst_caps_unref (tmp); + tmp = result; + } else { + tmp = gst_caps_ref (caps); + } + + result = + gst_gl_color_convert_transform_caps (context, direction, tmp, filter); + gst_caps_unref (tmp); + tmp = result; + + if (direction == GST_PAD_SINK) { + result = _set_caps_features (tmp, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); + gst_caps_unref (tmp); + tmp = result; + result = gst_caps_intersect_full (tmp, templ, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + tmp = result; + } + + if (filter) { + result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + } else { + result = tmp; + } + gst_caps_unref (templ); + gst_caps_unref (gl_templ); + + return result; +} + /** * gst_gl_download_perform_with_data: * @download: a #GstGLDownload diff --git a/gst-libs/gst/gl/gstgldownload.h b/gst-libs/gst/gl/gstgldownload.h index 828c01dc0e..2eeccf476e 100644 --- a/gst-libs/gst/gl/gstgldownload.h +++ b/gst-libs/gst/gl/gstgldownload.h @@ -73,6 +73,10 @@ struct _GstGLDownloadClass GstGLDownload * gst_gl_download_new (GstGLContext * context); void gst_gl_download_set_format (GstGLDownload * download, GstVideoInfo * out_info); +GstCaps * gst_gl_download_transform_caps (GstGLContext * convert, + GstPadDirection direction, + GstCaps * caps, + GstCaps * filter); gboolean gst_gl_download_perform_with_data (GstGLDownload * download, GLuint texture_id, GLuint texture_target, diff --git a/gst-libs/gst/gl/gstglfilter.c b/gst-libs/gst/gl/gstglfilter.c index d77dde9a20..4069657fbe 100644 --- a/gst-libs/gst/gl/gstglfilter.c +++ b/gst-libs/gst/gl/gstglfilter.c @@ -705,31 +705,13 @@ done: return othercaps; } - static GstCaps * gst_gl_filter_set_caps_features (const GstCaps * caps, const gchar * feature_name) { - GstCaps *tmp = gst_caps_copy (caps); - guint n = gst_caps_get_size (tmp); - guint i = 0; - - for (i = 0; i < n; i++) { - GstCapsFeatures *features = gst_caps_get_features (tmp, i); - if (features) { - guint n_f = gst_caps_features_get_size (features); - guint j = 0; - for (j = 0; j < n_f; j++) { - gst_caps_features_remove_id (features, - gst_caps_features_get_nth_id (features, j)); - } - } - - gst_caps_features_add (features, feature_name); - gst_caps_set_simple (tmp, "format", G_TYPE_STRING, "RGBA", NULL); - } - - return tmp; + GstCaps *ret = gst_gl_caps_replace_all_caps_features (caps, feature_name); + gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL); + return ret; } /* copies the given caps */ @@ -770,35 +752,79 @@ gst_gl_filter_caps_remove_format_info (GstCaps * caps) static GstCaps * gst_gl_filter_transform_caps (GstBaseTransform * bt, - GstPadDirection direction, GstCaps * caps, GstCaps * filter) + GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps) { + GstGLFilter *filter = GST_GL_FILTER (bt); GstCaps *tmp = NULL; GstCaps *result = NULL; - GstCaps *glcaps = gst_gl_filter_set_caps_features (caps, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY); -#if GST_GL_HAVE_PLATFORM_EGL - GstCaps *eglcaps = gst_gl_filter_set_caps_features (caps, - GST_CAPS_FEATURE_MEMORY_EGL_IMAGE); -#endif - GstCaps *uploadcaps = gst_gl_filter_set_caps_features (caps, - GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META); - GstCaps *raw_caps = - gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)); - tmp = gst_caps_new_empty (); + /* The following is the list of caps transformations performed. When + * direction == GST_PAD_SINK we start at the sinkpad and work toward the src + * pad and vice versa for direction == GST_PAD_SRC. + * + * sinkpad <-> (upload <-> convert) <-> filter (possible resize) <-> + * (convert <-> download/output) <-> srcpad + */ + if (direction == GST_PAD_SINK) { + result = + gst_gl_upload_transform_caps (filter->context, direction, caps, NULL); + tmp = result; - tmp = gst_caps_merge (tmp, gst_gl_filter_caps_remove_format_info (glcaps)); -#if GST_GL_HAVE_PLATFORM_EGL - tmp = gst_caps_merge (tmp, gst_gl_filter_caps_remove_format_info (eglcaps)); -#endif - tmp = - gst_caps_merge (tmp, gst_gl_filter_caps_remove_format_info (uploadcaps)); - tmp = gst_caps_merge (tmp, gst_gl_filter_caps_remove_format_info (raw_caps)); + result = + gst_gl_color_convert_transform_caps (filter->context, direction, tmp, + NULL); + gst_caps_unref (tmp); + } else { + GstCaps *gl_caps = + gst_caps_merge (gst_caps_merge (gst_gl_filter_set_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY), + gst_gl_filter_set_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_EGL_IMAGE)), + gst_gl_filter_set_caps_features (caps, + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META)); - tmp = gst_caps_merge (tmp, gst_gl_filter_caps_remove_format_info (caps)); + result = + gst_gl_download_transform_caps (filter->context, direction, caps, NULL); - if (filter) { - result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + result = gst_caps_merge (gl_caps, result); + } + tmp = result; + GST_DEBUG_OBJECT (bt, "transfer returned caps %" GST_PTR_FORMAT, tmp); + + result = gst_gl_filter_caps_remove_format_info (tmp); + gst_caps_unref (tmp); + tmp = result; + GST_DEBUG_OBJECT (bt, "remove format returned caps %" GST_PTR_FORMAT, tmp); + + if (direction == GST_PAD_SRC) { + result = + gst_gl_color_convert_transform_caps (filter->context, direction, tmp, + NULL); + gst_caps_unref (tmp); + tmp = result; + result = + gst_gl_upload_transform_caps (filter->context, direction, tmp, NULL); + } else { + GstCaps *gl_caps = + gst_caps_merge (gst_caps_merge (gst_gl_filter_set_caps_features (tmp, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY), + gst_gl_filter_set_caps_features (tmp, + GST_CAPS_FEATURE_MEMORY_EGL_IMAGE)), + gst_gl_filter_set_caps_features (tmp, + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META)); + + result = + gst_gl_download_transform_caps (filter->context, direction, tmp, NULL); + + result = gst_caps_merge (gl_caps, result); + } + gst_caps_unref (tmp); + tmp = result; + GST_DEBUG_OBJECT (bt, "transfer returned caps %" GST_PTR_FORMAT, tmp); + + if (filter_caps) { + result = + gst_caps_intersect_full (filter_caps, tmp, GST_CAPS_INTERSECT_FIRST); gst_caps_unref (tmp); } else { result = tmp; @@ -868,18 +894,11 @@ gst_gl_filter_propose_allocation (GstBaseTransform * trans, GstQuery * decide_query, GstQuery * query) { GstGLFilter *filter = GST_GL_FILTER (trans); - GstGLFilterClass *filter_class = GST_GL_FILTER_GET_CLASS (filter); GstBufferPool *pool; GstStructure *config; GstCaps *caps, *decide_caps; guint size; gboolean need_pool; - GError *error = NULL; - GstStructure *gl_context; - gchar *platform, *gl_apis; - gpointer handle; - GstAllocator *allocator = NULL; - GstAllocationParams params; gst_query_parse_allocation (query, &caps, &need_pool); @@ -906,19 +925,6 @@ gst_gl_filter_propose_allocation (GstBaseTransform * trans, gst_structure_free (config); } - if (!gst_gl_ensure_element_data (filter, &filter->display, - &filter->other_context)) - return FALSE; - - gst_gl_display_filter_gl_api (filter->display, - filter_class->supported_gl_api); - - if (!filter->context) { - filter->context = gst_gl_context_new (filter->display); - if (!gst_gl_context_create (filter->context, filter->other_context, &error)) - goto context_error; - } - if (pool == NULL && need_pool) { GstVideoInfo info; GstBufferPool *decide_pool = NULL; @@ -959,43 +965,11 @@ gst_gl_filter_propose_allocation (GstBaseTransform * trans, gst_object_unref (pool); } - /* we also support various metadata */ - gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); + gst_gl_upload_propose_allocation (filter->upload, decide_query, query); + if (filter->context->gl_vtable->FenceSync) gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); - gl_apis = gst_gl_api_to_string (gst_gl_context_get_gl_api (filter->context)); - platform = - gst_gl_platform_to_string (gst_gl_context_get_gl_platform - (filter->context)); - handle = (gpointer) gst_gl_context_get_gl_context (filter->context); - - gl_context = - gst_structure_new ("GstVideoGLTextureUploadMeta", "gst.gl.GstGLContext", - GST_GL_TYPE_CONTEXT, filter->context, "gst.gl.context.handle", - G_TYPE_POINTER, handle, "gst.gl.context.type", G_TYPE_STRING, platform, - "gst.gl.context.apis", G_TYPE_STRING, gl_apis, NULL); - gst_query_add_allocation_meta (query, - GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, gl_context); - - g_free (gl_apis); - g_free (platform); - gst_structure_free (gl_context); - - gst_allocation_params_init (¶ms); - - allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR); - gst_query_add_allocation_param (query, allocator, ¶ms); - gst_object_unref (allocator); - -#if GST_GL_HAVE_PLATFORM_EGL - if (gst_gl_context_check_feature (filter->context, "EGL_KHR_image_base")) { - allocator = gst_allocator_find (GST_EGL_IMAGE_MEMORY_TYPE); - gst_query_add_allocation_param (query, allocator, ¶ms); - gst_object_unref (allocator); - } -#endif - return TRUE; /* ERRORS */ @@ -1014,12 +988,6 @@ config_failed: GST_DEBUG_OBJECT (trans, "failed setting config"); return FALSE; } -context_error: - { - GST_ELEMENT_ERROR (trans, RESOURCE, NOT_FOUND, ("%s", error->message), - (NULL)); - return FALSE; - } } static gboolean @@ -1037,10 +1005,19 @@ gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query) guint in_width, in_height, out_width, out_height; GstGLContext *other_context = NULL; gboolean same_downstream_gl_context = FALSE; + GstCapsFeatures *uploaded_features; + GstCaps *uploaded_caps; + GstCapsFeatures *converted_features; + GstVideoInfo converted_info; + + gst_query_parse_allocation (query, &caps, NULL); + if (!caps) + return FALSE; if (!gst_gl_ensure_element_data (filter, &filter->display, - &filter->other_context)) + &filter->other_context)) { return FALSE; + } gst_gl_display_filter_gl_api (filter->display, filter_class->supported_gl_api); @@ -1105,6 +1082,41 @@ gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query) out_width = GST_VIDEO_INFO_WIDTH (&filter->out_info); out_height = GST_VIDEO_INFO_HEIGHT (&filter->out_info); + if (!filter->upload) { + filter->upload = gst_gl_upload_new (filter->context); + } + + uploaded_caps = gst_caps_copy (caps); + uploaded_features = + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + gst_caps_set_features (uploaded_caps, 0, uploaded_features); + + if (!gst_gl_upload_set_caps (filter->upload, caps, uploaded_caps)) { + gst_caps_unref (uploaded_caps); + return FALSE; + } + + if (!filter->in_convert) { + filter->in_convert = gst_gl_color_convert_new (filter->context); + } + + gst_video_info_set_format (&converted_info, GST_VIDEO_FORMAT_RGBA, + in_width, in_height); + converted_features = + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + + if (filter->in_converted_caps) + gst_caps_unref (filter->in_converted_caps); + filter->in_converted_caps = gst_video_info_to_caps (&converted_info); + gst_caps_set_features (filter->in_converted_caps, 0, converted_features); + + if (!gst_gl_color_convert_set_caps (filter->in_convert, uploaded_caps, + filter->in_converted_caps)) { + gst_caps_unref (uploaded_caps); + return FALSE; + } + gst_caps_unref (uploaded_caps); + if (filter->fbo) { gst_gl_context_del_fbo (filter->context, filter->fbo, filter->depthbuffer); filter->fbo = 0; @@ -1140,8 +1152,6 @@ gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query) goto error; } - gst_query_parse_allocation (query, &caps, NULL); - if (gst_query_get_n_allocation_pools (query) > 0) { gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); @@ -1178,11 +1188,6 @@ gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query) gst_buffer_pool_set_config (pool, config); - if (filter->upload) { - gst_object_unref (filter->upload); - filter->upload = NULL; - } - if (update_pool) gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); else @@ -1328,58 +1333,6 @@ gst_gl_filter_transform (GstBaseTransform * bt, GstBuffer * inbuf, if (!filter->display) return GST_FLOW_NOT_NEGOTIATED; - if (!filter->upload) { - GstCaps *in_caps = - gst_pad_get_current_caps (GST_BASE_TRANSFORM_SINK_PAD (bt)); - GstCaps *uploaded_caps; - GstCapsFeatures *out_features; - GstVideoInfo out_info; - - gst_video_info_set_format (&out_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&filter->in_info), - GST_VIDEO_INFO_HEIGHT (&filter->in_info)); - out_features = - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); - - uploaded_caps = gst_caps_copy (in_caps); - gst_caps_set_features (uploaded_caps, 0, out_features); - - filter->upload = gst_gl_upload_new (filter->context); - if (!gst_gl_upload_set_caps (filter->upload, in_caps, uploaded_caps)) { - gst_caps_unref (in_caps); - return GST_FLOW_ERROR; - } - - gst_caps_unref (in_caps); - - if (!filter->in_convert) { - GstCapsFeatures *converted_features; - GstVideoInfo converted_info; - - gst_video_info_set_format (&converted_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&filter->in_info), - GST_VIDEO_INFO_HEIGHT (&filter->in_info)); - converted_features = - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); - - if (filter->in_converted_caps) - gst_caps_unref (filter->in_converted_caps); - filter->in_converted_caps = gst_video_info_to_caps (&converted_info); - gst_caps_set_features (filter->in_converted_caps, 0, converted_features); - - filter->in_convert = gst_gl_color_convert_new (filter->context); - if (!gst_gl_color_convert_set_caps (filter->in_convert, uploaded_caps, - filter->in_converted_caps)) { - gst_caps_unref (uploaded_caps); - return GST_FLOW_ERROR; - } - } - - gst_caps_unref (uploaded_caps); - } - g_assert (filter_class->filter || filter_class->filter_texture); in_sync_meta = gst_buffer_get_gl_sync_meta (inbuf); diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c index 23142d554b..356ad47631 100644 --- a/gst-libs/gst/gl/gstglupload.c +++ b/gst-libs/gst/gl/gstglupload.c @@ -63,11 +63,39 @@ struct _GstGLUploadPrivate GstBuffer *outbuf; + /* all method impl pointers */ + gpointer *upload_impl; + + /* current method */ const UploadMethod *method; gpointer method_impl; int method_i; }; +static GstCaps * +_set_caps_features (const GstCaps * caps, const gchar * feature_name) +{ + GstCaps *tmp = gst_caps_copy (caps); + guint n = gst_caps_get_size (tmp); + guint i = 0; + + for (i = 0; i < n; i++) { + GstCapsFeatures *features = gst_caps_get_features (tmp, i); + if (features) { + guint n_f = gst_caps_features_get_size (features); + guint j = 0; + for (j = 0; j < n_f; j++) { + gst_caps_features_remove_id (features, + gst_caps_features_get_nth_id (features, j)); + } + } + + gst_caps_features_add (features, feature_name); + } + + return tmp; +} + typedef enum { METHOD_FLAG_CAN_SHARE_CONTEXT = 1, @@ -79,8 +107,12 @@ struct _UploadMethod GstGLUploadMethodFlags flags; gpointer (*new) (GstGLUpload * upload); + GstCaps *(*transform_caps) (GstGLContext * context, + GstPadDirection direction, GstCaps * caps); gboolean (*accept) (gpointer impl, GstBuffer * buffer, GstCaps * in_caps, GstCaps * out_caps); + void (*propose_allocation) (gpointer impl, GstQuery * decide_query, + GstQuery * query); GstGLUploadReturn (*perform) (gpointer impl, GstBuffer * buffer, GstBuffer ** outbuf); void (*release) (gpointer impl, GstBuffer * buffer); @@ -102,6 +134,13 @@ _gl_memory_upload_new (GstGLUpload * upload) return mem; } +static GstCaps * +_gl_memory_upload_transform_caps (GstGLContext * context, + GstPadDirection direction, GstCaps * caps) +{ + return _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); +} + static gboolean _gl_memory_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps, GstCaps * out_caps) @@ -137,6 +176,20 @@ _gl_memory_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps, return TRUE; } +static void +_gl_memory_upload_propose_allocation (gpointer impl, GstQuery * decide_query, + GstQuery * query) +{ + GstAllocationParams params; + GstAllocator *allocator; + + gst_allocation_params_init (¶ms); + + allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR); + gst_query_add_allocation_param (query, allocator, ¶ms); + gst_object_unref (allocator); +} + static GstGLUploadReturn _gl_memory_upload_perform (gpointer impl, GstBuffer * buffer, GstBuffer ** outbuf) @@ -173,7 +226,9 @@ static const UploadMethod _gl_memory_upload = { "GLMemory", METHOD_FLAG_CAN_SHARE_CONTEXT, &_gl_memory_upload_new, + &_gl_memory_upload_transform_caps, &_gl_memory_upload_accept, + &_gl_memory_upload_propose_allocation, &_gl_memory_upload_perform, &_gl_memory_upload_release, &_gl_memory_upload_free @@ -195,6 +250,21 @@ _egl_image_upload_new (GstGLUpload * upload) return image; } +static GstCaps * +_egl_image_upload_transform_caps (GstGLContext * context, + GstPadDirection direction, GstCaps * caps) +{ + GstCaps *ret; + + if (direction == GST_PAD_SINK) { + ret = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + } else { + ret = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_EGL_IMAGE); + } + + return ret; +} + static gboolean _egl_image_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps, GstCaps * out_caps) @@ -237,6 +307,24 @@ _egl_image_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps, return TRUE; } +static void +_egl_image_upload_propose_allocation (gpointer impl, GstQuery * decide_query, + GstQuery * query) +{ + struct EGLImageUpload *image = impl; + GstAllocationParams params; + GstAllocator *allocator; + + gst_allocation_params_init (¶ms); + + if (gst_gl_context_check_feature (image->upload->context, + "EGL_KHR_image_base")) { + allocator = gst_allocator_find (GST_EGL_IMAGE_MEMORY_TYPE); + gst_query_add_allocation_param (query, allocator, ¶ms); + gst_object_unref (allocator); + } +} + static GstGLUploadReturn _egl_image_upload_perform (gpointer impl, GstBuffer * buffer, GstBuffer ** outbuf) @@ -285,7 +373,9 @@ static const UploadMethod _egl_image_upload = { "EGLImage", 0, &_egl_image_upload_new, + &_egl_image_upload_transform_caps, &_egl_image_upload_accept, + &_egl_image_upload_propose_allocation, &_egl_image_upload_perform, &_egl_image_upload_release, &_egl_image_upload_free @@ -311,6 +401,23 @@ _upload_meta_upload_new (GstGLUpload * upload) return meta; } +static GstCaps * +_upload_meta_upload_transform_caps (GstGLContext * context, + GstPadDirection direction, GstCaps * caps) +{ + GstCaps *ret; + + if (direction == GST_PAD_SINK) { + ret = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + } else { + ret = + _set_caps_features (caps, + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META); + } + + return ret; +} + static gboolean _upload_meta_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps, GstCaps * out_caps) @@ -358,6 +465,36 @@ _upload_meta_upload_accept (gpointer impl, GstBuffer * buffer, return TRUE; } +static void +_upload_meta_upload_propose_allocation (gpointer impl, GstQuery * decide_query, + GstQuery * query) +{ + struct GLUploadMeta *upload = impl; + GstStructure *gl_context; + gchar *platform, *gl_apis; + gpointer handle; + + gl_apis = + gst_gl_api_to_string (gst_gl_context_get_gl_api (upload->upload-> + context)); + platform = + gst_gl_platform_to_string (gst_gl_context_get_gl_platform (upload-> + upload->context)); + handle = (gpointer) gst_gl_context_get_gl_context (upload->upload->context); + + gl_context = + gst_structure_new ("GstVideoGLTextureUploadMeta", "gst.gl.GstGLContext", + GST_GL_TYPE_CONTEXT, upload->upload->context, "gst.gl.context.handle", + G_TYPE_POINTER, handle, "gst.gl.context.type", G_TYPE_STRING, platform, + "gst.gl.context.apis", G_TYPE_STRING, gl_apis, NULL); + gst_query_add_allocation_meta (query, + GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, gl_context); + + g_free (gl_apis); + g_free (platform); + gst_structure_free (gl_context); +} + /* * Uploads using gst_video_gl_texture_upload_meta_upload(). * i.e. consumer of GstVideoGLTextureUploadMeta @@ -437,7 +574,9 @@ static const UploadMethod _upload_meta_upload = { "UploadMeta", METHOD_FLAG_CAN_SHARE_CONTEXT, &_upload_meta_upload_new, + &_upload_meta_upload_transform_caps, &_upload_meta_upload_accept, + &_upload_meta_upload_propose_allocation, &_upload_meta_upload_perform, &_upload_meta_upload_release, &_upload_meta_upload_free @@ -460,6 +599,21 @@ _raw_data_upload_new (GstGLUpload * upload) return raw; } +static GstCaps * +_raw_data_upload_transform_caps (GstGLContext * context, + GstPadDirection direction, GstCaps * caps) +{ + GstCaps *ret; + + if (direction == GST_PAD_SINK) { + ret = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + } else { + ret = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); + } + + return ret; +} + static gboolean _raw_data_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps, GstCaps * out_caps) @@ -488,6 +642,13 @@ _raw_data_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps, return TRUE; } +static void +_raw_data_upload_propose_allocation (gpointer impl, GstQuery * decide_query, + GstQuery * query) +{ + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); +} + static GstGLUploadReturn _raw_data_upload_perform (gpointer impl, GstBuffer * buffer, GstBuffer ** outbuf) @@ -540,7 +701,9 @@ static const UploadMethod _raw_data_upload = { "Raw Data", 0, &_raw_data_upload_new, + &_raw_data_upload_transform_caps, &_raw_data_upload_accept, + &_raw_data_upload_propose_allocation, &_raw_data_upload_perform, &_raw_data_upload_release, &_raw_data_upload_free @@ -588,9 +751,16 @@ GstGLUpload * gst_gl_upload_new (GstGLContext * context) { GstGLUpload *upload = g_object_new (GST_TYPE_GL_UPLOAD, NULL); + gint i, n; upload->context = gst_object_ref (context); + n = G_N_ELEMENTS (upload_methods); + upload->priv->upload_impl = g_malloc (sizeof (gpointer) * n); + for (i = 0; i < n; i++) { + upload->priv->upload_impl[i] = upload_methods[i]->new (upload); + } + return upload; } @@ -598,6 +768,7 @@ static void gst_gl_upload_finalize (GObject * object) { GstGLUpload *upload; + gint i, n; upload = GST_GL_UPLOAD (object); @@ -627,9 +798,62 @@ gst_gl_upload_finalize (GObject * object) upload->priv->out_caps = NULL; } + n = G_N_ELEMENTS (upload_methods); + for (i = 0; i < n; i++) { + if (upload->priv->upload_impl[i]) + upload_methods[i]->free (upload->priv->upload_impl[i]); + } + g_free (upload->priv->upload_impl); + G_OBJECT_CLASS (gst_gl_upload_parent_class)->finalize (object); } +GstCaps * +gst_gl_upload_transform_caps (GstGLContext * context, GstPadDirection direction, + GstCaps * caps, GstCaps * filter) +{ + GstCaps *result, *tmp; + gint i; + + tmp = gst_caps_new_empty (); + + for (i = 0; i < G_N_ELEMENTS (upload_methods); i++) { + GstCaps *tmp2 = + upload_methods[i]->transform_caps (context, direction, caps); + + if (tmp2) + tmp = gst_caps_merge (tmp, tmp2); + } + + if (filter) { + result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + } else { + result = tmp; + } + + return result; +} + +/** + * gst_gl_upload_propose_allocation: + * @upload: a #GstGLUpload + * @decide_query: (allow-none): a #GstQuery from a decide allocation + * @query: the proposed allocation query + * + * Adds the required allocation parameters to support uploading. + */ +void +gst_gl_upload_propose_allocation (GstGLUpload * upload, GstQuery * decide_query, + GstQuery * query) +{ + gint i; + + for (i = 0; i < G_N_ELEMENTS (upload_methods); i++) + upload_methods[i]->propose_allocation (upload->priv->upload_impl[i], + decide_query, query); +} + static gboolean _gst_gl_upload_set_caps_unlocked (GstGLUpload * upload, GstCaps * in_caps, GstCaps * out_caps) diff --git a/gst-libs/gst/gl/gstglupload.h b/gst-libs/gst/gl/gstglupload.h index 39666fce60..865289143d 100644 --- a/gst-libs/gst/gl/gstglupload.h +++ b/gst-libs/gst/gl/gstglupload.h @@ -78,13 +78,26 @@ struct _GstGLUploadClass GstObjectClass object_class; }; -GstGLUpload * gst_gl_upload_new (GstGLContext * context); +GstGLUpload * gst_gl_upload_new (GstGLContext * context); -gboolean gst_gl_upload_set_caps (GstGLUpload * upload, GstCaps * in_caps, GstCaps * out_caps); -void gst_gl_upload_get_caps (GstGLUpload * upload, GstCaps ** in_caps, GstCaps ** out_caps); +GstCaps * gst_gl_upload_transform_caps (GstGLContext * context, + GstPadDirection direction, + GstCaps * caps, + GstCaps * filter); +gboolean gst_gl_upload_set_caps (GstGLUpload * upload, + GstCaps * in_caps, + GstCaps * out_caps); +void gst_gl_upload_get_caps (GstGLUpload * upload, + GstCaps ** in_caps, + GstCaps ** out_caps); +void gst_gl_upload_propose_allocation (GstGLUpload * upload, + GstQuery * decide_query, + GstQuery * query); -GstGLUploadReturn gst_gl_upload_perform_with_buffer (GstGLUpload * upload, GstBuffer * buffer, GstBuffer ** outbuf); -void gst_gl_upload_release_buffer (GstGLUpload * upload); +GstGLUploadReturn gst_gl_upload_perform_with_buffer (GstGLUpload * upload, + GstBuffer * buffer, + GstBuffer ** outbuf); +void gst_gl_upload_release_buffer (GstGLUpload * upload); G_END_DECLS diff --git a/gst-libs/gst/gl/gstglutils.c b/gst-libs/gst/gl/gstglutils.c index b9e60ec2d3..3b9a829a35 100644 --- a/gst-libs/gst/gl/gstglutils.c +++ b/gst-libs/gst/gl/gstglutils.c @@ -872,3 +872,28 @@ gst_gl_get_plane_data_size (GstVideoInfo * info, GstVideoAlignment * align, return plane_size; } + +GstCaps * +gst_gl_caps_replace_all_caps_features (const GstCaps * caps, + const gchar * feature_name) +{ + GstCaps *tmp = gst_caps_copy (caps); + guint n = gst_caps_get_size (tmp); + guint i = 0; + + for (i = 0; i < n; i++) { + GstCapsFeatures *features = gst_caps_get_features (tmp, i); + if (features) { + guint n_f = gst_caps_features_get_size (features); + guint j = 0; + for (j = 0; j < n_f; j++) { + gst_caps_features_remove_id (features, + gst_caps_features_get_nth_id (features, j)); + } + } + + gst_caps_features_add (features, feature_name); + } + + return tmp; +} diff --git a/gst-libs/gst/gl/gstglutils.h b/gst-libs/gst/gl/gstglutils.h index b851612ee9..867887e5ca 100644 --- a/gst-libs/gst/gl/gstglutils.h +++ b/gst-libs/gst/gl/gstglutils.h @@ -101,6 +101,8 @@ gboolean gst_gl_handle_context_query (GstElement * element, GstQuery * query, gsize gst_gl_get_plane_data_size (GstVideoInfo * info, GstVideoAlignment * align, guint plane); +GstCaps * gst_gl_caps_replace_all_caps_features (const GstCaps * caps, + const gchar * feature_name); G_END_DECLS